feat: migrate semantic search to pgvector + full-text search
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s
Replace JSON-string embeddings with native pgvector(1536) storage and add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion for hybrid keyword + semantic ranking. Changes: - NoteEmbedding.embedding: String → vector(1536) via pgvector - NoteEmbedding: added updatedAt for reindex tracking - Note: added tsv (tsvector) with auto-update trigger for FTS - semantic-search.service: hybrid FTS + vector search with RRF fusion - embedding.service: toVectorString() for pgvector SQL literals - Removed JS-side cosine similarity loops (now DB-side via <=>) - Added HNSW index on NoteEmbedding.embedding (cosine distance) - Added GIN index on Note.tsv for FTS queries Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/ Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,16 +1,16 @@
|
||||
/**
|
||||
* Note Search Tool
|
||||
* Wraps semanticSearchService.searchAsUser()
|
||||
* Uses the unified SemanticSearchService (FTS + pgvector + RRF).
|
||||
*/
|
||||
|
||||
import { tool } from 'ai'
|
||||
import { z } from 'zod'
|
||||
import { toolRegistry } from './registry'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { semanticSearchService } from '@/lib/ai/services/semantic-search.service'
|
||||
|
||||
toolRegistry.register({
|
||||
name: 'note_search',
|
||||
description: 'Search the user\'s notes using semantic search. Returns matching notes with titles and content excerpts.',
|
||||
description: 'Search the user\'s notes using hybrid semantic + keyword search. Returns matching notes with titles and content excerpts.',
|
||||
isInternal: true,
|
||||
buildTool: (ctx) =>
|
||||
tool({
|
||||
@@ -21,34 +21,20 @@ toolRegistry.register({
|
||||
notebookId: z.string().optional().describe('Optional notebook ID to restrict search to a specific notebook'),
|
||||
}),
|
||||
execute: async ({ query, limit = 5, notebookId: explicitNotebookId }) => {
|
||||
// If no notebookId passed explicitly, fall back to the chat scope from context
|
||||
const notebookId = explicitNotebookId || ctx.notebookId
|
||||
try {
|
||||
// Keyword fallback search using Prisma
|
||||
const keywords = query.toLowerCase().split(/\s+/).filter(w => w.length > 2)
|
||||
const conditions = keywords.flatMap(term => [
|
||||
{ title: { contains: term } },
|
||||
{ content: { contains: term } }
|
||||
])
|
||||
|
||||
const notes = await prisma.note.findMany({
|
||||
where: {
|
||||
userId: ctx.userId,
|
||||
...(notebookId ? { notebookId } : {}),
|
||||
...(conditions.length > 0 ? { OR: conditions } : {}),
|
||||
isArchived: false,
|
||||
trashedAt: null,
|
||||
},
|
||||
select: { id: true, title: true, content: true, createdAt: true },
|
||||
take: limit,
|
||||
orderBy: { createdAt: 'desc' },
|
||||
const results = await semanticSearchService.searchAsUser(ctx.userId, query, {
|
||||
limit,
|
||||
threshold: 0.25,
|
||||
notebookId
|
||||
})
|
||||
|
||||
return notes.map(n => ({
|
||||
id: n.id,
|
||||
title: n.title || 'Untitled',
|
||||
excerpt: n.content.substring(0, 300),
|
||||
createdAt: n.createdAt.toISOString(),
|
||||
return results.map(r => ({
|
||||
id: r.noteId,
|
||||
title: r.title || 'Untitled',
|
||||
excerpt: r.content.substring(0, 300),
|
||||
score: r.score,
|
||||
matchType: r.matchType,
|
||||
}))
|
||||
} catch (e: any) {
|
||||
return { error: `Note search failed: ${e.message}` }
|
||||
|
||||
Reference in New Issue
Block a user