feat: migrate semantic search to pgvector + full-text search
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s
Replace JSON-string embeddings with native pgvector(1536) storage and add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion for hybrid keyword + semantic ranking. Changes: - NoteEmbedding.embedding: String → vector(1536) via pgvector - NoteEmbedding: added updatedAt for reindex tracking - Note: added tsv (tsvector) with auto-update trigger for FTS - semantic-search.service: hybrid FTS + vector search with RRF fusion - embedding.service: toVectorString() for pgvector SQL literals - Removed JS-side cosine similarity loops (now DB-side via <=>) - Added HNSW index on NoteEmbedding.embedding (cosine distance) - Added GIN index on Note.tsv for FTS queries Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/ Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { getAIProvider, getChatProvider } from '../factory'
|
||||
import { cosineSimilarity } from '@/lib/utils'
|
||||
import { embeddingService } from './embedding.service'
|
||||
import { getSystemConfig } from '@/lib/config'
|
||||
import prisma from '@/lib/prisma'
|
||||
|
||||
@@ -78,11 +79,15 @@ export class MemoryEchoService {
|
||||
try {
|
||||
const embedding = await provider.getEmbeddings(note.content)
|
||||
if (embedding && embedding.length > 0) {
|
||||
await prisma.noteEmbedding.upsert({
|
||||
where: { noteId: note.id },
|
||||
create: { noteId: note.id, embedding: JSON.stringify(embedding) },
|
||||
update: { embedding: JSON.stringify(embedding) }
|
||||
})
|
||||
const vecStr = `[${embedding.join(',')}]`
|
||||
await prisma.$executeRawUnsafe(
|
||||
`INSERT INTO "NoteEmbedding" ("id", "noteId", "embedding", "createdAt", "updatedAt")
|
||||
VALUES (gen_random_uuid(), $1, $2::vector, now(), now())
|
||||
ON CONFLICT ("noteId")
|
||||
DO UPDATE SET "embedding" = $2::vector, "updatedAt" = now()`,
|
||||
note.id,
|
||||
vecStr
|
||||
)
|
||||
}
|
||||
} catch {
|
||||
// Skip this note, continue with others
|
||||
@@ -122,11 +127,12 @@ export class MemoryEchoService {
|
||||
return [] // Need at least 2 notes to find connections
|
||||
}
|
||||
|
||||
// Parse embeddings (already native Json from PostgreSQL)
|
||||
const notesWithEmbeddings = notes
|
||||
.map(note => ({
|
||||
...note,
|
||||
embedding: note.noteEmbedding?.embedding ? JSON.parse(note.noteEmbedding.embedding) as number[] : null
|
||||
embedding: note.noteEmbedding?.embedding
|
||||
? embeddingService.fromVectorString(note.noteEmbedding.embedding as unknown as string)
|
||||
: null
|
||||
}))
|
||||
.filter(note => note.embedding && Array.isArray(note.embedding))
|
||||
|
||||
@@ -500,8 +506,9 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
|
||||
return []
|
||||
}
|
||||
|
||||
// Target note embedding (already native Json from PostgreSQL)
|
||||
const targetEmbedding = targetNote.noteEmbedding?.embedding ? JSON.parse(targetNote.noteEmbedding.embedding) as number[] : null
|
||||
const targetEmbedding = targetNote.noteEmbedding?.embedding
|
||||
? embeddingService.fromVectorString(targetNote.noteEmbedding.embedding as unknown as string)
|
||||
: null
|
||||
if (!targetEmbedding) return []
|
||||
|
||||
// Check if user has demo mode enabled
|
||||
@@ -535,7 +542,9 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
|
||||
for (const otherNote of otherNotes) {
|
||||
if (!otherNote.noteEmbedding) continue
|
||||
|
||||
const otherEmbedding = otherNote.noteEmbedding?.embedding ? JSON.parse(otherNote.noteEmbedding.embedding) as number[] : null
|
||||
const otherEmbedding = otherNote.noteEmbedding?.embedding
|
||||
? embeddingService.fromVectorString(otherNote.noteEmbedding.embedding as unknown as string)
|
||||
: null
|
||||
if (!otherEmbedding) continue
|
||||
|
||||
// Check if this connection was dismissed
|
||||
|
||||
Reference in New Issue
Block a user