feat: migrate semantic search to pgvector + full-text search
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s

Replace JSON-string embeddings with native pgvector(1536) storage and
add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion
for hybrid keyword + semantic ranking.

Changes:
- NoteEmbedding.embedding: String → vector(1536) via pgvector
- NoteEmbedding: added updatedAt for reindex tracking
- Note: added tsv (tsvector) with auto-update trigger for FTS
- semantic-search.service: hybrid FTS + vector search with RRF fusion
- embedding.service: toVectorString() for pgvector SQL literals
- Removed JS-side cosine similarity loops (now DB-side via <=>)
- Added HNSW index on NoteEmbedding.embedding (cosine distance)
- Added GIN index on Note.tsv for FTS queries

Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Antigravity
2026-05-12 07:03:56 +00:00
parent 92c3a6f307
commit 03e6a62b80
43 changed files with 4024 additions and 786 deletions

View File

@@ -1,5 +1,6 @@
import { getAIProvider, getChatProvider } from '../factory'
import { cosineSimilarity } from '@/lib/utils'
import { embeddingService } from './embedding.service'
import { getSystemConfig } from '@/lib/config'
import prisma from '@/lib/prisma'
@@ -78,11 +79,15 @@ export class MemoryEchoService {
try {
const embedding = await provider.getEmbeddings(note.content)
if (embedding && embedding.length > 0) {
await prisma.noteEmbedding.upsert({
where: { noteId: note.id },
create: { noteId: note.id, embedding: JSON.stringify(embedding) },
update: { embedding: JSON.stringify(embedding) }
})
const vecStr = `[${embedding.join(',')}]`
await prisma.$executeRawUnsafe(
`INSERT INTO "NoteEmbedding" ("id", "noteId", "embedding", "createdAt", "updatedAt")
VALUES (gen_random_uuid(), $1, $2::vector, now(), now())
ON CONFLICT ("noteId")
DO UPDATE SET "embedding" = $2::vector, "updatedAt" = now()`,
note.id,
vecStr
)
}
} catch {
// Skip this note, continue with others
@@ -122,11 +127,12 @@ export class MemoryEchoService {
return [] // Need at least 2 notes to find connections
}
// Parse embeddings (already native Json from PostgreSQL)
const notesWithEmbeddings = notes
.map(note => ({
...note,
embedding: note.noteEmbedding?.embedding ? JSON.parse(note.noteEmbedding.embedding) as number[] : null
embedding: note.noteEmbedding?.embedding
? embeddingService.fromVectorString(note.noteEmbedding.embedding as unknown as string)
: null
}))
.filter(note => note.embedding && Array.isArray(note.embedding))
@@ -500,8 +506,9 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
return []
}
// Target note embedding (already native Json from PostgreSQL)
const targetEmbedding = targetNote.noteEmbedding?.embedding ? JSON.parse(targetNote.noteEmbedding.embedding) as number[] : null
const targetEmbedding = targetNote.noteEmbedding?.embedding
? embeddingService.fromVectorString(targetNote.noteEmbedding.embedding as unknown as string)
: null
if (!targetEmbedding) return []
// Check if user has demo mode enabled
@@ -535,7 +542,9 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
for (const otherNote of otherNotes) {
if (!otherNote.noteEmbedding) continue
const otherEmbedding = otherNote.noteEmbedding?.embedding ? JSON.parse(otherNote.noteEmbedding.embedding) as number[] : null
const otherEmbedding = otherNote.noteEmbedding?.embedding
? embeddingService.fromVectorString(otherNote.noteEmbedding.embedding as unknown as string)
: null
if (!otherEmbedding) continue
// Check if this connection was dismissed