feat: migrate semantic search to pgvector + full-text search
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s

Replace JSON-string embeddings with native pgvector(1536) storage and
add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion
for hybrid keyword + semantic ranking.

Changes:
- NoteEmbedding.embedding: String → vector(1536) via pgvector
- NoteEmbedding: added updatedAt for reindex tracking
- Note: added tsv (tsvector) with auto-update trigger for FTS
- semantic-search.service: hybrid FTS + vector search with RRF fusion
- embedding.service: toVectorString() for pgvector SQL literals
- Removed JS-side cosine similarity loops (now DB-side via <=>)
- Added HNSW index on NoteEmbedding.embedding (cosine distance)
- Added GIN index on Note.tsv for FTS queries

Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Antigravity
2026-05-12 07:03:56 +00:00
parent 92c3a6f307
commit 03e6a62b80
43 changed files with 4024 additions and 786 deletions

View File

@@ -0,0 +1,52 @@
-- Phase 1: Enable pgvector extension
CREATE EXTENSION IF NOT EXISTS vector;
-- Phase 2: Add native vector column to NoteEmbedding
-- Convert existing JSON-string embeddings to native vector(1536)
ALTER TABLE "NoteEmbedding" ADD COLUMN "vec" vector(1536);
-- Migrate existing data: parse JSON arrays into pgvector format
UPDATE "NoteEmbedding"
SET "vec" = ("embedding"::jsonb)::text::vector(1536)
WHERE "embedding" IS NOT NULL;
-- Drop old string column, rename new one
ALTER TABLE "NoteEmbedding" DROP COLUMN "embedding";
ALTER TABLE "NoteEmbedding" RENAME COLUMN "vec" TO "embedding";
-- Add updatedAt column for tracking reindex freshness
ALTER TABLE "NoteEmbedding" ADD COLUMN "updatedAt" TIMESTAMP NOT NULL DEFAULT now();
-- HNSW index for fast approximate nearest neighbor search (cosine distance)
CREATE INDEX "NoteEmbedding_embedding_hnsw_idx" ON "NoteEmbedding"
USING hnsw ("embedding" vector_cosine_ops)
WITH (m = 16, ef_construction = 64);
-- Phase 3: Add full-text search tsvector column to Note
ALTER TABLE "Note" ADD COLUMN "tsv" tsvector;
-- Populate tsv from existing title + content
UPDATE "Note"
SET "tsv" =
setweight(to_tsvector('simple', COALESCE("title", '')), 'A') ||
setweight(to_tsvector('simple', COALESCE("content", '')), 'B');
-- GIN index for fast FTS queries
CREATE INDEX "Note_tsv_gin_idx" ON "Note" USING gin ("tsv");
-- Trigger function to auto-update tsv on INSERT or UPDATE of title/content
CREATE OR REPLACE FUNCTION "note_tsv_trigger"() RETURNS trigger AS $$
BEGIN
NEW."tsv" :=
setweight(to_tsvector('simple', COALESCE(NEW."title", '')), 'A') ||
setweight(to_tsvector('simple', COALESCE(NEW."content", '')), 'B');
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Attach trigger
DROP TRIGGER IF EXISTS "note_tsv_update" ON "Note";
CREATE TRIGGER "note_tsv_update"
BEFORE INSERT OR UPDATE OF "title", "content" ON "Note"
FOR EACH ROW
EXECUTE FUNCTION "note_tsv_trigger"();

View File

@@ -155,6 +155,7 @@ model Note {
languageConfidence Float?
lastAiAnalysis DateTime?
trashedAt DateTime?
tsv Unsupported("tsvector")?
aiFeedback AiFeedback[]
memoryEchoAsNote1 MemoryEchoInsight[] @relation("EchoNote1")
memoryEchoAsNote2 MemoryEchoInsight[] @relation("EchoNote2")
@@ -299,8 +300,9 @@ model UserAISettings {
model NoteEmbedding {
id String @id @default(cuid())
noteId String @unique
embedding String
embedding Unsupported("vector(1536)")
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
note Note @relation(fields: [noteId], references: [id], onDelete: Cascade)
@@index([noteId])