feat: migrate semantic search to pgvector + full-text search
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s
Replace JSON-string embeddings with native pgvector(1536) storage and add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion for hybrid keyword + semantic ranking. Changes: - NoteEmbedding.embedding: String → vector(1536) via pgvector - NoteEmbedding: added updatedAt for reindex tracking - Note: added tsv (tsvector) with auto-update trigger for FTS - semantic-search.service: hybrid FTS + vector search with RRF fusion - embedding.service: toVectorString() for pgvector SQL literals - Removed JS-side cosine similarity loops (now DB-side via <=>) - Added HNSW index on NoteEmbedding.embedding (cosine distance) - Added GIN index on Note.tsv for FTS queries Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/ Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
-- Phase 1: Enable pgvector extension
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
|
||||
-- Phase 2: Add native vector column to NoteEmbedding
|
||||
-- Convert existing JSON-string embeddings to native vector(1536)
|
||||
ALTER TABLE "NoteEmbedding" ADD COLUMN "vec" vector(1536);
|
||||
|
||||
-- Migrate existing data: parse JSON arrays into pgvector format
|
||||
UPDATE "NoteEmbedding"
|
||||
SET "vec" = ("embedding"::jsonb)::text::vector(1536)
|
||||
WHERE "embedding" IS NOT NULL;
|
||||
|
||||
-- Drop old string column, rename new one
|
||||
ALTER TABLE "NoteEmbedding" DROP COLUMN "embedding";
|
||||
ALTER TABLE "NoteEmbedding" RENAME COLUMN "vec" TO "embedding";
|
||||
|
||||
-- Add updatedAt column for tracking reindex freshness
|
||||
ALTER TABLE "NoteEmbedding" ADD COLUMN "updatedAt" TIMESTAMP NOT NULL DEFAULT now();
|
||||
|
||||
-- HNSW index for fast approximate nearest neighbor search (cosine distance)
|
||||
CREATE INDEX "NoteEmbedding_embedding_hnsw_idx" ON "NoteEmbedding"
|
||||
USING hnsw ("embedding" vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 64);
|
||||
|
||||
-- Phase 3: Add full-text search tsvector column to Note
|
||||
ALTER TABLE "Note" ADD COLUMN "tsv" tsvector;
|
||||
|
||||
-- Populate tsv from existing title + content
|
||||
UPDATE "Note"
|
||||
SET "tsv" =
|
||||
setweight(to_tsvector('simple', COALESCE("title", '')), 'A') ||
|
||||
setweight(to_tsvector('simple', COALESCE("content", '')), 'B');
|
||||
|
||||
-- GIN index for fast FTS queries
|
||||
CREATE INDEX "Note_tsv_gin_idx" ON "Note" USING gin ("tsv");
|
||||
|
||||
-- Trigger function to auto-update tsv on INSERT or UPDATE of title/content
|
||||
CREATE OR REPLACE FUNCTION "note_tsv_trigger"() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW."tsv" :=
|
||||
setweight(to_tsvector('simple', COALESCE(NEW."title", '')), 'A') ||
|
||||
setweight(to_tsvector('simple', COALESCE(NEW."content", '')), 'B');
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Attach trigger
|
||||
DROP TRIGGER IF EXISTS "note_tsv_update" ON "Note";
|
||||
CREATE TRIGGER "note_tsv_update"
|
||||
BEFORE INSERT OR UPDATE OF "title", "content" ON "Note"
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION "note_tsv_trigger"();
|
||||
Reference in New Issue
Block a user