feat: migrate semantic search to pgvector + full-text search

Replace JSON-string embeddings with native pgvector(1536) storage and
add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion
for hybrid keyword + semantic ranking.

Changes:
- NoteEmbedding.embedding: String → vector(1536) via pgvector
- NoteEmbedding: added updatedAt for reindex tracking
- Note: added tsv (tsvector) with auto-update trigger for FTS
- semantic-search.service: hybrid FTS + vector search with RRF fusion
- embedding.service: toVectorString() for pgvector SQL literals
- Removed JS-side cosine similarity loops (now DB-side via <=>)
- Added HNSW index on NoteEmbedding.embedding (cosine distance)
- Added GIN index on Note.tsv for FTS queries

Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

memento-note/scripts/migrate-embeddings.ts

@@ -1,59 +1,67 @@
 // scripts/migrate-embeddings.ts
-const { PrismaClient } = require('../prisma/client-generated')
+// Re-indexes all notes that lack a NoteEmbedding row using pgvector format.
+// Run with: npx tsx scripts/migrate-embeddings.ts
+
+const { PrismaClient } = require('../node_modules/.prisma/client')
 
 const prisma = new PrismaClient({
   datasources: {
     db: {
-      url: process.env.DATABASE_URL || "file:../prisma/dev.db"
+      url: process.env.DATABASE_URL
     }
   }
 })
 
 async function main() {
-  console.log("Fetching notes with embeddings...")
+  console.log('Fetching notes without embeddings...')
   const notes = await prisma.note.findMany({
     where: {
-      embedding: { not: null }
+      trashedAt: null,
+      noteEmbedding: { is: null }
     },
     select: {
       id: true,
-      embedding: true
+      content: true,
+      title: true
     }
   })
 
-  console.log(`Found ${notes.length} notes with an embedding.`)
-  
+  console.log(`Found ${notes.length} notes without an embedding.`)
+
   if (notes.length === 0) {
-    console.log("Nothing to migrate.")
+    console.log('Nothing to migrate.')
     return
   }
 
   let count = 0
+  let failed = 0
   for (const note of notes) {
-    if (!note.embedding) continue
-
-    await prisma.noteEmbedding.upsert({
-      where: { noteId: note.id },
-      create: {
-        noteId: note.id,
-        embedding: note.embedding
-      },
-      update: {
-        embedding: note.embedding
+    if (!note.content) continue
+    try {
+      // Embedding will be generated by the indexNote method which handles pgvector format
+      await prisma.$executeRawUnsafe(
+        `INSERT INTO "NoteEmbedding" ("id", "noteId", "embedding", "createdAt", "updatedAt")
+         VALUES (gen_random_uuid(), $1, '[0]'::vector(1536), now(), now())
+         ON CONFLICT ("noteId") DO NOTHING`,
+        note.id
+      )
+      count++
+      if (count % 10 === 0) {
+        console.log(`Placeholder for ${count}/${notes.length}...`)
       }
-    })
-    count++
-    if (count % 10 === 0) {
-      console.log(`Migrated ${count}/${notes.length}...`)
+    } catch (e) {
+      failed++
+      console.error(`Failed for note ${note.id}:`, e.message)
     }
   }
 
-  console.log(`✅ Successfully migrated ${count} note embeddings to the NoteEmbedding table.`)
+  console.log(`Created ${count} embedding placeholders (${failed} failed).`)
+  console.log('Run /api/notes/reindex to populate with real embeddings.')
 }
 
 main()
   .catch((e) => {
-    console.error("Migration failed:", e)
+    console.error('Migration failed:', e)
     process.exit(1)
   })
   .finally(async () => {

memento-note/scripts/test-backend-logic.ts

@@ -1,63 +1,40 @@
 
 import { prisma } from '../lib/prisma'
 
-// Copy of parseNote from app/actions/notes.ts (since it's not exported)
 function parseNote(dbNote: any) {
-    const embedding = dbNote.embedding ? JSON.parse(dbNote.embedding) : null
-
-    if (embedding && Array.isArray(embedding)) {
-        // Simplified validation check for test
-        if (embedding.length !== 1536 && embedding.length !== 768 && embedding.length !== 384) {
-            return {
-                ...dbNote,
-                checkItems: dbNote.checkItems ? JSON.parse(dbNote.checkItems) : null,
-                labels: dbNote.labels ? JSON.parse(dbNote.labels) : null,
-                images: dbNote.images ? JSON.parse(dbNote.images) : null,
-                links: dbNote.links ? JSON.parse(dbNote.links) : null,
-                embedding: null,
-                sharedWith: dbNote.sharedWith ? JSON.parse(dbNote.sharedWith) : [],
-                size: dbNote.size || 'small',
-            }
-        }
-    }
-
     return {
         ...dbNote,
         checkItems: dbNote.checkItems ? JSON.parse(dbNote.checkItems) : null,
         labels: dbNote.labels ? JSON.parse(dbNote.labels) : null,
         images: dbNote.images ? JSON.parse(dbNote.images) : null,
         links: dbNote.links ? JSON.parse(dbNote.links) : null,
-        embedding,
         sharedWith: dbNote.sharedWith ? JSON.parse(dbNote.sharedWith) : [],
         size: dbNote.size || 'small',
     }
 }
 
 async function main() {
-    console.log('🧪 Testing parseNote logic...')
+    console.log('Testing parseNote logic...')
 
-    // 1. Fetch a real note from DB that is KNOWN to be large
     const rawNote = await prisma.note.findFirst({
         where: { size: 'large' }
     })
 
     if (!rawNote) {
-        console.error('❌ No large note found in DB. Create one first.')
+        console.error('No large note found in DB.')
         return
     }
 
-    console.log('📊 Raw Note from DB:', { id: rawNote.id, size: rawNote.size })
+    console.log('Raw Note from DB:', { id: rawNote.id, size: rawNote.size })
 
-    // 2. Pass it through parseNote
     const parsed = parseNote(rawNote)
-    console.log('🔄 Parsed Note:', { id: parsed.id, size: parsed.size })
+    console.log('Parsed Note:', { id: parsed.id, size: parsed.size })
 
     if (parsed.size === 'large') {
-        console.log('✅ parseNote preserves size correctly.')
+        console.log('parseNote preserves size correctly.')
     } else {
-        console.error('❌ parseNote returned wrong size:', parsed.size)
+        console.error('parseNote returned wrong size:', parsed.size)
     }
-
 }
 
 main().catch(console.error).finally(() => prisma.$disconnect())