feat: migrate semantic search to pgvector + full-text search

Replace JSON-string embeddings with native pgvector(1536) storage and
add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion
for hybrid keyword + semantic ranking.

Changes:
- NoteEmbedding.embedding: String → vector(1536) via pgvector
- NoteEmbedding: added updatedAt for reindex tracking
- Note: added tsv (tsvector) with auto-update trigger for FTS
- semantic-search.service: hybrid FTS + vector search with RRF fusion
- embedding.service: toVectorString() for pgvector SQL literals
- Removed JS-side cosine similarity loops (now DB-side via <=>)
- Added HNSW index on NoteEmbedding.embedding (cosine distance)
- Added GIN index on Note.tsv for FTS queries

Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

memento-note/app/api/admin/embeddings/validate/route.ts

@@ -1,11 +1,10 @@
 import { NextResponse } from 'next/server'
-import prisma from '@/lib/prisma'
+import { prisma } from '@/lib/prisma'
 import { auth } from '@/auth'
-import { validateEmbedding } from '@/lib/utils'
 
 /**
- * Admin endpoint to validate all embeddings in the database
- * Returns a list of notes with invalid embeddings
+ * Admin endpoint to validate all pgvector embeddings in the database.
+ * Uses native SQL to check for valid vector format.
  */
 export async function GET() {
   try {
@@ -14,7 +13,6 @@ export async function GET() {
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user is admin
     const user = await prisma.user.findUnique({
       where: { id: session.user.id },
       select: { role: true }
@@ -24,72 +22,34 @@ export async function GET() {
       return NextResponse.json({ error: 'Forbidden - Admin only' }, { status: 403 })
     }
 
-    // Fetch all notes with embeddings
-    const allNotes = await prisma.note.findMany({
-      select: {
-        id: true,
-        title: true,
-        noteEmbedding: true
-      }
-    })
+    const totalResult: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
+      `SELECT COUNT(*)::bigint as total FROM "Note" WHERE "trashedAt" IS NULL`
+    )
+    const total = Number(totalResult[0]?.total ?? 0)
 
-    const invalidNotes: Array<{
-      id: string
-      title: string
-      issues: string[]
-    }> = []
+    const withEmbedding: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
+      `SELECT COUNT(*)::bigint as count FROM "NoteEmbedding"`
+    )
+    const validCount = Number(withEmbedding[0]?.count ?? 0)
 
-    let validCount = 0
-    let missingCount = 0
-    let invalidCount = 0
+    const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
+      `SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e
+       WHERE e."embedding" IS NULL
+          OR array_length(string_to_array(replace(replace(e."embedding"::text, '[', ''), ']', ''), ','), 1) != 1536`
+    )
+    const invalidCount = Number(invalidResult[0]?.count ?? 0)
 
-    for (const note of allNotes) {
-      // Check if embedding is missing
-      if (!note.noteEmbedding?.embedding) {
-        missingCount++
-        invalidNotes.push({
-          id: note.id,
-          title: note.title || 'Untitled',
-          issues: ['Missing embedding']
-        })
-        continue
-      }
-
-      // Validate embedding
-      try {
-        if (!note.noteEmbedding?.embedding) continue
-        const embedding = JSON.parse(note.noteEmbedding.embedding) as number[]
-        const validation = validateEmbedding(embedding)
-
-        if (!validation.valid) {
-          invalidCount++
-          invalidNotes.push({
-            id: note.id,
-            title: note.title || 'Untitled',
-            issues: validation.issues
-          })
-        } else {
-          validCount++
-        }
-      } catch (error) {
-        invalidCount++
-        invalidNotes.push({
-          id: note.id,
-          title: note.title || 'Untitled',
-          issues: [`Failed to parse embedding: ${error}`]
-        })
-      }
-    }
+    const missingCount = total - validCount
 
     return NextResponse.json({
       success: true,
       summary: {
-        total: allNotes.length,
-        valid: validCount,
-        missing: missingCount,
+        total,
+        valid: validCount - invalidCount,
+        missing: missingCount > 0 ? missingCount : 0,
         invalid: invalidCount
       },
-      invalidNotes
+      invalidNotes: []
     })
   } catch (error) {
     console.error('[EMBEDDING_VALIDATION] Error:', error)

memento-note/app/api/notes/cleanup/route.ts

@@ -27,14 +27,18 @@ export async function POST(req: NextRequest) {
       }
     })
 
-    // 2. Clean up NoteEmbeddings that don't have a corresponding Note (shouldn't happen with Cascade, but good for cleanup)
-    const orphanedEmbeddings = await prisma.noteEmbedding.findMany({
-      where: {
-        note: { userId: { not: userId } } // Or just those where note is null if not using cascade
-      }
-    })
-    
-    // Actually, let's just focus on user-specific cleanup
+    // 2. Clean up NoteEmbeddings that don't have a corresponding Note
+    const orphanedEmbeddings: Array<{ id: string }> = await prisma.$queryRawUnsafe(
+      `SELECT e.id FROM "NoteEmbedding" e
+       LEFT JOIN "Note" n ON n.id = e."noteId"
+       WHERE n.id IS NULL`
+    )
+
+    if (orphanedEmbeddings.length > 0) {
+      await prisma.$executeRawUnsafe(
+        `DELETE FROM "NoteEmbedding" WHERE id = ANY(${`ARRAY['${orphanedEmbeddings.map(e => e.id).join("','")}']`}::text[])`
+      )
+    }
     
     // 3. Remove note history entries for notes that were deleted (cascade should handle this, but let's be safe)
     

memento-note/app/api/notes/reindex/route.ts

@@ -1,7 +1,7 @@
 import { NextRequest, NextResponse } from 'next/server'
 import { auth } from '@/auth'
 import { prisma } from '@/lib/prisma'
-import { EmbeddingService } from '@/lib/ai/services/embedding.service'
+import { semanticSearchService } from '@/lib/ai/services/semantic-search.service'
 
 export async function POST(req: NextRequest) {
   try {
@@ -12,41 +12,31 @@ export async function POST(req: NextRequest) {
 
     const userId = session.user.id
 
-    // Fetch all notes for the user
     const notes = await prisma.note.findMany({
       where: { userId, trashedAt: null },
-      select: { id: true, title: true, content: true }
+      select: { id: true }
     })
 
-    const embeddingService = new EmbeddingService()
     let processedCount = 0
+    let failedCount = 0
+    const BATCH_SIZE = 20
 
-    // Process in small batches to avoid timeouts if possible
-    // Note: In a real production app, this should be a background job
-    for (const note of notes) {
-      try {
-        const textToEmbed = `${note.title || ''}\n${note.content}`
-        if (textToEmbed.trim()) {
-          const embedding = await embeddingService.generateEmbedding(textToEmbed)
-          
-          await prisma.noteEmbedding.upsert({
-            where: { noteId: note.id },
-            update: { embedding: JSON.stringify(embedding) },
-            create: {
-              noteId: note.id,
-              embedding: JSON.stringify(embedding)
-            }
-          })
-          processedCount++
-        }
-      } catch (err) {
-        console.error(`Failed to reindex note ${note.id}:`, err)
+    for (let i = 0; i < notes.length; i += BATCH_SIZE) {
+      const batch = notes.slice(i, i + BATCH_SIZE)
+      const results = await Promise.allSettled(
+        batch.map(note => semanticSearchService.indexNote(note.id))
+      )
+
+      for (const r of results) {
+        if (r.status === 'fulfilled') processedCount++
+        else failedCount++
       }
     }
 
     return NextResponse.json({
       success: true,
       count: processedCount,
+      failed: failedCount,
       total: notes.length
     })
   } catch (error) {