feat: migrate semantic search to pgvector + full-text search

Replace JSON-string embeddings with native pgvector(1536) storage and
add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion
for hybrid keyword + semantic ranking.

Changes:
- NoteEmbedding.embedding: String → vector(1536) via pgvector
- NoteEmbedding: added updatedAt for reindex tracking
- Note: added tsv (tsvector) with auto-update trigger for FTS
- semantic-search.service: hybrid FTS + vector search with RRF fusion
- embedding.service: toVectorString() for pgvector SQL literals
- Removed JS-side cosine similarity loops (now DB-side via <=>)
- Added HNSW index on NoteEmbedding.embedding (cosine distance)
- Added GIN index on Note.tsv for FTS queries

Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

memento-note/app/actions/notes.ts

@@ -5,9 +5,10 @@ import prisma from '@/lib/prisma'
 import { Note, CheckItem, NoteType } from '@/lib/types'
 import { auth } from '@/auth'
 import { getAIProvider } from '@/lib/ai/factory'
-import { parseNote as parseNoteUtil, cosineSimilarity, calculateRRFK, detectQueryType, getSearchWeights } from '@/lib/utils'
+import { parseNote as parseNoteUtil } from '@/lib/utils'
 import { getSystemConfig, getConfigNumber, getConfigBoolean, SEARCH_DEFAULTS } from '@/lib/config'
 import { contextualAutoTagService } from '@/lib/ai/services/contextual-auto-tag.service'
+import { semanticSearchService } from '@/lib/ai/services/semantic-search.service'
 import { cleanupNoteImages, parseImageUrls, deleteImageFileSafely } from '@/lib/image-cleanup'
 import { getAISettings } from '@/app/actions/ai-settings'
 import {
@@ -486,122 +487,54 @@ export async function enableNoteHistory(noteId: string) {
   })
 }
 
-// Search notes - DB-side filtering (fast) with optional semantic search
-// Supports contextual search within notebook (IA5)
-export async function searchNotes(query: string, useSemantic: boolean = false, notebookId?: string) {
+// Unified hybrid search — always uses FTS + pgvector with RRF fusion.
+// Supports contextual search within notebook (IA5).
+export async function searchNotes(query: string, _useSemantic: boolean = true, notebookId?: string) {
   const session = await auth();
   if (!session?.user?.id) return [];
 
   try {
-    // If query empty, return all notes
     if (!query || !query.trim()) {
       return await getAllNotes();
     }
 
-    // If semantic search is requested, use the full implementation
-    if (useSemantic) {
-      return await semanticSearch(query, session.user.id, notebookId);
-    }
+    const results = await semanticSearchService.searchAsUser(session.user.id, query, {
+      limit: 50,
+      threshold: 0.25,
+      notebookId
+    });
 
-    // DB-side keyword search using LIKE — much faster than loading all notes in memory
+    const noteIds = results.map(r => r.noteId);
     const notes = await prisma.note.findMany({
       where: {
+        id: { in: noteIds },
         userId: session.user.id,
         isArchived: false,
         trashedAt: null,
-        OR: [
-          { title: { contains: query } },
-          { content: { contains: query } },
-          { labels: { contains: query } },
-        ],
       },
       select: NOTE_LIST_SELECT,
-      orderBy: [
-        { isPinned: 'desc' },
-        { order: 'asc' },
-        { updatedAt: 'desc' }
-      ]
     });
 
-    return notes.map(parseNote);
+    const orderMap = new Map(results.map((r, i) => [r.noteId, i]));
+    const parsed = notes.map(parseNote);
+
+    parsed.sort((a, b) => (orderMap.get(a.id) ?? 999) - (orderMap.get(b.id) ?? 999));
+
+    if (parsed.length > 0) {
+      const topResult = results[0];
+      if (topResult) {
+        parsed[0].matchType = topResult.matchType;
+        parsed[0].searchScore = topResult.score;
+      }
+    }
+
+    return parsed;
   } catch (error) {
     console.error('Search error:', error);
     return [];
   }
 }
 
-// Semantic search with AI embeddings - SIMPLE VERSION
-// Supports contextual search within notebook (IA5)
-async function semanticSearch(query: string, userId: string, notebookId?: string) {
-  const allNotes = await prisma.note.findMany({
-    where: {
-      userId: userId,
-      isArchived: false,
-      trashedAt: null,
-      ...(notebookId !== undefined ? { notebookId } : {})
-    },
-    include: { noteEmbedding: true }
-  });
-
-  const queryLower = query.toLowerCase().trim();
-
-  // Get query embedding
-  let queryEmbedding: number[] | null = null;
-  try {
-    const provider = getAIProvider(await getSystemConfig());
-    queryEmbedding = await provider.getEmbeddings(query);
-  } catch (e) {
-    console.error('Failed to generate query embedding:', e);
-    // Fallback to simple keyword search
-    queryEmbedding = null;
-  }
-
-  // Filter notes: keyword match OR semantic match (threshold 30%)
-  const results = allNotes.map(note => {
-    const title = (note.title || '').toLowerCase();
-    const content = note.content.toLowerCase();
-    const labels = note.labels ? JSON.parse(note.labels) : [];
-
-    // Keyword match
-    const keywordMatch = title.includes(queryLower) ||
-      content.includes(queryLower) ||
-      labels.some((l: string) => l.toLowerCase().includes(queryLower));
-
-    // Semantic match (if embedding available)
-    let semanticMatch = false;
-    let similarity = 0;
-    if (queryEmbedding && note.noteEmbedding?.embedding) {
-      similarity = cosineSimilarity(queryEmbedding, JSON.parse(note.noteEmbedding.embedding));
-      semanticMatch = similarity > 0.3; // 30% threshold - works well for related concepts
-    }
-
-    return {
-      note,
-      keywordMatch,
-      semanticMatch,
-      similarity
-    };
-  }).filter(r => r.keywordMatch || r.semanticMatch);
-
-  // Parse and add match info
-  return results.map(r => {
-    const parsed = parseNote(r.note);
-
-    // Determine match type
-    let matchType: 'exact' | 'related' | null = null;
-    if (r.semanticMatch) {
-      matchType = 'related';
-    } else if (r.keywordMatch) {
-      matchType = 'exact';
-    }
-
-    return {
-      ...parsed,
-      matchType
-    };
-  });
-}
-
 // Create a new note
 export async function createNote(data: {
   title?: string
@@ -683,16 +616,19 @@ export async function createNote(data: {
       // Use setImmediate-like pattern to not block the response
       ; (async () => {
         try {
-          // Background task 1: Generate embedding
           const bgConfig = await getSystemConfig()
           const provider = getAIProvider(bgConfig)
           const embedding = await provider.getEmbeddings(content)
           if (embedding) {
-            await prisma.noteEmbedding.upsert({
-              where: { noteId: noteId },
-              create: { noteId: noteId, embedding: JSON.stringify(embedding) },
-              update: { embedding: JSON.stringify(embedding) }
-            })
+            const vecStr = `[${embedding.join(',')}]`
+            await prisma.$executeRawUnsafe(
+              `INSERT INTO "NoteEmbedding" ("id", "noteId", "embedding", "createdAt", "updatedAt")
+               VALUES (gen_random_uuid(), $1, $2::vector, now(), now())
+               ON CONFLICT ("noteId")
+               DO UPDATE SET "embedding" = $2::vector, "updatedAt" = now()`,
+              noteId,
+              vecStr
+            )
           }
         } catch (e) {
           console.error('[BG] Embedding generation failed:', e)
@@ -815,7 +751,6 @@ export async function updateNote(id: string, data: {
       }
     }
 
-    // Generate embedding in background — don't block the update
     if (data.content !== undefined) {
       const noteId = id
       const content = data.content
@@ -824,11 +759,15 @@ export async function updateNote(id: string, data: {
             const provider = getAIProvider(await getSystemConfig());
             const embedding = await provider.getEmbeddings(content);
             if (embedding) {
-              await prisma.noteEmbedding.upsert({
-                where: { noteId: noteId },
-                create: { noteId: noteId, embedding: JSON.stringify(embedding) },
-                update: { embedding: JSON.stringify(embedding) }
-              })
+              const vecStr = `[${embedding.join(',')}]`
+              await prisma.$executeRawUnsafe(
+                `INSERT INTO "NoteEmbedding" ("id", "noteId", "embedding", "createdAt", "updatedAt")
+                 VALUES (gen_random_uuid(), $1, $2::vector, now(), now())
+                 ON CONFLICT ("noteId")
+                 DO UPDATE SET "embedding" = $2::vector, "updatedAt" = now()`,
+                noteId,
+                vecStr
+              )
             }
           } catch (e) {
             console.error('[BG] Embedding regeneration failed:', e);
@@ -1409,11 +1348,15 @@ export async function syncAllEmbeddings() {
       try {
         const embedding = await provider.getEmbeddings(note.content);
         if (embedding) {
-          await prisma.noteEmbedding.upsert({
-            where: { noteId: note.id },
-            create: { noteId: note.id, embedding: JSON.stringify(embedding) },
-            update: { embedding: JSON.stringify(embedding) }
-          })
+          const vecStr = `[${embedding.join(',')}]`
+          await prisma.$executeRawUnsafe(
+            `INSERT INTO "NoteEmbedding" ("id", "noteId", "embedding", "createdAt", "updatedAt")
+             VALUES (gen_random_uuid(), $1, $2::vector, now(), now())
+             ON CONFLICT ("noteId")
+             DO UPDATE SET "embedding" = $2::vector, "updatedAt" = now()`,
+            note.id,
+            vecStr
+          )
           updatedCount++;
         }
       } catch (e) { }

memento-note/app/actions/semantic-search.ts

@@ -23,7 +23,7 @@ export async function semanticSearch(
   try {
     const results = await semanticSearchService.search(query, {
       limit: options?.limit || 20,
-      threshold: options?.threshold || 0.6,
+      threshold: options?.threshold || 0.3,
       notebookId: options?.notebookId // NEW: Pass notebook filter
     })
 

memento-note/app/api/admin/embeddings/validate/route.ts

@@ -1,11 +1,10 @@
 import { NextResponse } from 'next/server'
-import prisma from '@/lib/prisma'
+import { prisma } from '@/lib/prisma'
 import { auth } from '@/auth'
-import { validateEmbedding } from '@/lib/utils'
 
 /**
- * Admin endpoint to validate all embeddings in the database
- * Returns a list of notes with invalid embeddings
+ * Admin endpoint to validate all pgvector embeddings in the database.
+ * Uses native SQL to check for valid vector format.
  */
 export async function GET() {
   try {
@@ -14,7 +13,6 @@ export async function GET() {
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
     }
 
-    // Check if user is admin
     const user = await prisma.user.findUnique({
       where: { id: session.user.id },
       select: { role: true }
@@ -24,72 +22,34 @@ export async function GET() {
       return NextResponse.json({ error: 'Forbidden - Admin only' }, { status: 403 })
     }
 
-    // Fetch all notes with embeddings
-    const allNotes = await prisma.note.findMany({
-      select: {
-        id: true,
-        title: true,
-        noteEmbedding: true
-      }
-    })
+    const totalResult: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
+      `SELECT COUNT(*)::bigint as total FROM "Note" WHERE "trashedAt" IS NULL`
+    )
+    const total = Number(totalResult[0]?.total ?? 0)
 
-    const invalidNotes: Array<{
-      id: string
-      title: string
-      issues: string[]
-    }> = []
+    const withEmbedding: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
+      `SELECT COUNT(*)::bigint as count FROM "NoteEmbedding"`
+    )
+    const validCount = Number(withEmbedding[0]?.count ?? 0)
 
-    let validCount = 0
-    let missingCount = 0
-    let invalidCount = 0
+    const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
+      `SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e
+       WHERE e."embedding" IS NULL
+          OR array_length(string_to_array(replace(replace(e."embedding"::text, '[', ''), ']', ''), ','), 1) != 1536`
+    )
+    const invalidCount = Number(invalidResult[0]?.count ?? 0)
 
-    for (const note of allNotes) {
-      // Check if embedding is missing
-      if (!note.noteEmbedding?.embedding) {
-        missingCount++
-        invalidNotes.push({
-          id: note.id,
-          title: note.title || 'Untitled',
-          issues: ['Missing embedding']
-        })
-        continue
-      }
-
-      // Validate embedding
-      try {
-        if (!note.noteEmbedding?.embedding) continue
-        const embedding = JSON.parse(note.noteEmbedding.embedding) as number[]
-        const validation = validateEmbedding(embedding)
-
-        if (!validation.valid) {
-          invalidCount++
-          invalidNotes.push({
-            id: note.id,
-            title: note.title || 'Untitled',
-            issues: validation.issues
-          })
-        } else {
-          validCount++
-        }
-      } catch (error) {
-        invalidCount++
-        invalidNotes.push({
-          id: note.id,
-          title: note.title || 'Untitled',
-          issues: [`Failed to parse embedding: ${error}`]
-        })
-      }
-    }
+    const missingCount = total - validCount
 
     return NextResponse.json({
       success: true,
       summary: {
-        total: allNotes.length,
-        valid: validCount,
-        missing: missingCount,
+        total,
+        valid: validCount - invalidCount,
+        missing: missingCount > 0 ? missingCount : 0,
         invalid: invalidCount
       },
-      invalidNotes
+      invalidNotes: []
     })
   } catch (error) {
     console.error('[EMBEDDING_VALIDATION] Error:', error)

memento-note/app/api/notes/cleanup/route.ts

@@ -27,14 +27,18 @@ export async function POST(req: NextRequest) {
       }
     })
 
-    // 2. Clean up NoteEmbeddings that don't have a corresponding Note (shouldn't happen with Cascade, but good for cleanup)
-    const orphanedEmbeddings = await prisma.noteEmbedding.findMany({
-      where: {
-        note: { userId: { not: userId } } // Or just those where note is null if not using cascade
-      }
-    })
-    
-    // Actually, let's just focus on user-specific cleanup
+    // 2. Clean up NoteEmbeddings that don't have a corresponding Note
+    const orphanedEmbeddings: Array<{ id: string }> = await prisma.$queryRawUnsafe(
+      `SELECT e.id FROM "NoteEmbedding" e
+       LEFT JOIN "Note" n ON n.id = e."noteId"
+       WHERE n.id IS NULL`
+    )
+
+    if (orphanedEmbeddings.length > 0) {
+      await prisma.$executeRawUnsafe(
+        `DELETE FROM "NoteEmbedding" WHERE id = ANY(${`ARRAY['${orphanedEmbeddings.map(e => e.id).join("','")}']`}::text[])`
+      )
+    }
     
     // 3. Remove note history entries for notes that were deleted (cascade should handle this, but let's be safe)
     

memento-note/app/api/notes/reindex/route.ts

@@ -1,7 +1,7 @@
 import { NextRequest, NextResponse } from 'next/server'
 import { auth } from '@/auth'
 import { prisma } from '@/lib/prisma'
-import { EmbeddingService } from '@/lib/ai/services/embedding.service'
+import { semanticSearchService } from '@/lib/ai/services/semantic-search.service'
 
 export async function POST(req: NextRequest) {
   try {
@@ -12,41 +12,31 @@ export async function POST(req: NextRequest) {
 
     const userId = session.user.id
 
-    // Fetch all notes for the user
     const notes = await prisma.note.findMany({
       where: { userId, trashedAt: null },
-      select: { id: true, title: true, content: true }
+      select: { id: true }
     })
 
-    const embeddingService = new EmbeddingService()
     let processedCount = 0
+    let failedCount = 0
+    const BATCH_SIZE = 20
 
-    // Process in small batches to avoid timeouts if possible
-    // Note: In a real production app, this should be a background job
-    for (const note of notes) {
-      try {
-        const textToEmbed = `${note.title || ''}\n${note.content}`
-        if (textToEmbed.trim()) {
-          const embedding = await embeddingService.generateEmbedding(textToEmbed)
-          
-          await prisma.noteEmbedding.upsert({
-            where: { noteId: note.id },
-            update: { embedding: JSON.stringify(embedding) },
-            create: {
-              noteId: note.id,
-              embedding: JSON.stringify(embedding)
-            }
-          })
-          processedCount++
-        }
-      } catch (err) {
-        console.error(`Failed to reindex note ${note.id}:`, err)
+    for (let i = 0; i < notes.length; i += BATCH_SIZE) {
+      const batch = notes.slice(i, i + BATCH_SIZE)
+      const results = await Promise.allSettled(
+        batch.map(note => semanticSearchService.indexNote(note.id))
+      )
+
+      for (const r of results) {
+        if (r.status === 'fulfilled') processedCount++
+        else failedCount++
       }
     }
 
     return NextResponse.json({
       success: true,
       count: processedCount,
+      failed: failedCount,
       total: notes.length
     })
   } catch (error) {