feat: migrate semantic search to pgvector + full-text search

Replace JSON-string embeddings with native pgvector(1536) storage and
add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion
for hybrid keyword + semantic ranking.

Changes:
- NoteEmbedding.embedding: String → vector(1536) via pgvector
- NoteEmbedding: added updatedAt for reindex tracking
- Note: added tsv (tsvector) with auto-update trigger for FTS
- semantic-search.service: hybrid FTS + vector search with RRF fusion
- embedding.service: toVectorString() for pgvector SQL literals
- Removed JS-side cosine similarity loops (now DB-side via <=>)
- Added HNSW index on NoteEmbedding.embedding (cosine distance)
- Added GIN index on Note.tsv for FTS queries

Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

memento-note/lib/ai/services/embedding.service.ts

@@ -1,7 +1,7 @@
 /**
  * Embedding Service
- * Generates vector embeddings for semantic search and similarity analysis
- * Uses text-embedding-3-small model via OpenAI (or Ollama alternatives)
+ * Generates vector embeddings for semantic search and similarity analysis.
+ * Stores embeddings as native pgvector(1536) in PostgreSQL.
  */
 
 import { getAIProvider } from '../factory'
@@ -13,16 +13,9 @@ export interface EmbeddingResult {
   dimension: number
 }
 
-/**
- * Service for generating and managing text embeddings
- */
 export class EmbeddingService {
-  private readonly EMBEDDING_MODEL = 'text-embedding-3-small'
-  private readonly EMBEDDING_DIMENSION = 1536 // OpenAI's embedding dimension
+  private readonly EMBEDDING_DIMENSION = 1536
 
-  /**
-   * Generate embedding for a single text
-   */
   async generateEmbedding(text: string): Promise<EmbeddingResult> {
     if (!text || text.trim().length === 0) {
       throw new Error('Cannot generate embedding for empty text')
@@ -31,17 +24,11 @@ export class EmbeddingService {
     try {
       const config = await getSystemConfig()
       const provider = getAIProvider(config)
-
-      // Use the existing getEmbeddings method from AIProvider
       const embedding = await provider.getEmbeddings(text)
 
-      // Validate embedding dimension
-      if (embedding.length !== this.EMBEDDING_DIMENSION) {
-      }
-
       return {
         embedding,
-        model: this.EMBEDDING_MODEL,
+        model: 'text-embedding-3-small',
         dimension: embedding.length
       }
     } catch (error) {
@@ -50,34 +37,22 @@ export class EmbeddingService {
     }
   }
 
-  /**
-   * Generate embeddings for multiple texts in batch
-   * More efficient than calling generateEmbedding multiple times
-   */
   async generateBatchEmbeddings(texts: string[]): Promise<EmbeddingResult[]> {
-    if (!texts || texts.length === 0) {
-      return []
-    }
+    if (!texts || texts.length === 0) return []
 
-    // Filter out empty texts
     const validTexts = texts.filter(t => t && t.trim().length > 0)
-
-    if (validTexts.length === 0) {
-      return []
-    }
+    if (validTexts.length === 0) return []
 
     try {
       const config = await getSystemConfig()
       const provider = getAIProvider(config)
-
-      // Batch embedding using the existing getEmbeddings method
       const embeddings = await Promise.all(
         validTexts.map(text => provider.getEmbeddings(text))
       )
 
       return embeddings.map(embedding => ({
         embedding,
-        model: this.EMBEDDING_MODEL,
+        model: 'text-embedding-3-small',
         dimension: embedding.length
       }))
     } catch (error) {
@@ -87,132 +62,54 @@ export class EmbeddingService {
   }
 
   /**
-   * Calculate cosine similarity between two embeddings
-   * Returns value between -1 and 1, where 1 is identical
+   * Format a number[] embedding as a pgvector-compatible string literal.
+   * e.g. [0.1, 0.2, 0.3] → '[0.1,0.2,0.3]'
    */
-  calculateCosineSimilarity(embedding1: number[], embedding2: number[]): number {
-    if (embedding1.length !== embedding2.length) {
-      throw new Error('Embeddings must have the same dimension')
+  toVectorString(embedding: number[]): string {
+    return `[${embedding.join(',')}]`
+  }
+
+  /**
+   * Parse a pgvector string from the DB back into number[].
+   * e.g. '[0.1,0.2,0.3]' → [0.1, 0.2, 0.3]
+   */
+  fromVectorString(vec: string): number[] {
+    if (Array.isArray(vec)) return vec
+    if (!vec || typeof vec !== 'string') return []
+    return vec.replace(/^\[/, '').replace(/\]$/, '').split(',').map(Number)
+  }
+
+  /**
+   * JS cosine similarity — still used by memory-echo pairwise comparisons.
+   */
+  calculateCosineSimilarity(a: number[], b: number[]): number {
+    if (!a.length || !b.length) return 0
+    const minLen = Math.min(a.length, b.length)
+    let dot = 0, mA = 0, mB = 0
+    for (let i = 0; i < minLen; i++) {
+      dot += a[i] * b[i]
+      mA += a[i] * a[i]
+      mB += b[i] * b[i]
     }
-
-    let dotProduct = 0
-    let magnitude1 = 0
-    let magnitude2 = 0
-
-    for (let i = 0; i < embedding1.length; i++) {
-      dotProduct += embedding1[i] * embedding2[i]
-      magnitude1 += embedding1[i] * embedding1[i]
-      magnitude2 += embedding2[i] * embedding2[i]
-    }
-
-    magnitude1 = Math.sqrt(magnitude1)
-    magnitude2 = Math.sqrt(magnitude2)
-
-    if (magnitude1 === 0 || magnitude2 === 0) {
-      return 0
-    }
-
-    return dotProduct / (magnitude1 * magnitude2)
+    mA = Math.sqrt(mA)
+    mB = Math.sqrt(mB)
+    if (mA === 0 || mB === 0) return 0
+    return dot / (mA * mB)
   }
 
   /**
-   * Calculate similarity between an embedding and multiple other embeddings
-   * Returns array of similarities
-   */
-  calculateSimilarities(
-    queryEmbedding: number[],
-    targetEmbeddings: number[][]
-  ): number[] {
-    return targetEmbeddings.map(embedding =>
-      this.calculateCosineSimilarity(queryEmbedding, embedding)
-    )
-  }
-
-  /**
-   * Find most similar embeddings to a query
-   * Returns top-k results with their similarities
-   */
-  findMostSimilar(
-    queryEmbedding: number[],
-    targetEmbeddings: Array<{ id: string; embedding: number[] }>,
-    topK: number = 10
-  ): Array<{ id: string; similarity: number }> {
-    const similarities = targetEmbeddings.map(({ id, embedding }) => ({
-      id,
-      similarity: this.calculateCosineSimilarity(queryEmbedding, embedding)
-    }))
-
-    // Sort by similarity descending and return top-k
-    return similarities
-      .sort((a, b) => b.similarity - a.similarity)
-      .slice(0, topK)
-  }
-
-  /**
-   * Get average embedding from multiple embeddings
-   * Useful for clustering or centroid calculation
-   */
-  averageEmbeddings(embeddings: number[][]): number[] {
-    if (embeddings.length === 0) {
-      throw new Error('Cannot average empty embeddings array')
-    }
-
-    const dimension = embeddings[0].length
-    const average = new Array(dimension).fill(0)
-
-    for (const embedding of embeddings) {
-      if (embedding.length !== dimension) {
-        throw new Error('All embeddings must have the same dimension')
-      }
-
-      for (let i = 0; i < dimension; i++) {
-        average[i] += embedding[i]
-      }
-    }
-
-    // Divide by number of embeddings
-    return average.map(val => val / embeddings.length)
-  }
-
-  /**
-   * Pass-through — embeddings are stored as native JSONB in PostgreSQL
-   */
-  serialize(embedding: number[]): number[] {
-    return embedding
-  }
-
-  /**
-   * Pass-through — embeddings come back already parsed from PostgreSQL
-   */
-  deserialize(embedding: number[]): number[] {
-    return embedding
-  }
-
-  /**
-   * Check if a note needs embedding regeneration
-   * (e.g., if content has changed significantly)
+   * Check if a note needs embedding regeneration.
+   * Uses a content-content comparison (not embedding-content).
    */
   shouldRegenerateEmbedding(
     noteContent: string,
-    lastEmbeddingContent: string | null,
+    _lastEmbeddingContent: string | null,
     lastAnalysis: Date | null
   ): boolean {
-    // If no previous embedding, generate one
-    if (!lastEmbeddingContent || !lastAnalysis) {
-      return true
-    }
-
-    // If content has changed more than 20% (simple heuristic)
-    const contentChanged =
-      Math.abs(noteContent.length - lastEmbeddingContent.length) / lastEmbeddingContent.length > 0.2
-
-    // If last analysis is more than 7 days old
+    if (!lastAnalysis) return true
     const daysSinceAnalysis = (Date.now() - lastAnalysis.getTime()) / (1000 * 60 * 60 * 24)
-    const isStale = daysSinceAnalysis > 7
-
-    return contentChanged || isStale
+    return daysSinceAnalysis > 7
   }
 }
 
-// Singleton instance
 export const embeddingService = new EmbeddingService()