feat(insights): fix DBSCAN, Persian embeddings crash, D3 physics layouts, and D3 node not found runtime error

memento-note/lib/ai/services/embedding.service.ts

@@ -7,61 +7,96 @@
 import { withAiProviderFallback } from '../fallback'
 import { getSystemConfig } from '@/lib/config'
 import { prisma } from '@/lib/prisma'
+import {
+  meanPoolEmbeddingVectors,
+  prepareNoteTextForEmbedding,
+  prepareTextForEmbedding,
+  splitPlainTextForEmbeddingChunks,
+} from '@/lib/text/plain-text'
 
 export interface EmbeddingResult {
   embedding: number[]
   model: string
   dimension: number
+  /** Nombre de caractères plain text indexés */
+  indexedChars?: number
+  /** Nombre de chunks API utilisés */
+  chunkCount?: number
 }
 
 export class EmbeddingService {
-  private readonly MAX_CHARS = 15000
-
-  private truncateForEmbedding(text: string): string {
-    if (text.length <= this.MAX_CHARS) return text
-    return text.slice(0, this.MAX_CHARS)
+  prepareTextForEmbedding(content: string): string {
+    return prepareTextForEmbedding(content)
   }
 
+  private async embedPlainText(plain: string): Promise<number[]> {
+    const config = await getSystemConfig()
+    return withAiProviderFallback('embedding', config, (provider) =>
+      provider.getEmbeddings(plain)
+    )
+  }
+
+  /**
+   * Embedding d'une note complète : titre + corps, multi-chunks si l'article dépasse la fenêtre API.
+   * Ex. 17 679 caractères → 3 chunks → vecteur moyenné (aucune perte de contenu).
+   */
+  async generateNoteEmbedding(
+    title: string | null | undefined,
+    content: string,
+  ): Promise<EmbeddingResult> {
+    const plain = prepareNoteTextForEmbedding(title, content)
+    if (!plain.trim()) {
+      throw new Error('Cannot generate embedding for empty note')
+    }
+
+    const chunks = splitPlainTextForEmbeddingChunks(plain)
+    const vectors = await Promise.all(chunks.map((chunk) => this.embedPlainText(chunk)))
+    const embedding = meanPoolEmbeddingVectors(vectors)
+
+    return {
+      embedding,
+      model: 'text-embedding-3-small',
+      dimension: embedding.length,
+      indexedChars: plain.length,
+      chunkCount: chunks.length,
+    }
+  }
+
+  /** Embedding d'une requête courte (recherche). */
   async generateEmbedding(text: string): Promise<EmbeddingResult> {
     if (!text || text.trim().length === 0) {
       throw new Error('Cannot generate embedding for empty text')
     }
 
-    const truncated = this.truncateForEmbedding(text)
+    const plain = prepareTextForEmbedding(text)
+    const embedding = await this.embedPlainText(plain)
 
-    try {
-      const config = await getSystemConfig()
-      const embedding = await withAiProviderFallback('embedding', config, (provider) =>
-        provider.getEmbeddings(truncated)
-      )
-
-      return {
-        embedding,
-        model: 'text-embedding-3-small',
-        dimension: embedding.length
-      }
-    } catch (error) {
-      console.error('Error generating embedding:', error)
-      throw new Error(`Failed to generate embedding: ${error}`)
+    return {
+      embedding,
+      model: 'text-embedding-3-small',
+      dimension: embedding.length,
+      indexedChars: plain.length,
+      chunkCount: 1,
     }
   }
 
   async generateBatchEmbeddings(texts: string[]): Promise<EmbeddingResult[]> {
     if (!texts || texts.length === 0) return []
 
-    const validTexts = texts.filter(t => t && t.trim().length > 0).map(t => this.truncateForEmbedding(t))
+    const validTexts = texts
+      .filter((t) => t && t.trim().length > 0)
+      .map((t) => prepareTextForEmbedding(t))
     if (validTexts.length === 0) return []
 
     try {
-      const config = await getSystemConfig()
-      const embeddings = await withAiProviderFallback('embedding', config, (provider) =>
-        Promise.all(validTexts.map((text) => provider.getEmbeddings(text)))
-      )
+      const embeddings = await Promise.all(validTexts.map((text) => this.embedPlainText(text)))
 
-      return embeddings.map(embedding => ({
+      return embeddings.map((embedding, i) => ({
         embedding,
         model: 'text-embedding-3-small',
-        dimension: embedding.length
+        dimension: embedding.length,
+        indexedChars: validTexts[i].length,
+        chunkCount: 1,
       }))
     } catch (error) {
       console.error('Error generating batch embeddings:', error)
@@ -69,31 +104,22 @@ export class EmbeddingService {
     }
   }
 
-  /**
-   * Format a number[] embedding as a pgvector-compatible string literal.
-   * e.g. [0.1, 0.2, 0.3] → '[0.1,0.2,0.3]'
-   */
   toVectorString(embedding: number[]): string {
     return `[${embedding.join(',')}]`
   }
 
-  /**
-   * Parse a pgvector string from the DB back into number[].
-   * e.g. '[0.1,0.2,0.3]' → [0.1, 0.2, 0.3]
-   */
   fromVectorString(vec: string): number[] {
     if (Array.isArray(vec)) return vec
     if (!vec || typeof vec !== 'string') return []
     return vec.replace(/^\[/, '').replace(/\]$/, '').split(',').map(Number)
   }
 
-  /**
-   * JS cosine similarity — still used by memory-echo pairwise comparisons.
-   */
   calculateCosineSimilarity(a: number[], b: number[]): number {
     if (!a.length || !b.length) return 0
     const minLen = Math.min(a.length, b.length)
-    let dot = 0, mA = 0, mB = 0
+    let dot = 0
+    let mA = 0
+    let mB = 0
     for (let i = 0; i < minLen; i++) {
       dot += a[i] * b[i]
       mA += a[i] * a[i]
@@ -105,10 +131,6 @@ export class EmbeddingService {
     return dot / (mA * mB)
   }
 
-  /**
-   * Check if a note needs embedding regeneration.
-   * Uses a content-content comparison (not embedding-content).
-   */
   async getDbDimension(): Promise<number | null> {
     try {
       const result: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
@@ -142,10 +164,13 @@ export class EmbeddingService {
   }
 
   shouldRegenerateEmbedding(
-    noteContent: string,
+    _noteContent: string,
     _lastEmbeddingContent: string | null,
-    lastAnalysis: Date | null
+    lastAnalysis: Date | null,
+    options?: { force?: boolean; isClip?: boolean },
   ): boolean {
+    if (options?.force) return true
+    if (options?.isClip) return true
     if (!lastAnalysis) return true
     const daysSinceAnalysis = (Date.now() - lastAnalysis.getTime()) / (1000 * 60 * 60 * 24)
     return daysSinceAnalysis > 7