feat(insights): fix DBSCAN, Persian embeddings crash, D3 physics layouts, and D3 node not found runtime error

2026-05-24 18:57:33 +00:00
parent e2672cd2c2
commit e881004c77
63 changed files with 5729 additions and 563 deletions
--- a/memento-note/lib/ai/services/memory-echo.service.ts
+++ b/memento-note/lib/ai/services/memory-echo.service.ts
@@ -1,10 +1,20 @@
-import { getAIProvider, getChatProvider } from '../factory'
+import { getChatProvider } from '../factory'
 import { cosineSimilarity } from '@/lib/utils'
 import { embeddingService } from './embedding.service'
 import { getSystemConfig } from '@/lib/config'
 import prisma from '@/lib/prisma'
 import { Prisma } from '@prisma/client'
 import { upsertNoteEmbedding } from '@/lib/embeddings'
+import {
+  excerptPlainNoteContent,
+  prepareNoteTextForEmbedding,
+} from '@/lib/text/plain-text'
+import { detectTextDirection } from '@/lib/clip/rtl-content'
+import {
+  SEMANTIC_SIMILARITY_FLOOR_CLIP,
+  SEMANTIC_SIMILARITY_FLOOR_DEMO,
+  SEMANTIC_SIMILARITY_FLOOR,
+} from '@/lib/ai/semantic-proximity'

 export interface NoteConnection {
  note1: {
@@ -50,45 +60,109 @@ export interface MemoryEchoInsight {
 * "I didn't search, it found me"
 */
 export class MemoryEchoService {
-  private readonly SIMILARITY_THRESHOLD = 0.75 // High threshold for quality connections
-  private readonly SIMILARITY_THRESHOLD_DEMO = 0.50 // Lower threshold for demo mode
+  private readonly SIMILARITY_THRESHOLD = SEMANTIC_SIMILARITY_FLOOR
+  private readonly SIMILARITY_THRESHOLD_DEMO = SEMANTIC_SIMILARITY_FLOOR_DEMO
+  private readonly SIMILARITY_THRESHOLD_CLIP = SEMANTIC_SIMILARITY_FLOOR_CLIP
  private readonly MIN_DAYS_APART = 7 // Notes must be at least 7 days apart
+  private readonly MIN_DAYS_APART_CLIP = 0 // Notes clippées (sourceUrl) : même jour OK
  private readonly MIN_DAYS_APART_DEMO = 0 // No delay for demo mode
  private readonly MAX_INSIGHTS_PER_USER = 100 // Prevent spam

+  private isClippedNote(note: { sourceUrl?: string | null }): boolean {
+    return Boolean(note.sourceUrl?.trim())
+  }
+
+  private passesTimeDiversityFilter(
+    daysApart: number,
+    noteA: { sourceUrl?: string | null },
+    noteB: { sourceUrl?: string | null },
+    demoMode: boolean,
+  ): boolean {
+    if (demoMode) return true
+    const minDays =
+      this.isClippedNote(noteA) || this.isClippedNote(noteB)
+        ? this.MIN_DAYS_APART_CLIP
+        : this.MIN_DAYS_APART
+    return daysApart >= minDays
+  }
+
+  private isRtlOrClipNote(note: {
+    sourceUrl?: string | null
+    content?: string
+    title?: string | null
+  }): boolean {
+    if (this.isClippedNote(note)) return true
+    if (note.content?.includes('clip-article--rtl')) return true
+    const sample = prepareNoteTextForEmbedding(note.title, note.content || '')
+    return detectTextDirection(sample) === 'rtl'
+  }
+
+  private pairSimilarityThreshold(
+    noteA: { sourceUrl?: string | null; content?: string; title?: string | null },
+    noteB: { sourceUrl?: string | null; content?: string; title?: string | null },
+    demoMode: boolean,
+  ): number {
+    if (demoMode) return this.SIMILARITY_THRESHOLD_DEMO
+    if (this.isRtlOrClipNote(noteA) || this.isRtlOrClipNote(noteB)) {
+      return this.SIMILARITY_THRESHOLD_CLIP
+    }
+    return this.SIMILARITY_THRESHOLD
+  }
+
+  /** Texte plain complet envoyé à l'API / résolution de blocs (pas de troncature). */
+  private connectionPlainText(
+    title: string | null,
+    content: string,
+  ): string {
+    return prepareNoteTextForEmbedding(title, content)
+  }
+
+  private async upsertNoteEmbeddingFromNote(note: {
+    id: string
+    title: string | null
+    content: string
+  }): Promise<number[] | null> {
+    const text = prepareNoteTextForEmbedding(note.title, note.content)
+    if (!text.trim()) return null
+    try {
+      const { embedding } = await embeddingService.generateNoteEmbedding(note.title, note.content)
+      if (embedding?.length) {
+        await upsertNoteEmbedding(note.id, embedding)
+        return embedding
+      }
+    } catch (error) {
+      console.error(`[MemoryEcho] embedding failed for note ${note.id}:`, error)
+    }
+    return null
+  }
+
  /**
   * Generate embeddings for notes that don't have one yet
   */
  private async ensureEmbeddings(userId: string): Promise<void> {
-    const notesWithoutEmbeddings = await prisma.note.findMany({
+    const notes = await prisma.note.findMany({
      where: {
        userId,
        isArchived: false,
        trashedAt: null,
-        noteEmbedding: { is: null }
      },
-      select: { id: true, content: true }
+      select: {
+        id: true,
+        title: true,
+        content: true,
+        sourceUrl: true,
+        noteEmbedding: { select: { noteId: true } },
+      },
    })

-    if (notesWithoutEmbeddings.length === 0) return
+    if (notes.length === 0) return

-    try {
-      const config = await getSystemConfig()
-      const provider = getAIProvider(config)
-
-      for (const note of notesWithoutEmbeddings) {
-        if (!note.content || note.content.trim().length === 0) continue
-        try {
-          const embedding = await provider.getEmbeddings(note.content.slice(0, 15000))
-          if (embedding && embedding.length > 0) {
-            await upsertNoteEmbedding(note.id, embedding)
-          }
-        } catch {
-          // Skip this note, continue with others
-        }
-      }
-    } catch {
-      // Provider not configured — nothing we can do
+    for (const note of notes) {
+      if (!note.content?.trim()) continue
+      const isClip = this.isClippedNote(note)
+      const missing = !note.noteEmbedding
+      if (!missing && !isClip) continue
+      await this.upsertNoteEmbeddingFromNote(note)
    }
  }

@@ -121,6 +195,7 @@ export class MemoryEchoService {
        id: true,
        title: true,
        content: true,
+        sourceUrl: true,
        noteEmbedding: true,
        createdAt: true
      },
@@ -151,10 +226,6 @@ export class MemoryEchoService {

    const connections: NoteConnection[] = []

-    // Use demo mode parameters if enabled
-    const minDaysApart = demoMode ? this.MIN_DAYS_APART_DEMO : this.MIN_DAYS_APART
-    const similarityThreshold = demoMode ? this.SIMILARITY_THRESHOLD_DEMO : this.SIMILARITY_THRESHOLD
-
    // Load user feedback to adjust thresholds per note
    const feedbackInsights = await prisma.memoryEchoInsight.findMany({
      where: { userId, feedback: { not: null } },
@@ -183,8 +254,8 @@ export class MemoryEchoService {
          Math.floor((note1.createdAt.getTime() - note2.createdAt.getTime()) / (1000 * 60 * 60 * 24))
        )

-        // Time diversity filter: notes must be from different time periods
-        if (daysApart < minDaysApart) {
+        // Time diversity filter: notes must be from different time periods (sauf clips récents)
+        if (!this.passesTimeDiversityFilter(daysApart, note1, note2, demoMode)) {
          continue
        }

@@ -192,7 +263,8 @@ export class MemoryEchoService {
        const similarity = cosineSimilarity(note1.embedding!, note2.embedding!)

        // Similarity threshold for meaningful connections (adjusted by feedback)
-        const adjustedThreshold = similarityThreshold
+        const baseThreshold = this.pairSimilarityThreshold(note1, note2, demoMode)
+        const adjustedThreshold = baseThreshold
          + (notePenalty.get(note1.id) || 0)
          + (notePenalty.get(note2.id) || 0)
        if (similarity >= adjustedThreshold) {
@@ -200,13 +272,13 @@ export class MemoryEchoService {
            note1: {
              id: note1.id,
              title: note1.title,
-              content: note1.content.substring(0, 200) + (note1.content.length > 200 ? '...' : ''),
+              content: this.connectionPlainText(note1.title, note1.content),
              createdAt: note1.createdAt
            },
            note2: {
              id: note2.id,
              title: note2.title,
-              content: note2.content ? note2.content.substring(0, 200) + (note2.content.length > 200 ? '...' : '') : '',
+              content: this.connectionPlainText(note2.title, note2.content || ''),
              createdAt: note2.createdAt
            },
            similarityScore: similarity,
@@ -239,30 +311,52 @@ export class MemoryEchoService {

      const note1Desc = note1Title || 'Untitled note'
      const note2Desc = note2Title || 'Untitled note'
+      const excerpt1 = excerptPlainNoteContent(note1Title, note1Content, 1200)
+      const excerpt2 = excerptPlainNoteContent(note2Title, note2Content, 1200)
+      const directionSample = `${note1Desc}\n${excerpt1}\n${note2Desc}\n${excerpt2}`
+      const isRtl = detectTextDirection(directionSample) === 'rtl'

-      const prompt = `You are a helpful assistant analyzing connections between notes.
+      const prompt = isRtl
+        ? `تو یک دستیار هستی که ارتباط بین یادداشت‌ها را تحلیل می‌کنی.
+
+یادداشت ۱: «${note1Desc}»
+متن: ${excerpt1}
+
+یادداشت ۲: «${note2Desc}»
+متن: ${excerpt2}
+
+در یک جمله کوتاه (حداکثر ۱۵ کلمه) به فارسی توضیح بده چرا این دو یادداشت به هم مرتبط‌اند. فقط رابطه معنایی را بگو.`
+        : `You are a helpful assistant analyzing connections between notes.

 Note 1: "${note1Desc}"
-Content: ${note1Content.substring(0, 300)}
+Content: ${excerpt1}

 Note 2: "${note2Desc}"
-Content: ${note2Content.substring(0, 300)}
+Content: ${excerpt2}

 Explain in one brief sentence (max 15 words) why these notes are connected. Focus on the semantic relationship.`

      const response = await provider.generateText(prompt)

-      // Clean up response
      const insight = response
-        .replace(/^["']|["']$/g, '') // Remove quotes
-        .replace(/^[^.]+\.\s*/, '') // Remove "Here is..." prefix
+        .replace(/^["'«»]|["'«»]$/g, '')
+        .replace(/^[^.]+\.\s*/, '')
        .trim()
-        .substring(0, 150) // Max length
+        .substring(0, 150)

-      return insight || 'These notes appear to be semantically related.'
+      const fallback = isRtl
+        ? 'این یادداشت‌ها از نظر معنایی به هم مرتبط به نظر می‌رسند.'
+        : 'These notes appear to be semantically related.'
+
+      return insight || fallback

    } catch (error) {
      console.error('[MemoryEcho] Failed to generate insight:', error)
+      const sample = excerptPlainNoteContent(note1Title, note1Content, 200)
+        + excerptPlainNoteContent(note2Title, note2Content, 200)
+      if (detectTextDirection(sample) === 'rtl') {
+        return 'این یادداشت‌ها از نظر معنایی به هم مرتبط به نظر می‌رسند.'
+      }
      return 'These notes appear to be semantically related.'
    }
  }
@@ -459,6 +553,7 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
        id: true,
        title: true,
        content: true,
+        sourceUrl: true,
        createdAt: true,
        userId: true
      }
@@ -475,8 +570,16 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
    )
    const targetEmbeddingStr = embeddingResult[0]?.embedding

-    if (!targetEmbeddingStr) {
-      return [] // Note has no embedding
+    let targetEmbedding = targetEmbeddingStr
+      ? embeddingService.fromVectorString(targetEmbeddingStr)
+      : null
+
+    if (!targetEmbedding && targetNote.content?.trim()) {
+      targetEmbedding = await this.upsertNoteEmbeddingFromNote(targetNote)
+    }
+
+    if (!targetEmbedding) {
+      return []
    }

    // Get dismissed connections for this note (to filter them out)
@@ -514,6 +617,7 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
        id: true,
        title: true,
        content: true,
+        sourceUrl: true,
        createdAt: true
      },
      orderBy: { createdAt: 'desc' }
@@ -523,11 +627,6 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
      return []
    }

-    const targetEmbedding = targetEmbeddingStr
-      ? embeddingService.fromVectorString(targetEmbeddingStr)
-      : null
-    if (!targetEmbedding) return []
-
    // Fetch all other embeddings
    const otherNoteIds = otherNotes.map(n => n.id)
    const otherEmbeddings = otherNoteIds.length === 0 ? [] : await prisma.$queryRaw<Array<{ noteId: string, embedding: any }>>(
@@ -541,9 +640,6 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
    })
    const demoMode = settings?.demoMode || false

-    const minDaysApart = demoMode ? this.MIN_DAYS_APART_DEMO : this.MIN_DAYS_APART
-    const similarityThreshold = demoMode ? this.SIMILARITY_THRESHOLD_DEMO : this.SIMILARITY_THRESHOLD
-
    // Load user feedback to adjust thresholds
    const feedbackInsights = await prisma.memoryEchoInsight.findMany({
      where: { userId, feedback: { not: null } },
@@ -565,9 +661,13 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
    // Compare target note with all other notes
    for (const otherNote of otherNotes) {
      const otherEmbeddingStr = otherEmbeddingMap.get(otherNote.id)
-      if (!otherEmbeddingStr) continue
+      let otherEmbedding = otherEmbeddingStr
+        ? embeddingService.fromVectorString(otherEmbeddingStr)
+        : null

-      const otherEmbedding = embeddingService.fromVectorString(otherEmbeddingStr)
+      if (!otherEmbedding && otherNote.content?.trim()) {
+        otherEmbedding = await this.upsertNoteEmbeddingFromNote(otherNote)
+      }
      if (!otherEmbedding) continue

      // Check if this connection was dismissed
@@ -582,8 +682,8 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
        Math.floor((targetNote.createdAt.getTime() - otherNote.createdAt.getTime()) / (1000 * 60 * 60 * 24))
      )

-      // Time diversity filter
-      if (daysApart < minDaysApart) {
+      // Time diversity filter (clips récents autorisés sans délai de 7 jours)
+      if (!this.passesTimeDiversityFilter(daysApart, targetNote, otherNote, demoMode)) {
        continue
      }

@@ -591,7 +691,8 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
      const similarity = cosineSimilarity(targetEmbedding, otherEmbedding)

      // Similarity threshold (adjusted by feedback)
-      const adjustedThreshold = similarityThreshold
+      const baseThreshold = this.pairSimilarityThreshold(targetNote, otherNote, demoMode)
+      const adjustedThreshold = baseThreshold
        + (notePenalty.get(targetNote.id) || 0)
        + (notePenalty.get(otherNote.id) || 0)
      if (similarity >= adjustedThreshold) {
@@ -599,13 +700,13 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
          note1: {
            id: targetNote.id,
            title: targetNote.title,
-            content: targetNote.content.substring(0, 200) + (targetNote.content.length > 200 ? '...' : ''),
+            content: this.connectionPlainText(targetNote.title, targetNote.content),
            createdAt: targetNote.createdAt
          },
          note2: {
            id: otherNote.id,
            title: otherNote.title,
-            content: otherNote.content ? otherNote.content.substring(0, 200) + (otherNote.content.length > 200 ? '...' : '') : '',
+            content: this.connectionPlainText(otherNote.title, otherNote.content || ''),
            createdAt: otherNote.createdAt
          },
          similarityScore: similarity,