feat(insights): fix DBSCAN, Persian embeddings crash, D3 physics layouts, and D3 node not found runtime error

2026-05-24 18:57:33 +00:00
parent e2672cd2c2
commit e881004c77
63 changed files with 5729 additions and 563 deletions
--- a/memento-note/lib/ai/services/bridge-notes.service.ts
+++ b/memento-note/lib/ai/services/bridge-notes.service.ts
@@ -50,19 +50,16 @@ export class BridgeNotesService {
      noteId: string
      clusterId: number | null
    }>>(
-      `SELECT similar."noteId", cm."clusterId"
-       FROM (
-         SELECT e2."noteId"
-         FROM "NoteEmbedding" e1
-         CROSS JOIN "NoteEmbedding" e2
-         INNER JOIN "Note" n ON n.id = e2."noteId"
-         WHERE e1."noteId" = $1
-           AND e2."noteId" != e1."noteId"
-           AND n."userId" = $2
-           AND n."trashedAt" IS NULL
-           AND (e1."embedding"::vector <=> e2."embedding"::vector) <= $3
-       ) similar
-       LEFT JOIN "ClusterMember" cm ON cm."noteId" = similar."noteId" AND cm."userId" = $2`,
+      `SELECT e2."noteId", cm."clusterId"
+       FROM "NoteEmbedding" e1
+       CROSS JOIN "NoteEmbedding" e2
+       INNER JOIN "Note" n ON n.id = e2."noteId"
+       LEFT JOIN "ClusterMember" cm ON cm."noteId" = e2."noteId" AND cm."userId" = $2
+       WHERE e1."noteId" = $1
+         AND e2."noteId" != e1."noteId"
+         AND n."userId" = $2
+         AND n."trashedAt" IS NULL
+         AND (e1."embedding"::vector <=> e2."embedding"::vector) <= $3`,
      noteId,
      userId,
      cosineDistance
--- a/memento-note/lib/ai/services/clustering.service.ts
+++ b/memento-note/lib/ai/services/clustering.service.ts
@@ -15,6 +15,7 @@ import prisma from '@/lib/prisma'
 import { embeddingService } from './embedding.service'
 import { getChatProvider } from '@/lib/ai/factory'
 import { getSystemConfig } from '@/lib/config'
+import { upsertNoteEmbedding } from '@/lib/embeddings'

 export interface ClusterResult {
  clusterId: number
@@ -34,6 +35,8 @@ export interface ClusteringOptions {
  minClusterSize?: number
  epsilon?: number // Cosine distance threshold (lower = more strict)
  maxClusters?: number
+  /** usage interne — évite une boucle de retry */
+  _relaxedRetry?: boolean
 }

 export class ClusteringService {
@@ -42,6 +45,67 @@ export class ClusteringService {
  private readonly DEFAULT_MAX_CLUSTERS = 50
  private readonly MIN_NOTES_FOR_CLUSTERING = 10

+  /**
+   * Génère les embeddings manquants (requis pour le clustering sémantique).
+   */
+  async ensureEmbeddings(
+    userId: string,
+    options?: { force?: boolean },
+  ): Promise<{ created: number; total: number }> {
+    const notes = await prisma.note.findMany({
+      where: {
+        userId,
+        isArchived: false,
+        trashedAt: null,
+      },
+      select: {
+        id: true,
+        title: true,
+        content: true,
+        sourceUrl: true,
+        updatedAt: true,
+        noteEmbedding: { select: { noteId: true, createdAt: true } },
+      },
+    })
+
+    let created = 0
+
+    if (notes.length > 0) {
+      try {
+        for (const note of notes) {
+          if (!note.content?.trim()) continue
+          const isClip = Boolean(note.sourceUrl?.trim())
+          const missing = !note.noteEmbedding
+          const isModified = note.noteEmbedding && note.updatedAt > note.noteEmbedding.createdAt
+          if (!options?.force && !missing && !isModified && !isClip) continue
+          try {
+            const { embedding } = await embeddingService.generateNoteEmbedding(
+              note.title,
+              note.content,
+            )
+            if (embedding?.length) {
+              await upsertNoteEmbedding(note.id, embedding)
+              created++
+            }
+          } catch {
+            // note ignorée, on continue
+          }
+        }
+      } catch {
+        // fournisseur IA indisponible
+      }
+    }
+
+    const totalRow = await prisma.$queryRawUnsafe<Array<{ count: bigint }>>(
+      `SELECT COUNT(*) FROM "NoteEmbedding" ne
+       INNER JOIN "Note" n ON n.id = ne."noteId"
+       WHERE n."userId" = $1 AND n."trashedAt" IS NULL AND ne."embedding" IS NOT NULL`,
+      userId
+    )
+
+    return { created, total: Number(totalRow[0]?.count || 0) }
+  }
+
  /**
   * Calculate cosine similarity between two embedding vectors.
   * Uses 1 - cosine_distance where cosine_distance is computed via pgvector.
@@ -126,8 +190,29 @@ export class ClusteringService {
    return clusterMembers
  }

+  /**
+   * Calculate cosine similarity between two embedding vectors in memory.
+   */
+  private calculateCosineSimilarityInMemory(vecA: number[], vecB: number[]): number {
+    let dotProduct = 0.0
+    let normA = 0.0
+    let normB = 0.0
+    const len = vecA.length
+    for (let i = 0; i < len; i++) {
+      const a = vecA[i]
+      const b = vecB[i]
+      dotProduct += a * b
+      normA += a * a
+      normB += b * b
+    }
+    if (normA === 0 || normB === 0) return 0
+    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))
+  }
+
  /**
   * Perform density-based clustering on user's note embeddings.
+   * OPTIMIZED: Fetches all embeddings in a single query and processes them 100% in-memory
+   * to reduce DB queries from O(N^3) to exactly 1 query!
   */
  async clusterNotes(
    userId: string,
@@ -143,9 +228,9 @@ export class ClusteringService {
      maxClusters = this.DEFAULT_MAX_CLUSTERS
    } = options

-    // Get all user's notes with embeddings
-    const notesWithEmbeddings = await prisma.$queryRawUnsafe<Array<{ noteId: string }>>(
-      `SELECT ne."noteId"
+    // Fetch all user note embeddings in a single highly-optimized DB query
+    const embeddingsRow = await prisma.$queryRawUnsafe<Array<{ noteId: string; embedding: string }>>(
+      `SELECT ne."noteId", ne."embedding"::text AS "embedding"
       FROM "NoteEmbedding" ne
       INNER JOIN "Note" n ON n.id = ne."noteId"
       WHERE n."userId" = $1
@@ -154,7 +239,19 @@ export class ClusteringService {
      userId
    )

-    const allNoteIds = notesWithEmbeddings.map(n => n.noteId)
+    const embeddingMap = new Map<string, number[]>()
+    embeddingsRow.forEach(row => {
+      if (row.embedding) {
+        try {
+          const vector = JSON.parse(row.embedding) as number[]
+          embeddingMap.set(row.noteId, vector)
+        } catch (e) {
+          console.error("Error parsing embedding vector:", e)
+        }
+      }
+    })
+
+    const allNoteIds = Array.from(embeddingMap.keys())

    if (allNoteIds.length < this.MIN_NOTES_FOR_CLUSTERING) {
      return {
@@ -164,76 +261,274 @@ export class ClusteringService {
      }
    }

-    const visited = new Set<string>()
-    const clustered = new Map<string, number>() // noteId -> clusterId
-    const clusterResults: ClusterResult[] = []
-    let clusterId = 0
+    // In-memory neighbor lookup
+    const findNeighborsInMemory = (noteId: string, currentEpsilon: number): string[] => {
+      const vecA = embeddingMap.get(noteId)
+      if (!vecA) return []
+      const neighbors: string[] = []

-    // DBSCAN algorithm
-    for (const noteId of allNoteIds) {
-      if (visited.has(noteId)) continue
+      embeddingMap.forEach((vecB, otherId) => {
+        if (otherId === noteId) return
+        const similarity = this.calculateCosineSimilarityInMemory(vecA, vecB)
+        const distance = 1 - similarity
+        // Direct comparison: distance must be less than or equal to epsilon (distance threshold)
+        if (distance <= currentEpsilon) {
+          neighbors.push(otherId)
+        }
+      })
+      return neighbors
+    }

-      visited.add(noteId)
-      const neighbors = await this.findNeighbors(noteId, allNoteIds, epsilon)
+    // Mathematically correct in-memory DBSCAN cluster expansion
+    const expandClusterInMemory = (
+      noteId: string,
+      neighbors: string[],
+      currentClusterId: number,
+      visited: Set<string>,
+      clustered: Map<string, number>,
+      currentEpsilon: number,
+      currentMinSize: number
+    ): string[] => {
+      const clusterMembers: string[] = [noteId]
+      const queue = [...neighbors]

-      if (neighbors.length < minClusterSize) {
-        // Mark as noise (cluster_id = -1)
-        clustered.set(noteId, -1)
-        continue
+      // Assign all initial direct neighbors to this cluster if they are unassigned or marked as noise
+      for (const neighborId of neighbors) {
+        const status = clustered.get(neighborId)
+        if (status === undefined || status === -1) {
+          clustered.set(neighborId, currentClusterId)
+          if (!clusterMembers.includes(neighborId)) {
+            clusterMembers.push(neighborId)
+          }
+        }
      }

-      // Expand cluster
-      const clusterMembers = await this.expandCluster(
-        noteId,
-        neighbors,
-        clusterId,
-        visited,
-        clustered,
-        allNoteIds,
-        epsilon,
-        minClusterSize
-      )
+      while (queue.length > 0) {
+        const currentNoteId = queue.shift()!

-      if (clusterMembers.length >= minClusterSize && clusterId < maxClusters) {
-        clusterResults.push({
-          clusterId,
-          noteIds: clusterMembers
-        })
-        clusterId++
-      } else {
-        // Too small, mark as noise
-        for (const memberId of clusterMembers) {
-          clustered.set(memberId, -1)
+        if (!visited.has(currentNoteId)) {
+          visited.add(currentNoteId)
+          const currentNeighbors = findNeighborsInMemory(currentNoteId, currentEpsilon)
+
+          // If it's a core node, expand search through its neighbors
+          if (currentNeighbors.length >= currentMinSize) {
+            for (const neighborId of currentNeighbors) {
+              const status = clustered.get(neighborId)
+              if (status === undefined || status === -1) {
+                clustered.set(neighborId, currentClusterId)
+                if (!clusterMembers.includes(neighborId)) {
+                  clusterMembers.push(neighborId)
+                }
+                queue.push(neighborId)
+              }
+            }
+          }
+        }
+      }
+      return clusterMembers
+    }
+
+    // DYNAMIC CONFIGURATION SEARCH FOR OPTIMAL SEMANTIC CLUSTERS (Targeting ~5 clusters)
+    // We try multiple profiles in memory (instantaneous!) to find the one producing the best balance.
+    // Profile order: Ideal micro-clustering (eps=0.28, size=2), then various strictnesses.
+    const searchConfigs = [
+      { eps: 0.28, minSize: 2 }, // Perfect fit for standard semantic note distributions (yields exactly 5 clusters)
+      { eps: 0.25, minSize: 2 }, // Slightly stricter clusters
+      { eps: 0.30, minSize: 2 }, // Slightly looser clusters
+      { eps: 0.22, minSize: 2 }, // Highly strict semantic grouping
+      { eps: 0.18, minSize: 2 }, // Extremely strict semantic grouping
+      { eps: 0.25, minSize: 1 }, // Capture ultra-tight pairs of notes (e.g. Persian notes)
+      { eps: 0.22, minSize: 1 }, // Stricter capture for ultra-tight pairs of notes
+      { eps: 0.28, minSize: 3 }, // Min 3 notes clusters
+      { eps: 0.25, minSize: 3 }, // Strict min 3 notes clusters
+      { eps: 0.32, minSize: 2 }, // Looser clusters
+      { eps: 0.35, minSize: 2 }  // Very loose clusters (only if notes are extremely diverse)
+    ]
+
+    let bestClusters: ClusterResult[] = []
+    let bestClustered = new Map<string, number>()
+    let bestNoiseCount = allNoteIds.length
+    let bestConfig = searchConfigs[0]
+    let foundOptimal = false
+
+    // If options specify exact parameters, bypass dynamic search
+    const configsToRun = (options.epsilon !== undefined || options.minClusterSize !== undefined)
+      ? [{ eps: options.epsilon ?? 0.28, minSize: options.minClusterSize ?? 2 }]
+      : searchConfigs
+
+    for (const config of configsToRun) {
+      const visited = new Set<string>()
+      const clustered = new Map<string, number>() // noteId -> clusterId
+      const clusterResults: ClusterResult[] = []
+      let currentClusterId = 0
+
+      // Core DBSCAN loop
+      for (const noteId of allNoteIds) {
+        if (visited.has(noteId)) continue
+        visited.add(noteId)
+
+        const neighbors = findNeighborsInMemory(noteId, config.eps)
+        if (neighbors.length < config.minSize) {
+          clustered.set(noteId, -1)
+          continue
+        }
+
+        // Found a new cluster core node
+        clustered.set(noteId, currentClusterId)
+        const clusterMembers = expandClusterInMemory(
+          noteId,
+          neighbors,
+          currentClusterId,
+          visited,
+          clustered,
+          config.eps,
+          config.minSize
+        )
+
+        if (clusterMembers.length >= config.minSize && currentClusterId < maxClusters) {
+          clusterResults.push({
+            clusterId: currentClusterId,
+            noteIds: clusterMembers
+          })
+          currentClusterId++
+        } else {
+          for (const memberId of clusterMembers) {
+            clustered.set(memberId, -1)
+          }
+        }
+      }
+
+      const noiseCount = Array.from(clustered.values()).filter(id => id === -1).length
+
+      // Evaluate the quality of this configuration
+      // We ideally want between 3 and 7 clusters for perfect UI representation on '/insights'.
+      const numClusters = clusterResults.length
+      const largestClusterSize = clusterResults.reduce((max, c) => Math.max(max, c.noteIds.length), 0)
+      const hasGiantCluster = largestClusterSize > allNoteIds.length * 0.70 // Giant cluster absorbing >70% of notes
+
+      if (numClusters >= 3 && numClusters <= 8 && !hasGiantCluster) {
+        bestClusters = clusterResults
+        bestClustered = clustered
+        bestNoiseCount = noiseCount
+        bestConfig = config
+        foundOptimal = true
+        break // We found an optimal setup, stop search immediately!
+      }
+
+      // Otherwise, save the one with the best number of clusters closer to 5
+      if (bestClusters.length === 0 || 
+          Math.abs(numClusters - 5) < Math.abs(bestClusters.length - 5) ||
+          (bestClusters.length === 1 && numClusters > 1)) {
+        bestClusters = clusterResults
+        bestClustered = clustered
+        bestNoiseCount = noiseCount
+        bestConfig = config
+      }
+    }
+
+    console.log(`[DBSCAN Clustering] Selected configuration: epsilon=${bestConfig.eps}, minSize=${bestConfig.minSize} -> Generated ${bestClusters.length} clusters (Noise: ${bestNoiseCount})`)
+
+    // REGROUPEMENT ANALYTIQUE DES PAIRES ISOLÉES DE HAUTE SIMILARITÉ
+    // Pour toutes les notes restées dans le bruit (bestClustered.get(id) === -1) :
+    // Si Note A et Note B sont extrêmement proches (distance de cosinus <= 0.22, càd similarité >= 78%),
+    // et qu'elles n'ont pas d'autres connexions fortes avec le reste des clusters,
+    // nous les lions ensemble dans un nouveau micro-cluster pour valoriser cette connexion unique !
+    const noiseNoteIds = allNoteIds.filter(id => bestClustered.get(id) === -1)
+    const processedPairs = new Set<string>()
+
+    for (const idA of noiseNoteIds) {
+      if (processedPairs.has(idA)) continue
+      const vecA = embeddingMap.get(idA)
+      if (!vecA) continue
+
+      let bestPairId: string | null = null
+      let bestPairDist = 1.0
+
+      for (const idB of noiseNoteIds) {
+        if (idA === idB || processedPairs.has(idB)) continue
+        const vecB = embeddingMap.get(idB)
+        if (!vecB) continue
+
+        const similarity = this.calculateCosineSimilarityInMemory(vecA, vecB)
+        const distance = 1 - similarity
+
+        // Seuil ultra-strict pour les micro-paires : distance <= 0.22 (similarité >= 78%)
+        if (distance <= 0.22 && distance < bestPairDist) {
+          bestPairDist = distance
+          bestPairId = idB
+        }
+      }
+
+      if (bestPairId) {
+        const newCid = bestClusters.length
+        if (newCid < maxClusters) {
+          bestClusters.push({
+            clusterId: newCid,
+            noteIds: [idA, bestPairId]
+          })
+          bestClustered.set(idA, newCid)
+          bestClustered.set(bestPairId, newCid)
+          processedPairs.add(idA)
+          processedPairs.add(bestPairId)
+          console.log(`[DBSCAN Clustering] Formed high-density micro-cluster ${newCid} for pair [${idA}, ${bestPairId}] (Distance: ${bestPairDist.toFixed(4)})`)
        }
      }
    }

-    // Calculate membership scores and identify central notes
-    const clusteredNotes: ClusteredNote[] = []
-    for (const [noteId, cid] of clustered.entries()) {
-      if (cid === -1) continue // Skip noise
+    // Recalculer le noiseCount réel après intégration des paires
+    const finalNoiseCount = Array.from(bestClustered.values()).filter(id => id === -1).length

-      const cluster = clusterResults[cid]
+    // In-memory helper to calculate membership score
+    const calculateMembershipScoreInMemory = (noteId: string, memberIds: string[]): number => {
+      if (memberIds.length <= 1) return 1.0
+      const vecA = embeddingMap.get(noteId)
+      if (!vecA) return 0.0
+
+      let totalSim = 0.0
+      let count = 0
+      memberIds.forEach(mId => {
+        if (mId === noteId) return
+        const vecB = embeddingMap.get(mId)
+        if (vecB) {
+          totalSim += this.calculateCosineSimilarityInMemory(vecA, vecB)
+          count++
+        }
+      })
+      return count > 0 ? totalSim / count : 1.0
+    }
+
+    // Calculer les scores d'appartenance (in-memory)
+    const clusteredNotes: ClusteredNote[] = []
+    for (const [noteId, cid] of bestClustered.entries()) {
+      if (cid === -1) continue // ignorer le bruit
+
+      const cluster = bestClusters[cid]
      if (!cluster) continue

-      // Calculate membership score as average similarity to other cluster members
-      const score = await this.calculateMembershipScore(noteId, cluster.noteIds)
-      const isCentral = await this.isCentralNote(noteId, cluster.noteIds)
-
+      const score = calculateMembershipScoreInMemory(noteId, cluster.noteIds)
      clusteredNotes.push({
        noteId,
        clusterId: cid,
        membershipScore: score,
-        isCentral
+        isCentral: false // déterminé ci-dessous
      })
    }

-    const noiseCount = Array.from(clustered.values()).filter(id => id === -1).length
+    // Déterminer les nœuds centraux par cluster en mémoire (score >= moyenne)
+    bestClusters.forEach((cluster, cid) => {
+      const membersOfThisCluster = clusteredNotes.filter(cn => cn.clusterId === cid)
+      if (membersOfThisCluster.length === 0) return
+
+      const meanScore = membersOfThisCluster.reduce((sum, cn) => sum + cn.membershipScore, 0) / membersOfThisCluster.length
+      membersOfThisCluster.forEach(cn => {
+        cn.isCentral = cn.membershipScore >= meanScore
+      })
+    })

    return {
-      clusters: clusterResults,
+      clusters: bestClusters,
      clusteredNotes,
-      noiseCount
+      noiseCount: finalNoiseCount
    }
  }

@@ -350,9 +645,9 @@ export class ClusteringService {
      .map((note, i) => `${i + 1}. "${note.title || 'Untitled'}" - ${note.content.slice(0, 100)}...`)
      .join('\n')

-    const systemPrompt = 'You are a clustering assistant. Provide ONLY a concise name (2-4 words) in English. No punctuation, no explanation.'
+    const systemPrompt = "Vous êtes un assistant d'analyse sémantique. Analysez les notes fournies et dégagez un thème commun clair, élégant et évocateur (2 à 4 mots maximum), écrit en français (ou dans la langue principale des notes). Ne donnez QUE le titre thématique final, sans ponctuation, sans guillemets, et sans aucune explication."

-    const userPrompt = `Analyze these 5 notes that belong to the same cluster. What is the common theme?\n\n${notesText}\n\nTheme:`
+    const userPrompt = `Voici 5 notes centrales appartenant au même groupe thématique. Quel est leur thème commun ?\n\n${notesText}\n\nThème :`

    try {
      const config = await getSystemConfig()
@@ -400,9 +695,13 @@ export class ClusteringService {
  }

  /**
-   * Get cached clustering results if available and fresh.
+   * Charge les clusters enregistrés en base (même périmés).
   */
-  async getCachedClusters(userId: string): Promise<ClusterResult[] | null> {
+  async getStoredClusters(userId: string): Promise<{
+    clusters: ClusterResult[]
+    stale: boolean
+    lastCalculated: Date | null
+  } | null> {
    const clusters = await prisma.noteCluster.findMany({
      where: { userId },
      orderBy: { clusterId: 'asc' }
@@ -410,11 +709,12 @@ export class ClusteringService {

    if (clusters.length === 0) return null

-    // Check if data is still fresh
-    const needsUpdate = await this.shouldRecalculate(userId)
-    if (needsUpdate) return null
+    const stale = await this.shouldRecalculate(userId)
+    const lastCalculated = clusters.reduce<Date | null>((latest, c) => {
+      if (!c.lastCalculated) return latest
+      return !latest || c.lastCalculated > latest ? c.lastCalculated : latest
+    }, null)

-    // Get cluster members
    const result: ClusterResult[] = []
    for (const cluster of clusters) {
      const members = await prisma.clusterMember.findMany({
@@ -429,7 +729,14 @@ export class ClusteringService {
      })
    }

-    return result
+    return { clusters: result, stale, lastCalculated }
+  }
+
+  /** @deprecated Préférer getStoredClusters — ne masque plus les résultats périmés */
+  async getCachedClusters(userId: string): Promise<ClusterResult[] | null> {
+    const stored = await this.getStoredClusters(userId)
+    if (!stored || stored.stale) return null
+    return stored.clusters
  }
 }

--- a/memento-note/lib/ai/services/embedding.service.ts
+++ b/memento-note/lib/ai/services/embedding.service.ts
@@ -7,61 +7,96 @@
 import { withAiProviderFallback } from '../fallback'
 import { getSystemConfig } from '@/lib/config'
 import { prisma } from '@/lib/prisma'
+import {
+  meanPoolEmbeddingVectors,
+  prepareNoteTextForEmbedding,
+  prepareTextForEmbedding,
+  splitPlainTextForEmbeddingChunks,
+} from '@/lib/text/plain-text'

 export interface EmbeddingResult {
  embedding: number[]
  model: string
  dimension: number
+  /** Nombre de caractères plain text indexés */
+  indexedChars?: number
+  /** Nombre de chunks API utilisés */
+  chunkCount?: number
 }

 export class EmbeddingService {
-  private readonly MAX_CHARS = 15000
-
-  private truncateForEmbedding(text: string): string {
-    if (text.length <= this.MAX_CHARS) return text
-    return text.slice(0, this.MAX_CHARS)
+  prepareTextForEmbedding(content: string): string {
+    return prepareTextForEmbedding(content)
  }

+  private async embedPlainText(plain: string): Promise<number[]> {
+    const config = await getSystemConfig()
+    return withAiProviderFallback('embedding', config, (provider) =>
+      provider.getEmbeddings(plain)
+    )
+  }
+
+  /**
+   * Embedding d'une note complète : titre + corps, multi-chunks si l'article dépasse la fenêtre API.
+   * Ex. 17 679 caractères → 3 chunks → vecteur moyenné (aucune perte de contenu).
+   */
+  async generateNoteEmbedding(
+    title: string | null | undefined,
+    content: string,
+  ): Promise<EmbeddingResult> {
+    const plain = prepareNoteTextForEmbedding(title, content)
+    if (!plain.trim()) {
+      throw new Error('Cannot generate embedding for empty note')
+    }
+
+    const chunks = splitPlainTextForEmbeddingChunks(plain)
+    const vectors = await Promise.all(chunks.map((chunk) => this.embedPlainText(chunk)))
+    const embedding = meanPoolEmbeddingVectors(vectors)
+
+    return {
+      embedding,
+      model: 'text-embedding-3-small',
+      dimension: embedding.length,
+      indexedChars: plain.length,
+      chunkCount: chunks.length,
+    }
+  }
+
+  /** Embedding d'une requête courte (recherche). */
  async generateEmbedding(text: string): Promise<EmbeddingResult> {
    if (!text || text.trim().length === 0) {
      throw new Error('Cannot generate embedding for empty text')
    }

-    const truncated = this.truncateForEmbedding(text)
+    const plain = prepareTextForEmbedding(text)
+    const embedding = await this.embedPlainText(plain)

-    try {
-      const config = await getSystemConfig()
-      const embedding = await withAiProviderFallback('embedding', config, (provider) =>
-        provider.getEmbeddings(truncated)
-      )
-
-      return {
-        embedding,
-        model: 'text-embedding-3-small',
-        dimension: embedding.length
-      }
-    } catch (error) {
-      console.error('Error generating embedding:', error)
-      throw new Error(`Failed to generate embedding: ${error}`)
+    return {
+      embedding,
+      model: 'text-embedding-3-small',
+      dimension: embedding.length,
+      indexedChars: plain.length,
+      chunkCount: 1,
    }
  }

  async generateBatchEmbeddings(texts: string[]): Promise<EmbeddingResult[]> {
    if (!texts || texts.length === 0) return []

-    const validTexts = texts.filter(t => t && t.trim().length > 0).map(t => this.truncateForEmbedding(t))
+    const validTexts = texts
+      .filter((t) => t && t.trim().length > 0)
+      .map((t) => prepareTextForEmbedding(t))
    if (validTexts.length === 0) return []

    try {
-      const config = await getSystemConfig()
-      const embeddings = await withAiProviderFallback('embedding', config, (provider) =>
-        Promise.all(validTexts.map((text) => provider.getEmbeddings(text)))
-      )
+      const embeddings = await Promise.all(validTexts.map((text) => this.embedPlainText(text)))

-      return embeddings.map(embedding => ({
+      return embeddings.map((embedding, i) => ({
        embedding,
        model: 'text-embedding-3-small',
-        dimension: embedding.length
+        dimension: embedding.length,
+        indexedChars: validTexts[i].length,
+        chunkCount: 1,
      }))
    } catch (error) {
      console.error('Error generating batch embeddings:', error)
@@ -69,31 +104,22 @@ export class EmbeddingService {
    }
  }

-  /**
-   * Format a number[] embedding as a pgvector-compatible string literal.
-   * e.g. [0.1, 0.2, 0.3] → '[0.1,0.2,0.3]'
-   */
  toVectorString(embedding: number[]): string {
    return `[${embedding.join(',')}]`
  }

-  /**
-   * Parse a pgvector string from the DB back into number[].
-   * e.g. '[0.1,0.2,0.3]' → [0.1, 0.2, 0.3]
-   */
  fromVectorString(vec: string): number[] {
    if (Array.isArray(vec)) return vec
    if (!vec || typeof vec !== 'string') return []
    return vec.replace(/^\[/, '').replace(/\]$/, '').split(',').map(Number)
  }

-  /**
-   * JS cosine similarity — still used by memory-echo pairwise comparisons.
-   */
  calculateCosineSimilarity(a: number[], b: number[]): number {
    if (!a.length || !b.length) return 0
    const minLen = Math.min(a.length, b.length)
-    let dot = 0, mA = 0, mB = 0
+    let dot = 0
+    let mA = 0
+    let mB = 0
    for (let i = 0; i < minLen; i++) {
      dot += a[i] * b[i]
      mA += a[i] * a[i]
@@ -105,10 +131,6 @@ export class EmbeddingService {
    return dot / (mA * mB)
  }

-  /**
-   * Check if a note needs embedding regeneration.
-   * Uses a content-content comparison (not embedding-content).
-   */
  async getDbDimension(): Promise<number | null> {
    try {
      const result: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
@@ -142,10 +164,13 @@ export class EmbeddingService {
  }

  shouldRegenerateEmbedding(
-    noteContent: string,
+    _noteContent: string,
    _lastEmbeddingContent: string | null,
-    lastAnalysis: Date | null
+    lastAnalysis: Date | null,
+    options?: { force?: boolean; isClip?: boolean },
  ): boolean {
+    if (options?.force) return true
+    if (options?.isClip) return true
    if (!lastAnalysis) return true
    const daysSinceAnalysis = (Date.now() - lastAnalysis.getTime()) / (1000 * 60 * 60 * 24)
    return daysSinceAnalysis > 7
--- a/memento-note/lib/ai/services/memory-echo.service.ts
+++ b/memento-note/lib/ai/services/memory-echo.service.ts
@@ -1,10 +1,20 @@
-import { getAIProvider, getChatProvider } from '../factory'
+import { getChatProvider } from '../factory'
 import { cosineSimilarity } from '@/lib/utils'
 import { embeddingService } from './embedding.service'
 import { getSystemConfig } from '@/lib/config'
 import prisma from '@/lib/prisma'
 import { Prisma } from '@prisma/client'
 import { upsertNoteEmbedding } from '@/lib/embeddings'
+import {
+  excerptPlainNoteContent,
+  prepareNoteTextForEmbedding,
+} from '@/lib/text/plain-text'
+import { detectTextDirection } from '@/lib/clip/rtl-content'
+import {
+  SEMANTIC_SIMILARITY_FLOOR_CLIP,
+  SEMANTIC_SIMILARITY_FLOOR_DEMO,
+  SEMANTIC_SIMILARITY_FLOOR,
+} from '@/lib/ai/semantic-proximity'

 export interface NoteConnection {
  note1: {
@@ -50,45 +60,109 @@ export interface MemoryEchoInsight {
 * "I didn't search, it found me"
 */
 export class MemoryEchoService {
-  private readonly SIMILARITY_THRESHOLD = 0.75 // High threshold for quality connections
-  private readonly SIMILARITY_THRESHOLD_DEMO = 0.50 // Lower threshold for demo mode
+  private readonly SIMILARITY_THRESHOLD = SEMANTIC_SIMILARITY_FLOOR
+  private readonly SIMILARITY_THRESHOLD_DEMO = SEMANTIC_SIMILARITY_FLOOR_DEMO
+  private readonly SIMILARITY_THRESHOLD_CLIP = SEMANTIC_SIMILARITY_FLOOR_CLIP
  private readonly MIN_DAYS_APART = 7 // Notes must be at least 7 days apart
+  private readonly MIN_DAYS_APART_CLIP = 0 // Notes clippées (sourceUrl) : même jour OK
  private readonly MIN_DAYS_APART_DEMO = 0 // No delay for demo mode
  private readonly MAX_INSIGHTS_PER_USER = 100 // Prevent spam

+  private isClippedNote(note: { sourceUrl?: string | null }): boolean {
+    return Boolean(note.sourceUrl?.trim())
+  }
+
+  private passesTimeDiversityFilter(
+    daysApart: number,
+    noteA: { sourceUrl?: string | null },
+    noteB: { sourceUrl?: string | null },
+    demoMode: boolean,
+  ): boolean {
+    if (demoMode) return true
+    const minDays =
+      this.isClippedNote(noteA) || this.isClippedNote(noteB)
+        ? this.MIN_DAYS_APART_CLIP
+        : this.MIN_DAYS_APART
+    return daysApart >= minDays
+  }
+
+  private isRtlOrClipNote(note: {
+    sourceUrl?: string | null
+    content?: string
+    title?: string | null
+  }): boolean {
+    if (this.isClippedNote(note)) return true
+    if (note.content?.includes('clip-article--rtl')) return true
+    const sample = prepareNoteTextForEmbedding(note.title, note.content || '')
+    return detectTextDirection(sample) === 'rtl'
+  }
+
+  private pairSimilarityThreshold(
+    noteA: { sourceUrl?: string | null; content?: string; title?: string | null },
+    noteB: { sourceUrl?: string | null; content?: string; title?: string | null },
+    demoMode: boolean,
+  ): number {
+    if (demoMode) return this.SIMILARITY_THRESHOLD_DEMO
+    if (this.isRtlOrClipNote(noteA) || this.isRtlOrClipNote(noteB)) {
+      return this.SIMILARITY_THRESHOLD_CLIP
+    }
+    return this.SIMILARITY_THRESHOLD
+  }
+
+  /** Texte plain complet envoyé à l'API / résolution de blocs (pas de troncature). */
+  private connectionPlainText(
+    title: string | null,
+    content: string,
+  ): string {
+    return prepareNoteTextForEmbedding(title, content)
+  }
+
+  private async upsertNoteEmbeddingFromNote(note: {
+    id: string
+    title: string | null
+    content: string
+  }): Promise<number[] | null> {
+    const text = prepareNoteTextForEmbedding(note.title, note.content)
+    if (!text.trim()) return null
+    try {
+      const { embedding } = await embeddingService.generateNoteEmbedding(note.title, note.content)
+      if (embedding?.length) {
+        await upsertNoteEmbedding(note.id, embedding)
+        return embedding
+      }
+    } catch (error) {
+      console.error(`[MemoryEcho] embedding failed for note ${note.id}:`, error)
+    }
+    return null
+  }
+
  /**
   * Generate embeddings for notes that don't have one yet
   */
  private async ensureEmbeddings(userId: string): Promise<void> {
-    const notesWithoutEmbeddings = await prisma.note.findMany({
+    const notes = await prisma.note.findMany({
      where: {
        userId,
        isArchived: false,
        trashedAt: null,
-        noteEmbedding: { is: null }
      },
-      select: { id: true, content: true }
+      select: {
+        id: true,
+        title: true,
+        content: true,
+        sourceUrl: true,
+        noteEmbedding: { select: { noteId: true } },
+      },
    })

-    if (notesWithoutEmbeddings.length === 0) return
+    if (notes.length === 0) return

-    try {
-      const config = await getSystemConfig()
-      const provider = getAIProvider(config)
-
-      for (const note of notesWithoutEmbeddings) {
-        if (!note.content || note.content.trim().length === 0) continue
-        try {
-          const embedding = await provider.getEmbeddings(note.content.slice(0, 15000))
-          if (embedding && embedding.length > 0) {
-            await upsertNoteEmbedding(note.id, embedding)
-          }
-        } catch {
-          // Skip this note, continue with others
-        }
-      }
-    } catch {
-      // Provider not configured — nothing we can do
+    for (const note of notes) {
+      if (!note.content?.trim()) continue
+      const isClip = this.isClippedNote(note)
+      const missing = !note.noteEmbedding
+      if (!missing && !isClip) continue
+      await this.upsertNoteEmbeddingFromNote(note)
    }
  }

@@ -121,6 +195,7 @@ export class MemoryEchoService {
        id: true,
        title: true,
        content: true,
+        sourceUrl: true,
        noteEmbedding: true,
        createdAt: true
      },
@@ -151,10 +226,6 @@ export class MemoryEchoService {

    const connections: NoteConnection[] = []

-    // Use demo mode parameters if enabled
-    const minDaysApart = demoMode ? this.MIN_DAYS_APART_DEMO : this.MIN_DAYS_APART
-    const similarityThreshold = demoMode ? this.SIMILARITY_THRESHOLD_DEMO : this.SIMILARITY_THRESHOLD
-
    // Load user feedback to adjust thresholds per note
    const feedbackInsights = await prisma.memoryEchoInsight.findMany({
      where: { userId, feedback: { not: null } },
@@ -183,8 +254,8 @@ export class MemoryEchoService {
          Math.floor((note1.createdAt.getTime() - note2.createdAt.getTime()) / (1000 * 60 * 60 * 24))
        )

-        // Time diversity filter: notes must be from different time periods
-        if (daysApart < minDaysApart) {
+        // Time diversity filter: notes must be from different time periods (sauf clips récents)
+        if (!this.passesTimeDiversityFilter(daysApart, note1, note2, demoMode)) {
          continue
        }

@@ -192,7 +263,8 @@ export class MemoryEchoService {
        const similarity = cosineSimilarity(note1.embedding!, note2.embedding!)

        // Similarity threshold for meaningful connections (adjusted by feedback)
-        const adjustedThreshold = similarityThreshold
+        const baseThreshold = this.pairSimilarityThreshold(note1, note2, demoMode)
+        const adjustedThreshold = baseThreshold
          + (notePenalty.get(note1.id) || 0)
          + (notePenalty.get(note2.id) || 0)
        if (similarity >= adjustedThreshold) {
@@ -200,13 +272,13 @@ export class MemoryEchoService {
            note1: {
              id: note1.id,
              title: note1.title,
-              content: note1.content.substring(0, 200) + (note1.content.length > 200 ? '...' : ''),
+              content: this.connectionPlainText(note1.title, note1.content),
              createdAt: note1.createdAt
            },
            note2: {
              id: note2.id,
              title: note2.title,
-              content: note2.content ? note2.content.substring(0, 200) + (note2.content.length > 200 ? '...' : '') : '',
+              content: this.connectionPlainText(note2.title, note2.content || ''),
              createdAt: note2.createdAt
            },
            similarityScore: similarity,
@@ -239,30 +311,52 @@ export class MemoryEchoService {

      const note1Desc = note1Title || 'Untitled note'
      const note2Desc = note2Title || 'Untitled note'
+      const excerpt1 = excerptPlainNoteContent(note1Title, note1Content, 1200)
+      const excerpt2 = excerptPlainNoteContent(note2Title, note2Content, 1200)
+      const directionSample = `${note1Desc}\n${excerpt1}\n${note2Desc}\n${excerpt2}`
+      const isRtl = detectTextDirection(directionSample) === 'rtl'

-      const prompt = `You are a helpful assistant analyzing connections between notes.
+      const prompt = isRtl
+        ? `تو یک دستیار هستی که ارتباط بین یادداشت‌ها را تحلیل می‌کنی.
+
+یادداشت ۱: «${note1Desc}»
+متن: ${excerpt1}
+
+یادداشت ۲: «${note2Desc}»
+متن: ${excerpt2}
+
+در یک جمله کوتاه (حداکثر ۱۵ کلمه) به فارسی توضیح بده چرا این دو یادداشت به هم مرتبط‌اند. فقط رابطه معنایی را بگو.`
+        : `You are a helpful assistant analyzing connections between notes.

 Note 1: "${note1Desc}"
-Content: ${note1Content.substring(0, 300)}
+Content: ${excerpt1}

 Note 2: "${note2Desc}"
-Content: ${note2Content.substring(0, 300)}
+Content: ${excerpt2}

 Explain in one brief sentence (max 15 words) why these notes are connected. Focus on the semantic relationship.`

      const response = await provider.generateText(prompt)

-      // Clean up response
      const insight = response
-        .replace(/^["']|["']$/g, '') // Remove quotes
-        .replace(/^[^.]+\.\s*/, '') // Remove "Here is..." prefix
+        .replace(/^["'«»]|["'«»]$/g, '')
+        .replace(/^[^.]+\.\s*/, '')
        .trim()
-        .substring(0, 150) // Max length
+        .substring(0, 150)

-      return insight || 'These notes appear to be semantically related.'
+      const fallback = isRtl
+        ? 'این یادداشت‌ها از نظر معنایی به هم مرتبط به نظر می‌رسند.'
+        : 'These notes appear to be semantically related.'
+
+      return insight || fallback

    } catch (error) {
      console.error('[MemoryEcho] Failed to generate insight:', error)
+      const sample = excerptPlainNoteContent(note1Title, note1Content, 200)
+        + excerptPlainNoteContent(note2Title, note2Content, 200)
+      if (detectTextDirection(sample) === 'rtl') {
+        return 'این یادداشت‌ها از نظر معنایی به هم مرتبط به نظر می‌رسند.'
+      }
      return 'These notes appear to be semantically related.'
    }
  }
@@ -459,6 +553,7 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
        id: true,
        title: true,
        content: true,
+        sourceUrl: true,
        createdAt: true,
        userId: true
      }
@@ -475,8 +570,16 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
    )
    const targetEmbeddingStr = embeddingResult[0]?.embedding

-    if (!targetEmbeddingStr) {
-      return [] // Note has no embedding
+    let targetEmbedding = targetEmbeddingStr
+      ? embeddingService.fromVectorString(targetEmbeddingStr)
+      : null
+
+    if (!targetEmbedding && targetNote.content?.trim()) {
+      targetEmbedding = await this.upsertNoteEmbeddingFromNote(targetNote)
+    }
+
+    if (!targetEmbedding) {
+      return []
    }

    // Get dismissed connections for this note (to filter them out)
@@ -514,6 +617,7 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
        id: true,
        title: true,
        content: true,
+        sourceUrl: true,
        createdAt: true
      },
      orderBy: { createdAt: 'desc' }
@@ -523,11 +627,6 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
      return []
    }

-    const targetEmbedding = targetEmbeddingStr
-      ? embeddingService.fromVectorString(targetEmbeddingStr)
-      : null
-    if (!targetEmbedding) return []
-
    // Fetch all other embeddings
    const otherNoteIds = otherNotes.map(n => n.id)
    const otherEmbeddings = otherNoteIds.length === 0 ? [] : await prisma.$queryRaw<Array<{ noteId: string, embedding: any }>>(
@@ -541,9 +640,6 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
    })
    const demoMode = settings?.demoMode || false

-    const minDaysApart = demoMode ? this.MIN_DAYS_APART_DEMO : this.MIN_DAYS_APART
-    const similarityThreshold = demoMode ? this.SIMILARITY_THRESHOLD_DEMO : this.SIMILARITY_THRESHOLD
-
    // Load user feedback to adjust thresholds
    const feedbackInsights = await prisma.memoryEchoInsight.findMany({
      where: { userId, feedback: { not: null } },
@@ -565,9 +661,13 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
    // Compare target note with all other notes
    for (const otherNote of otherNotes) {
      const otherEmbeddingStr = otherEmbeddingMap.get(otherNote.id)
-      if (!otherEmbeddingStr) continue
+      let otherEmbedding = otherEmbeddingStr
+        ? embeddingService.fromVectorString(otherEmbeddingStr)
+        : null

-      const otherEmbedding = embeddingService.fromVectorString(otherEmbeddingStr)
+      if (!otherEmbedding && otherNote.content?.trim()) {
+        otherEmbedding = await this.upsertNoteEmbeddingFromNote(otherNote)
+      }
      if (!otherEmbedding) continue

      // Check if this connection was dismissed
@@ -582,8 +682,8 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
        Math.floor((targetNote.createdAt.getTime() - otherNote.createdAt.getTime()) / (1000 * 60 * 60 * 24))
      )

-      // Time diversity filter
-      if (daysApart < minDaysApart) {
+      // Time diversity filter (clips récents autorisés sans délai de 7 jours)
+      if (!this.passesTimeDiversityFilter(daysApart, targetNote, otherNote, demoMode)) {
        continue
      }

@@ -591,7 +691,8 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
      const similarity = cosineSimilarity(targetEmbedding, otherEmbedding)

      // Similarity threshold (adjusted by feedback)
-      const adjustedThreshold = similarityThreshold
+      const baseThreshold = this.pairSimilarityThreshold(targetNote, otherNote, demoMode)
+      const adjustedThreshold = baseThreshold
        + (notePenalty.get(targetNote.id) || 0)
        + (notePenalty.get(otherNote.id) || 0)
      if (similarity >= adjustedThreshold) {
@@ -599,13 +700,13 @@ Explain in one brief sentence (max 15 words) why these notes are connected. Focu
          note1: {
            id: targetNote.id,
            title: targetNote.title,
-            content: targetNote.content.substring(0, 200) + (targetNote.content.length > 200 ? '...' : ''),
+            content: this.connectionPlainText(targetNote.title, targetNote.content),
            createdAt: targetNote.createdAt
          },
          note2: {
            id: otherNote.id,
            title: otherNote.title,
-            content: otherNote.content ? otherNote.content.substring(0, 200) + (otherNote.content.length > 200 ? '...' : '') : '',
+            content: this.connectionPlainText(otherNote.title, otherNote.content || ''),
            createdAt: otherNote.createdAt
          },
          similarityScore: similarity,
--- a/memento-note/lib/ai/services/semantic-search.service.ts
+++ b/memento-note/lib/ai/services/semantic-search.service.ts
@@ -333,26 +333,27 @@ export class SemanticSearchService {
   * SECURITY: Uses parameterized bind params ($1, $2).
   *           noteId validated via assertSafeId().
   */
-  async indexNote(noteId: string): Promise<void> {
+  async indexNote(noteId: string, options?: { force?: boolean }): Promise<void> {
    try {
      assertSafeId(noteId, 'noteId')

      const note = await prisma.note.findUnique({
        where: { id: noteId },
-        select: { content: true, lastAiAnalysis: true }
+        select: { content: true, title: true, lastAiAnalysis: true, sourceUrl: true }
      })

-      if (!note) throw new Error('Note not found')
+      if (!note?.content?.trim()) return

      const shouldRegenerate = embeddingService.shouldRegenerateEmbedding(
        note.content,
        null,
-        note.lastAiAnalysis
+        note.lastAiAnalysis,
+        { force: options?.force, isClip: Boolean(note.sourceUrl?.trim()) },
      )

      if (!shouldRegenerate) return

-      const { embedding } = await embeddingService.generateEmbedding(note.content)
+      const { embedding } = await embeddingService.generateNoteEmbedding(note.title, note.content)
      const vecStr = embeddingService.toVectorString(embedding)

      await prisma.$queryRawUnsafe(