feat(insights): fix DBSCAN, Persian embeddings crash, D3 physics layouts, and D3 node not found runtime error

memento-note/app/api/clip/analyze/route.ts

@@ -0,0 +1,101 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { auth } from '@/auth'
+import { extractArticleFromHtml } from '@/lib/clip/extract-article'
+import { analyzeClipContent } from '@/lib/clip/analyze-clip'
+import { resolveClipLocale, wrapClipPlainParagraph } from '@/lib/clip/rtl-content'
+
+function isBlockedUrl(url: string): boolean {
+  try {
+    const parsed = new URL(url)
+    const hostname = parsed.hostname.toLowerCase()
+    const blocked = ['localhost', '127.0.0.1', '0.0.0.0', '::1', '169.254.169.254']
+    if (blocked.includes(hostname)) return true
+    if (hostname.startsWith('10.') || hostname.startsWith('172.') || hostname.startsWith('192.168.')) return true
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return true
+    return false
+  } catch {
+    return true
+  }
+}
+
+async function fetchPageHtml(url: string): Promise<string | null> {
+  const controller = new AbortController()
+  const timeoutId = setTimeout(() => controller.abort(), 15000)
+  try {
+    const response = await fetch(url, {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (compatible; MementoClipper/1.0)',
+        Accept: 'text/html,application/xhtml+xml',
+      },
+      signal: controller.signal,
+      redirect: 'follow',
+    })
+    if (!response.ok) return null
+    const ct = response.headers.get('content-type') || ''
+    if (!ct.includes('text/html') && !ct.includes('application/xhtml')) return null
+    return await response.text()
+  } catch {
+    return null
+  } finally {
+    clearTimeout(timeoutId)
+  }
+}
+
+export async function POST(request: NextRequest) {
+  try {
+    const session = await auth()
+    if (!session?.user?.id) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    const body = await request.json()
+    const url = typeof body.url === 'string' ? body.url.trim() : ''
+    const htmlInput = typeof body.html === 'string' ? body.html : ''
+    const selection = typeof body.selection === 'string' ? body.selection.trim() : ''
+
+    if (!url || isBlockedUrl(url)) {
+      return NextResponse.json({ error: 'Invalid URL' }, { status: 400 })
+    }
+
+    let title = ''
+    let textContent = ''
+    let contentHtml = ''
+
+    if (body.mode === 'link') {
+      title = new URL(url).hostname
+      textContent = url
+      contentHtml = `<p><a href="${url}" rel="noopener noreferrer">${url}</a></p>`
+    } else if (body.mode === 'selection' && selection) {
+      title = typeof body.title === 'string' ? body.title : new URL(url).hostname
+      textContent = selection
+      const locale = resolveClipLocale(url, title, selection)
+      contentHtml = wrapClipPlainParagraph(selection, locale)
+    } else {
+      const html = htmlInput || (await fetchPageHtml(url))
+      if (!html) {
+        return NextResponse.json({ error: 'Could not fetch page content' }, { status: 422 })
+      }
+      const extracted = extractArticleFromHtml(html, url)
+      if (!extracted) {
+        return NextResponse.json({ error: 'Could not extract readable article' }, { status: 422 })
+      }
+      title = extracted.title || (typeof body.title === 'string' ? body.title : '')
+      textContent = extracted.textContent
+      contentHtml = extracted.content
+    }
+
+    const analysis = await analyzeClipContent({ url, title, textContent })
+
+    return NextResponse.json({
+      title: analysis.title,
+      summary: analysis.summary,
+      tags: analysis.tags,
+      readingTime: analysis.readingTimeMinutes,
+      content: contentHtml,
+      excerpt: textContent.slice(0, 500),
+    })
+  } catch (error) {
+    console.error('[POST /api/clip/analyze]', error)
+    return NextResponse.json({ error: 'Analysis failed' }, { status: 500 })
+  }
+}

memento-note/app/api/clip/notebooks/route.ts

@@ -0,0 +1,24 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { auth } from '@/auth'
+import prisma from '@/lib/prisma'
+
+/** Liste hiérarchique des carnets pour le clipper (extension). */
+export async function GET(_request: NextRequest) {
+  try {
+    const session = await auth()
+    if (!session?.user?.id) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    const notebooks = await prisma.notebook.findMany({
+      where: { userId: session.user.id, trashedAt: null },
+      select: { id: true, name: true, parentId: true, color: true },
+      orderBy: [{ order: 'asc' }, { name: 'asc' }],
+    })
+
+    return NextResponse.json({ notebooks })
+  } catch (error) {
+    console.error('[GET /api/clip/notebooks]', error)
+    return NextResponse.json({ error: 'Failed to load notebooks' }, { status: 500 })
+  }
+}

memento-note/app/api/clip/save/route.ts

@@ -0,0 +1,120 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { auth } from '@/auth'
+import prisma from '@/lib/prisma'
+import { syncNoteLabels } from '@/app/actions/notes'
+import { createNotification } from '@/app/actions/notifications'
+import { buildClipSourceFooter, clipFooterLocaleTag } from '@/lib/clip/extract-article'
+import { resolveClipLocale, wrapClipArticleHtml, applyRtlToHtmlBlocks } from '@/lib/clip/rtl-content'
+import { embeddingService } from '@/lib/ai/services/embedding.service'
+import { upsertNoteEmbedding } from '@/lib/embeddings'
+import DOMPurify from 'isomorphic-dompurify'
+
+function isBlockedUrl(url: string): boolean {
+  try {
+    const parsed = new URL(url)
+    const hostname = parsed.hostname.toLowerCase()
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return true
+    return ['localhost', '127.0.0.1', '0.0.0.0', '::1'].includes(hostname)
+  } catch {
+    return true
+  }
+}
+
+export async function POST(request: NextRequest) {
+  try {
+    const session = await auth()
+    if (!session?.user?.id) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    const body = await request.json()
+    const url = typeof body.url === 'string' ? body.url.trim() : ''
+    const title = typeof body.title === 'string' ? body.title.trim().slice(0, 300) : null
+    const rawContent = typeof body.content === 'string' ? body.content : ''
+    const summary = typeof body.summary === 'string' ? body.summary.trim() : ''
+    const notebookId = typeof body.notebookId === 'string' ? body.notebookId : null
+    const tags = Array.isArray(body.tags)
+      ? body.tags.filter((t: unknown): t is string => typeof t === 'string').slice(0, 5)
+      : []
+
+    if (!url || isBlockedUrl(url)) {
+      return NextResponse.json({ error: 'Invalid URL' }, { status: 400 })
+    }
+    if (!rawContent.trim()) {
+      return NextResponse.json({ error: 'Content required' }, { status: 400 })
+    }
+
+    await prisma.user.upsert({
+      where: { id: session.user.id },
+      update: {
+        ...(session.user.email ? { email: session.user.email } : {}),
+        ...(session.user.name !== undefined ? { name: session.user.name } : {}),
+      },
+      create: {
+        id: session.user.id,
+        email: session.user.email || `user-${session.user.id}@local.momento`,
+        name: session.user.name || null,
+      },
+    })
+
+    const domain = new URL(url).hostname.replace(/^www\./, '')
+    const locale = resolveClipLocale(url, title || '', summary, rawContent.replace(/<[^>]+>/g, ' '))
+    const sanitizedContent = DOMPurify.sanitize(rawContent)
+    const rtlBlocks = applyRtlToHtmlBlocks(sanitizedContent, locale)
+    const summaryBlock = summary
+      ? `<p dir="${locale.direction}"${locale.lang ? ` lang="${locale.lang}"` : ''}><em>${DOMPurify.sanitize(summary)}</em></p>`
+      : ''
+    const footer = buildClipSourceFooter(domain, new Date(), clipFooterLocaleTag(locale.lang))
+    const bodyHtml = rtlBlocks.includes('clip-article--rtl')
+      ? rtlBlocks
+      : wrapClipArticleHtml(rtlBlocks, locale)
+    const fullContent = `${summaryBlock}${bodyHtml}${footer}`
+
+    const note = await prisma.note.create({
+      data: {
+        userId: session.user.id,
+        title: title || domain,
+        content: fullContent,
+        type: 'richtext',
+        notebookId,
+        sourceUrl: url,
+        autoGenerated: true,
+        ...(locale.lang ? { language: locale.lang } : {}),
+      },
+    })
+
+    void (async () => {
+      try {
+        const { embedding } = await embeddingService.generateNoteEmbedding(
+          title || domain,
+          fullContent,
+        )
+        if (embedding?.length) {
+          await upsertNoteEmbedding(note.id, embedding)
+        }
+      } catch (error) {
+        console.error('[clip/save] embedding generation failed:', error)
+      }
+    })()
+
+    if (tags.length > 0) {
+      await syncNoteLabels(note.id, tags, notebookId, session.user.id)
+    }
+
+    const noteUrl = `/home?openNote=${encodeURIComponent(note.id)}`
+
+    await createNotification({
+      userId: session.user.id,
+      type: 'clip',
+      title: title || domain,
+      message: summary || undefined,
+      actionUrl: noteUrl,
+      relatedId: note.id,
+    })
+
+    return NextResponse.json({ noteId: note.id, noteUrl })
+  } catch (error) {
+    console.error('[POST /api/clip/save]', error)
+    return NextResponse.json({ error: 'Save failed' }, { status: 500 })
+  }
+}

memento-note/app/api/clusters/route.ts

@@ -7,6 +7,7 @@ import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
 /**
  * GET /api/clusters
  * Get all clusters for the current user.
+ * Returns cached clusters + bridge notes enriched with cluster names.
  */
 export async function GET(request: NextRequest) {
   try {
@@ -17,9 +18,10 @@ export async function GET(request: NextRequest) {
 
     const userId = session.user.id
 
-    // Check for cached results
-    const cached = await clusteringService.getCachedClusters(userId)
-    if (cached) {
+    // Check for stored results (even if stale/périmés)
+    const stored = await clusteringService.getStoredClusters(userId)
+    if (stored) {
+      const cached = stored.clusters
       // Fetch notes with their cluster assignments
       const notes = await prisma.note.findMany({
         where: { userId, trashedAt: null },
@@ -29,20 +31,65 @@ export async function GET(request: NextRequest) {
       // Get cluster member mappings
       const clusterMembers = await prisma.clusterMember.findMany({
         where: { userId },
-        select: { noteId: true, clusterId: true }
+        select: { noteId: true, clusterId: true, isCentral: true }
       })
 
-      const noteClusterMap = new Map(clusterMembers.map(cm => [cm.noteId, cm.clusterId]))
-      const notesWithClusters = notes.map(n => ({
-        ...n,
-        clusterId: noteClusterMap.get(n.id)
-      }))
+      const noteClusterMap = new Map(clusterMembers.map(cm => [cm.noteId, { clusterId: cm.clusterId, isCentral: cm.isCentral }]))
+      const notesWithClusters = notes.map(n => {
+        const mapping = noteClusterMap.get(n.id)
+        return {
+          ...n,
+          clusterId: mapping?.clusterId,
+          isCentral: mapping?.isCentral || false
+        }
+      })
+
+      // Fetch bridge notes with enrichment (cluster names + note details)
+      const bridgeNotesData = await prisma.bridgeNote.findMany({
+        where: { userId },
+        orderBy: { bridgeScore: 'desc' },
+        take: 10
+      })
+
+      let enrichedBridgeNotes: object[] = []
+      if (bridgeNotesData.length > 0) {
+        const bridgeNoteIds = bridgeNotesData.map(b => b.noteId)
+        const bridgeNoteDetails = await prisma.note.findMany({
+          where: { id: { in: bridgeNoteIds } },
+          select: { id: true, title: true, content: true }
+        })
+        const bridgeNoteDetailsMap = new Map(bridgeNoteDetails.map(n => [n.id, n]))
+
+        enrichedBridgeNotes = bridgeNotesData.map(b => {
+          const clustersConnected = JSON.parse(b.clustersConnected) as number[]
+          return {
+            noteId: b.noteId,
+            bridgeScore: b.bridgeScore,
+            clustersConnected,
+            clusterNames: clustersConnected.map(
+              cid => cached.find(c => c.clusterId === cid)?.name || `Cluster ${cid}`
+            ),
+            note: bridgeNoteDetailsMap.get(b.noteId)
+          }
+        })
+      }
+
+      const embeddingCountRow = await prisma.$queryRawUnsafe<Array<{ count: bigint }>>(
+        `SELECT COUNT(*) FROM "NoteEmbedding" ne
+         INNER JOIN "Note" n ON n.id = ne."noteId"
+         WHERE n."userId" = $1 AND n."trashedAt" IS NULL`,
+        userId
+      )
 
       return NextResponse.json({
         clusters: cached,
         notes: notesWithClusters,
+        bridgeNotes: enrichedBridgeNotes,
         cached: true,
-        totalNotes: cached.reduce((sum, c) => sum + c.noteIds.length, 0)
+        stale: stored.stale,
+        lastCalculated: stored.lastCalculated,
+        totalNotes: notes.length,
+        embeddingCount: Number(embeddingCountRow[0]?.count || 0),
       })
     }
 
@@ -61,6 +108,7 @@ export async function GET(request: NextRequest) {
     return NextResponse.json({
       clusters: [],
       notes: [],
+      bridgeNotes: [],
       totalNotes: notesCount,
       embeddingCount: Number(embeddingCount[0]?.count || 0),
       needsCalculation: true
@@ -77,6 +125,7 @@ export async function GET(request: NextRequest) {
 /**
  * POST /api/clusters
  * Trigger a full recalculation of clusters and bridge notes.
+ * Returns clusters + bridge notes enriched for immediate display.
  */
 export async function POST(request: NextRequest) {
   try {
@@ -86,66 +135,77 @@ export async function POST(request: NextRequest) {
     }
 
     const userId = session.user.id
+    const body = await request.json().catch(() => ({}))
+    const force = Boolean(body?.force)
 
-    // Use the PROPER clustering service (DBSCAN algorithm)
+    // 0. Indexer / réindexer les embeddings (texte complet, multi-chunks)
+    await clusteringService.ensureEmbeddings(userId, { force: force || true })
+
+    // 1. Run DBSCAN clustering
     const results = await clusteringService.clusterNotes(userId)
 
     if (results.clusters.length === 0) {
       return NextResponse.json({
         clusters: [],
+        bridgeNotes: [],
         message: 'Could not generate clusters. Notes may be too diverse or not enough.',
         noiseCount: results.noiseCount
       })
     }
 
-    // Generate cluster names using AI
+    // 2. Generate cluster names with AI
     for (const cluster of results.clusters) {
       cluster.name = await clusteringService.generateClusterName(cluster.clusterId, userId)
     }
 
-    // Save clustering results
+    // 3. Save clustering results
     await clusteringService.saveClusteringResults(userId, results)
 
-    // Detect and save bridge notes
+    // 4. Detect and save bridge notes
     const bridgeNotes = await bridgeNotesService.detectBridgeNotes(userId)
     await bridgeNotesService.saveBridgeNotes(userId, bridgeNotes)
 
-    // Generate and save bridge suggestions
+    // 5. Generate and save bridge suggestions
     const suggestions = await bridgeNotesService.generateBridgeSuggestions(userId)
     await bridgeNotesService.saveBridgeSuggestions(userId, suggestions)
 
-    // Fetch notes with their cluster assignments
+    // 6. Fetch notes with cluster assignments
     const notes = await prisma.note.findMany({
       where: { userId, trashedAt: null },
       select: { id: true, title: true, content: true }
     })
 
-    // Get cluster member mappings
     const clusterMembers = await prisma.clusterMember.findMany({
       where: { userId },
-      select: { noteId: true, clusterId: true }
+      select: { noteId: true, clusterId: true, isCentral: true }
     })
 
-    const noteClusterMap = new Map(clusterMembers.map(cm => [cm.noteId, cm.clusterId]))
-    const notesWithClusters = notes.map(n => ({
-      ...n,
-      clusterId: noteClusterMap.get(n.id)
-    }))
+    const noteClusterMap = new Map(clusterMembers.map(cm => [cm.noteId, { clusterId: cm.clusterId, isCentral: cm.isCentral }]))
+    const notesWithClusters = notes.map(n => {
+      const mapping = noteClusterMap.get(n.id)
+      return {
+        ...n,
+        clusterId: mapping?.clusterId,
+        isCentral: mapping?.isCentral || false
+      }
+    })
 
-    // Get enriched bridge notes with note details
+    // 7. Enrich bridge notes with cluster names + note details
+    const noteMap = new Map(notes.map(n => [n.id, n]))
     const enrichedBridgeNotes = bridgeNotes.slice(0, 10).map(b => ({
       noteId: b.noteId,
       bridgeScore: b.bridgeScore,
       clustersConnected: b.clustersConnected,
       clusterNames: b.clusterNames,
-      note: notes.find(n => n.id === b.noteId)
+      note: noteMap.get(b.noteId)
     }))
 
     return NextResponse.json({
       clusters: results.clusters,
       notes: notesWithClusters,
       bridgeNotes: enrichedBridgeNotes,
-      totalNotes: results.clusters.reduce((sum, c) => sum + c.noteIds.length, 0) + results.noiseCount,
+      totalNotes:
+        results.clusters.reduce((sum, c) => sum + c.noteIds.length, 0) + results.noiseCount,
       noiseCount: results.noiseCount,
       message: `Generated ${results.clusters.length} clusters with ${bridgeNotes.length} bridge notes`
     })

memento-note/app/api/insights/graph/route.ts

@@ -0,0 +1,134 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { auth } from '@/auth'
+import prisma from '@/lib/prisma'
+
+/**
+ * GET /api/insights/graph
+ *
+ * Retourne les similarités cosinus pairwise calculées depuis les embeddings pgvector
+ * pour TOUS les membres des clusters (intra-cluster) + les échos Memory Echo (inter-cluster).
+ *
+ * Structure de réponse :
+ * {
+ *   pairs: [{ sourceId, targetId, similarity, type: 'cluster' | 'echo' }],
+ *   membershipScores: { [noteId]: number }
+ * }
+ *
+ * - pairs.cluster : paires au sein du même cluster, score = similarité cosinus réelle
+ * - pairs.echo    : paires Memory Echo non-rejetées, score = similarityScore stocké
+ * - membershipScores : score de centralité de chaque note dans son cluster (de ClusterMember)
+ */
+export async function GET(request: NextRequest) {
+  try {
+    const session = await auth()
+    if (!session?.user?.id) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    const userId = session.user.id
+
+    // 1. Charger les membres de clusters avec leur score de centralité
+    const clusterMembers = await prisma.clusterMember.findMany({
+      where: { userId },
+      select: { noteId: true, clusterId: true, membershipScore: true }
+    })
+
+    if (clusterMembers.length === 0) {
+      return NextResponse.json({ pairs: [], membershipScores: {} })
+    }
+
+    // Construire la map noteId -> clusterId
+    const noteToCluster = new Map<string, number>()
+    const membershipScores: Record<string, number> = {}
+    const clusterToNotes = new Map<number, string[]>()
+
+    for (const m of clusterMembers) {
+      noteToCluster.set(m.noteId, m.clusterId)
+      membershipScores[m.noteId] = m.membershipScore
+      if (!clusterToNotes.has(m.clusterId)) clusterToNotes.set(m.clusterId, [])
+      clusterToNotes.get(m.clusterId)!.push(m.noteId)
+    }
+
+    const allNoteIds = clusterMembers.map(m => m.noteId)
+
+    // 2. Calculer les similarités cosinus pairwise intra-cluster via pgvector
+    // On utilise une requête SQL qui calcule toutes les paires d'un même cluster en une fois
+    const intraClusterPairs = await prisma.$queryRawUnsafe<
+      Array<{ sourceId: string; targetId: string; similarity: number; clusterId: number }>
+    >(
+      `SELECT
+        e1."noteId" AS "sourceId",
+        e2."noteId" AS "targetId",
+        1 - (e1.embedding::vector <=> e2.embedding::vector) AS similarity,
+        cm1."clusterId" AS "clusterId"
+      FROM "NoteEmbedding" e1
+      INNER JOIN "NoteEmbedding" e2 ON e1."noteId" < e2."noteId"
+      INNER JOIN "ClusterMember" cm1 ON cm1."noteId" = e1."noteId" AND cm1."userId" = $1
+      INNER JOIN "ClusterMember" cm2 ON cm2."noteId" = e2."noteId" AND cm2."userId" = $1
+      WHERE cm1."clusterId" = cm2."clusterId"
+        AND e1."noteId" = ANY($2::text[])
+        AND e2."noteId" = ANY($2::text[])`,
+      userId,
+      allNoteIds
+    )
+
+    // 3. Récupérer les échos Memory Echo non-rejetés entre notes clusterisées
+    const echoInsights = await prisma.memoryEchoInsight.findMany({
+      where: {
+        userId,
+        dismissed: false,
+        note1Id: { in: allNoteIds },
+        note2Id: { in: allNoteIds }
+      },
+      select: {
+        note1Id: true,
+        note2Id: true,
+        similarityScore: true
+      }
+    })
+
+    // 4. Construire la liste finale des paires
+    const pairs: Array<{
+      sourceId: string
+      targetId: string
+      similarity: number
+      type: 'cluster' | 'echo'
+      clusterId?: number
+    }> = []
+
+    // Paires intra-cluster
+    for (const p of intraClusterPairs) {
+      pairs.push({
+        sourceId: p.sourceId,
+        targetId: p.targetId,
+        similarity: Math.max(0, Math.min(1, p.similarity)),
+        type: 'cluster',
+        clusterId: p.clusterId
+      })
+    }
+
+    // Paires Memory Echo (entre clusters différents souvent, mais peut être intra aussi)
+    const existingPairKeys = new Set(pairs.map(p => `${p.sourceId}--${p.targetId}`))
+    for (const echo of echoInsights) {
+      const key1 = `${echo.note1Id}--${echo.note2Id}`
+      const key2 = `${echo.note2Id}--${echo.note1Id}`
+      // Ajouter uniquement si pas déjà couvert par intra-cluster
+      if (!existingPairKeys.has(key1) && !existingPairKeys.has(key2)) {
+        pairs.push({
+          sourceId: echo.note1Id,
+          targetId: echo.note2Id,
+          similarity: Math.max(0, Math.min(1, echo.similarityScore)),
+          type: 'echo'
+        })
+      }
+    }
+
+    return NextResponse.json({ pairs, membershipScores })
+  } catch (error) {
+    console.error('[/api/insights/graph] Error:', error)
+    return NextResponse.json(
+      { error: 'Failed to compute semantic graph', details: String(error) },
+      { status: 500 }
+    )
+  }
+}

memento-note/app/api/notes/reindex/route.ts

@@ -24,7 +24,7 @@ export async function POST(req: NextRequest) {
     for (let i = 0; i < notes.length; i += BATCH_SIZE) {
       const batch = notes.slice(i, i + BATCH_SIZE)
       const results = await Promise.allSettled(
-        batch.map(note => semanticSearchService.indexNote(note.id))
+        batch.map(note => semanticSearchService.indexNote(note.id, { force: true }))
       )
 
       for (const r of results) {