feat(insights): fix DBSCAN, Persian embeddings crash, D3 physics layouts, and D3 node not found runtime error

memento-note/lib/clip/analyze-clip.ts

@@ -0,0 +1,86 @@
+import { getSystemConfig } from '@/lib/config'
+import { getChatProvider } from '@/lib/ai/factory'
+
+export interface ClipAnalysis {
+  title: string
+  summary: string
+  tags: string[]
+  readingTimeMinutes: number
+}
+
+function parseAnalysisJson(raw: string): ClipAnalysis | null {
+  const trimmed = raw.trim()
+  const jsonMatch = trimmed.match(/\{[\s\S]*\}/)
+  if (!jsonMatch) return null
+  try {
+    const parsed = JSON.parse(jsonMatch[0]) as Partial<ClipAnalysis>
+    const tags = Array.isArray(parsed.tags)
+      ? parsed.tags.filter((t): t is string => typeof t === 'string').slice(0, 5)
+      : []
+    const readingTime = typeof parsed.readingTimeMinutes === 'number'
+      ? Math.max(1, Math.min(120, Math.round(parsed.readingTimeMinutes)))
+      : 5
+    return {
+      title: typeof parsed.title === 'string' && parsed.title.trim() ? parsed.title.trim().slice(0, 200) : 'Web clip',
+      summary: typeof parsed.summary === 'string' ? parsed.summary.trim().slice(0, 800) : '',
+      tags,
+      readingTimeMinutes: readingTime,
+    }
+  } catch {
+    return null
+  }
+}
+
+function estimateReadingMinutes(text: string): number {
+  const words = text.split(/\s+/).filter(Boolean).length
+  return Math.max(1, Math.round(words / 200))
+}
+
+export async function analyzeClipContent(params: {
+  url: string
+  title: string
+  textContent: string
+}): Promise<ClipAnalysis> {
+  const excerpt = params.textContent.slice(0, 6000)
+  const fallbackReading = estimateReadingMinutes(params.textContent)
+
+  try {
+    const config = await getSystemConfig()
+    const provider = getChatProvider(config)
+    const prompt = `You analyze web articles for a personal knowledge base. URL: ${params.url}
+Page title: ${params.title}
+
+Content excerpt:
+${excerpt}
+
+Respond with ONLY valid JSON (no markdown):
+{
+  "title": "concise improved title",
+  "summary": "max 3 sentences in the same language as the content",
+  "tags": ["tag1", "tag2"],
+  "readingTimeMinutes": ${fallbackReading}
+}
+
+Rules: tags max 5, short lowercase labels, summary factual.`
+
+    const raw = await provider.generateText(prompt)
+    const parsed = parseAnalysisJson(raw)
+    if (parsed) {
+      if (!parsed.title) parsed.title = params.title || 'Web clip'
+      if (!parsed.summary && params.textContent) {
+        parsed.summary = params.textContent.slice(0, 400)
+      }
+      if (parsed.tags.length === 0) parsed.tags = []
+      return parsed
+    }
+  } catch (error) {
+    console.error('[ClipAnalyze] AI failed:', error)
+  }
+
+  return {
+    title: params.title || 'Web clip',
+    summary: params.textContent.slice(0, 400),
+    tags: [],
+    readingTimeMinutes: fallbackReading,
+  }
+}

memento-note/lib/clip/extract-article.ts

@@ -0,0 +1,78 @@
+import { Readability } from '@mozilla/readability'
+import { JSDOM } from 'jsdom'
+import DOMPurify from 'isomorphic-dompurify'
+import {
+  applyRtlToHtmlBlocks,
+  readPageLocaleFromHtml,
+  resolveClipLocale,
+  wrapClipArticleHtml,
+  type ClipLocaleHint,
+} from '@/lib/clip/rtl-content'
+
+export interface ExtractedArticle {
+  title: string
+  content: string
+  textContent: string
+  excerpt: string
+  locale: ClipLocaleHint
+}
+
+export function extractArticleFromHtml(html: string, pageUrl: string): ExtractedArticle | null {
+  const dom = new JSDOM(html, { url: pageUrl })
+  const reader = new Readability(dom.window.document)
+  const article = reader.parse()
+  if (!article) return null
+
+  const pageLocale = readPageLocaleFromHtml(html)
+  const readabilityDir = article.dir?.toLowerCase() === 'rtl' ? 'rtl' : 'ltr'
+  const readabilityLang = article.lang?.split('-')[0]?.toLowerCase()
+  const locale = resolveClipLocale(
+    pageUrl,
+    article.title || '',
+    article.textContent || '',
+  )
+  const mergedLocale: ClipLocaleHint = {
+    direction:
+      readabilityDir === 'rtl' || pageLocale.direction === 'rtl' || locale.direction === 'rtl'
+        ? 'rtl'
+        : 'ltr',
+    lang:
+      (readabilityLang === 'fa' || readabilityLang === 'ar' || readabilityLang === 'he'
+        ? readabilityLang
+        : undefined) ||
+      locale.lang ||
+      pageLocale.lang,
+  }
+
+  const sanitized = DOMPurify.sanitize(article.content || '')
+  const rtlBlocks = applyRtlToHtmlBlocks(sanitized, mergedLocale)
+  const content = wrapClipArticleHtml(rtlBlocks, mergedLocale)
+
+  return {
+    title: (article.title || '').trim(),
+    content,
+    textContent: (article.textContent || '').trim(),
+    excerpt: (article.excerpt || '').trim(),
+    locale: mergedLocale,
+  }
+}
+
+export function clipFooterLocaleTag(lang?: string): string {
+  if (lang === 'fa') return 'fa-IR'
+  if (lang === 'ar') return 'ar'
+  if (lang === 'he') return 'he-IL'
+  return 'fr-FR'
+}
+
+export function buildClipSourceFooter(domain: string, date: Date, localeTag = 'fr-FR'): string {
+  const formatted = date.toLocaleDateString(localeTag, { day: 'numeric', month: 'long', year: 'numeric' })
+  const isRtl = localeTag.startsWith('fa') || localeTag.startsWith('ar') || localeTag.startsWith('he')
+  const label =
+    localeTag.startsWith('fa')
+      ? `برگرفته از ${domain} — ${formatted}`
+      : localeTag.startsWith('ar')
+        ? `مقتبس من ${domain} — ${formatted}`
+        : `Extrait de ${domain} le ${formatted}`
+  const dirAttr = isRtl ? ' dir="rtl"' : ''
+  return `<hr/><p${dirAttr}><small>${DOMPurify.sanitize(label)}</small></p>`
+}

memento-note/lib/clip/rtl-content.ts

@@ -0,0 +1,112 @@
+/** Détection RTL et enveloppe HTML pour contenus clippés (persan, arabe, hébreu). */
+
+const RTL_CHAR = /[\u0590-\u05FF\u0600-\u06FF\u0700-\u074F\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF]/
+const LTR_CHAR = /[A-Za-z0-9]/
+
+export type ClipTextDirection = 'rtl' | 'ltr'
+
+export interface ClipLocaleHint {
+  direction: ClipTextDirection
+  lang?: 'fa' | 'ar' | 'he'
+}
+
+export function inferLangFromUrl(url: string): ClipLocaleHint['lang'] | undefined {
+  const lower = url.toLowerCase()
+  if (/\/persian\b|\/fa\b|lang=fa|bbc\.com\/persian/.test(lower)) return 'fa'
+  if (/\/arabic\b|\/ar\b|lang=ar/.test(lower)) return 'ar'
+  if (/\/hebrew\b|\/he\b|lang=he/.test(lower)) return 'he'
+  return undefined
+}
+
+export function detectTextDirection(text: string): ClipTextDirection {
+  const sample = text.replace(/\s+/g, '').slice(0, 4000)
+  if (!sample) return 'ltr'
+
+  let rtl = 0
+  let ltr = 0
+  for (const ch of sample) {
+    if (RTL_CHAR.test(ch)) rtl++
+    else if (LTR_CHAR.test(ch)) ltr++
+  }
+  if (rtl === 0) return 'ltr'
+  return rtl >= ltr ? 'rtl' : 'ltr'
+}
+
+/** Direction du titre de note (éviter dir="auto" qui casse les chiffres persans). */
+export function resolveTitleDirection(title: string, sourceUrl?: string | null): ClipTextDirection {
+  if (sourceUrl && inferLangFromUrl(sourceUrl)) return 'rtl'
+  return detectTextDirection(title)
+}
+
+export function resolveTitleLang(
+  title: string,
+  sourceUrl?: string | null,
+): ClipLocaleHint['lang'] | undefined {
+  const urlLang = sourceUrl ? inferLangFromUrl(sourceUrl) : undefined
+  if (urlLang) return urlLang
+  if (detectTextDirection(title) !== 'rtl') return undefined
+  return resolveClipLocale(sourceUrl || '', title).lang
+}
+
+export function resolveClipLocale(url: string, ...texts: string[]): ClipLocaleHint {
+  const combined = texts.filter(Boolean).join('\n')
+  const direction = detectTextDirection(combined)
+  const urlLang = inferLangFromUrl(url)
+  let lang = urlLang
+
+  if (!lang && direction === 'rtl') {
+    if (/[\u06AF\u06CC\u06A9\u067E\u0686\u0698\u200C]/.test(combined)) lang = 'fa'
+    else if (/[\u0590-\u05FF]/.test(combined)) lang = 'he'
+    else lang = 'ar'
+  }
+
+  return { direction, lang }
+}
+
+/** Applique dir/lang sur les blocs HTML extraits (Readability ne les conserve pas toujours). */
+export function applyRtlToHtmlBlocks(html: string, hint: ClipLocaleHint): string {
+  if (hint.direction !== 'rtl') return html
+
+  const langAttr = hint.lang ? ` lang="${hint.lang}"` : ''
+  const blockTags = ['p', 'h1', 'h2', 'h3', 'h4', 'li', 'ul', 'ol', 'blockquote', 'figcaption']
+
+  let out = html
+  for (const tag of blockTags) {
+    out = out.replace(new RegExp(`<${tag}(\\s[^>]*)?>`, 'gi'), (match, attrs = '') => {
+      if (/dir\s*=/.test(attrs)) return match
+      return `<${tag}${attrs} dir="rtl"${langAttr}>`
+    })
+  }
+
+  return out
+}
+
+export function wrapClipArticleHtml(innerHtml: string, hint: ClipLocaleHint): string {
+  if (hint.direction !== 'rtl') return innerHtml
+  const langAttr = hint.lang ? ` lang="${hint.lang}"` : ''
+  return `<div class="clip-article clip-article--rtl" dir="rtl"${langAttr}>${innerHtml}</div>`
+}
+
+export function wrapClipPlainParagraph(text: string, hint: ClipLocaleHint): string {
+  const escaped = text
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+  const langAttr = hint.lang ? ` lang="${hint.lang}"` : ''
+  const dirAttr = hint.direction === 'rtl' ? ' dir="rtl"' : ''
+  return `<p${dirAttr}${langAttr}>${escaped}</p>`
+}
+
+export function readPageLocaleFromHtml(html: string): Pick<ClipLocaleHint, 'direction' | 'lang'> {
+  const dirMatch =
+    html.match(/<html[^>]*\sdir=["'](rtl|ltr)["']/i) ||
+    html.match(/<body[^>]*\sdir=["'](rtl|ltr)["']/i)
+  const langMatch =
+    html.match(/<html[^>]*\slang=["']([^"']+)["']/i) ||
+    html.match(/<body[^>]*\slang=["']([^"']+)["']/i)
+  const direction: ClipTextDirection = dirMatch?.[1]?.toLowerCase() === 'rtl' ? 'rtl' : 'ltr'
+  const rawLang = langMatch?.[1]?.split('-')[0]?.toLowerCase()
+  const lang =
+    rawLang === 'fa' || rawLang === 'ar' || rawLang === 'he' ? (rawLang as ClipLocaleHint['lang']) : undefined
+  return { direction, lang }
+}