feat(insights): fix DBSCAN, Persian embeddings crash, D3 physics layouts, and D3 node not found runtime error

memento-note/lib/blocks/extract-blocks.ts

@@ -1,3 +1,5 @@
+import { stripHtmlToPlainText, tokenizeForSimilarity } from '@/lib/text/plain-text'
+
 export interface ExtractedBlock {
   blockId: string
   content: string
@@ -9,7 +11,7 @@ export function extractBlocksFromHtml(html: string): ExtractedBlock[] {
   let match
   while ((match = regex.exec(html)) !== null) {
     const blockId = match[1]
-    const content = match[2].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim()
+    const content = stripHtmlToPlainText(match[2])
     if (content.length >= 10) {
       blocks.push({ blockId, content })
     }
@@ -18,16 +20,8 @@ export function extractBlocksFromHtml(html: string): ExtractedBlock[] {
 }
 
 export function jaccardSimilarity(a: string, b: string): number {
-  const tokenize = (s: string) =>
-    new Set(
-      s
-        .toLowerCase()
-        .replace(/[^\w\s]/g, '')
-        .split(/\s+/)
-        .filter(w => w.length > 3)
-    )
-  const A = tokenize(a)
-  const B = tokenize(b)
+  const A = tokenizeForSimilarity(a)
+  const B = tokenizeForSimilarity(b)
   if (A.size === 0 || B.size === 0) return 0
   let intersection = 0
   A.forEach(w => { if (B.has(w)) intersection++ })
@@ -39,7 +33,7 @@ function extractPlainBlocksFromHtml(html: string): ExtractedBlock[] {
   const regex = /<(?:p|h[1-6]|blockquote|li|td|th|div)[^>]*>([\s\S]*?)<\/(?:p|h[1-6]|blockquote|li|td|th|div)>/gi
   let match
   while ((match = regex.exec(html)) !== null) {
-    const content = match[1].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim()
+    const content = stripHtmlToPlainText(match[1])
     if (content.length >= 10) {
       blocks.push({ blockId: '', content })
     }