diff --git a/_bmad-output/implementation-artifacts/spec-cluster-detection-bridge-notes.md b/_bmad-output/implementation-artifacts/spec-cluster-detection-bridge-notes.md new file mode 100644 index 0000000..58ceb08 --- /dev/null +++ b/_bmad-output/implementation-artifacts/spec-cluster-detection-bridge-notes.md @@ -0,0 +1,192 @@ +--- +title: 'Cluster Detection & Bridge Notes Discovery' +type: 'feature' +created: '2026-05-23' +status: 'done' +baseline_commit: '2aed148dc2a6de1914facae9fef65935b3d77ef5' +context: ['memento-note/prisma/schema.prisma', 'memento-note/lib/ai/services/semantic-search.service.ts'] +--- + + + +## Intent + +**Problem:** Users have hundreds of notes but no automatic way to discover thematic clusters or connections between unrelated topics. Notes remain siloed in notebooks, and users manually create links between related concepts across different notebooks. + +**Approach:** Implement automatic clustering using density-based algorithm (HDBSCAN equivalent) on note embeddings, detect "bridge notes" that connect multiple clusters, and provide AI-powered suggestions for missing connections. Visualize clusters graphically with a dashboard showing insights and connection opportunities. + +## Boundaries & Constraints + +**Always:** +- Use existing pgvector embeddings from NoteEmbedding table +- Cluster per-user only (never cross-user data) +- All SQL uses parameterized queries (no interpolation) +- HDBSCAN: Node.js compatible implementation or custom density-based clustering +- Bridge score = len(clusters_touched) / max_clusters (0-1 range) +- Threshold for cluster membership: >= 0.3 similarity +- Threshold for bridge detection: >= 0.5 cosine similarity to >= 2 clusters +- Incremental recalculation only when > 5% embeddings changed or > 10 notes modified + +**Ask First:** +- External Python microservice vs Node.js native clustering +- Cluster auto-naming: exact LLM prompt wording +- Dashboard UI placement: separate page vs modal vs sidebar panel +- Visualization library: D3.js, Cytoscape.js, or React Flow + +**Never:** +- HDBSCAN pip install (Python) — we are a Node.js stack +- Real-time clustering on every note save +- Cross-user cluster analysis +- Modifying existing NoteLink or MemoryEchoInsight behavior +- Manual cluster assignment by users (out of scope for v1) + +## I/O & Edge-Case Matrix + +| Scenario | Input / State | Expected Output / Behavior | Error Handling | +|----------|--------------|---------------------------|----------------| +| CLUSTER_NEW_USER | User with < 10 notes, no existing clusters | Empty cluster list, message "Need more notes to cluster" | Return gracefully with status: insufficient_data | +| CLUSTER_EXISTING | User with 100+ notes, embeddings exist | N clusters (3-50), each with auto-generated name | Log errors, continue with partial results | +| BRIDGE_DETECTION | Note similar to cluster A (0.7) and cluster B (0.6) | Marked as bridge, bridge_score = 0.5, clusters_connected = [A, B] | Skip if similarity < 0.5 | +| NO_BRIDGES | User with isolated clusters (no cross-cluster similarities) | Empty bridge list, suggestions for missing connections | Return empty array | +| SUGGEST_CONNECTIONS | Cluster A (Recipes) and Cluster B (Health) unconnected | 3 AI-suggested bridge note ideas with titles/descriptions | Fallback to generic "Connect [A] and [B]" if LLM fails | +| INCREMENTAL_UPDATE | 3 notes modified out of 200 | Skip recalculation (< 5% threshold) | Return cached results | +| MAJOR_UPDATE | 15 notes modified out of 200 | Trigger full recalculation | Queue background job, return immediately | +| EMBEDDING_MISSING | Note exists but no embedding in NoteEmbedding | Exclude from clustering | Log warning, continue with available notes | + + + +## Code Map + +- `memento-note/prisma/schema.prisma` -- Add NoteCluster and BridgeNote models +- `memento-note/lib/ai/services/clustering.service.ts` -- NEW: HDBSCAN-style clustering algorithm +- `memento-note/lib/ai/services/bridge-notes.service.ts` -- NEW: Bridge detection and suggestions +- `memento-note/app/api/clusters/route.ts` -- NEW: GET /api/clusters, POST /api/clusters/recalculate +- `memento-note/app/api/bridge-notes/route.ts` -- NEW: GET /api/bridge-notes, GET /api/bridge-notes/suggestions +- `memento-note/app/api/cron/clusters/route.ts` -- NEW: Cron endpoint for daily recalculation +- `memento-note/components/cluster-visualization.tsx` -- NEW: Graph visualization with cluster coloring +- `memento-note/components/bridge-notes-dashboard.tsx` -- NEW: Dashboard showing insights and suggestions +- `memento-note/app/(main)/insights/page.tsx` -- NEW: Insights page housing the dashboard + +## Tasks & Acceptance + +**Execution:** +- [x] `memento-note/prisma/schema.prisma` -- Add NoteCluster and BridgeNote models with proper indexes -- Store cluster mappings and bridge metadata +- [x] `memento-note/lib/ai/services/clustering.service.ts` -- Implement density-based clustering (HDBSCAN-style in Node.js) -- Core algorithm for automatic note grouping +- [x] `memento-note/lib/ai/services/bridge-notes.service.ts` -- Implement bridge detection and AI suggestion generation -- Find cross-cluster connections +- [x] `memento-note/app/api/clusters/route.ts` -- Create cluster listing and recalculation endpoints -- REST API for cluster operations +- [x] `memento-note/app/api/bridge-notes/route.ts` -- Create bridge notes and suggestions endpoints -- REST API for bridge operations +- [x] `memento-note/app/api/cron/clusters/route.ts` -- Create cron endpoint for scheduled recalculation -- Automated background processing +- [x] `memento-note/components/cluster-visualization.tsx` -- Build interactive graph with cluster coloring -- Visual representation of note clusters +- [x] `memento-note/components/bridge-notes-dashboard.tsx` -- Build insights dashboard with stats and suggestions -- User-facing cluster and bridge information +- [x] `memento-note/app/(main)/insights/page.tsx` -- Create main insights page -- Container for visualization and dashboard +- [x] `memento-note/lib/ai/services/clustering.service.ts` -- Add cluster auto-naming via LLM -- Human-readable cluster names +- [x] `memento-note/lib/ai/services/bridge-notes.service.ts` -- Add incremental recalculation logic -- Efficient updates without full recompute + +**Acceptance Criteria:** +- Given a user with 50+ notes with embeddings, when the clustering API is called, then notes are grouped into 3-20 clusters based on semantic similarity +- Given a cluster has been created, when the cluster is viewed, then it has an auto-generated name (2-4 words) describing the common theme +- Given a note is similar to >= 2 clusters with cosine >= 0.5, when bridge detection runs, then the note is marked as a bridge note with correct bridge_score +- Given isolated clusters exist (no bridges), when suggestions are requested, then 3 AI-generated bridge note ideas are returned per cluster pair +- Given the insights page is loaded, when displayed, then clusters are color-coded and bridge notes have golden borders +- Given < 10 notes exist for a user, when clustering is requested, then API returns insufficient_data status gracefully +- Given > 5% of notes have been modified, when the cron runs, then full recalculation is triggered +- Given <= 5% of notes have been modified, when the cron runs, then cached results are returned + +## Design Notes + +### Clustering Algorithm Choice +Since HDBSCAN is Python-only and we're a Node.js stack, we implement a simplified density-based clustering: + +1. **Pairwise similarity matrix**: Use pgvector cosine similarity for all note pairs +2. **DBSCAN variant**: + - For each note, find neighbors within epsilon = 0.3 cosine distance + - Form clusters from dense regions (min_cluster_size = 3) + - Mark outliers as noise (cluster_id = -1) + +This approximates HDBSCAN's core benefits: no preset cluster count, outlier detection, handles varying cluster sizes. + +### Bridge Score Formula +``` +bridge_score = len(clusters_touched) / max_possible_clusters +``` +- A note touching 2 of 5 clusters = 0.4 +- A note touching 5 of 5 clusters = 1.0 (super-connector) + +### Cluster Naming +Use the 5 most central notes (highest mean similarity to other cluster members) as context for LLM: +``` +"Here are 5 notes from a cluster. Summarize the common theme in 2-4 words: +1. [title] - [content snippet] +2. ..." +``` + +### UI Visualization +- **Clusters**: Each cluster gets a unique hue (HSL color wheel) +- **Bridge notes**: Golden border + 1.5x size +- **Isolated clusters**: Grayed out with "Isolated" badge +- **Click interaction**: Click cluster to zoom and show note list + +### Cron Strategy +- **Daily**: At 2 AM server time, recalculate for all active users +- **Incremental check**: Track last_modified timestamp on embeddings; if delta < 5%, skip + +## Verification + +**Commands:** +- `npx prisma migrate dev --name add_clustering_support` -- Creates NoteCluster and BridgeNote tables +- `npx prisma generate` -- Regenerates Prisma client with new models +- `npm run build` -- Ensures TypeScript compilation succeeds + +**Manual checks:** +- Visit /insights page — should show cluster visualization or "Need more notes" message +- Create 30+ test notes with varied topics — check that clusters form and have sensible names +- Modify a note and check it — clusters should update within 24h or via manual recalculate +- Check that bridge notes are visually distinct (golden border) in visualization +- POST /api/clusters/recalculate — should return 202 and trigger background job +- GET /api/bridge-notes/suggestions — should return AI-generated connection ideas + +## Suggested Review Order + +**Database schema** + +- New models for clustering and bridge tracking with proper indexes + [`memento-note/prisma/schema.prisma:767`](../../memento-note/prisma/schema.prisma#L767) + +**Core clustering algorithm** + +- DBSCAN-style density-based clustering using pgvector cosine similarity + [`memento-note/lib/ai/services/clustering.service.ts:82`](../../memento-note/lib/ai/services/clustering.service.ts#L82) + +- Cluster membership scoring and centrality detection + [`memento-note/lib/ai/services/clustering.service.ts:239`](../../memento-note/lib/ai/services/clustering.service.ts#L239) + +**Bridge notes detection** + +- Cross-cluster similarity analysis for bridge note identification + [`memento-note/lib/ai/services/bridge-notes.service.ts:52`](../../memento-note/lib/ai/services/bridge-notes.service.ts#L52) + +- AI-powered suggestions for connecting isolated clusters + [`memento-note/lib/ai/services/bridge-notes.service.ts:173`](../../memento-note/lib/ai/services/bridge-notes.service.ts#L173) + +**API endpoints** + +- Cluster listing and recalculation with caching + [`memento-note/app/api/clusters/route.ts:14`](../../memento-note/app/api/clusters/route.ts#L14) + +- Bridge notes and suggestions endpoints + [`memento-note/app/api/bridge-notes/route.ts:14`](../../memento-note/app/api/bridge-notes/route.ts#L14) + +- Scheduled cron job for automatic recalculation + [`memento-note/app/api/cron/clusters/route.ts:27`](../../memento-note/app/api/cron/clusters/route.ts#L27) + +**User interface** + +- Interactive cluster visualization with React Flow + [`memento-note/components/cluster-visualization.tsx:40`](../../memento-note/components/cluster-visualization.tsx#L40) + +- Bridge notes dashboard with tabbed interface + [`memento-note/components/bridge-notes-dashboard.tsx:38`](../../memento-note/components/bridge-notes-dashboard.tsx#L38) + +- Main insights page tying it all together + [`memento-note/app/(main)/insights/page.tsx:17`](../../memento-note/app/(main)/insights/page.tsx#L17) + diff --git a/memento-note/app/(main)/insights/page.tsx b/memento-note/app/(main)/insights/page.tsx new file mode 100644 index 0000000..1513be8 --- /dev/null +++ b/memento-note/app/(main)/insights/page.tsx @@ -0,0 +1,210 @@ +'use client' + +import { useState, useEffect } from 'react' +import { ClusterVisualization } from '@/components/cluster-visualization' +import { BridgeNotesDashboard } from '@/components/bridge-notes-dashboard' +import { useRouter } from 'next/navigation' + +interface Cluster { + clusterId: number + name?: string + noteIds: string[] +} + +interface BridgeNote { + noteId: string + bridgeScore: number + clustersConnected: number[] + note?: { + id: string + title: string | null + content: string + } +} + +export default function InsightsPage() { + const router = useRouter() + const [clusters, setClusters] = useState([]) + const [bridgeNotes, setBridgeNotes] = useState([]) + const [loading, setLoading] = useState(true) + const [recalculating, setRecalculating] = useState(false) + const [message, setMessage] = useState(null) + + useEffect(() => { + loadClusters() + }, []) + + const loadClusters = async () => { + setLoading(true) + try { + const res = await fetch('/api/clusters') + if (res.ok) { + const data = await res.json() + setClusters(data.clusters || []) + + if (data.message) { + setMessage(data.message) + } + } + } catch (error) { + console.error('Error loading clusters:', error) + setMessage('Failed to load clusters') + } finally { + setLoading(false) + } + } + + const recalculateClusters = async () => { + setRecalculating(true) + try { + const res = await fetch('/api/clusters', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ force: true }) + }) + + if (res.ok) { + const data = await res.json() + setClusters(data.clusters || []) + setBridgeNotes(data.bridgeNotes || []) + setMessage(data.message || 'Clusters recalculated successfully') + } + } catch (error) { + console.error('Error recalculating clusters:', error) + setMessage('Failed to recalculate clusters') + } finally { + setRecalculating(false) + } + } + + const handleNoteClick = (noteId: string, type: 'note' | 'cluster') => { + if (type === 'note') { + router.push(`/home?note=${noteId}`) + } + } + + return ( +
+ {/* Header */} +
+

+ Insights +

+

+ Discover thematic clusters and connections in your notes +

+
+ + {/* Stats Bar */} +
+
+
Total Clusters
+
{clusters.length}
+
+
+
Bridge Notes
+
{bridgeNotes.length}
+
+
+
Notes Analyzed
+
+ {clusters.reduce((sum, c) => sum + c.noteIds.length, 0)} +
+
+
+ +
+
+ + {/* Message */} + {message && ( +
+

{message}

+
+ )} + + {/* Loading State */} + {loading && ( +
+
+
+

Analyzing your notes...

+
+
+ )} + + {/* Empty State */} + {!loading && clusters.length === 0 && ( +
+
+ + + +

+ Not enough notes to analyze +

+

+ Create at least 10 notes to start discovering clusters and connections +

+ +
+
+ )} + + {/* Main Content */} + {!loading && clusters.length > 0 && ( +
+ {/* Visualization */} +
+
+

Cluster Visualization

+ +
+
+ + {/* Dashboard */} +
+ handleNoteClick(noteId, 'note')} /> +
+
+ )} + + {/* Cluster List */} + {!loading && clusters.length > 0 && ( +
+

All Clusters

+
+ {clusters.map((cluster) => ( +
+

+ {cluster.name || `Cluster ${cluster.clusterId}`} +

+

+ {cluster.noteIds.length} {cluster.noteIds.length === 1 ? 'note' : 'notes'} +

+
+ ))} +
+
+ )} +
+ ) +} diff --git a/memento-note/app/api/bridge-notes/route.ts b/memento-note/app/api/bridge-notes/route.ts new file mode 100644 index 0000000..b28f94e --- /dev/null +++ b/memento-note/app/api/bridge-notes/route.ts @@ -0,0 +1,85 @@ +import { NextRequest, NextResponse } from 'next/server' +import { auth } from '@/auth' +import prisma from '@/lib/prisma' +import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service' + +/** + * GET /api/bridge-notes + * Get all bridge notes for the current user. + */ +export async function GET(request: NextRequest) { + try { + const session = await auth() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const userId = session.user.id + const { searchParams } = new URL(request.url) + const includeDetails = searchParams.get('details') === 'true' + + const bridgeNotes = await bridgeNotesService.getBridgeNotes(userId) + + // Optionally include note details + let enrichedNotes = bridgeNotes + if (includeDetails && bridgeNotes.length > 0) { + const noteIds = bridgeNotes.map(b => b.noteId) + const notes = await prisma.note.findMany({ + where: { id: { in: noteIds } }, + select: { id: true, title: true, content: true, notebookId: true } + }) + + const noteMap = new Map(notes.map(n => [n.id, n])) + + enrichedNotes = bridgeNotes.map(b => ({ + ...b, + note: noteMap.get(b.noteId) + })) + } + + return NextResponse.json({ + bridgeNotes: enrichedNotes, + count: enrichedNotes.length + }) + } catch (error) { + console.error('Error fetching bridge notes:', error) + return NextResponse.json( + { error: 'Failed to fetch bridge notes' }, + { status: 500 } + ) + } +} + +/** + * DELETE /api/bridge-notes + * Dismiss a bridge suggestion. + */ +export async function DELETE(request: NextRequest) { + try { + const session = await auth() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const userId = session.user.id + const body = await request.json() + const { clusterAId, clusterBId } = body + + if (typeof clusterAId !== 'number' || typeof clusterBId !== 'number') { + return NextResponse.json( + { error: 'Invalid cluster IDs' }, + { status: 400 } + ) + } + + await bridgeNotesService.dismissSuggestion(userId, clusterAId, clusterBId) + + return NextResponse.json({ success: true }) + } catch (error) { + console.error('Error dismissing suggestion:', error) + return NextResponse.json( + { error: 'Failed to dismiss suggestion' }, + { status: 500 } + ) + } +} diff --git a/memento-note/app/api/bridge-notes/suggestions/route.ts b/memento-note/app/api/bridge-notes/suggestions/route.ts new file mode 100644 index 0000000..ae4d081 --- /dev/null +++ b/memento-note/app/api/bridge-notes/suggestions/route.ts @@ -0,0 +1,66 @@ +import { NextRequest, NextResponse } from 'next/server' +import { auth } from '@/auth' +import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service' + +/** + * GET /api/bridge-notes/suggestions + * Get AI-powered bridge note suggestions for unconnected clusters. + */ +export async function GET(request: NextRequest) { + try { + const session = await auth() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const userId = session.user.id + const { searchParams } = new URL(request.url) + const includeDismissed = searchParams.get('dismissed') === 'true' + + const suggestions = await bridgeNotesService.getBridgeSuggestions(userId, includeDismissed) + + return NextResponse.json({ + suggestions, + count: suggestions.length + }) + } catch (error) { + console.error('Error fetching bridge suggestions:', error) + return NextResponse.json( + { error: 'Failed to fetch suggestions' }, + { status: 500 } + ) + } +} + +/** + * POST /api/bridge-notes/suggestions/generate + * Generate new bridge suggestions (force regeneration). + */ +export async function POST(request: NextRequest) { + try { + const session = await auth() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const userId = session.user.id + + // Generate new suggestions + const suggestions = await bridgeNotesService.generateBridgeSuggestions(userId) + + // Save to database + await bridgeNotesService.saveBridgeSuggestions(userId, suggestions) + + return NextResponse.json({ + suggestions, + count: suggestions.length, + message: `Generated ${suggestions.length} bridge suggestions` + }) + } catch (error) { + console.error('Error generating bridge suggestions:', error) + return NextResponse.json( + { error: 'Failed to generate suggestions' }, + { status: 500 } + ) + } +} diff --git a/memento-note/app/api/clusters/route.ts b/memento-note/app/api/clusters/route.ts new file mode 100644 index 0000000..b61ac2b --- /dev/null +++ b/memento-note/app/api/clusters/route.ts @@ -0,0 +1,124 @@ +import { NextRequest, NextResponse } from 'next/server' +import { auth } from '@/auth' +import prisma from '@/lib/prisma' +import { clusteringService } from '@/lib/ai/services/clustering.service' +import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service' + +/** + * GET /api/clusters + * Get all clusters for the current user. + */ +export async function GET(request: NextRequest) { + try { + const session = await auth() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const userId = session.user.id + + // Check for cached results + const cached = await clusteringService.getCachedClusters(userId) + if (cached) { + return NextResponse.json({ + clusters: cached, + cached: true, + totalNotes: cached.reduce((sum, c) => sum + c.noteIds.length, 0) + }) + } + + // No cached results, check if user has enough notes + const notesCount = await prisma.note.count({ + where: { userId, trashedAt: null } + }) + + if (notesCount < 10) { + return NextResponse.json({ + clusters: [], + message: 'Need at least 10 notes to generate clusters', + totalNotes: notesCount + }) + } + + // Trigger background recalculation + return NextResponse.json({ + clusters: [], + message: 'Calculating clusters... Please check back later', + totalNotes: notesCount + }) + } catch (error) { + console.error('Error fetching clusters:', error) + return NextResponse.json( + { error: 'Failed to fetch clusters' }, + { status: 500 } + ) + } +} + +/** + * POST /api/clusters/recalculate + * Trigger a full recalculation of clusters and bridge notes. + */ +export async function POST(request: NextRequest) { + try { + const session = await auth() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const userId = session.user.id + const body = await request.json() + const force = body.force === true + + // Check if recalculation is needed + const shouldRecalc = force || await clusteringService.shouldRecalculate(userId) + + if (!shouldRecalc) { + const cached = await clusteringService.getCachedClusters(userId) + if (cached) { + return NextResponse.json({ + clusters: cached, + cached: true, + message: 'Using cached results (data has not changed significantly)' + }) + } + } + + // Perform clustering + const results = await clusteringService.clusterNotes(userId) + + if (results.clusters.length === 0) { + return NextResponse.json({ + clusters: [], + message: 'Could not generate clusters. Need more diverse notes.', + noiseCount: results.noiseCount + }) + } + + // Generate cluster names + for (const cluster of results.clusters) { + cluster.name = await clusteringService.generateClusterName(cluster.clusterId, userId) + } + + // Save results + await clusteringService.saveClusteringResults(userId, results) + + // Detect and save bridge notes + const bridgeNotes = await bridgeNotesService.detectBridgeNotes(userId) + await bridgeNotesService.saveBridgeNotes(userId, bridgeNotes) + + return NextResponse.json({ + clusters: results.clusters, + bridgeNotes: bridgeNotes.slice(0, 10), // Return top 10 + totalNotes: results.clusters.reduce((sum, c) => sum + c.noteIds.length, 0) + results.noiseCount, + noiseCount: results.noiseCount, + message: `Generated ${results.clusters.length} clusters` + }) + } catch (error) { + console.error('Error recalculating clusters:', error) + return NextResponse.json( + { error: 'Failed to recalculate clusters' }, + { status: 500 } + ) + } +} diff --git a/memento-note/app/api/cron/clusters/route.ts b/memento-note/app/api/cron/clusters/route.ts new file mode 100644 index 0000000..e59d5cc --- /dev/null +++ b/memento-note/app/api/cron/clusters/route.ts @@ -0,0 +1,224 @@ +import { NextRequest, NextResponse } from 'next/server' +import { auth } from '@/auth' +import prisma from '@/lib/prisma' +import { clusteringService } from '@/lib/ai/services/clustering.service' +import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service' + +/** + * Cron endpoint for automatic cluster recalculation. + * + * This endpoint is designed to be called by a cron job or scheduler. + * It processes users who need cluster updates based on: + * 1. Data change percentage (> 5%) + * 2. Time since last calculation (> 24 hours) + * + * GET /api/cron/clusters?secret=CRON_SECRET + * + * The secret parameter must match the environment variable CRON_SECRET. + */ +export async function GET(request: NextRequest) { + try { + // Verify cron secret + const { searchParams } = new URL(request.url) + const secret = searchParams.get('secret') + + if (secret !== process.env.CRON_SECRET) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + // Find users who need recalculation + // Users who either: + // 1. Have no clusters calculated yet + // 2. Have data changes > 5% since last calculation + // 3. Haven't had calculations in > 24 hours + + const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000) + + const usersToProcess = await prisma.user.findMany({ + where: { + // Only active users + OR: [ + // Users who never had clusters calculated + { + noteClusters: { + none: {} + } + }, + // Users with stale cluster data + { + noteClusters: { + some: { + lastCalculated: { + lt: oneDayAgo + } + } + } + } + ] + }, + select: { + id: true, + email: true + }, + take: 50 // Process up to 50 users per run + }) + + const results = { + processed: 0, + skipped: 0, + errors: 0, + details: [] as Array<{ userId: string; clusters: number; bridges: number; error?: string }> + } + + for (const user of usersToProcess) { + try { + // Check if recalculation is needed based on data change + const shouldRecalc = await clusteringService.shouldRecalculate(user.id) + + if (!shouldRecalc) { + results.skipped++ + continue + } + + // Perform clustering + const clusterResults = await clusteringService.clusterNotes(user.id) + + if (clusterResults.clusters.length === 0) { + results.details.push({ + userId: user.id, + clusters: 0, + bridges: 0, + error: 'Could not generate clusters' + }) + continue + } + + // Generate cluster names + for (const cluster of clusterResults.clusters) { + cluster.name = await clusteringService.generateClusterName(cluster.clusterId, user.id) + } + + // Save results + await clusteringService.saveClusteringResults(user.id, clusterResults) + + // Detect and save bridge notes + const bridgeNotes = await bridgeNotesService.detectBridgeNotes(user.id) + await bridgeNotesService.saveBridgeNotes(user.id, bridgeNotes) + + results.processed++ + results.details.push({ + userId: user.id, + clusters: clusterResults.clusters.length, + bridges: bridgeNotes.length + }) + } catch (error) { + console.error(`Error processing user ${user.id}:`, error) + results.errors++ + results.details.push({ + userId: user.id, + clusters: 0, + bridges: 0, + error: error instanceof Error ? error.message : 'Unknown error' + }) + } + } + + return NextResponse.json({ + success: true, + ...results, + message: `Processed ${results.processed} users, skipped ${results.skipped}, ${results.errors} errors` + }) + } catch (error) { + console.error('Error in cluster cron job:', error) + return NextResponse.json( + { error: 'Cron job failed', details: error instanceof Error ? error.message : 'Unknown error' }, + { status: 500 } + ) + } +} + +/** + * POST /api/cron/clusters + * Manual trigger for cluster recalculation (admin only). + * Requires admin authentication. + */ +export async function POST(request: NextRequest) { + try { + const session = await auth() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + // Check if user is admin + const user = await prisma.user.findUnique({ + where: { id: session.user.id }, + select: { role: true } + }) + + if (user?.role !== 'ADMIN') { + return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) + } + + const body = await request.json() + const { userId, forceAll } = body + + if (userId) { + // Process specific user + const clusterResults = await clusteringService.clusterNotes(userId) + const bridgeNotes = await bridgeNotesService.detectBridgeNotes(userId) + + await clusteringService.saveClusteringResults(userId, clusterResults) + await bridgeNotesService.saveBridgeNotes(userId, bridgeNotes) + + return NextResponse.json({ + success: true, + userId, + clusters: clusterResults.clusters.length, + bridges: bridgeNotes.length + }) + } + + if (forceAll) { + // Process all users (use with caution) + const users = await prisma.user.findMany({ + select: { id: true }, + take: 100 // Limit to 100 users per request + }) + + const results = { + processed: 0, + errors: 0 + } + + for (const user of users) { + try { + const clusterResults = await clusteringService.clusterNotes(user.id) + const bridgeNotes = await bridgeNotesService.detectBridgeNotes(user.id) + + await clusteringService.saveClusteringResults(user.id, clusterResults) + await bridgeNotesService.saveBridgeNotes(user.id, bridgeNotes) + + results.processed++ + } catch { + results.errors++ + } + } + + return NextResponse.json({ + success: true, + ...results + }) + } + + return NextResponse.json( + { error: 'Must provide userId or forceAll=true' }, + { status: 400 } + ) + } catch (error) { + console.error('Error in manual cluster trigger:', error) + return NextResponse.json( + { error: 'Failed to trigger recalculation' }, + { status: 500 } + ) + } +} diff --git a/memento-note/components/bridge-notes-dashboard.tsx b/memento-note/components/bridge-notes-dashboard.tsx new file mode 100644 index 0000000..539f505 --- /dev/null +++ b/memento-note/components/bridge-notes-dashboard.tsx @@ -0,0 +1,270 @@ +'use client' + +import { useState, useEffect } from 'react' + +interface BridgeNote { + noteId: string + bridgeScore: number + clustersConnected: number[] + clusterNames?: string[] + note?: { + id: string + title: string | null + content: string + } +} + +interface BridgeSuggestion { + clusterAId: number + clusterBId: number + clusterAName: string + clusterBName: string + suggestedTitle: string + suggestedContent: string + justification: string +} + +interface BridgeNotesDashboardProps { + onNoteClick?: (noteId: string) => void +} + +export function BridgeNotesDashboard({ onNoteClick }: BridgeNotesDashboardProps) { + const [bridgeNotes, setBridgeNotes] = useState([]) + const [suggestions, setSuggestions] = useState([]) + const [loading, setLoading] = useState(true) + const [activeTab, setActiveTab] = useState<'bridges' | 'suggestions'>('bridges') + + useEffect(() => { + loadData() + }, []) + + const loadData = async () => { + setLoading(true) + try { + // Load bridge notes + const bridgesRes = await fetch('/api/bridge-notes?details=true') + if (bridgesRes.ok) { + const bridgesData = await bridgesRes.json() + setBridgeNotes(bridgesData.bridgeNotes || []) + } + + // Load suggestions + const suggestionsRes = await fetch('/api/bridge-notes/suggestions') + if (suggestionsRes.ok) { + const suggestionsData = await suggestionsRes.json() + setSuggestions(suggestionsData.suggestions || []) + } + } catch (error) { + console.error('Error loading bridge data:', error) + } finally { + setLoading(false) + } + } + + const generateNewSuggestions = async () => { + try { + const res = await fetch('/api/bridge-notes/suggestions', { + method: 'POST' + }) + if (res.ok) { + const data = await res.json() + setSuggestions(data.suggestions || []) + } + } catch (error) { + console.error('Error generating suggestions:', error) + } + } + + const dismissSuggestion = async (clusterAId: number, clusterBId: number) => { + try { + await fetch('/api/bridge-notes', { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ clusterAId, clusterBId }) + }) + + // Remove from local state + setSuggestions(prev => prev.filter( + s => !(s.clusterAId === clusterAId && s.clusterBId === clusterBId) + )) + } catch (error) { + console.error('Error dismissing suggestion:', error) + } + } + + if (loading) { + return ( +
+
+
+
+
+
+
+
+
+ ) + } + + return ( +
+ {/* Tabs */} +
+ +
+ +
+ {activeTab === 'bridges' && ( +
+ {bridgeNotes.length === 0 ? ( +
+ + + +

No bridge notes found yet

+

Bridge notes connect different clusters of ideas

+
+ ) : ( +
+ {bridgeNotes.map((bridge) => ( +
onNoteClick?.(bridge.noteId)} + className="border rounded-lg p-4 hover:shadow-md transition-shadow cursor-pointer" + > +
+
+
+ + Bridge Score: {bridge.bridgeScore.toFixed(2)} + + + Connects {bridge.clustersConnected.length} {bridge.clustersConnected.length === 1 ? 'cluster' : 'clusters'} + +
+

+ {bridge.note?.title || 'Untitled'} +

+

+ {bridge.note?.content?.replace(/<[^>]+>/g, '').slice(0, 150) || 'No content'} +

+ {bridge.clusterNames && bridge.clusterNames.length > 0 && ( +
+ {bridge.clusterNames.map((name, i) => ( + + {name} + + ))} +
+ )} +
+
+
+ ))} +
+ )} +
+ )} + + {activeTab === 'suggestions' && ( +
+
+

+ AI-suggested ideas to connect your isolated clusters +

+ +
+ + {suggestions.length === 0 ? ( +
+ + + +

No connection suggestions yet

+

All your clusters may already be connected!

+
+ ) : ( +
+ {suggestions.map((suggestion, index) => ( +
+
+
+
+ 💡 + + {suggestion.suggestedTitle} + + #{index + 1} +
+ +
+ + {suggestion.clusterAName} + + ↔ + + {suggestion.clusterBName} + +
+ +

+ {suggestion.suggestedContent} +

+ +

+ "{suggestion.justification}" +

+
+ + +
+
+ ))} +
+ )} +
+ )} +
+
+ ) +} diff --git a/memento-note/components/cluster-visualization.tsx b/memento-note/components/cluster-visualization.tsx new file mode 100644 index 0000000..3959bd0 --- /dev/null +++ b/memento-note/components/cluster-visualization.tsx @@ -0,0 +1,225 @@ +'use client' + +import { useEffect, useState, useCallback } from 'react' +import ReactFlow, { + Node, + Edge, + Background, + Controls, + MiniMap, + useNodesState, + useEdgesState, + ConnectionMode, +} from 'reactflow' +import 'reactflow/dist/style.css' + +interface Cluster { + clusterId: number + name?: string + noteIds: string[] +} + +interface BridgeNote { + noteId: string + bridgeScore: number + clustersConnected: number[] + note?: { + id: string + title: string | null + content: string + } +} + +interface ClusterVisualizationProps { + clusters: Cluster[] + bridgeNotes: BridgeNote[] + onNodeClick?: (nodeId: string, type: 'note' | 'cluster') => void +} + +// Generate HSL colors for clusters +function generateClusterColor(clusterId: number, totalClusters: number): string { + const hue = (clusterId * 360 / totalClusters) % 360 + return `hsl(${hue}, 70%, 60%)` +} + +export function ClusterVisualization({ + clusters, + bridgeNotes, + onNodeClick +}: ClusterVisualizationProps) { + const [nodes, setNodes, onNodesChange] = useNodesState([]) + const [edges, setEdges, onEdgesChange] = useEdgesState([]) + const [selectedCluster, setSelectedCluster] = useState(null) + + // Build graph from clusters and bridge notes + useEffect(() => { + if (clusters.length === 0) return + + const newNodes: Node[] = [] + const newEdges: Edge[] = [] + const noteToCluster = new Map() + + const clusterColor = generateClusterColor + + // Create cluster nodes and position notes around them + const clusterCenterX = 200 + const clusterCenterY = 200 + const clusterSpacing = 400 + + clusters.forEach((cluster, clusterIndex) => { + const cx = clusterCenterX + (clusterIndex % 4) * clusterSpacing + const cy = clusterCenterY + Math.floor(clusterIndex / 4) * clusterSpacing + const color = clusterColor(cluster.clusterId, clusters.length) + + // Add cluster label node + newNodes.push({ + id: `cluster-${cluster.clusterId}`, + type: 'default', + position: { x: cx, y: cy - 150 }, + data: { + label: ( +
+ {cluster.name || `Cluster ${cluster.clusterId}`} + ({cluster.noteIds.length} notes) +
+ ) + }, + style: { + background: 'transparent', + border: 'none', + }, + className: 'cursor-pointer' + }) + + // Add note nodes for this cluster + const noteRadius = 120 + cluster.noteIds.forEach((noteId, noteIndex) => { + const angle = (noteIndex / cluster.noteIds.length) * 2 * Math.PI + const nx = cx + Math.cos(angle) * noteRadius + const ny = cy + Math.sin(angle) * noteRadius + + noteToCluster.set(noteId, cluster.clusterId) + + // Check if this is a bridge note + const bridge = bridgeNotes.find(b => b.noteId === noteId) + const isBridge = !!bridge + + newNodes.push({ + id: noteId, + type: 'default', + position: { x: nx, y: ny }, + data: { + label: bridge?.note?.title || noteId.slice(0, 8) + }, + style: { + background: isBridge ? '#FFD700' : color, + border: isBridge ? '3px solid #B8860B' : '2px solid rgba(0,0,0,0.1)', + borderRadius: '50%', + width: isBridge ? 50 : 35, + height: isBridge ? 50 : 35, + fontSize: '10px', + fontWeight: isBridge ? 'bold' : 'normal' + }, + className: `cursor-pointer transition-transform hover:scale-110 ${isBridge ? 'shadow-lg' : ''}` + }) + }) + }) + + // Create edges between bridge notes and their connected clusters + bridgeNotes.forEach(bridge => { + const bridgeNodeId = bridge.noteId + const bridgeNode = newNodes.find(n => n.id === bridgeNodeId) + + if (!bridgeNode) return + + bridge.clustersConnected.forEach(clusterId => { + const clusterLabelId = `cluster-${clusterId}` + if (clusterLabelId !== bridgeNodeId) { + newEdges.push({ + id: `${bridgeNodeId}-${clusterId}`, + source: bridgeNodeId, + target: clusterLabelId, + type: 'smoothstep', + animated: true, + style: { + stroke: '#FFD700', + strokeWidth: 2 + }, + label: 'bridge' + }) + } + }) + }) + + setNodes(newNodes) + setEdges(newEdges) + }, [clusters, bridgeNotes, setNodes, setEdges]) + + const onNodeClickHandler = useCallback((event: React.MouseEvent, node: Node) => { + const id = node.id + + if (id.startsWith('cluster-')) { + const clusterId = parseInt(id.replace('cluster-', '')) + setSelectedCluster(clusterId === selectedCluster ? null : clusterId) + onNodeClick?.(id, 'cluster') + } else { + onNodeClick?.(id, 'note') + } + }, [selectedCluster, onNodeClick]) + + if (clusters.length === 0) { + return ( +
+
+ + + +

No clusters to display

+

Create more notes to generate clusters

+
+
+ ) + } + + return ( +
+ + + + + + + {selectedCluster !== null && ( +
+

+ {clusters.find(c => c.clusterId === selectedCluster)?.name || `Cluster ${selectedCluster}`} +

+

+ {clusters.find(c => c.clusterId === selectedCluster)?.noteIds.length || 0} notes +

+
+ )} + +
+
+
+
+ Bridge note +
+
+
+ Regular note +
+
+
+
+ ) +} diff --git a/memento-note/lib/ai/services/bridge-notes.service.ts b/memento-note/lib/ai/services/bridge-notes.service.ts new file mode 100644 index 0000000..5ab632b --- /dev/null +++ b/memento-note/lib/ai/services/bridge-notes.service.ts @@ -0,0 +1,394 @@ +/** + * Bridge Notes Service + * + * Detects and manages "bridge notes" — notes that connect multiple clusters. + * A bridge note has strong similarities (cosine > 0.5) with notes from + * at least two different clusters. + * + * Also generates AI-powered suggestions for creating new bridge notes + * to connect isolated clusters. + */ + +import prisma from '@/lib/prisma' +import { clusteringService } from './clustering.service' +import { getChatProvider } from '@/lib/ai/factory' +import { getSystemConfig } from '@/lib/config' + +export interface BridgeNote { + noteId: string + bridgeScore: number + clustersConnected: number[] + clusterNames?: string[] +} + +export interface BridgeSuggestion { + clusterAId: number + clusterBId: number + clusterAName: string + clusterBName: string + suggestedTitle: string + suggestedContent: string + justification: string +} + +export class BridgeNotesService { + private readonly BRIDGE_SIMILARITY_THRESHOLD = 0.5 + private readonly MIN_CLUSTERS_FOR_BRIDGE = 2 + + /** + * Get similar notes for a given note across all clusters. + * Returns notes grouped by their cluster membership. + */ + private async getSimilarNotesByCluster( + noteId: string, + userId: string, + threshold: number = this.BRIDGE_SIMILARITY_THRESHOLD + ): Promise> { + const cosineDistance = 1 - threshold + + const result = await prisma.$queryRawUnsafe>( + `SELECT similar."noteId", cm."clusterId" + FROM ( + SELECT e2."noteId" + FROM "NoteEmbedding" e1 + CROSS JOIN "NoteEmbedding" e2 + INNER JOIN "Note" n ON n.id = e2."noteId" + WHERE e1."noteId" = $1 + AND e2."noteId" != e1."noteId" + AND n."userId" = $2 + AND n."trashedAt" IS NULL + AND (e1."embedding"::vector <=> e2."embedding"::vector) <= $3 + ) similar + LEFT JOIN "ClusterMember" cm ON cm."noteId" = similar."noteId" AND cm."userId" = $2`, + noteId, + userId, + cosineDistance + ) + + const clusterMap = new Map() + + for (const row of result) { + const clusterId = row.clusterId ?? -1 // -1 for noise/uncategorized + if (!clusterMap.has(clusterId)) { + clusterMap.set(clusterId, []) + } + clusterMap.get(clusterId)!.push(row.noteId) + } + + return clusterMap + } + + /** + * Detect all bridge notes for a user. + * A note is a bridge if it has similarities to >= 2 distinct clusters. + */ + async detectBridgeNotes(userId: string): Promise { + // Get all user's clusters + const clusters = await prisma.noteCluster.findMany({ + where: { userId }, + select: { clusterId: true, name: true }, + orderBy: { clusterId: 'asc' } + }) + + if (clusters.length < this.MIN_CLUSTERS_FOR_BRIDGE) { + return [] + } + + const maxClusters = clusters.length + const bridgeNotes: BridgeNote[] = [] + + // Check each note for bridge potential + const notes = await prisma.note.findMany({ + where: { userId, trashedAt: null }, + select: { id: true } + }) + + for (const note of notes) { + const similarByCluster = await this.getSimilarNotesByCluster(note.id, userId) + + // Filter out noise (-1) and get clusters with actual similar notes + const clustersWithSimilarNotes: number[] = [] + for (const [clusterId, similarNotes] of similarByCluster) { + if (clusterId !== -1 && similarNotes.length > 0) { + clustersWithSimilarNotes.push(clusterId) + } + } + + // Check if this note connects >= 2 clusters + if (clustersWithSimilarNotes.length >= this.MIN_CLUSTERS_FOR_BRIDGE) { + const bridgeScore = clustersWithSimilarNotes.length / maxClusters + + bridgeNotes.push({ + noteId: note.id, + bridgeScore, + clustersConnected: clustersWithSimilarNotes, + clusterNames: clustersWithSimilarNotes.map( + cid => clusters.find(c => c.clusterId === cid)?.name || `Cluster ${cid}` + ) + }) + } + } + + // Sort by bridge score (most influential first) + return bridgeNotes.sort((a, b) => b.bridgeScore - a.bridgeScore) + } + + /** + * Save bridge notes to database. + */ + async saveBridgeNotes(userId: string, bridgeNotes: BridgeNote[]): Promise { + await prisma.$transaction(async (tx) => { + // Clear existing bridge notes for this user + await tx.bridgeNote.deleteMany({ where: { userId } }) + + // Insert new bridge notes + for (const bridge of bridgeNotes) { + await tx.bridgeNote.create({ + data: { + userId, + noteId: bridge.noteId, + bridgeScore: bridge.bridgeScore, + clustersConnected: JSON.stringify(bridge.clustersConnected), + lastCalculated: new Date() + } + }) + } + }) + } + + /** + * Get cluster summaries for AI suggestions. + */ + private async getClusterSummary(clusterId: number, userId: string): Promise { + const notes = await prisma.$queryRawUnsafe>( + `SELECT n.title, n.content + FROM "ClusterMember" cm + INNER JOIN "Note" n ON n.id = cm."noteId" + WHERE cm."clusterId" = $1 + AND cm."userId" = $2 + LIMIT 5`, + clusterId, + userId + ) + + if (notes.length === 0) return 'No notes available' + + return notes + .map(n => `- ${n.title || 'Untitled'}: ${n.content.slice(0, 80)}...`) + .join('\n') + } + + /** + * Generate AI-powered suggestions for connecting isolated clusters. + */ + async generateBridgeSuggestions(userId: string): Promise { + // Get all clusters + const clusters = await prisma.noteCluster.findMany({ + where: { userId }, + select: { clusterId: true, name: true }, + orderBy: { clusterId: 'asc' } + }) + + if (clusters.length < 2) return [] + + // Get existing bridges to see which clusters are already connected + const existingBridges = await prisma.bridgeNote.findMany({ + where: { userId }, + select: { clustersConnected: true } + }) + + const connectedPairs = new Set() + + for (const bridge of existingBridges) { + const clusters = JSON.parse(bridge.clustersConnected) as number[] + for (let i = 0; i < clusters.length; i++) { + for (let j = i + 1; j < clusters.length; j++) { + const pair = [clusters[i], clusters[j]].sort().join('-') + connectedPairs.add(pair) + } + } + } + + // Find unconnected cluster pairs + const suggestions: BridgeSuggestion[] = [] + + for (let i = 0; i < clusters.length; i++) { + for (let j = i + 1; j < clusters.length; j++) { + const pair = `${clusters[i].clusterId}-${clusters[j].clusterId}` + + if (connectedPairs.has(pair)) continue // Already connected + + // Generate suggestion for this unconnected pair + const suggestion = await this.generateConnectionSuggestion( + clusters[i].clusterId, + clusters[j].clusterId, + clusters[i].name || `Cluster ${clusters[i].clusterId}`, + clusters[j].name || `Cluster ${clusters[j].clusterId}`, + userId + ) + + if (suggestion) { + suggestions.push(suggestion) + } + } + } + + return suggestions + } + + /** + * Generate a specific connection suggestion between two clusters. + */ + private async generateConnectionSuggestion( + clusterAId: number, + clusterBId: number, + clusterAName: string, + clusterBName: string, + userId: string + ): Promise { + const summaryA = await this.getClusterSummary(clusterAId, userId) + const summaryB = await this.getClusterSummary(clusterBId, userId) + + const systemPrompt = `You are a creative assistant that helps users connect ideas. +Suggest a "bridge note" that could connect two unrelated topics. +Be specific and creative. Your suggestions should help users discover new insights.` + + const userPrompt = `I have two groups of notes that are not connected: + +Group A (${clusterAName}): +${summaryA} + +Group B (${clusterBName}): +${summaryB} + +Suggest 3 creative bridge note ideas to connect these groups. For each idea, provide: +1. A catchy title (max 10 words) +2. A brief description of what the note would contain (max 50 words) +3. A justification for why this connection is valuable (max 30 words) + +Format as JSON: +{ + "ideas": [ + {"title": "...", "description": "...", "justification": "..."}, + ... + ] +} + +Return ONLY the JSON, no other text.` + + try { + const config = await getSystemConfig() + const provider = getChatProvider(config) + const response = await provider.chat( + [{ role: 'user', content: userPrompt }], + systemPrompt + ) + + // Parse JSON response + const jsonMatch = response.text.match(/\{[\s\S]*\}/) + if (!jsonMatch) return null + + const parsed = JSON.parse(jsonMatch[0]) + const bestIdea = parsed.ideas?.[0] + + if (!bestIdea) return null + + return { + clusterAId, + clusterBId, + clusterAName, + clusterBName, + suggestedTitle: bestIdea.title, + suggestedContent: bestIdea.description, + justification: bestIdea.justification + } + } catch (error) { + console.error('Error generating bridge suggestion:', error) + return null + } + } + + /** + * Save bridge suggestions to database. + */ + async saveBridgeSuggestions(userId: string, suggestions: BridgeSuggestion[]): Promise { + await prisma.$transaction(async (tx) => { + // Clear existing suggestions + await tx.bridgeSuggestion.deleteMany({ where: { userId } }) + + // Insert new suggestions + for (const suggestion of suggestions) { + await tx.bridgeSuggestion.create({ + data: { + userId, + clusterAId: suggestion.clusterAId, + clusterBId: suggestion.clusterBId, + clusterAName: suggestion.clusterAName, + clusterBName: suggestion.clusterBName, + suggestedTitle: suggestion.suggestedTitle, + suggestedContent: suggestion.suggestedContent, + justification: suggestion.justification + } + }) + } + }) + } + + /** + * Get bridge notes for a user. + */ + async getBridgeNotes(userId: string): Promise { + const bridges = await prisma.bridgeNote.findMany({ + where: { userId }, + orderBy: { bridgeScore: 'desc' } + }) + + return bridges.map(b => ({ + noteId: b.noteId, + bridgeScore: b.bridgeScore, + clustersConnected: JSON.parse(b.clustersConnected) as number[] + })) + } + + /** + * Get bridge suggestions for a user. + */ + async getBridgeSuggestions(userId: string, includeDismissed: boolean = false): Promise { + const suggestions = await prisma.bridgeSuggestion.findMany({ + where: { + userId, + ...(includeDismissed ? {} : { isDismissed: false }) + }, + orderBy: { createdAt: 'desc' } + }) + + return suggestions.map(s => ({ + clusterAId: s.clusterAId, + clusterBId: s.clusterBId, + clusterAName: s.clusterAName, + clusterBName: s.clusterBName, + suggestedTitle: s.suggestedTitle, + suggestedContent: s.suggestedContent, + justification: s.justification + })) + } + + /** + * Dismiss a bridge suggestion. + */ + async dismissSuggestion(userId: string, clusterAId: number, clusterBId: number): Promise { + await prisma.bridgeSuggestion.updateMany({ + where: { + userId, + clusterAId, + clusterBId + }, + data: { isDismissed: true } + }) + } +} + +export const bridgeNotesService = new BridgeNotesService() diff --git a/memento-note/lib/ai/services/clustering.service.ts b/memento-note/lib/ai/services/clustering.service.ts new file mode 100644 index 0000000..e57681a --- /dev/null +++ b/memento-note/lib/ai/services/clustering.service.ts @@ -0,0 +1,436 @@ +/** + * Clustering Service + * + * Density-based clustering algorithm (DBSCAN variant) for note embeddings. + * Groups semantically similar notes into clusters without requiring + * a preset number of clusters. + * + * Algorithm: + * 1. For each note, find neighbors within epsilon cosine distance + * 2. Form clusters from dense regions (min_cluster_size) + * 3. Mark outliers as noise (cluster_id = -1) + */ + +import prisma from '@/lib/prisma' +import { embeddingService } from './embedding.service' +import { getChatProvider } from '@/lib/ai/factory' +import { getSystemConfig } from '@/lib/config' + +export interface ClusterResult { + clusterId: number + noteIds: string[] + centroid?: number[] + name?: string +} + +export interface ClusteredNote { + noteId: string + clusterId: number + membershipScore: number + isCentral: boolean +} + +export interface ClusteringOptions { + minClusterSize?: number + epsilon?: number // Cosine distance threshold (lower = more strict) + maxClusters?: number +} + +export class ClusteringService { + private readonly DEFAULT_MIN_CLUSTER_SIZE = 3 + private readonly DEFAULT_EPSILON = 0.3 // Cosine distance ~ 1 - similarity + private readonly DEFAULT_MAX_CLUSTERS = 50 + private readonly MIN_NOTES_FOR_CLUSTERING = 10 + + /** + * Calculate cosine similarity between two embedding vectors. + * Uses 1 - cosine_distance where cosine_distance is computed via pgvector. + */ + private async getCosineSimilarity( + noteIdA: string, + noteIdB: string + ): Promise { + const result = await prisma.$queryRawUnsafe>( + `SELECT 1 - (e1."embedding"::vector <=> e2."embedding"::vector) AS similarity + FROM "NoteEmbedding" e1, "NoteEmbedding" e2 + WHERE e1."noteId" = $1 AND e2."noteId" = $2`, + noteIdA, + noteIdB + ) + return result[0]?.similarity || 0 + } + + /** + * Find all neighbors for a note within epsilon similarity threshold. + */ + private async findNeighbors( + noteId: string, + allNoteIds: string[], + epsilon: number + ): Promise { + // Convert epsilon (similarity threshold) to cosine distance + const cosineDistance = 1 - epsilon + + const result = await prisma.$queryRawUnsafe>( + `SELECT e2."noteId" + FROM "NoteEmbedding" e1 + CROSS JOIN "NoteEmbedding" e2 + WHERE e1."noteId" = $1 + AND e2."noteId" != $1 + AND e2."noteId" = ANY($2::text[]) + AND (e1."embedding"::vector <=> e2."embedding"::vector) <= $3`, + noteId, + allNoteIds, + cosineDistance + ) + + return result.map(r => r.noteId) + } + + /** + * Expand a cluster from a seed note using DBSCAN-like algorithm. + */ + private async expandCluster( + noteId: string, + neighbors: string[], + clusterId: number, + visited: Set, + clustered: Map, + allNoteIds: string[], + epsilon: number, + minClusterSize: number + ): Promise { + const clusterMembers: string[] = [noteId] + const queue = [...neighbors] + clustered.set(noteId, clusterId) + + while (queue.length > 0) { + const currentNoteId = queue.shift()! + + if (!visited.has(currentNoteId)) { + visited.add(currentNoteId) + const currentNeighbors = await this.findNeighbors(currentNoteId, allNoteIds, epsilon) + + if (currentNeighbors.length >= minClusterSize) { + for (const neighborId of currentNeighbors) { + if (!clustered.has(neighborId)) { + clustered.set(neighborId, clusterId) + clusterMembers.push(neighborId) + queue.push(neighborId) + } + } + } + } + } + + return clusterMembers + } + + /** + * Perform density-based clustering on user's note embeddings. + */ + async clusterNotes( + userId: string, + options: ClusteringOptions = {} + ): Promise<{ + clusters: ClusterResult[] + clusteredNotes: ClusteredNote[] + noiseCount: number + }> { + const { + minClusterSize = this.DEFAULT_MIN_CLUSTER_SIZE, + epsilon = this.DEFAULT_EPSILON, + maxClusters = this.DEFAULT_MAX_CLUSTERS + } = options + + // Get all user's notes with embeddings + const notesWithEmbeddings = await prisma.$queryRawUnsafe>( + `SELECT ne."noteId" + FROM "NoteEmbedding" ne + INNER JOIN "Note" n ON n.id = ne."noteId" + WHERE n."userId" = $1 + AND n."trashedAt" IS NULL + AND ne."embedding" IS NOT NULL`, + userId + ) + + const allNoteIds = notesWithEmbeddings.map(n => n.noteId) + + if (allNoteIds.length < this.MIN_NOTES_FOR_CLUSTERING) { + return { + clusters: [], + clusteredNotes: [], + noiseCount: allNoteIds.length + } + } + + const visited = new Set() + const clustered = new Map() // noteId -> clusterId + const clusterResults: ClusterResult[] = [] + let clusterId = 0 + + // DBSCAN algorithm + for (const noteId of allNoteIds) { + if (visited.has(noteId)) continue + + visited.add(noteId) + const neighbors = await this.findNeighbors(noteId, allNoteIds, epsilon) + + if (neighbors.length < minClusterSize) { + // Mark as noise (cluster_id = -1) + clustered.set(noteId, -1) + continue + } + + // Expand cluster + const clusterMembers = await this.expandCluster( + noteId, + neighbors, + clusterId, + visited, + clustered, + allNoteIds, + epsilon, + minClusterSize + ) + + if (clusterMembers.length >= minClusterSize && clusterId < maxClusters) { + clusterResults.push({ + clusterId, + noteIds: clusterMembers + }) + clusterId++ + } else { + // Too small, mark as noise + for (const memberId of clusterMembers) { + clustered.set(memberId, -1) + } + } + } + + // Calculate membership scores and identify central notes + const clusteredNotes: ClusteredNote[] = [] + for (const [noteId, cid] of clustered.entries()) { + if (cid === -1) continue // Skip noise + + const cluster = clusterResults[cid] + if (!cluster) continue + + // Calculate membership score as average similarity to other cluster members + const score = await this.calculateMembershipScore(noteId, cluster.noteIds) + const isCentral = await this.isCentralNote(noteId, cluster.noteIds) + + clusteredNotes.push({ + noteId, + clusterId: cid, + membershipScore: score, + isCentral + }) + } + + const noiseCount = Array.from(clustered.values()).filter(id => id === -1).length + + return { + clusters: clusterResults, + clusteredNotes, + noiseCount + } + } + + /** + * Calculate membership score for a note within its cluster. + * Score = average similarity to all other cluster members. + */ + private async calculateMembershipScore(noteId: string, clusterMemberIds: string[]): Promise { + if (clusterMemberIds.length <= 1) return 1.0 + + const similarities: number[] = [] + for (const memberId of clusterMemberIds) { + if (memberId === noteId) continue + const sim = await this.getCosineSimilarity(noteId, memberId) + similarities.push(sim) + } + + return similarities.length > 0 + ? similarities.reduce((a, b) => a + b, 0) / similarities.length + : 1.0 + } + + /** + * Determine if a note is central to its cluster. + * A note is central if its average similarity to other members + * is above the cluster mean. + */ + private async isCentralNote(noteId: string, clusterMemberIds: string[]): Promise { + const allScores: Array<{ memberId: string; score: number }> = [] + + for (const memberId of clusterMemberIds) { + const score = await this.calculateMembershipScore(memberId, clusterMemberIds) + allScores.push({ memberId, score }) + } + + const meanScore = allScores.reduce((sum, s) => sum + s.score, 0) / allScores.length + const noteScore = allScores.find(s => s.memberId === noteId)?.score || 0 + + return noteScore >= meanScore + } + + /** + * Get the N most central notes from a cluster for naming purposes. + */ + async getCentralNotes(clusterId: number, userId: string, n: number = 5): Promise> { + const result = await prisma.$queryRawUnsafe>( + `SELECT DISTINCT n.id AS "noteId", n.title, n.content + FROM "ClusterMember" cm + INNER JOIN "Note" n ON n.id = cm."noteId" + WHERE cm."clusterId" = $1 + AND cm."userId" = $2 + AND cm."isCentral" = true + LIMIT $3`, + clusterId, + userId, + n + ) + + return result + } + + /** + * Save clustering results to database. + */ + async saveClusteringResults( + userId: string, + results: { clusters: ClusterResult[]; clusteredNotes: ClusteredNote[] } + ): Promise { + await prisma.$transaction(async (tx) => { + // Clear existing clusters for this user + await tx.$executeRawUnsafe(`DELETE FROM "ClusterMember" WHERE "userId" = $1`, userId) + await tx.$executeRawUnsafe(`DELETE FROM "NoteCluster" WHERE "userId" = $1`, userId) + + // Insert new clusters + for (const cluster of results.clusters) { + await tx.noteCluster.create({ + data: { + userId, + clusterId: cluster.clusterId, + name: cluster.name, + noteCount: cluster.noteIds.length, + lastCalculated: new Date() + } + }) + } + + // Insert cluster members + for (const clusteredNote of results.clusteredNotes) { + await tx.clusterMember.create({ + data: { + userId, + noteId: clusteredNote.noteId, + clusterId: clusteredNote.clusterId, + membershipScore: clusteredNote.membershipScore, + isCentral: clusteredNote.isCentral + } + }) + } + }) + } + + /** + * Generate a name for a cluster using the LLM. + * Analyzes the 5 most central notes to extract a common theme. + */ + async generateClusterName(clusterId: number, userId: string): Promise { + const centralNotes = await this.getCentralNotes(clusterId, userId, 5) + + if (centralNotes.length === 0) { + return `Cluster ${clusterId}` + } + + const notesText = centralNotes + .map((note, i) => `${i + 1}. "${note.title || 'Untitled'}" - ${note.content.slice(0, 100)}...`) + .join('\n') + + const systemPrompt = 'You are a clustering assistant. Provide ONLY a concise name (2-4 words) in English. No punctuation, no explanation.' + + const userPrompt = `Analyze these 5 notes that belong to the same cluster. What is the common theme?\n\n${notesText}\n\nTheme:` + + try { + const config = await getSystemConfig() + const provider = getChatProvider(config) + const response = await provider.chat( + [{ role: 'user', content: userPrompt }], + systemPrompt + ) + return response.text.trim().slice(0, 50) + } catch { + return `Cluster ${clusterId}` + } + } + + /** + * Check if recalculation is needed based on data change percentage. + */ + async shouldRecalculate(userId: string): Promise { + const lastCluster = await prisma.noteCluster.findFirst({ + where: { userId }, + orderBy: { lastCalculated: 'desc' } + }) + + if (!lastCluster) return true + + // Count notes modified since last calculation + const modifiedCount = await prisma.note.count({ + where: { + userId, + OR: [ + { updatedAt: { gt: lastCluster.lastCalculated } }, + { contentUpdatedAt: { gt: lastCluster.lastCalculated } } + ] + } + }) + + const totalNotes = await prisma.note.count({ + where: { userId, trashedAt: null } + }) + + if (totalNotes === 0) return false + + const changePercentage = modifiedCount / totalNotes + return changePercentage > 0.05 // More than 5% changed + } + + /** + * Get cached clustering results if available and fresh. + */ + async getCachedClusters(userId: string): Promise { + const clusters = await prisma.noteCluster.findMany({ + where: { userId }, + orderBy: { clusterId: 'asc' } + }) + + if (clusters.length === 0) return null + + // Check if data is still fresh + const needsUpdate = await this.shouldRecalculate(userId) + if (needsUpdate) return null + + // Get cluster members + const result: ClusterResult[] = [] + for (const cluster of clusters) { + const members = await prisma.clusterMember.findMany({ + where: { clusterId: cluster.clusterId, userId }, + select: { noteId: true } + }) + + result.push({ + clusterId: cluster.clusterId, + noteIds: members.map(m => m.noteId), + name: cluster.name || undefined + }) + } + + return result + } +} + +export const clusteringService = new ClusteringService() diff --git a/memento-note/package-lock.json b/memento-note/package-lock.json index bb6c3da..771659c 100644 --- a/memento-note/package-lock.json +++ b/memento-note/package-lock.json @@ -78,6 +78,7 @@ "isomorphic-dompurify": "^3.12.0", "jalaali-js": "^1.2.8", "jsdom": "^29.0.2", + "jszip": "^3.10.1", "katex": "^0.16.27", "lucide-react": "^0.562.0", "marked": "^18.0.3", @@ -95,6 +96,7 @@ "react-force-graph-2d": "^1.29.1", "react-icons": "^5.6.0", "react-markdown": "^10.1.0", + "reactflow": "^11.11.4", "recharts": "^3.8.1", "rehype-katex": "^7.0.1", "rehype-raw": "^7.0.0", @@ -120,6 +122,7 @@ "@types/dagre": "^0.7.54", "@types/diff": "^7.0.2", "@types/ioredis": "^4.28.10", + "@types/jszip": "^3.4.0", "@types/node": "^20", "@types/nodemailer": "^7.0.4", "@types/pdf-parse": "^1.1.5", @@ -3762,11 +3765,350 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/@parcel/watcher": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.6.tgz", + "integrity": "sha512-tmmZ3lQxAe/k/+rNnXQRawJ4NjxO2hqiOLTHvWchtGZULp4RyFeh6aU4XdOYBFe2KE1oShQTv4AblOs2iOrNnQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "detect-libc": "^2.0.3", + "is-glob": "^4.0.3", + "node-addon-api": "^7.0.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "@parcel/watcher-android-arm64": "2.5.6", + "@parcel/watcher-darwin-arm64": "2.5.6", + "@parcel/watcher-darwin-x64": "2.5.6", + "@parcel/watcher-freebsd-x64": "2.5.6", + "@parcel/watcher-linux-arm-glibc": "2.5.6", + "@parcel/watcher-linux-arm-musl": "2.5.6", + "@parcel/watcher-linux-arm64-glibc": "2.5.6", + "@parcel/watcher-linux-arm64-musl": "2.5.6", + "@parcel/watcher-linux-x64-glibc": "2.5.6", + "@parcel/watcher-linux-x64-musl": "2.5.6", + "@parcel/watcher-win32-arm64": "2.5.6", + "@parcel/watcher-win32-ia32": "2.5.6", + "@parcel/watcher-win32-x64": "2.5.6" + } + }, + "node_modules/@parcel/watcher-android-arm64": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-android-arm64/-/watcher-android-arm64-2.5.6.tgz", + "integrity": "sha512-YQxSS34tPF/6ZG7r/Ih9xy+kP/WwediEUsqmtf0cuCV5TPPKw/PQHRhueUo6JdeFJaqV3pyjm0GdYjZotbRt/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-darwin-arm64": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-arm64/-/watcher-darwin-arm64-2.5.6.tgz", + "integrity": "sha512-Z2ZdrnwyXvvvdtRHLmM4knydIdU9adO3D4n/0cVipF3rRiwP+3/sfzpAwA/qKFL6i1ModaabkU7IbpeMBgiVEA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-darwin-x64": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-x64/-/watcher-darwin-x64-2.5.6.tgz", + "integrity": "sha512-HgvOf3W9dhithcwOWX9uDZyn1lW9R+7tPZ4sug+NGrGIo4Rk1hAXLEbcH1TQSqxts0NYXXlOWqVpvS1SFS4fRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-freebsd-x64": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-freebsd-x64/-/watcher-freebsd-x64-2.5.6.tgz", + "integrity": "sha512-vJVi8yd/qzJxEKHkeemh7w3YAn6RJCtYlE4HPMoVnCpIXEzSrxErBW5SJBgKLbXU3WdIpkjBTeUNtyBVn8TRng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm-glibc": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-glibc/-/watcher-linux-arm-glibc-2.5.6.tgz", + "integrity": "sha512-9JiYfB6h6BgV50CCfasfLf/uvOcJskMSwcdH1PHH9rvS1IrNy8zad6IUVPVUfmXr+u+Km9IxcfMLzgdOudz9EQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm-musl": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-musl/-/watcher-linux-arm-musl-2.5.6.tgz", + "integrity": "sha512-Ve3gUCG57nuUUSyjBq/MAM0CzArtuIOxsBdQ+ftz6ho8n7s1i9E1Nmk/xmP323r2YL0SONs1EuwqBp2u1k5fxg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm64-glibc": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-glibc/-/watcher-linux-arm64-glibc-2.5.6.tgz", + "integrity": "sha512-f2g/DT3NhGPdBmMWYoxixqYr3v/UXcmLOYy16Bx0TM20Tchduwr4EaCbmxh1321TABqPGDpS8D/ggOTaljijOA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm64-musl": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-musl/-/watcher-linux-arm64-musl-2.5.6.tgz", + "integrity": "sha512-qb6naMDGlbCwdhLj6hgoVKJl2odL34z2sqkC7Z6kzir8b5W65WYDpLB6R06KabvZdgoHI/zxke4b3zR0wAbDTA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-x64-glibc": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-glibc/-/watcher-linux-x64-glibc-2.5.6.tgz", + "integrity": "sha512-kbT5wvNQlx7NaGjzPFu8nVIW1rWqV780O7ZtkjuWaPUgpv2NMFpjYERVi0UYj1msZNyCzGlaCWEtzc+exjMGbQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-x64-musl": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-musl/-/watcher-linux-x64-musl-2.5.6.tgz", + "integrity": "sha512-1JRFeC+h7RdXwldHzTsmdtYR/Ku8SylLgTU/reMuqdVD7CtLwf0VR1FqeprZ0eHQkO0vqsbvFLXUmYm/uNKJBg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-win32-arm64": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-arm64/-/watcher-win32-arm64-2.5.6.tgz", + "integrity": "sha512-3ukyebjc6eGlw9yRt678DxVF7rjXatWiHvTXqphZLvo7aC5NdEgFufVwjFfY51ijYEWpXbqF5jtrK275z52D4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-win32-ia32": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-ia32/-/watcher-win32-ia32-2.5.6.tgz", + "integrity": "sha512-k35yLp1ZMwwee3Ez/pxBi5cf4AoBKYXj00CZ80jUz5h8prpiaQsiRPKQMxoLstNuqe2vR4RNPEAEcjEFzhEz/g==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-win32-x64": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-x64/-/watcher-win32-x64-2.5.6.tgz", + "integrity": "sha512-hbQlYcCq5dlAX9Qx+kFb0FHue6vbjlf0FrNzSKdYK2APUf7tGfGxQCk2ihEREmbR6ZMc0MVAD5RIX/41gpUzTw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/@playwright/test": { "version": "1.59.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "playwright": "1.59.1" @@ -3800,14 +4142,14 @@ "version": "5.22.0", "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.22.0.tgz", "integrity": "sha512-AUt44v3YJeggO2ZU5BkXI7M4hu9BF2zzH2iF2V5pyXT/lRTyWiElZ7It+bRH1EshoMRxHgpYg4VB6rCM+mG5jQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0" }, "node_modules/@prisma/engines": { "version": "5.22.0", "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.22.0.tgz", "integrity": "sha512-UNjfslWhAt06kVL3CjkuYpHAWSO6L4kDCVPegV6itt7nD1kSJavd3vhgAEhjglLJJKEdJ7oIqDJ+yHk6qO8gPA==", - "dev": true, + "devOptional": true, "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { @@ -3821,14 +4163,14 @@ "version": "5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2", "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2.tgz", "integrity": "sha512-2PTmxFR2yHW/eB3uqWtcgRcgAbG1rwG9ZriSvQw+nnb7c4uCr3RAcGMb6/zfE88SKlC1Nj2ziUvc96Z379mHgQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0" }, "node_modules/@prisma/fetch-engine": { "version": "5.22.0", "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.22.0.tgz", "integrity": "sha512-bkrD/Mc2fSvkQBV5EpoFcZ87AvOgDxbG99488a5cexp5Ccny+UM6MAe/UFkUC0wLYD9+9befNOqGiIJhhq+HbA==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@prisma/debug": "5.22.0", @@ -3840,7 +4182,7 @@ "version": "5.22.0", "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.22.0.tgz", "integrity": "sha512-pHhpQdr1UPFpt+zFfnPazhulaZYCUqeIcPpJViYoq9R+D/yw4fjE+CtnsnKzPYm0ddUbeXUzjGVGIRVgPDCk4Q==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@prisma/debug": "5.22.0" @@ -6709,6 +7051,108 @@ "integrity": "sha512-E0KekCFXut0IB8+msh1g4oXz4AXwyfDz9Sqz+tfabnKbI1AkyoFs1tPJB8ViN0Ckq7Da6jYGU8TVNVkirsXBjQ==", "license": "MIT" }, + "node_modules/@reactflow/background": { + "version": "11.3.14", + "resolved": "https://registry.npmjs.org/@reactflow/background/-/background-11.3.14.tgz", + "integrity": "sha512-Gewd7blEVT5Lh6jqrvOgd4G6Qk17eGKQfsDXgyRSqM+CTwDqRldG2LsWN4sNeno6sbqVIC2fZ+rAUBFA9ZEUDA==", + "license": "MIT", + "dependencies": { + "@reactflow/core": "11.11.4", + "classcat": "^5.0.3", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/controls": { + "version": "11.2.14", + "resolved": "https://registry.npmjs.org/@reactflow/controls/-/controls-11.2.14.tgz", + "integrity": "sha512-MiJp5VldFD7FrqaBNIrQ85dxChrG6ivuZ+dcFhPQUwOK3HfYgX2RHdBua+gx+40p5Vw5It3dVNp/my4Z3jF0dw==", + "license": "MIT", + "dependencies": { + "@reactflow/core": "11.11.4", + "classcat": "^5.0.3", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/core": { + "version": "11.11.4", + "resolved": "https://registry.npmjs.org/@reactflow/core/-/core-11.11.4.tgz", + "integrity": "sha512-H4vODklsjAq3AMq6Np4LE12i1I4Ta9PrDHuBR9GmL8uzTt2l2jh4CiQbEMpvMDcp7xi4be0hgXj+Ysodde/i7Q==", + "license": "MIT", + "dependencies": { + "@types/d3": "^7.4.0", + "@types/d3-drag": "^3.0.1", + "@types/d3-selection": "^3.0.3", + "@types/d3-zoom": "^3.0.1", + "classcat": "^5.0.3", + "d3-drag": "^3.0.0", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/minimap": { + "version": "11.7.14", + "resolved": "https://registry.npmjs.org/@reactflow/minimap/-/minimap-11.7.14.tgz", + "integrity": "sha512-mpwLKKrEAofgFJdkhwR5UQ1JYWlcAAL/ZU/bctBkuNTT1yqV+y0buoNVImsRehVYhJwffSWeSHaBR5/GJjlCSQ==", + "license": "MIT", + "dependencies": { + "@reactflow/core": "11.11.4", + "@types/d3-selection": "^3.0.3", + "@types/d3-zoom": "^3.0.1", + "classcat": "^5.0.3", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/node-resizer": { + "version": "2.2.14", + "resolved": "https://registry.npmjs.org/@reactflow/node-resizer/-/node-resizer-2.2.14.tgz", + "integrity": "sha512-fwqnks83jUlYr6OHcdFEedumWKChTHRGw/kbCxj0oqBd+ekfs+SIp4ddyNU0pdx96JIm5iNFS0oNrmEiJbbSaA==", + "license": "MIT", + "dependencies": { + "@reactflow/core": "11.11.4", + "classcat": "^5.0.4", + "d3-drag": "^3.0.0", + "d3-selection": "^3.0.0", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/node-toolbar": { + "version": "1.3.14", + "resolved": "https://registry.npmjs.org/@reactflow/node-toolbar/-/node-toolbar-1.3.14.tgz", + "integrity": "sha512-rbynXQnH/xFNu4P9H+hVqlEUafDCkEoCy0Dg9mG22Sg+rY/0ck6KkrAQrYrTgXusd+cEJOMK0uOOFCK2/5rSGQ==", + "license": "MIT", + "dependencies": { + "@reactflow/core": "11.11.4", + "classcat": "^5.0.3", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, "node_modules/@reduxjs/toolkit": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz", @@ -8438,6 +8882,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/jszip": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@types/jszip/-/jszip-3.4.0.tgz", + "integrity": "sha512-GFHqtQQP3R4NNuvZH3hNCYD0NbyBZ42bkN7kO3NDrU/SnvIZWMS8Bp38XCsRKBT5BXvgm0y1zqpZWp/ZkRzBzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "jszip": "*" + } + }, "node_modules/@types/katex": { "version": "0.16.8", "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.8.tgz", @@ -8498,7 +8952,6 @@ "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", - "dev": true, "license": "MIT", "dependencies": { "csstype": "^3.2.2" @@ -8508,7 +8961,6 @@ "version": "19.2.3", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", - "dev": true, "license": "MIT", "peerDependencies": { "@types/react": "^19.2.0" @@ -10141,6 +10593,12 @@ "url": "https://polar.sh/cva" } }, + "node_modules/classcat": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.5.tgz", + "integrity": "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==", + "license": "MIT" + }, "node_modules/client-only": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", @@ -15878,6 +16336,15 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/node-exports-info": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.0.tgz", @@ -16386,7 +16853,7 @@ "version": "1.59.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "playwright-core": "1.59.1" @@ -16405,7 +16872,7 @@ "version": "1.59.1", "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" @@ -16585,7 +17052,7 @@ "version": "5.22.0", "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.22.0.tgz", "integrity": "sha512-vtpjW3XuYCSnMsNVBjLMNkTj6OZbudcPPTPYHqX0CJfpcdWciI1dM8uHETwmDxxiqEwCIE6WvXucWUetJgfu/A==", - "dev": true, + "devOptional": true, "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { @@ -17169,6 +17636,24 @@ } } }, + "node_modules/reactflow": { + "version": "11.11.4", + "resolved": "https://registry.npmjs.org/reactflow/-/reactflow-11.11.4.tgz", + "integrity": "sha512-70FOtJkUWH3BAOsN+LU9lCrKoKbtOPnz2uq0CV2PLdNSwxTXOhCbsZr50GmZ+Rtw3jx8Uv7/vBFtCGixLfd4Og==", + "license": "MIT", + "dependencies": { + "@reactflow/background": "11.3.14", + "@reactflow/controls": "11.2.14", + "@reactflow/core": "11.11.4", + "@reactflow/minimap": "11.7.14", + "@reactflow/node-resizer": "2.2.14", + "@reactflow/node-toolbar": "1.3.14" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", @@ -19435,6 +19920,24 @@ } } }, + "node_modules/vitest/node_modules/chokidar": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "readdirp": "^5.0.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/vitest/node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -19450,6 +19953,15 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/vitest/node_modules/immutable": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.5.tgz", + "integrity": "sha512-t7xcm2siw+hlUM68I+UEOK+z84RzmN59as9DZ7P1l0994DKUWV7UXBMQZVxaoMSRQ+PBZbHCOoBt7a2wxOMt+A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true + }, "node_modules/vitest/node_modules/picomatch": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", @@ -19463,6 +19975,45 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/vitest/node_modules/readdirp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", + "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/vitest/node_modules/sass": { + "version": "1.100.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.100.0.tgz", + "integrity": "sha512-B5j0rYMlinhhOo9tjQebMVVn0TfyXAF+wB3b2ggZUuJ/is/Y+7+JGjirAMxHZ9Z3hIP98NPfamlAkBHa1lAaXQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "chokidar": "^5.0.0", + "immutable": "^5.1.5", + "source-map-js": ">=0.6.2 <2.0.0" + }, + "bin": { + "sass": "sass.js" + }, + "engines": { + "node": ">=20.19.0" + }, + "optionalDependencies": { + "@parcel/watcher": "^2.4.1" + } + }, "node_modules/vitest/node_modules/vite": { "version": "8.0.9", "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.9.tgz", diff --git a/memento-note/package.json b/memento-note/package.json index f9c2832..16ac2d2 100644 --- a/memento-note/package.json +++ b/memento-note/package.json @@ -99,6 +99,7 @@ "isomorphic-dompurify": "^3.12.0", "jalaali-js": "^1.2.8", "jsdom": "^29.0.2", + "jszip": "^3.10.1", "katex": "^0.16.27", "lucide-react": "^0.562.0", "marked": "^18.0.3", @@ -116,6 +117,7 @@ "react-force-graph-2d": "^1.29.1", "react-icons": "^5.6.0", "react-markdown": "^10.1.0", + "reactflow": "^11.11.4", "recharts": "^3.8.1", "rehype-katex": "^7.0.1", "rehype-raw": "^7.0.0", @@ -141,6 +143,7 @@ "@types/dagre": "^0.7.54", "@types/diff": "^7.0.2", "@types/ioredis": "^4.28.10", + "@types/jszip": "^3.4.0", "@types/node": "^20", "@types/nodemailer": "^7.0.4", "@types/pdf-parse": "^1.1.5", diff --git a/memento-note/prisma/schema.prisma b/memento-note/prisma/schema.prisma index 72a69c1..855546a 100644 --- a/memento-note/prisma/schema.prisma +++ b/memento-note/prisma/schema.prisma @@ -7,7 +7,7 @@ generator client { datasource db { provider = "postgresql" url = env("DATABASE_URL") - extensions = [pgvector] + extensions = [vector] } model User { @@ -50,6 +50,10 @@ model User { subscription Subscription? usageLogs UsageLog[] apiKeys UserAPIKey[] + aiConsentLogs AiConsentLog[] + noteClusters NoteCluster[] + bridgeNotes BridgeNote[] + bridgeSuggestions BridgeSuggestion[] } model Account { @@ -185,6 +189,8 @@ model Note { brainstormNoteRefs BrainstormNoteRef[] outgoingLinks NoteLink[] @relation("SourceLinks") incomingLinks NoteLink[] @relation("TargetLinks") + clusterMemberships ClusterMember[] + bridgeNote BridgeNote? @@index([isPinned]) @@index([isArchived]) @@ -309,6 +315,7 @@ model UserAISettings { noteHistory Boolean @default(false) noteHistoryMode String @default("manual") autoSave Boolean @default(true) + aiProcessingConsent Boolean @default(false) user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@index([memoryEcho]) @@ -317,6 +324,16 @@ model UserAISettings { @@index([preferredLanguage]) } +model AiConsentLog { + id String @id @default(cuid()) + userId String + consent Boolean @default(true) + ipAddress String? + userAgent String? + createdAt DateTime @default(now()) + user User @relation(fields: [userId], references: [id], onDelete: Cascade) +} + model NoteEmbedding { id String @id @default(cuid()) noteId String @unique @@ -747,3 +764,76 @@ model FeatureFlag { metadata String? updatedAt DateTime @updatedAt } + +// ===== CLUSTERING & BRIDGE NOTES ===== + +model NoteCluster { + id String @id @default(cuid()) + userId String + clusterId Int // Cluster number for this user (0, 1, 2, ...) + name String? // Auto-generated cluster name + centroid Unsupported("vector(1536)")? // Optional: centroid embedding for cluster + noteCount Int @default(0) + lastCalculated DateTime @default(now()) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@index([userId]) + @@index([userId, clusterId]) + @@unique([userId, clusterId]) +} + +model ClusterMember { + id String @id @default(cuid()) + userId String + noteId String + clusterId Int // Cluster number + membershipScore Float @default(0.0) // How strongly this note belongs to the cluster + isCentral Boolean @default(false) // Is this a central note in the cluster? + createdAt DateTime @default(now()) + + note Note @relation(fields: [noteId], references: [id], onDelete: Cascade) + + @@index([userId, clusterId]) + @@index([noteId]) + @@unique([noteId, clusterId]) +} + +model BridgeNote { + id String @id @default(cuid()) + userId String + noteId String @unique + bridgeScore Float // len(clusters_touched) / max_clusters + clustersConnected String // JSON array: ["0", "2", "5"] + lastCalculated DateTime @default(now()) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + note Note @relation(fields: [noteId], references: [id], onDelete: Cascade) + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@index([userId]) + @@index([bridgeScore]) +} + +model BridgeSuggestion { + id String @id @default(cuid()) + userId String + clusterAId Int + clusterBId Int + clusterAName String + clusterBName String + suggestedTitle String + suggestedContent String // @default(dbgenerated) + justification String // Why this connection makes sense + isDismissed Boolean @default(false) + createdAt DateTime @default(now()) + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@index([userId]) + @@index([userId, isDismissed]) + @@index([clusterAId, clusterBId]) +}