feat(cluster): implement cluster detection and bridge notes discovery

Add automatic note clustering using density-based algorithm (DBSCAN variant)
and bridge notes detection for connecting different thematic clusters.

Features:
- NoteCluster, ClusterMember, BridgeNote, BridgeSuggestion models
- Clustering service with pgvector cosine similarity
- Bridge notes detection (notes connecting >=2 clusters)
- AI-powered suggestions for missing cluster connections
- /insights page with React Flow visualization
- Cron endpoint for automatic recalculation

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Antigravity
2026-05-23 20:26:25 +00:00
parent 2aed148dc2
commit 077e665dfc
13 changed files with 2882 additions and 12 deletions

View File

@@ -0,0 +1,210 @@
'use client'
import { useState, useEffect } from 'react'
import { ClusterVisualization } from '@/components/cluster-visualization'
import { BridgeNotesDashboard } from '@/components/bridge-notes-dashboard'
import { useRouter } from 'next/navigation'
interface Cluster {
clusterId: number
name?: string
noteIds: string[]
}
interface BridgeNote {
noteId: string
bridgeScore: number
clustersConnected: number[]
note?: {
id: string
title: string | null
content: string
}
}
export default function InsightsPage() {
const router = useRouter()
const [clusters, setClusters] = useState<Cluster[]>([])
const [bridgeNotes, setBridgeNotes] = useState<BridgeNote[]>([])
const [loading, setLoading] = useState(true)
const [recalculating, setRecalculating] = useState(false)
const [message, setMessage] = useState<string | null>(null)
useEffect(() => {
loadClusters()
}, [])
const loadClusters = async () => {
setLoading(true)
try {
const res = await fetch('/api/clusters')
if (res.ok) {
const data = await res.json()
setClusters(data.clusters || [])
if (data.message) {
setMessage(data.message)
}
}
} catch (error) {
console.error('Error loading clusters:', error)
setMessage('Failed to load clusters')
} finally {
setLoading(false)
}
}
const recalculateClusters = async () => {
setRecalculating(true)
try {
const res = await fetch('/api/clusters', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ force: true })
})
if (res.ok) {
const data = await res.json()
setClusters(data.clusters || [])
setBridgeNotes(data.bridgeNotes || [])
setMessage(data.message || 'Clusters recalculated successfully')
}
} catch (error) {
console.error('Error recalculating clusters:', error)
setMessage('Failed to recalculate clusters')
} finally {
setRecalculating(false)
}
}
const handleNoteClick = (noteId: string, type: 'note' | 'cluster') => {
if (type === 'note') {
router.push(`/home?note=${noteId}`)
}
}
return (
<div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
{/* Header */}
<div className="mb-8">
<h1 className="text-3xl font-bold text-gray-900">
Insights
</h1>
<p className="mt-2 text-gray-600">
Discover thematic clusters and connections in your notes
</p>
</div>
{/* Stats Bar */}
<div className="grid grid-cols-1 md:grid-cols-4 gap-4 mb-8">
<div className="bg-white rounded-lg shadow p-4">
<div className="text-sm text-gray-500">Total Clusters</div>
<div className="text-2xl font-bold text-gray-900">{clusters.length}</div>
</div>
<div className="bg-white rounded-lg shadow p-4">
<div className="text-sm text-gray-500">Bridge Notes</div>
<div className="text-2xl font-bold text-gray-900">{bridgeNotes.length}</div>
</div>
<div className="bg-white rounded-lg shadow p-4">
<div className="text-sm text-gray-500">Notes Analyzed</div>
<div className="text-2xl font-bold text-gray-900">
{clusters.reduce((sum, c) => sum + c.noteIds.length, 0)}
</div>
</div>
<div className="bg-white rounded-lg shadow p-4">
<button
onClick={recalculateClusters}
disabled={recalculating || loading}
className="w-full px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 disabled:bg-gray-400 disabled:cursor-not-allowed transition-colors"
>
{recalculating ? 'Calculating...' : 'Recalculate'}
</button>
</div>
</div>
{/* Message */}
{message && (
<div className="mb-6 p-4 bg-blue-50 border border-blue-200 rounded-lg">
<p className="text-sm text-blue-800">{message}</p>
</div>
)}
{/* Loading State */}
{loading && (
<div className="bg-white rounded-lg shadow p-12">
<div className="flex flex-col items-center justify-center">
<div className="animate-spin rounded-full h-12 w-12 border-b-2 border-blue-600 mb-4"></div>
<p className="text-gray-600">Analyzing your notes...</p>
</div>
</div>
)}
{/* Empty State */}
{!loading && clusters.length === 0 && (
<div className="bg-white rounded-lg shadow p-12">
<div className="flex flex-col items-center justify-center text-center">
<svg className="w-24 h-24 text-gray-400 mb-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={1} d="M19 11H5m14 0a2 2 0 012 2v6a2 2 0 01-2 2H5a2 2 0 01-2-2v-6a2 2 0 012-2m14 0V9a2 2 0 00-2-2M5 11V9a2 2 0 012-2m0 0V5a2 2 0 012-2h6a2 2 0 012 2v2M7 7h10" />
</svg>
<h3 className="text-xl font-semibold text-gray-900 mb-2">
Not enough notes to analyze
</h3>
<p className="text-gray-600 mb-6">
Create at least 10 notes to start discovering clusters and connections
</p>
<button
onClick={() => router.push('/home')}
className="px-6 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition-colors"
>
Create Notes
</button>
</div>
</div>
)}
{/* Main Content */}
{!loading && clusters.length > 0 && (
<div className="grid grid-cols-1 lg:grid-cols-3 gap-6">
{/* Visualization */}
<div className="lg:col-span-2">
<div className="bg-white rounded-lg shadow p-4">
<h2 className="text-lg font-semibold mb-4">Cluster Visualization</h2>
<ClusterVisualization
clusters={clusters}
bridgeNotes={bridgeNotes}
onNodeClick={handleNoteClick}
/>
</div>
</div>
{/* Dashboard */}
<div className="lg:col-span-1">
<BridgeNotesDashboard onNoteClick={(noteId) => handleNoteClick(noteId, 'note')} />
</div>
</div>
)}
{/* Cluster List */}
{!loading && clusters.length > 0 && (
<div className="mt-8">
<h2 className="text-lg font-semibold mb-4">All Clusters</h2>
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
{clusters.map((cluster) => (
<div
key={cluster.clusterId}
className="bg-white rounded-lg shadow p-4 hover:shadow-md transition-shadow cursor-pointer"
>
<h3 className="font-semibold text-gray-900 mb-1">
{cluster.name || `Cluster ${cluster.clusterId}`}
</h3>
<p className="text-sm text-gray-600">
{cluster.noteIds.length} {cluster.noteIds.length === 1 ? 'note' : 'notes'}
</p>
</div>
))}
</div>
</div>
)}
</div>
)
}

View File

@@ -0,0 +1,85 @@
import { NextRequest, NextResponse } from 'next/server'
import { auth } from '@/auth'
import prisma from '@/lib/prisma'
import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
/**
* GET /api/bridge-notes
* Get all bridge notes for the current user.
*/
export async function GET(request: NextRequest) {
try {
const session = await auth()
if (!session?.user?.id) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const userId = session.user.id
const { searchParams } = new URL(request.url)
const includeDetails = searchParams.get('details') === 'true'
const bridgeNotes = await bridgeNotesService.getBridgeNotes(userId)
// Optionally include note details
let enrichedNotes = bridgeNotes
if (includeDetails && bridgeNotes.length > 0) {
const noteIds = bridgeNotes.map(b => b.noteId)
const notes = await prisma.note.findMany({
where: { id: { in: noteIds } },
select: { id: true, title: true, content: true, notebookId: true }
})
const noteMap = new Map(notes.map(n => [n.id, n]))
enrichedNotes = bridgeNotes.map(b => ({
...b,
note: noteMap.get(b.noteId)
}))
}
return NextResponse.json({
bridgeNotes: enrichedNotes,
count: enrichedNotes.length
})
} catch (error) {
console.error('Error fetching bridge notes:', error)
return NextResponse.json(
{ error: 'Failed to fetch bridge notes' },
{ status: 500 }
)
}
}
/**
* DELETE /api/bridge-notes
* Dismiss a bridge suggestion.
*/
export async function DELETE(request: NextRequest) {
try {
const session = await auth()
if (!session?.user?.id) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const userId = session.user.id
const body = await request.json()
const { clusterAId, clusterBId } = body
if (typeof clusterAId !== 'number' || typeof clusterBId !== 'number') {
return NextResponse.json(
{ error: 'Invalid cluster IDs' },
{ status: 400 }
)
}
await bridgeNotesService.dismissSuggestion(userId, clusterAId, clusterBId)
return NextResponse.json({ success: true })
} catch (error) {
console.error('Error dismissing suggestion:', error)
return NextResponse.json(
{ error: 'Failed to dismiss suggestion' },
{ status: 500 }
)
}
}

View File

@@ -0,0 +1,66 @@
import { NextRequest, NextResponse } from 'next/server'
import { auth } from '@/auth'
import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
/**
* GET /api/bridge-notes/suggestions
* Get AI-powered bridge note suggestions for unconnected clusters.
*/
export async function GET(request: NextRequest) {
try {
const session = await auth()
if (!session?.user?.id) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const userId = session.user.id
const { searchParams } = new URL(request.url)
const includeDismissed = searchParams.get('dismissed') === 'true'
const suggestions = await bridgeNotesService.getBridgeSuggestions(userId, includeDismissed)
return NextResponse.json({
suggestions,
count: suggestions.length
})
} catch (error) {
console.error('Error fetching bridge suggestions:', error)
return NextResponse.json(
{ error: 'Failed to fetch suggestions' },
{ status: 500 }
)
}
}
/**
* POST /api/bridge-notes/suggestions/generate
* Generate new bridge suggestions (force regeneration).
*/
export async function POST(request: NextRequest) {
try {
const session = await auth()
if (!session?.user?.id) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const userId = session.user.id
// Generate new suggestions
const suggestions = await bridgeNotesService.generateBridgeSuggestions(userId)
// Save to database
await bridgeNotesService.saveBridgeSuggestions(userId, suggestions)
return NextResponse.json({
suggestions,
count: suggestions.length,
message: `Generated ${suggestions.length} bridge suggestions`
})
} catch (error) {
console.error('Error generating bridge suggestions:', error)
return NextResponse.json(
{ error: 'Failed to generate suggestions' },
{ status: 500 }
)
}
}

View File

@@ -0,0 +1,124 @@
import { NextRequest, NextResponse } from 'next/server'
import { auth } from '@/auth'
import prisma from '@/lib/prisma'
import { clusteringService } from '@/lib/ai/services/clustering.service'
import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
/**
* GET /api/clusters
* Get all clusters for the current user.
*/
export async function GET(request: NextRequest) {
try {
const session = await auth()
if (!session?.user?.id) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const userId = session.user.id
// Check for cached results
const cached = await clusteringService.getCachedClusters(userId)
if (cached) {
return NextResponse.json({
clusters: cached,
cached: true,
totalNotes: cached.reduce((sum, c) => sum + c.noteIds.length, 0)
})
}
// No cached results, check if user has enough notes
const notesCount = await prisma.note.count({
where: { userId, trashedAt: null }
})
if (notesCount < 10) {
return NextResponse.json({
clusters: [],
message: 'Need at least 10 notes to generate clusters',
totalNotes: notesCount
})
}
// Trigger background recalculation
return NextResponse.json({
clusters: [],
message: 'Calculating clusters... Please check back later',
totalNotes: notesCount
})
} catch (error) {
console.error('Error fetching clusters:', error)
return NextResponse.json(
{ error: 'Failed to fetch clusters' },
{ status: 500 }
)
}
}
/**
* POST /api/clusters/recalculate
* Trigger a full recalculation of clusters and bridge notes.
*/
export async function POST(request: NextRequest) {
try {
const session = await auth()
if (!session?.user?.id) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const userId = session.user.id
const body = await request.json()
const force = body.force === true
// Check if recalculation is needed
const shouldRecalc = force || await clusteringService.shouldRecalculate(userId)
if (!shouldRecalc) {
const cached = await clusteringService.getCachedClusters(userId)
if (cached) {
return NextResponse.json({
clusters: cached,
cached: true,
message: 'Using cached results (data has not changed significantly)'
})
}
}
// Perform clustering
const results = await clusteringService.clusterNotes(userId)
if (results.clusters.length === 0) {
return NextResponse.json({
clusters: [],
message: 'Could not generate clusters. Need more diverse notes.',
noiseCount: results.noiseCount
})
}
// Generate cluster names
for (const cluster of results.clusters) {
cluster.name = await clusteringService.generateClusterName(cluster.clusterId, userId)
}
// Save results
await clusteringService.saveClusteringResults(userId, results)
// Detect and save bridge notes
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(userId)
await bridgeNotesService.saveBridgeNotes(userId, bridgeNotes)
return NextResponse.json({
clusters: results.clusters,
bridgeNotes: bridgeNotes.slice(0, 10), // Return top 10
totalNotes: results.clusters.reduce((sum, c) => sum + c.noteIds.length, 0) + results.noiseCount,
noiseCount: results.noiseCount,
message: `Generated ${results.clusters.length} clusters`
})
} catch (error) {
console.error('Error recalculating clusters:', error)
return NextResponse.json(
{ error: 'Failed to recalculate clusters' },
{ status: 500 }
)
}
}

View File

@@ -0,0 +1,224 @@
import { NextRequest, NextResponse } from 'next/server'
import { auth } from '@/auth'
import prisma from '@/lib/prisma'
import { clusteringService } from '@/lib/ai/services/clustering.service'
import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
/**
* Cron endpoint for automatic cluster recalculation.
*
* This endpoint is designed to be called by a cron job or scheduler.
* It processes users who need cluster updates based on:
* 1. Data change percentage (> 5%)
* 2. Time since last calculation (> 24 hours)
*
* GET /api/cron/clusters?secret=CRON_SECRET
*
* The secret parameter must match the environment variable CRON_SECRET.
*/
export async function GET(request: NextRequest) {
try {
// Verify cron secret
const { searchParams } = new URL(request.url)
const secret = searchParams.get('secret')
if (secret !== process.env.CRON_SECRET) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
// Find users who need recalculation
// Users who either:
// 1. Have no clusters calculated yet
// 2. Have data changes > 5% since last calculation
// 3. Haven't had calculations in > 24 hours
const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000)
const usersToProcess = await prisma.user.findMany({
where: {
// Only active users
OR: [
// Users who never had clusters calculated
{
noteClusters: {
none: {}
}
},
// Users with stale cluster data
{
noteClusters: {
some: {
lastCalculated: {
lt: oneDayAgo
}
}
}
}
]
},
select: {
id: true,
email: true
},
take: 50 // Process up to 50 users per run
})
const results = {
processed: 0,
skipped: 0,
errors: 0,
details: [] as Array<{ userId: string; clusters: number; bridges: number; error?: string }>
}
for (const user of usersToProcess) {
try {
// Check if recalculation is needed based on data change
const shouldRecalc = await clusteringService.shouldRecalculate(user.id)
if (!shouldRecalc) {
results.skipped++
continue
}
// Perform clustering
const clusterResults = await clusteringService.clusterNotes(user.id)
if (clusterResults.clusters.length === 0) {
results.details.push({
userId: user.id,
clusters: 0,
bridges: 0,
error: 'Could not generate clusters'
})
continue
}
// Generate cluster names
for (const cluster of clusterResults.clusters) {
cluster.name = await clusteringService.generateClusterName(cluster.clusterId, user.id)
}
// Save results
await clusteringService.saveClusteringResults(user.id, clusterResults)
// Detect and save bridge notes
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(user.id)
await bridgeNotesService.saveBridgeNotes(user.id, bridgeNotes)
results.processed++
results.details.push({
userId: user.id,
clusters: clusterResults.clusters.length,
bridges: bridgeNotes.length
})
} catch (error) {
console.error(`Error processing user ${user.id}:`, error)
results.errors++
results.details.push({
userId: user.id,
clusters: 0,
bridges: 0,
error: error instanceof Error ? error.message : 'Unknown error'
})
}
}
return NextResponse.json({
success: true,
...results,
message: `Processed ${results.processed} users, skipped ${results.skipped}, ${results.errors} errors`
})
} catch (error) {
console.error('Error in cluster cron job:', error)
return NextResponse.json(
{ error: 'Cron job failed', details: error instanceof Error ? error.message : 'Unknown error' },
{ status: 500 }
)
}
}
/**
* POST /api/cron/clusters
* Manual trigger for cluster recalculation (admin only).
* Requires admin authentication.
*/
export async function POST(request: NextRequest) {
try {
const session = await auth()
if (!session?.user?.id) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
// Check if user is admin
const user = await prisma.user.findUnique({
where: { id: session.user.id },
select: { role: true }
})
if (user?.role !== 'ADMIN') {
return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
}
const body = await request.json()
const { userId, forceAll } = body
if (userId) {
// Process specific user
const clusterResults = await clusteringService.clusterNotes(userId)
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(userId)
await clusteringService.saveClusteringResults(userId, clusterResults)
await bridgeNotesService.saveBridgeNotes(userId, bridgeNotes)
return NextResponse.json({
success: true,
userId,
clusters: clusterResults.clusters.length,
bridges: bridgeNotes.length
})
}
if (forceAll) {
// Process all users (use with caution)
const users = await prisma.user.findMany({
select: { id: true },
take: 100 // Limit to 100 users per request
})
const results = {
processed: 0,
errors: 0
}
for (const user of users) {
try {
const clusterResults = await clusteringService.clusterNotes(user.id)
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(user.id)
await clusteringService.saveClusteringResults(user.id, clusterResults)
await bridgeNotesService.saveBridgeNotes(user.id, bridgeNotes)
results.processed++
} catch {
results.errors++
}
}
return NextResponse.json({
success: true,
...results
})
}
return NextResponse.json(
{ error: 'Must provide userId or forceAll=true' },
{ status: 400 }
)
} catch (error) {
console.error('Error in manual cluster trigger:', error)
return NextResponse.json(
{ error: 'Failed to trigger recalculation' },
{ status: 500 }
)
}
}