feat(cluster): implement cluster detection and bridge notes discovery
Add automatic note clustering using density-based algorithm (DBSCAN variant) and bridge notes detection for connecting different thematic clusters. Features: - NoteCluster, ClusterMember, BridgeNote, BridgeSuggestion models - Clustering service with pgvector cosine similarity - Bridge notes detection (notes connecting >=2 clusters) - AI-powered suggestions for missing cluster connections - /insights page with React Flow visualization - Cron endpoint for automatic recalculation Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Antigravity
85
memento-note/app/api/bridge-notes/route.ts
Normal file
85
memento-note/app/api/bridge-notes/route.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { auth } from '@/auth'
|
||||
import prisma from '@/lib/prisma'
|
||||
import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
|
||||
|
||||
/**
|
||||
* GET /api/bridge-notes
|
||||
* Get all bridge notes for the current user.
|
||||
*/
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const userId = session.user.id
|
||||
const { searchParams } = new URL(request.url)
|
||||
const includeDetails = searchParams.get('details') === 'true'
|
||||
|
||||
const bridgeNotes = await bridgeNotesService.getBridgeNotes(userId)
|
||||
|
||||
// Optionally include note details
|
||||
let enrichedNotes = bridgeNotes
|
||||
if (includeDetails && bridgeNotes.length > 0) {
|
||||
const noteIds = bridgeNotes.map(b => b.noteId)
|
||||
const notes = await prisma.note.findMany({
|
||||
where: { id: { in: noteIds } },
|
||||
select: { id: true, title: true, content: true, notebookId: true }
|
||||
})
|
||||
|
||||
const noteMap = new Map(notes.map(n => [n.id, n]))
|
||||
|
||||
enrichedNotes = bridgeNotes.map(b => ({
|
||||
...b,
|
||||
note: noteMap.get(b.noteId)
|
||||
}))
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
bridgeNotes: enrichedNotes,
|
||||
count: enrichedNotes.length
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error fetching bridge notes:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch bridge notes' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* DELETE /api/bridge-notes
|
||||
* Dismiss a bridge suggestion.
|
||||
*/
|
||||
export async function DELETE(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const userId = session.user.id
|
||||
const body = await request.json()
|
||||
const { clusterAId, clusterBId } = body
|
||||
|
||||
if (typeof clusterAId !== 'number' || typeof clusterBId !== 'number') {
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid cluster IDs' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
await bridgeNotesService.dismissSuggestion(userId, clusterAId, clusterBId)
|
||||
|
||||
return NextResponse.json({ success: true })
|
||||
} catch (error) {
|
||||
console.error('Error dismissing suggestion:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to dismiss suggestion' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
66
memento-note/app/api/bridge-notes/suggestions/route.ts
Normal file
66
memento-note/app/api/bridge-notes/suggestions/route.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { auth } from '@/auth'
|
||||
import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
|
||||
|
||||
/**
|
||||
* GET /api/bridge-notes/suggestions
|
||||
* Get AI-powered bridge note suggestions for unconnected clusters.
|
||||
*/
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const userId = session.user.id
|
||||
const { searchParams } = new URL(request.url)
|
||||
const includeDismissed = searchParams.get('dismissed') === 'true'
|
||||
|
||||
const suggestions = await bridgeNotesService.getBridgeSuggestions(userId, includeDismissed)
|
||||
|
||||
return NextResponse.json({
|
||||
suggestions,
|
||||
count: suggestions.length
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error fetching bridge suggestions:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch suggestions' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/bridge-notes/suggestions/generate
|
||||
* Generate new bridge suggestions (force regeneration).
|
||||
*/
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const userId = session.user.id
|
||||
|
||||
// Generate new suggestions
|
||||
const suggestions = await bridgeNotesService.generateBridgeSuggestions(userId)
|
||||
|
||||
// Save to database
|
||||
await bridgeNotesService.saveBridgeSuggestions(userId, suggestions)
|
||||
|
||||
return NextResponse.json({
|
||||
suggestions,
|
||||
count: suggestions.length,
|
||||
message: `Generated ${suggestions.length} bridge suggestions`
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error generating bridge suggestions:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to generate suggestions' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
124
memento-note/app/api/clusters/route.ts
Normal file
124
memento-note/app/api/clusters/route.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { auth } from '@/auth'
|
||||
import prisma from '@/lib/prisma'
|
||||
import { clusteringService } from '@/lib/ai/services/clustering.service'
|
||||
import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
|
||||
|
||||
/**
|
||||
* GET /api/clusters
|
||||
* Get all clusters for the current user.
|
||||
*/
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const userId = session.user.id
|
||||
|
||||
// Check for cached results
|
||||
const cached = await clusteringService.getCachedClusters(userId)
|
||||
if (cached) {
|
||||
return NextResponse.json({
|
||||
clusters: cached,
|
||||
cached: true,
|
||||
totalNotes: cached.reduce((sum, c) => sum + c.noteIds.length, 0)
|
||||
})
|
||||
}
|
||||
|
||||
// No cached results, check if user has enough notes
|
||||
const notesCount = await prisma.note.count({
|
||||
where: { userId, trashedAt: null }
|
||||
})
|
||||
|
||||
if (notesCount < 10) {
|
||||
return NextResponse.json({
|
||||
clusters: [],
|
||||
message: 'Need at least 10 notes to generate clusters',
|
||||
totalNotes: notesCount
|
||||
})
|
||||
}
|
||||
|
||||
// Trigger background recalculation
|
||||
return NextResponse.json({
|
||||
clusters: [],
|
||||
message: 'Calculating clusters... Please check back later',
|
||||
totalNotes: notesCount
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error fetching clusters:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch clusters' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/clusters/recalculate
|
||||
* Trigger a full recalculation of clusters and bridge notes.
|
||||
*/
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const userId = session.user.id
|
||||
const body = await request.json()
|
||||
const force = body.force === true
|
||||
|
||||
// Check if recalculation is needed
|
||||
const shouldRecalc = force || await clusteringService.shouldRecalculate(userId)
|
||||
|
||||
if (!shouldRecalc) {
|
||||
const cached = await clusteringService.getCachedClusters(userId)
|
||||
if (cached) {
|
||||
return NextResponse.json({
|
||||
clusters: cached,
|
||||
cached: true,
|
||||
message: 'Using cached results (data has not changed significantly)'
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Perform clustering
|
||||
const results = await clusteringService.clusterNotes(userId)
|
||||
|
||||
if (results.clusters.length === 0) {
|
||||
return NextResponse.json({
|
||||
clusters: [],
|
||||
message: 'Could not generate clusters. Need more diverse notes.',
|
||||
noiseCount: results.noiseCount
|
||||
})
|
||||
}
|
||||
|
||||
// Generate cluster names
|
||||
for (const cluster of results.clusters) {
|
||||
cluster.name = await clusteringService.generateClusterName(cluster.clusterId, userId)
|
||||
}
|
||||
|
||||
// Save results
|
||||
await clusteringService.saveClusteringResults(userId, results)
|
||||
|
||||
// Detect and save bridge notes
|
||||
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(userId)
|
||||
await bridgeNotesService.saveBridgeNotes(userId, bridgeNotes)
|
||||
|
||||
return NextResponse.json({
|
||||
clusters: results.clusters,
|
||||
bridgeNotes: bridgeNotes.slice(0, 10), // Return top 10
|
||||
totalNotes: results.clusters.reduce((sum, c) => sum + c.noteIds.length, 0) + results.noiseCount,
|
||||
noiseCount: results.noiseCount,
|
||||
message: `Generated ${results.clusters.length} clusters`
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error recalculating clusters:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to recalculate clusters' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
224
memento-note/app/api/cron/clusters/route.ts
Normal file
224
memento-note/app/api/cron/clusters/route.ts
Normal file
@@ -0,0 +1,224 @@
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { auth } from '@/auth'
|
||||
import prisma from '@/lib/prisma'
|
||||
import { clusteringService } from '@/lib/ai/services/clustering.service'
|
||||
import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
|
||||
|
||||
/**
|
||||
* Cron endpoint for automatic cluster recalculation.
|
||||
*
|
||||
* This endpoint is designed to be called by a cron job or scheduler.
|
||||
* It processes users who need cluster updates based on:
|
||||
* 1. Data change percentage (> 5%)
|
||||
* 2. Time since last calculation (> 24 hours)
|
||||
*
|
||||
* GET /api/cron/clusters?secret=CRON_SECRET
|
||||
*
|
||||
* The secret parameter must match the environment variable CRON_SECRET.
|
||||
*/
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// Verify cron secret
|
||||
const { searchParams } = new URL(request.url)
|
||||
const secret = searchParams.get('secret')
|
||||
|
||||
if (secret !== process.env.CRON_SECRET) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
// Find users who need recalculation
|
||||
// Users who either:
|
||||
// 1. Have no clusters calculated yet
|
||||
// 2. Have data changes > 5% since last calculation
|
||||
// 3. Haven't had calculations in > 24 hours
|
||||
|
||||
const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000)
|
||||
|
||||
const usersToProcess = await prisma.user.findMany({
|
||||
where: {
|
||||
// Only active users
|
||||
OR: [
|
||||
// Users who never had clusters calculated
|
||||
{
|
||||
noteClusters: {
|
||||
none: {}
|
||||
}
|
||||
},
|
||||
// Users with stale cluster data
|
||||
{
|
||||
noteClusters: {
|
||||
some: {
|
||||
lastCalculated: {
|
||||
lt: oneDayAgo
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
email: true
|
||||
},
|
||||
take: 50 // Process up to 50 users per run
|
||||
})
|
||||
|
||||
const results = {
|
||||
processed: 0,
|
||||
skipped: 0,
|
||||
errors: 0,
|
||||
details: [] as Array<{ userId: string; clusters: number; bridges: number; error?: string }>
|
||||
}
|
||||
|
||||
for (const user of usersToProcess) {
|
||||
try {
|
||||
// Check if recalculation is needed based on data change
|
||||
const shouldRecalc = await clusteringService.shouldRecalculate(user.id)
|
||||
|
||||
if (!shouldRecalc) {
|
||||
results.skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
// Perform clustering
|
||||
const clusterResults = await clusteringService.clusterNotes(user.id)
|
||||
|
||||
if (clusterResults.clusters.length === 0) {
|
||||
results.details.push({
|
||||
userId: user.id,
|
||||
clusters: 0,
|
||||
bridges: 0,
|
||||
error: 'Could not generate clusters'
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// Generate cluster names
|
||||
for (const cluster of clusterResults.clusters) {
|
||||
cluster.name = await clusteringService.generateClusterName(cluster.clusterId, user.id)
|
||||
}
|
||||
|
||||
// Save results
|
||||
await clusteringService.saveClusteringResults(user.id, clusterResults)
|
||||
|
||||
// Detect and save bridge notes
|
||||
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(user.id)
|
||||
await bridgeNotesService.saveBridgeNotes(user.id, bridgeNotes)
|
||||
|
||||
results.processed++
|
||||
results.details.push({
|
||||
userId: user.id,
|
||||
clusters: clusterResults.clusters.length,
|
||||
bridges: bridgeNotes.length
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error processing user ${user.id}:`, error)
|
||||
results.errors++
|
||||
results.details.push({
|
||||
userId: user.id,
|
||||
clusters: 0,
|
||||
bridges: 0,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
...results,
|
||||
message: `Processed ${results.processed} users, skipped ${results.skipped}, ${results.errors} errors`
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error in cluster cron job:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Cron job failed', details: error instanceof Error ? error.message : 'Unknown error' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/cron/clusters
|
||||
* Manual trigger for cluster recalculation (admin only).
|
||||
* Requires admin authentication.
|
||||
*/
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
// Check if user is admin
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: session.user.id },
|
||||
select: { role: true }
|
||||
})
|
||||
|
||||
if (user?.role !== 'ADMIN') {
|
||||
return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
|
||||
}
|
||||
|
||||
const body = await request.json()
|
||||
const { userId, forceAll } = body
|
||||
|
||||
if (userId) {
|
||||
// Process specific user
|
||||
const clusterResults = await clusteringService.clusterNotes(userId)
|
||||
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(userId)
|
||||
|
||||
await clusteringService.saveClusteringResults(userId, clusterResults)
|
||||
await bridgeNotesService.saveBridgeNotes(userId, bridgeNotes)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
userId,
|
||||
clusters: clusterResults.clusters.length,
|
||||
bridges: bridgeNotes.length
|
||||
})
|
||||
}
|
||||
|
||||
if (forceAll) {
|
||||
// Process all users (use with caution)
|
||||
const users = await prisma.user.findMany({
|
||||
select: { id: true },
|
||||
take: 100 // Limit to 100 users per request
|
||||
})
|
||||
|
||||
const results = {
|
||||
processed: 0,
|
||||
errors: 0
|
||||
}
|
||||
|
||||
for (const user of users) {
|
||||
try {
|
||||
const clusterResults = await clusteringService.clusterNotes(user.id)
|
||||
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(user.id)
|
||||
|
||||
await clusteringService.saveClusteringResults(user.id, clusterResults)
|
||||
await bridgeNotesService.saveBridgeNotes(user.id, bridgeNotes)
|
||||
|
||||
results.processed++
|
||||
} catch {
|
||||
results.errors++
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
...results
|
||||
})
|
||||
}
|
||||
|
||||
return NextResponse.json(
|
||||
{ error: 'Must provide userId or forceAll=true' },
|
||||
{ status: 400 }
|
||||
)
|
||||
} catch (error) {
|
||||
console.error('Error in manual cluster trigger:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to trigger recalculation' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user