feat(cluster): implement cluster detection and bridge notes discovery
Add automatic note clustering using density-based algorithm (DBSCAN variant) and bridge notes detection for connecting different thematic clusters. Features: - NoteCluster, ClusterMember, BridgeNote, BridgeSuggestion models - Clustering service with pgvector cosine similarity - Bridge notes detection (notes connecting >=2 clusters) - AI-powered suggestions for missing cluster connections - /insights page with React Flow visualization - Cron endpoint for automatic recalculation Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Antigravity
224
memento-note/app/api/cron/clusters/route.ts
Normal file
224
memento-note/app/api/cron/clusters/route.ts
Normal file
@@ -0,0 +1,224 @@
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { auth } from '@/auth'
|
||||
import prisma from '@/lib/prisma'
|
||||
import { clusteringService } from '@/lib/ai/services/clustering.service'
|
||||
import { bridgeNotesService } from '@/lib/ai/services/bridge-notes.service'
|
||||
|
||||
/**
|
||||
* Cron endpoint for automatic cluster recalculation.
|
||||
*
|
||||
* This endpoint is designed to be called by a cron job or scheduler.
|
||||
* It processes users who need cluster updates based on:
|
||||
* 1. Data change percentage (> 5%)
|
||||
* 2. Time since last calculation (> 24 hours)
|
||||
*
|
||||
* GET /api/cron/clusters?secret=CRON_SECRET
|
||||
*
|
||||
* The secret parameter must match the environment variable CRON_SECRET.
|
||||
*/
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
// Verify cron secret
|
||||
const { searchParams } = new URL(request.url)
|
||||
const secret = searchParams.get('secret')
|
||||
|
||||
if (secret !== process.env.CRON_SECRET) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
// Find users who need recalculation
|
||||
// Users who either:
|
||||
// 1. Have no clusters calculated yet
|
||||
// 2. Have data changes > 5% since last calculation
|
||||
// 3. Haven't had calculations in > 24 hours
|
||||
|
||||
const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000)
|
||||
|
||||
const usersToProcess = await prisma.user.findMany({
|
||||
where: {
|
||||
// Only active users
|
||||
OR: [
|
||||
// Users who never had clusters calculated
|
||||
{
|
||||
noteClusters: {
|
||||
none: {}
|
||||
}
|
||||
},
|
||||
// Users with stale cluster data
|
||||
{
|
||||
noteClusters: {
|
||||
some: {
|
||||
lastCalculated: {
|
||||
lt: oneDayAgo
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
email: true
|
||||
},
|
||||
take: 50 // Process up to 50 users per run
|
||||
})
|
||||
|
||||
const results = {
|
||||
processed: 0,
|
||||
skipped: 0,
|
||||
errors: 0,
|
||||
details: [] as Array<{ userId: string; clusters: number; bridges: number; error?: string }>
|
||||
}
|
||||
|
||||
for (const user of usersToProcess) {
|
||||
try {
|
||||
// Check if recalculation is needed based on data change
|
||||
const shouldRecalc = await clusteringService.shouldRecalculate(user.id)
|
||||
|
||||
if (!shouldRecalc) {
|
||||
results.skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
// Perform clustering
|
||||
const clusterResults = await clusteringService.clusterNotes(user.id)
|
||||
|
||||
if (clusterResults.clusters.length === 0) {
|
||||
results.details.push({
|
||||
userId: user.id,
|
||||
clusters: 0,
|
||||
bridges: 0,
|
||||
error: 'Could not generate clusters'
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// Generate cluster names
|
||||
for (const cluster of clusterResults.clusters) {
|
||||
cluster.name = await clusteringService.generateClusterName(cluster.clusterId, user.id)
|
||||
}
|
||||
|
||||
// Save results
|
||||
await clusteringService.saveClusteringResults(user.id, clusterResults)
|
||||
|
||||
// Detect and save bridge notes
|
||||
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(user.id)
|
||||
await bridgeNotesService.saveBridgeNotes(user.id, bridgeNotes)
|
||||
|
||||
results.processed++
|
||||
results.details.push({
|
||||
userId: user.id,
|
||||
clusters: clusterResults.clusters.length,
|
||||
bridges: bridgeNotes.length
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error processing user ${user.id}:`, error)
|
||||
results.errors++
|
||||
results.details.push({
|
||||
userId: user.id,
|
||||
clusters: 0,
|
||||
bridges: 0,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
...results,
|
||||
message: `Processed ${results.processed} users, skipped ${results.skipped}, ${results.errors} errors`
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error in cluster cron job:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Cron job failed', details: error instanceof Error ? error.message : 'Unknown error' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/cron/clusters
|
||||
* Manual trigger for cluster recalculation (admin only).
|
||||
* Requires admin authentication.
|
||||
*/
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
// Check if user is admin
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: session.user.id },
|
||||
select: { role: true }
|
||||
})
|
||||
|
||||
if (user?.role !== 'ADMIN') {
|
||||
return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
|
||||
}
|
||||
|
||||
const body = await request.json()
|
||||
const { userId, forceAll } = body
|
||||
|
||||
if (userId) {
|
||||
// Process specific user
|
||||
const clusterResults = await clusteringService.clusterNotes(userId)
|
||||
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(userId)
|
||||
|
||||
await clusteringService.saveClusteringResults(userId, clusterResults)
|
||||
await bridgeNotesService.saveBridgeNotes(userId, bridgeNotes)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
userId,
|
||||
clusters: clusterResults.clusters.length,
|
||||
bridges: bridgeNotes.length
|
||||
})
|
||||
}
|
||||
|
||||
if (forceAll) {
|
||||
// Process all users (use with caution)
|
||||
const users = await prisma.user.findMany({
|
||||
select: { id: true },
|
||||
take: 100 // Limit to 100 users per request
|
||||
})
|
||||
|
||||
const results = {
|
||||
processed: 0,
|
||||
errors: 0
|
||||
}
|
||||
|
||||
for (const user of users) {
|
||||
try {
|
||||
const clusterResults = await clusteringService.clusterNotes(user.id)
|
||||
const bridgeNotes = await bridgeNotesService.detectBridgeNotes(user.id)
|
||||
|
||||
await clusteringService.saveClusteringResults(user.id, clusterResults)
|
||||
await bridgeNotesService.saveBridgeNotes(user.id, bridgeNotes)
|
||||
|
||||
results.processed++
|
||||
} catch {
|
||||
results.errors++
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
...results
|
||||
})
|
||||
}
|
||||
|
||||
return NextResponse.json(
|
||||
{ error: 'Must provide userId or forceAll=true' },
|
||||
{ status: 400 }
|
||||
)
|
||||
} catch (error) {
|
||||
console.error('Error in manual cluster trigger:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to trigger recalculation' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user