Add automatic note clustering using density-based algorithm (DBSCAN variant) and bridge notes detection for connecting different thematic clusters. Features: - NoteCluster, ClusterMember, BridgeNote, BridgeSuggestion models - Clustering service with pgvector cosine similarity - Bridge notes detection (notes connecting >=2 clusters) - AI-powered suggestions for missing cluster connections - /insights page with React Flow visualization - Cron endpoint for automatic recalculation Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
395 lines
11 KiB
TypeScript
395 lines
11 KiB
TypeScript
/**
|
|
* Bridge Notes Service
|
|
*
|
|
* Detects and manages "bridge notes" — notes that connect multiple clusters.
|
|
* A bridge note has strong similarities (cosine > 0.5) with notes from
|
|
* at least two different clusters.
|
|
*
|
|
* Also generates AI-powered suggestions for creating new bridge notes
|
|
* to connect isolated clusters.
|
|
*/
|
|
|
|
import prisma from '@/lib/prisma'
|
|
import { clusteringService } from './clustering.service'
|
|
import { getChatProvider } from '@/lib/ai/factory'
|
|
import { getSystemConfig } from '@/lib/config'
|
|
|
|
export interface BridgeNote {
|
|
noteId: string
|
|
bridgeScore: number
|
|
clustersConnected: number[]
|
|
clusterNames?: string[]
|
|
}
|
|
|
|
export interface BridgeSuggestion {
|
|
clusterAId: number
|
|
clusterBId: number
|
|
clusterAName: string
|
|
clusterBName: string
|
|
suggestedTitle: string
|
|
suggestedContent: string
|
|
justification: string
|
|
}
|
|
|
|
export class BridgeNotesService {
|
|
private readonly BRIDGE_SIMILARITY_THRESHOLD = 0.5
|
|
private readonly MIN_CLUSTERS_FOR_BRIDGE = 2
|
|
|
|
/**
|
|
* Get similar notes for a given note across all clusters.
|
|
* Returns notes grouped by their cluster membership.
|
|
*/
|
|
private async getSimilarNotesByCluster(
|
|
noteId: string,
|
|
userId: string,
|
|
threshold: number = this.BRIDGE_SIMILARITY_THRESHOLD
|
|
): Promise<Map<number, string[]>> {
|
|
const cosineDistance = 1 - threshold
|
|
|
|
const result = await prisma.$queryRawUnsafe<Array<{
|
|
noteId: string
|
|
clusterId: number | null
|
|
}>>(
|
|
`SELECT similar."noteId", cm."clusterId"
|
|
FROM (
|
|
SELECT e2."noteId"
|
|
FROM "NoteEmbedding" e1
|
|
CROSS JOIN "NoteEmbedding" e2
|
|
INNER JOIN "Note" n ON n.id = e2."noteId"
|
|
WHERE e1."noteId" = $1
|
|
AND e2."noteId" != e1."noteId"
|
|
AND n."userId" = $2
|
|
AND n."trashedAt" IS NULL
|
|
AND (e1."embedding"::vector <=> e2."embedding"::vector) <= $3
|
|
) similar
|
|
LEFT JOIN "ClusterMember" cm ON cm."noteId" = similar."noteId" AND cm."userId" = $2`,
|
|
noteId,
|
|
userId,
|
|
cosineDistance
|
|
)
|
|
|
|
const clusterMap = new Map<number, string[]>()
|
|
|
|
for (const row of result) {
|
|
const clusterId = row.clusterId ?? -1 // -1 for noise/uncategorized
|
|
if (!clusterMap.has(clusterId)) {
|
|
clusterMap.set(clusterId, [])
|
|
}
|
|
clusterMap.get(clusterId)!.push(row.noteId)
|
|
}
|
|
|
|
return clusterMap
|
|
}
|
|
|
|
/**
|
|
* Detect all bridge notes for a user.
|
|
* A note is a bridge if it has similarities to >= 2 distinct clusters.
|
|
*/
|
|
async detectBridgeNotes(userId: string): Promise<BridgeNote[]> {
|
|
// Get all user's clusters
|
|
const clusters = await prisma.noteCluster.findMany({
|
|
where: { userId },
|
|
select: { clusterId: true, name: true },
|
|
orderBy: { clusterId: 'asc' }
|
|
})
|
|
|
|
if (clusters.length < this.MIN_CLUSTERS_FOR_BRIDGE) {
|
|
return []
|
|
}
|
|
|
|
const maxClusters = clusters.length
|
|
const bridgeNotes: BridgeNote[] = []
|
|
|
|
// Check each note for bridge potential
|
|
const notes = await prisma.note.findMany({
|
|
where: { userId, trashedAt: null },
|
|
select: { id: true }
|
|
})
|
|
|
|
for (const note of notes) {
|
|
const similarByCluster = await this.getSimilarNotesByCluster(note.id, userId)
|
|
|
|
// Filter out noise (-1) and get clusters with actual similar notes
|
|
const clustersWithSimilarNotes: number[] = []
|
|
for (const [clusterId, similarNotes] of similarByCluster) {
|
|
if (clusterId !== -1 && similarNotes.length > 0) {
|
|
clustersWithSimilarNotes.push(clusterId)
|
|
}
|
|
}
|
|
|
|
// Check if this note connects >= 2 clusters
|
|
if (clustersWithSimilarNotes.length >= this.MIN_CLUSTERS_FOR_BRIDGE) {
|
|
const bridgeScore = clustersWithSimilarNotes.length / maxClusters
|
|
|
|
bridgeNotes.push({
|
|
noteId: note.id,
|
|
bridgeScore,
|
|
clustersConnected: clustersWithSimilarNotes,
|
|
clusterNames: clustersWithSimilarNotes.map(
|
|
cid => clusters.find(c => c.clusterId === cid)?.name || `Cluster ${cid}`
|
|
)
|
|
})
|
|
}
|
|
}
|
|
|
|
// Sort by bridge score (most influential first)
|
|
return bridgeNotes.sort((a, b) => b.bridgeScore - a.bridgeScore)
|
|
}
|
|
|
|
/**
|
|
* Save bridge notes to database.
|
|
*/
|
|
async saveBridgeNotes(userId: string, bridgeNotes: BridgeNote[]): Promise<void> {
|
|
await prisma.$transaction(async (tx) => {
|
|
// Clear existing bridge notes for this user
|
|
await tx.bridgeNote.deleteMany({ where: { userId } })
|
|
|
|
// Insert new bridge notes
|
|
for (const bridge of bridgeNotes) {
|
|
await tx.bridgeNote.create({
|
|
data: {
|
|
userId,
|
|
noteId: bridge.noteId,
|
|
bridgeScore: bridge.bridgeScore,
|
|
clustersConnected: JSON.stringify(bridge.clustersConnected),
|
|
lastCalculated: new Date()
|
|
}
|
|
})
|
|
}
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Get cluster summaries for AI suggestions.
|
|
*/
|
|
private async getClusterSummary(clusterId: number, userId: string): Promise<string> {
|
|
const notes = await prisma.$queryRawUnsafe<Array<{ title: string | null; content: string }>>(
|
|
`SELECT n.title, n.content
|
|
FROM "ClusterMember" cm
|
|
INNER JOIN "Note" n ON n.id = cm."noteId"
|
|
WHERE cm."clusterId" = $1
|
|
AND cm."userId" = $2
|
|
LIMIT 5`,
|
|
clusterId,
|
|
userId
|
|
)
|
|
|
|
if (notes.length === 0) return 'No notes available'
|
|
|
|
return notes
|
|
.map(n => `- ${n.title || 'Untitled'}: ${n.content.slice(0, 80)}...`)
|
|
.join('\n')
|
|
}
|
|
|
|
/**
|
|
* Generate AI-powered suggestions for connecting isolated clusters.
|
|
*/
|
|
async generateBridgeSuggestions(userId: string): Promise<BridgeSuggestion[]> {
|
|
// Get all clusters
|
|
const clusters = await prisma.noteCluster.findMany({
|
|
where: { userId },
|
|
select: { clusterId: true, name: true },
|
|
orderBy: { clusterId: 'asc' }
|
|
})
|
|
|
|
if (clusters.length < 2) return []
|
|
|
|
// Get existing bridges to see which clusters are already connected
|
|
const existingBridges = await prisma.bridgeNote.findMany({
|
|
where: { userId },
|
|
select: { clustersConnected: true }
|
|
})
|
|
|
|
const connectedPairs = new Set<string>()
|
|
|
|
for (const bridge of existingBridges) {
|
|
const clusters = JSON.parse(bridge.clustersConnected) as number[]
|
|
for (let i = 0; i < clusters.length; i++) {
|
|
for (let j = i + 1; j < clusters.length; j++) {
|
|
const pair = [clusters[i], clusters[j]].sort().join('-')
|
|
connectedPairs.add(pair)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Find unconnected cluster pairs
|
|
const suggestions: BridgeSuggestion[] = []
|
|
|
|
for (let i = 0; i < clusters.length; i++) {
|
|
for (let j = i + 1; j < clusters.length; j++) {
|
|
const pair = `${clusters[i].clusterId}-${clusters[j].clusterId}`
|
|
|
|
if (connectedPairs.has(pair)) continue // Already connected
|
|
|
|
// Generate suggestion for this unconnected pair
|
|
const suggestion = await this.generateConnectionSuggestion(
|
|
clusters[i].clusterId,
|
|
clusters[j].clusterId,
|
|
clusters[i].name || `Cluster ${clusters[i].clusterId}`,
|
|
clusters[j].name || `Cluster ${clusters[j].clusterId}`,
|
|
userId
|
|
)
|
|
|
|
if (suggestion) {
|
|
suggestions.push(suggestion)
|
|
}
|
|
}
|
|
}
|
|
|
|
return suggestions
|
|
}
|
|
|
|
/**
|
|
* Generate a specific connection suggestion between two clusters.
|
|
*/
|
|
private async generateConnectionSuggestion(
|
|
clusterAId: number,
|
|
clusterBId: number,
|
|
clusterAName: string,
|
|
clusterBName: string,
|
|
userId: string
|
|
): Promise<BridgeSuggestion | null> {
|
|
const summaryA = await this.getClusterSummary(clusterAId, userId)
|
|
const summaryB = await this.getClusterSummary(clusterBId, userId)
|
|
|
|
const systemPrompt = `You are a creative assistant that helps users connect ideas.
|
|
Suggest a "bridge note" that could connect two unrelated topics.
|
|
Be specific and creative. Your suggestions should help users discover new insights.`
|
|
|
|
const userPrompt = `I have two groups of notes that are not connected:
|
|
|
|
Group A (${clusterAName}):
|
|
${summaryA}
|
|
|
|
Group B (${clusterBName}):
|
|
${summaryB}
|
|
|
|
Suggest 3 creative bridge note ideas to connect these groups. For each idea, provide:
|
|
1. A catchy title (max 10 words)
|
|
2. A brief description of what the note would contain (max 50 words)
|
|
3. A justification for why this connection is valuable (max 30 words)
|
|
|
|
Format as JSON:
|
|
{
|
|
"ideas": [
|
|
{"title": "...", "description": "...", "justification": "..."},
|
|
...
|
|
]
|
|
}
|
|
|
|
Return ONLY the JSON, no other text.`
|
|
|
|
try {
|
|
const config = await getSystemConfig()
|
|
const provider = getChatProvider(config)
|
|
const response = await provider.chat(
|
|
[{ role: 'user', content: userPrompt }],
|
|
systemPrompt
|
|
)
|
|
|
|
// Parse JSON response
|
|
const jsonMatch = response.text.match(/\{[\s\S]*\}/)
|
|
if (!jsonMatch) return null
|
|
|
|
const parsed = JSON.parse(jsonMatch[0])
|
|
const bestIdea = parsed.ideas?.[0]
|
|
|
|
if (!bestIdea) return null
|
|
|
|
return {
|
|
clusterAId,
|
|
clusterBId,
|
|
clusterAName,
|
|
clusterBName,
|
|
suggestedTitle: bestIdea.title,
|
|
suggestedContent: bestIdea.description,
|
|
justification: bestIdea.justification
|
|
}
|
|
} catch (error) {
|
|
console.error('Error generating bridge suggestion:', error)
|
|
return null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save bridge suggestions to database.
|
|
*/
|
|
async saveBridgeSuggestions(userId: string, suggestions: BridgeSuggestion[]): Promise<void> {
|
|
await prisma.$transaction(async (tx) => {
|
|
// Clear existing suggestions
|
|
await tx.bridgeSuggestion.deleteMany({ where: { userId } })
|
|
|
|
// Insert new suggestions
|
|
for (const suggestion of suggestions) {
|
|
await tx.bridgeSuggestion.create({
|
|
data: {
|
|
userId,
|
|
clusterAId: suggestion.clusterAId,
|
|
clusterBId: suggestion.clusterBId,
|
|
clusterAName: suggestion.clusterAName,
|
|
clusterBName: suggestion.clusterBName,
|
|
suggestedTitle: suggestion.suggestedTitle,
|
|
suggestedContent: suggestion.suggestedContent,
|
|
justification: suggestion.justification
|
|
}
|
|
})
|
|
}
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Get bridge notes for a user.
|
|
*/
|
|
async getBridgeNotes(userId: string): Promise<BridgeNote[]> {
|
|
const bridges = await prisma.bridgeNote.findMany({
|
|
where: { userId },
|
|
orderBy: { bridgeScore: 'desc' }
|
|
})
|
|
|
|
return bridges.map(b => ({
|
|
noteId: b.noteId,
|
|
bridgeScore: b.bridgeScore,
|
|
clustersConnected: JSON.parse(b.clustersConnected) as number[]
|
|
}))
|
|
}
|
|
|
|
/**
|
|
* Get bridge suggestions for a user.
|
|
*/
|
|
async getBridgeSuggestions(userId: string, includeDismissed: boolean = false): Promise<BridgeSuggestion[]> {
|
|
const suggestions = await prisma.bridgeSuggestion.findMany({
|
|
where: {
|
|
userId,
|
|
...(includeDismissed ? {} : { isDismissed: false })
|
|
},
|
|
orderBy: { createdAt: 'desc' }
|
|
})
|
|
|
|
return suggestions.map(s => ({
|
|
clusterAId: s.clusterAId,
|
|
clusterBId: s.clusterBId,
|
|
clusterAName: s.clusterAName,
|
|
clusterBName: s.clusterBName,
|
|
suggestedTitle: s.suggestedTitle,
|
|
suggestedContent: s.suggestedContent,
|
|
justification: s.justification
|
|
}))
|
|
}
|
|
|
|
/**
|
|
* Dismiss a bridge suggestion.
|
|
*/
|
|
async dismissSuggestion(userId: string, clusterAId: number, clusterBId: number): Promise<void> {
|
|
await prisma.bridgeSuggestion.updateMany({
|
|
where: {
|
|
userId,
|
|
clusterAId,
|
|
clusterBId
|
|
},
|
|
data: { isDismissed: true }
|
|
})
|
|
}
|
|
}
|
|
|
|
export const bridgeNotesService = new BridgeNotesService()
|