sepehr/Momento
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: migrate semantic search to pgvector + full-text search
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s
Details

Browse Source 
Replace JSON-string embeddings with native pgvector(1536) storage and
add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion
for hybrid keyword + semantic ranking.

Changes:
- NoteEmbedding.embedding: String → vector(1536) via pgvector
- NoteEmbedding: added updatedAt for reindex tracking
- Note: added tsv (tsvector) with auto-update trigger for FTS
- semantic-search.service: hybrid FTS + vector search with RRF fusion
- embedding.service: toVectorString() for pgvector SQL literals
- Removed JS-side cosine similarity loops (now DB-side via <=>)
- Added HNSW index on NoteEmbedding.embedding (cosine distance)
- Added GIN index on Note.tsv for FTS queries

Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Antigravity
2026-05-12 07:03:56 +00:00
parent 92c3a6f307
commit 03e6a62b80
43 changed files with 4024 additions and 786 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
memento-note/app/actions/notes.ts
View File

@@ -5,9 +5,10 @@ import prisma from '@/lib/prisma'
import { Note, CheckItem, NoteType } from '@/lib/types'
import { auth } from '@/auth'
import { getAIProvider } from '@/lib/ai/factory'
import { parseNote as parseNoteUtil, cosineSimilarity, calculateRRFK, detectQueryType, getSearchWeights } from '@/lib/utils'
import { parseNote as parseNoteUtil } from '@/lib/utils'
import { getSystemConfig, getConfigNumber, getConfigBoolean, SEARCH_DEFAULTS } from '@/lib/config'
import { contextualAutoTagService } from '@/lib/ai/services/contextual-auto-tag.service'
import { semanticSearchService } from '@/lib/ai/services/semantic-search.service'
import { cleanupNoteImages, parseImageUrls, deleteImageFileSafely } from '@/lib/image-cleanup'
import { getAISettings } from '@/app/actions/ai-settings'
import {
@@ -486,122 +487,54 @@ export async function enableNoteHistory(noteId: string) {
})
}
// Search notes - DB-side filtering (fast) with optional semantic search
// Supports contextual search within notebook (IA5)
export async function searchNotes(query: string, useSemantic: boolean = false, notebookId?: string) {
// Unified hybrid search — always uses FTS + pgvector with RRF fusion.
// Supports contextual search within notebook (IA5).
export async function searchNotes(query: string, _useSemantic: boolean = true, notebookId?: string) {
const session = await auth();
if (!session?.user?.id) return [];
try {
// If query empty, return all notes
if (!query || !query.trim()) {
return await getAllNotes();
}
// If semantic search is requested, use the full implementation
if (useSemantic) {
return await semanticSearch(query, session.user.id, notebookId);
}
const results = await semanticSearchService.searchAsUser(session.user.id, query, {
limit: 50,
threshold: 0.25,
notebookId
});
// DB-side keyword search using LIKE — much faster than loading all notes in memory
const noteIds = results.map(r => r.noteId);
const notes = await prisma.note.findMany({
where: {
id: { in: noteIds },
userId: session.user.id,
isArchived: false,
trashedAt: null,
OR: [
{ title: { contains: query } },
{ content: { contains: query } },
{ labels: { contains: query } },
],
},
select: NOTE_LIST_SELECT,
orderBy: [
{ isPinned: 'desc' },
{ order: 'asc' },
{ updatedAt: 'desc' }
]
});
return notes.map(parseNote);
const orderMap = new Map(results.map((r, i) => [r.noteId, i]));
const parsed = notes.map(parseNote);
parsed.sort((a, b) => (orderMap.get(a.id) ?? 999) - (orderMap.get(b.id) ?? 999));
if (parsed.length > 0) {
const topResult = results[0];
if (topResult) {
parsed[0].matchType = topResult.matchType;
parsed[0].searchScore = topResult.score;
}
}
return parsed;
} catch (error) {
console.error('Search error:', error);
return [];
}
}
// Semantic search with AI embeddings - SIMPLE VERSION
// Supports contextual search within notebook (IA5)
async function semanticSearch(query: string, userId: string, notebookId?: string) {
const allNotes = await prisma.note.findMany({
where: {
userId: userId,
isArchived: false,
trashedAt: null,
...(notebookId !== undefined ? { notebookId } : {})
},
include: { noteEmbedding: true }
});
const queryLower = query.toLowerCase().trim();
// Get query embedding
let queryEmbedding: number[] | null = null;
try {
const provider = getAIProvider(await getSystemConfig());
queryEmbedding = await provider.getEmbeddings(query);
} catch (e) {
console.error('Failed to generate query embedding:', e);
// Fallback to simple keyword search
queryEmbedding = null;
}
// Filter notes: keyword match OR semantic match (threshold 30%)
const results = allNotes.map(note => {
const title = (note.title || '').toLowerCase();
const content = note.content.toLowerCase();
const labels = note.labels ? JSON.parse(note.labels) : [];
// Keyword match
const keywordMatch = title.includes(queryLower) ||
content.includes(queryLower) ||
labels.some((l: string) => l.toLowerCase().includes(queryLower));
// Semantic match (if embedding available)
let semanticMatch = false;
let similarity = 0;
if (queryEmbedding && note.noteEmbedding?.embedding) {
similarity = cosineSimilarity(queryEmbedding, JSON.parse(note.noteEmbedding.embedding));
semanticMatch = similarity > 0.3; // 30% threshold - works well for related concepts
}
return {
note,
keywordMatch,
semanticMatch,
similarity
};
}).filter(r => r.keywordMatch || r.semanticMatch);
// Parse and add match info
return results.map(r => {
const parsed = parseNote(r.note);
// Determine match type
let matchType: 'exact' | 'related' | null = null;
if (r.semanticMatch) {
matchType = 'related';
} else if (r.keywordMatch) {
matchType = 'exact';
}
return {
...parsed,
matchType
};
});
}
// Create a new note
export async function createNote(data: {
title?: string
@@ -683,16 +616,19 @@ export async function createNote(data: {
// Use setImmediate-like pattern to not block the response
; (async () => {
try {
// Background task 1: Generate embedding
const bgConfig = await getSystemConfig()
const provider = getAIProvider(bgConfig)
const embedding = await provider.getEmbeddings(content)
if (embedding) {
await prisma.noteEmbedding.upsert({
where: { noteId: noteId },
create: { noteId: noteId, embedding: JSON.stringify(embedding) },
update: { embedding: JSON.stringify(embedding) }
})
const vecStr = `[${embedding.join(',')}]`
await prisma.$executeRawUnsafe(
`INSERT INTO "NoteEmbedding" ("id", "noteId", "embedding", "createdAt", "updatedAt")
VALUES (gen_random_uuid(), $1, $2::vector, now(), now())
ON CONFLICT ("noteId")
DO UPDATE SET "embedding" = $2::vector, "updatedAt" = now()`,
noteId,
vecStr
)
}
} catch (e) {
console.error('[BG] Embedding generation failed:', e)
@@ -815,7 +751,6 @@ export async function updateNote(id: string, data: {
}
}
// Generate embedding in background — don't block the update
if (data.content !== undefined) {
const noteId = id
const content = data.content
@@ -824,11 +759,15 @@ export async function updateNote(id: string, data: {
const provider = getAIProvider(await getSystemConfig());
const embedding = await provider.getEmbeddings(content);
if (embedding) {
await prisma.noteEmbedding.upsert({
where: { noteId: noteId },
create: { noteId: noteId, embedding: JSON.stringify(embedding) },
update: { embedding: JSON.stringify(embedding) }
})
const vecStr = `[${embedding.join(',')}]`
await prisma.$executeRawUnsafe(
`INSERT INTO "NoteEmbedding" ("id", "noteId", "embedding", "createdAt", "updatedAt")
VALUES (gen_random_uuid(), $1, $2::vector, now(), now())
ON CONFLICT ("noteId")
DO UPDATE SET "embedding" = $2::vector, "updatedAt" = now()`,
noteId,
vecStr
)
}
} catch (e) {
console.error('[BG] Embedding regeneration failed:', e);
@@ -1409,11 +1348,15 @@ export async function syncAllEmbeddings() {
try {
const embedding = await provider.getEmbeddings(note.content);
if (embedding) {
await prisma.noteEmbedding.upsert({
where: { noteId: note.id },
create: { noteId: note.id, embedding: JSON.stringify(embedding) },
update: { embedding: JSON.stringify(embedding) }
})
const vecStr = `[${embedding.join(',')}]`
await prisma.$executeRawUnsafe(
`INSERT INTO "NoteEmbedding" ("id", "noteId", "embedding", "createdAt", "updatedAt")
VALUES (gen_random_uuid(), $1, $2::vector, now(), now())
ON CONFLICT ("noteId")
DO UPDATE SET "embedding" = $2::vector, "updatedAt" = now()`,
note.id,
vecStr
)
updatedCount++;
}
} catch (e) { }