03e6a62b80
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 2m12s
Replace JSON-string embeddings with native pgvector(1536) storage and add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion for hybrid keyword + semantic ranking. Changes: - NoteEmbedding.embedding: String → vector(1536) via pgvector - NoteEmbedding: added updatedAt for reindex tracking - Note: added tsv (tsvector) with auto-update trigger for FTS - semantic-search.service: hybrid FTS + vector search with RRF fusion - embedding.service: toVectorString() for pgvector SQL literals - Removed JS-side cosine similarity loops (now DB-side via <=>) - Added HNSW index on NoteEmbedding.embedding (cosine distance) - Added GIN index on Note.tsv for FTS queries Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/ Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
116 lines
3.3 KiB
TypeScript
116 lines
3.3 KiB
TypeScript
/**
|
|
* Embedding Service
|
|
* Generates vector embeddings for semantic search and similarity analysis.
|
|
* Stores embeddings as native pgvector(1536) in PostgreSQL.
|
|
*/
|
|
|
|
import { getAIProvider } from '../factory'
|
|
import { getSystemConfig } from '@/lib/config'
|
|
|
|
export interface EmbeddingResult {
|
|
embedding: number[]
|
|
model: string
|
|
dimension: number
|
|
}
|
|
|
|
export class EmbeddingService {
|
|
private readonly EMBEDDING_DIMENSION = 1536
|
|
|
|
async generateEmbedding(text: string): Promise<EmbeddingResult> {
|
|
if (!text || text.trim().length === 0) {
|
|
throw new Error('Cannot generate embedding for empty text')
|
|
}
|
|
|
|
try {
|
|
const config = await getSystemConfig()
|
|
const provider = getAIProvider(config)
|
|
const embedding = await provider.getEmbeddings(text)
|
|
|
|
return {
|
|
embedding,
|
|
model: 'text-embedding-3-small',
|
|
dimension: embedding.length
|
|
}
|
|
} catch (error) {
|
|
console.error('Error generating embedding:', error)
|
|
throw new Error(`Failed to generate embedding: ${error}`)
|
|
}
|
|
}
|
|
|
|
async generateBatchEmbeddings(texts: string[]): Promise<EmbeddingResult[]> {
|
|
if (!texts || texts.length === 0) return []
|
|
|
|
const validTexts = texts.filter(t => t && t.trim().length > 0)
|
|
if (validTexts.length === 0) return []
|
|
|
|
try {
|
|
const config = await getSystemConfig()
|
|
const provider = getAIProvider(config)
|
|
const embeddings = await Promise.all(
|
|
validTexts.map(text => provider.getEmbeddings(text))
|
|
)
|
|
|
|
return embeddings.map(embedding => ({
|
|
embedding,
|
|
model: 'text-embedding-3-small',
|
|
dimension: embedding.length
|
|
}))
|
|
} catch (error) {
|
|
console.error('Error generating batch embeddings:', error)
|
|
throw error
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Format a number[] embedding as a pgvector-compatible string literal.
|
|
* e.g. [0.1, 0.2, 0.3] → '[0.1,0.2,0.3]'
|
|
*/
|
|
toVectorString(embedding: number[]): string {
|
|
return `[${embedding.join(',')}]`
|
|
}
|
|
|
|
/**
|
|
* Parse a pgvector string from the DB back into number[].
|
|
* e.g. '[0.1,0.2,0.3]' → [0.1, 0.2, 0.3]
|
|
*/
|
|
fromVectorString(vec: string): number[] {
|
|
if (Array.isArray(vec)) return vec
|
|
if (!vec || typeof vec !== 'string') return []
|
|
return vec.replace(/^\[/, '').replace(/\]$/, '').split(',').map(Number)
|
|
}
|
|
|
|
/**
|
|
* JS cosine similarity — still used by memory-echo pairwise comparisons.
|
|
*/
|
|
calculateCosineSimilarity(a: number[], b: number[]): number {
|
|
if (!a.length || !b.length) return 0
|
|
const minLen = Math.min(a.length, b.length)
|
|
let dot = 0, mA = 0, mB = 0
|
|
for (let i = 0; i < minLen; i++) {
|
|
dot += a[i] * b[i]
|
|
mA += a[i] * a[i]
|
|
mB += b[i] * b[i]
|
|
}
|
|
mA = Math.sqrt(mA)
|
|
mB = Math.sqrt(mB)
|
|
if (mA === 0 || mB === 0) return 0
|
|
return dot / (mA * mB)
|
|
}
|
|
|
|
/**
|
|
* Check if a note needs embedding regeneration.
|
|
* Uses a content-content comparison (not embedding-content).
|
|
*/
|
|
shouldRegenerateEmbedding(
|
|
noteContent: string,
|
|
_lastEmbeddingContent: string | null,
|
|
lastAnalysis: Date | null
|
|
): boolean {
|
|
if (!lastAnalysis) return true
|
|
const daysSinceAnalysis = (Date.now() - lastAnalysis.getTime()) / (1000 * 60 * 60 * 24)
|
|
return daysSinceAnalysis > 7
|
|
}
|
|
}
|
|
|
|
export const embeddingService = new EmbeddingService()
|