/** * Embedding Service * Generates vector embeddings for semantic search and similarity analysis. * Stores embeddings as native pgvector in PostgreSQL. */ import { withAiProviderFallback } from '../fallback' import { getSystemConfig } from '@/lib/config' import { prisma } from '@/lib/prisma' export interface EmbeddingResult { embedding: number[] model: string dimension: number } export class EmbeddingService { private readonly MAX_CHARS = 15000 private truncateForEmbedding(text: string): string { if (text.length <= this.MAX_CHARS) return text return text.slice(0, this.MAX_CHARS) } async generateEmbedding(text: string): Promise { if (!text || text.trim().length === 0) { throw new Error('Cannot generate embedding for empty text') } const truncated = this.truncateForEmbedding(text) try { const config = await getSystemConfig() const embedding = await withAiProviderFallback('embedding', config, (provider) => provider.getEmbeddings(truncated) ) return { embedding, model: 'text-embedding-3-small', dimension: embedding.length } } catch (error) { console.error('Error generating embedding:', error) throw new Error(`Failed to generate embedding: ${error}`) } } async generateBatchEmbeddings(texts: string[]): Promise { if (!texts || texts.length === 0) return [] const validTexts = texts.filter(t => t && t.trim().length > 0).map(t => this.truncateForEmbedding(t)) if (validTexts.length === 0) return [] try { const config = await getSystemConfig() const embeddings = await withAiProviderFallback('embedding', config, (provider) => Promise.all(validTexts.map((text) => provider.getEmbeddings(text))) ) return embeddings.map(embedding => ({ embedding, model: 'text-embedding-3-small', dimension: embedding.length })) } catch (error) { console.error('Error generating batch embeddings:', error) throw error } } /** * Format a number[] embedding as a pgvector-compatible string literal. * e.g. [0.1, 0.2, 0.3] → '[0.1,0.2,0.3]' */ toVectorString(embedding: number[]): string { return `[${embedding.join(',')}]` } /** * Parse a pgvector string from the DB back into number[]. * e.g. '[0.1,0.2,0.3]' → [0.1, 0.2, 0.3] */ fromVectorString(vec: string): number[] { if (Array.isArray(vec)) return vec if (!vec || typeof vec !== 'string') return [] return vec.replace(/^\[/, '').replace(/\]$/, '').split(',').map(Number) } /** * JS cosine similarity — still used by memory-echo pairwise comparisons. */ calculateCosineSimilarity(a: number[], b: number[]): number { if (!a.length || !b.length) return 0 const minLen = Math.min(a.length, b.length) let dot = 0, mA = 0, mB = 0 for (let i = 0; i < minLen; i++) { dot += a[i] * b[i] mA += a[i] * a[i] mB += b[i] * b[i] } mA = Math.sqrt(mA) mB = Math.sqrt(mB) if (mA === 0 || mB === 0) return 0 return dot / (mA * mB) } /** * Check if a note needs embedding regeneration. * Uses a content-content comparison (not embedding-content). */ async getDbDimension(): Promise { try { const result: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe( `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'` ) return result[0]?.dim ?? null } catch { return null } } async getModelDimension(): Promise { try { const { dimension } = await this.generateEmbedding('dimension test') return dimension } catch { return null } } async validateDimension(): Promise<{ dbDimension: number | null; modelDimension: number | null; match: boolean }> { const [dbDimension, modelDimension] = await Promise.all([ this.getDbDimension(), this.getModelDimension(), ]) return { dbDimension, modelDimension, match: dbDimension !== null && modelDimension !== null && dbDimension === modelDimension, } } shouldRegenerateEmbedding( noteContent: string, _lastEmbeddingContent: string | null, lastAnalysis: Date | null ): boolean { if (!lastAnalysis) return true const daysSinceAnalysis = (Date.now() - lastAnalysis.getTime()) / (1000 * 60 * 60 * 24) return daysSinceAnalysis > 7 } } export const embeddingService = new EmbeddingService()