feat: embedding dimension validation + migration system
- Add /api/admin/embeddings/dimension (GET column dim, POST test model dim) - Add /api/admin/embeddings/migrate (alter column, clear, re-index) - Admin form warns on dimension mismatch after save, offers migrate button - Remove hardcoded 1536 from validate endpoint and embedding service - Add validateDimension() utility to EmbeddingService - Fix health route: import prisma correctly, use router instead of missing registry - i18n keys for dimension warning (EN/FR)
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
|
||||
import { withAiProviderFallback } from '../fallback'
|
||||
import { getSystemConfig } from '@/lib/config'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
|
||||
export interface EmbeddingResult {
|
||||
embedding: number[]
|
||||
@@ -14,7 +15,6 @@ export interface EmbeddingResult {
|
||||
}
|
||||
|
||||
export class EmbeddingService {
|
||||
private readonly EMBEDDING_DIMENSION = 1536
|
||||
private readonly MAX_CHARS = 15000
|
||||
|
||||
private truncateForEmbedding(text: string): string {
|
||||
@@ -109,6 +109,38 @@ export class EmbeddingService {
|
||||
* Check if a note needs embedding regeneration.
|
||||
* Uses a content-content comparison (not embedding-content).
|
||||
*/
|
||||
async getDbDimension(): Promise<number | null> {
|
||||
try {
|
||||
const result: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||
)
|
||||
return result[0]?.dim ?? null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async getModelDimension(): Promise<number | null> {
|
||||
try {
|
||||
const { dimension } = await this.generateEmbedding('dimension test')
|
||||
return dimension
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async validateDimension(): Promise<{ dbDimension: number | null; modelDimension: number | null; match: boolean }> {
|
||||
const [dbDimension, modelDimension] = await Promise.all([
|
||||
this.getDbDimension(),
|
||||
this.getModelDimension(),
|
||||
])
|
||||
return {
|
||||
dbDimension,
|
||||
modelDimension,
|
||||
match: dbDimension !== null && modelDimension !== null && dbDimension === modelDimension,
|
||||
}
|
||||
}
|
||||
|
||||
shouldRegenerateEmbedding(
|
||||
noteContent: string,
|
||||
_lastEmbeddingContent: string | null,
|
||||
|
||||
Reference in New Issue
Block a user