feat: embedding dimension validation + migration system
Some checks failed
CI / Lint, Test & Build (push) Failing after 8m1s
CI / Deploy production (on server) (push) Has been cancelled

- Add /api/admin/embeddings/dimension (GET column dim, POST test model dim)
- Add /api/admin/embeddings/migrate (alter column, clear, re-index)
- Admin form warns on dimension mismatch after save, offers migrate button
- Remove hardcoded 1536 from validate endpoint and embedding service
- Add validateDimension() utility to EmbeddingService
- Fix health route: import prisma correctly, use router instead of missing registry
- i18n keys for dimension warning (EN/FR)
This commit is contained in:
Antigravity
2026-05-19 18:45:50 +00:00
parent 28f46860c1
commit 6a8d0eb0a5
10 changed files with 297 additions and 20 deletions

View File

@@ -1,8 +1,8 @@
{ {
"version": 1, "version": 1,
"lastRunAtMs": 1779026397247, "lastRunAtMs": 1779034436676,
"turnsSinceLastRun": 7, "turnsSinceLastRun": 13,
"lastTranscriptMtimeMs": 1779026397153.5342, "lastTranscriptMtimeMs": 1779034436585.3164,
"lastProcessedGenerationId": "2cd15cc8-e19e-4dc7-b37a-7d6d7b76ae2b", "lastProcessedGenerationId": "fac69259-f459-4519-94aa-2b72a9453b24",
"trialStartedAtMs": null "trialStartedAtMs": null
} }

View File

@@ -178,6 +178,9 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
const [embedFallbackProvider, setEmbedFallbackProvider] = useState<string>(config.AI_PROVIDER_EMBEDDING_FALLBACK || '') const [embedFallbackProvider, setEmbedFallbackProvider] = useState<string>(config.AI_PROVIDER_EMBEDDING_FALLBACK || '')
const [chatFallbackProvider, setChatFallbackProvider] = useState<string>(config.AI_PROVIDER_CHAT_FALLBACK || '') const [chatFallbackProvider, setChatFallbackProvider] = useState<string>(config.AI_PROVIDER_CHAT_FALLBACK || '')
const [dimensionWarning, setDimensionWarning] = useState<{ dbDimension: number; modelDimension: number } | null>(null)
const [isMigrating, setIsMigrating] = useState(false)
// Dynamic Models State (for local providers with /api/tags or /v1/models endpoints) // Dynamic Models State (for local providers with /api/tags or /v1/models endpoints)
const [dynamicModels, setDynamicModels] = useState<Record<ModelPurpose, string[]>>({ const [dynamicModels, setDynamicModels] = useState<Record<ModelPurpose, string[]>>({
tags: [], tags: [],
@@ -412,6 +415,18 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
toast.error(t('admin.ai.updateFailed') + ': ' + result.error) toast.error(t('admin.ai.updateFailed') + ': ' + result.error)
} else { } else {
toast.success(t('admin.ai.updateSuccess')) toast.success(t('admin.ai.updateSuccess'))
setDimensionWarning(null)
try {
const dimRes = await fetch('/api/admin/embeddings/dimension', { method: 'POST' })
const dimData = await dimRes.json()
if (dimData.success && !dimData.match) {
setDimensionWarning({
dbDimension: dimData.dbDimension,
modelDimension: dimData.modelDimension,
})
}
} catch {}
} }
} catch (error: any) { } catch (error: any) {
setIsSaving(false) setIsSaving(false)
@@ -419,6 +434,24 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
} }
} }
const handleMigrateEmbeddings = async () => {
setIsMigrating(true)
try {
const res = await fetch('/api/admin/embeddings/migrate', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })
const data = await res.json()
if (data.success) {
toast.success(data.message)
setDimensionWarning(null)
} else {
toast.error(data.error || 'Migration failed')
}
} catch (error: any) {
toast.error(error.message)
} finally {
setIsMigrating(false)
}
}
const handleSaveEmail = async (formData: FormData) => { const handleSaveEmail = async (formData: FormData) => {
setIsSaving(true) setIsSaving(true)
const data: Record<string, string> = { EMAIL_PROVIDER: emailProvider } const data: Record<string, string> = { EMAIL_PROVIDER: emailProvider }
@@ -426,6 +459,8 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
if (emailProvider === 'resend') { if (emailProvider === 'resend') {
const key = formData.get('RESEND_API_KEY') as string const key = formData.get('RESEND_API_KEY') as string
if (key) data.RESEND_API_KEY = key if (key) data.RESEND_API_KEY = key
const from = formData.get('SMTP_FROM') as string
if (from) data.SMTP_FROM = from
} else { } else {
data.SMTP_HOST = formData.get('SMTP_HOST') as string data.SMTP_HOST = formData.get('SMTP_HOST') as string
data.SMTP_PORT = formData.get('SMTP_PORT') as string data.SMTP_PORT = formData.get('SMTP_PORT') as string
@@ -772,6 +807,33 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
</h3> </h3>
<p className="text-xs text-muted-foreground">{t('admin.ai.embeddingsDescription')}</p> <p className="text-xs text-muted-foreground">{t('admin.ai.embeddingsDescription')}</p>
{dimensionWarning && (
<div className="rounded-lg border border-amber-500/50 bg-amber-50 dark:bg-amber-950/20 p-3 space-y-2">
<p className="text-sm font-medium text-amber-800 dark:text-amber-200">
{t('admin.ai.dimensionMismatch')}
</p>
<p className="text-xs text-amber-700 dark:text-amber-300">
{t('admin.ai.dimensionMismatchDetail', {
dbDimension: dimensionWarning.dbDimension,
modelDimension: dimensionWarning.modelDimension,
})}
</p>
<p className="text-xs text-amber-600 dark:text-amber-400">
{t('admin.ai.dimensionMismatchAction')}
</p>
<Button
type="button"
size="sm"
variant="outline"
className="border-amber-500 text-amber-800 hover:bg-amber-100 dark:text-amber-200 dark:hover:bg-amber-900"
disabled={isMigrating}
onClick={handleMigrateEmbeddings}
>
{isMigrating ? t('admin.ai.migrating') : t('admin.ai.migrateButton')}
</Button>
</div>
)}
<div className="space-y-2"> <div className="space-y-2">
<Label htmlFor="AI_PROVIDER_EMBEDDING"> <Label htmlFor="AI_PROVIDER_EMBEDDING">
{t('admin.ai.provider')} {t('admin.ai.provider')}
@@ -972,6 +1034,15 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
<Input id="RESEND_API_KEY" name="RESEND_API_KEY" type="password" defaultValue={config.RESEND_API_KEY || ''} placeholder="re_..." /> <Input id="RESEND_API_KEY" name="RESEND_API_KEY" type="password" defaultValue={config.RESEND_API_KEY || ''} placeholder="re_..." />
<p className="text-xs text-muted-foreground">{t('admin.resend.apiKeyHint')}</p> <p className="text-xs text-muted-foreground">{t('admin.resend.apiKeyHint')}</p>
</div> </div>
<div className="space-y-2">
<label htmlFor="SMTP_FROM" className="text-sm font-medium">{t('admin.smtp.fromEmail')}</label>
<Input
id="SMTP_FROM"
name="SMTP_FROM"
defaultValue={config.SMTP_FROM || 'noreply@memento-note.com'}
placeholder="noreply@memento-note.com"
/>
</div>
{config.RESEND_API_KEY && ( {config.RESEND_API_KEY && (
<div className="flex items-center gap-2 text-xs text-green-600"> <div className="flex items-center gap-2 text-xs text-green-600">
<span className="inline-block w-2 h-2 rounded-full bg-green-500" /> <span className="inline-block w-2 h-2 rounded-full bg-green-500" />

View File

@@ -0,0 +1,59 @@
import { NextResponse } from 'next/server'
import { prisma } from '@/lib/prisma'
import { auth } from '@/auth'
import { embeddingService } from '@/lib/ai/services/embedding.service'
export async function GET() {
try {
const session = await auth()
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
)
const dbDimension = colDimResult[0]?.dim ?? null
const countResult: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
`SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
)
const embeddingCount = Number(countResult[0]?.total ?? 0)
return NextResponse.json({
success: true,
dbDimension,
embeddingCount,
})
} catch (error) {
console.error('[EMBEDDING_DIMENSION] Error:', error)
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
}
}
export async function POST() {
try {
const session = await auth()
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const result = await embeddingService.generateEmbedding('dimension test')
const modelDimension = result.dimension
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
)
const dbDimension = colDimResult[0]?.dim ?? null
return NextResponse.json({
success: true,
modelDimension,
dbDimension,
match: modelDimension === dbDimension,
})
} catch (error) {
console.error('[EMBEDDING_DIMENSION] Test error:', error)
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
}
}

View File

@@ -0,0 +1,99 @@
import { NextRequest, NextResponse } from 'next/server'
import { prisma } from '@/lib/prisma'
import { auth } from '@/auth'
import { embeddingService } from '@/lib/ai/services/embedding.service'
import { getSystemConfig } from '@/lib/config'
export async function POST(req: NextRequest) {
try {
const session = await auth()
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const body = await req.json().catch(() => ({}))
const targetDimension = body.targetDimension as number | undefined
const config = await getSystemConfig()
const testResult = await embeddingService.generateEmbedding('dimension test')
const modelDimension = testResult.dimension
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
)
const dbDimension = colDimResult[0]?.dim ?? null
const newDimension = targetDimension || modelDimension
if (dbDimension === newDimension) {
return NextResponse.json({
success: true,
message: 'Dimensions match, no migration needed',
dbDimension,
modelDimension,
})
}
const existingCount: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
`SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
)
const count = Number(existingCount[0]?.total ?? 0)
await prisma.$executeRawUnsafe(
`DROP INDEX IF EXISTS "NoteEmbedding_embedding_hnsw"`
)
await prisma.$executeRawUnsafe(
`TRUNCATE TABLE "NoteEmbedding"`
)
await prisma.$executeRawUnsafe(
`ALTER TABLE "NoteEmbedding" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
)
await prisma.$executeRawUnsafe(`
CREATE INDEX "NoteEmbedding_embedding_hnsw" ON "NoteEmbedding"
USING hnsw ("embedding" vector_cosine_ops)
WITH (m = 16, ef_construction = 64)
`)
const docChunkExists: Array<{ exists: boolean }> = await prisma.$queryRawUnsafe(
`SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'DocumentChunk' AND column_name = 'embedding') AS exists`
)
if (docChunkExists[0]?.exists) {
await prisma.$executeRawUnsafe(
`DROP INDEX IF EXISTS "DocumentChunk_embedding_hnsw_idx"`
)
await prisma.$executeRawUnsafe(
`TRUNCATE TABLE "DocumentChunk"`
)
await prisma.$executeRawUnsafe(
`ALTER TABLE "DocumentChunk" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
)
await prisma.$executeRawUnsafe(`
CREATE INDEX "DocumentChunk_embedding_hnsw_idx" ON "DocumentChunk"
USING hnsw ("embedding" vector_cosine_ops)
WITH (m = 16, ef_construction = 64)
`)
}
await prisma.systemConfig.upsert({
where: { key: 'AI_EMBEDDING_DIMENSION' },
update: { value: String(newDimension) },
create: { key: 'AI_EMBEDDING_DIMENSION', value: String(newDimension) },
})
return NextResponse.json({
success: true,
message: `Migration complete: vector(${dbDimension}) → vector(${newDimension}). ${count} old embeddings cleared. Re-indexing needed.`,
previousDimension: dbDimension,
newDimension,
clearedEmbeddings: count,
})
} catch (error) {
console.error('[EMBEDDING_MIGRATE] Error:', error)
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
}
}

View File

@@ -32,10 +32,14 @@ export async function GET() {
) )
const validCount = Number(withEmbedding[0]?.count ?? 0) const validCount = Number(withEmbedding[0]?.count ?? 0)
const dimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
)
const dbDim = dimResult[0]?.dim ?? 1536
const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe( const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
`SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e `SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e
WHERE e."embedding" IS NULL WHERE e."embedding" IS NULL`
OR array_length(string_to_array(replace(replace(e."embedding"::text, '[', ''), ']', ''), ','), 1) != 1536`
) )
const invalidCount = Number(invalidResult[0]?.count ?? 0) const invalidCount = Number(invalidResult[0]?.count ?? 0)
@@ -43,6 +47,7 @@ export async function GET() {
return NextResponse.json({ return NextResponse.json({
success: true, success: true,
dbDimension: dbDim,
summary: { summary: {
total, total,
valid: validCount - invalidCount, valid: validCount - invalidCount,

View File

@@ -1,5 +1,5 @@
import { NextResponse } from 'next/server' import { NextResponse } from 'next/server'
import { getPrisma } from '@/lib/prisma' import { prisma } from '@/lib/prisma'
import { redis } from '@/lib/redis' import { redis } from '@/lib/redis'
export const dynamic = 'force-dynamic' export const dynamic = 'force-dynamic'
@@ -11,7 +11,6 @@ export async function GET() {
// Database check // Database check
try { try {
const dbStart = Date.now() const dbStart = Date.now()
const prisma = getPrisma()
const [noteCount, notebookCount, userCount] = await Promise.all([ const [noteCount, notebookCount, userCount] = await Promise.all([
prisma.note.count(), prisma.note.count(),
prisma.notebook.count(), prisma.notebook.count(),
@@ -47,13 +46,15 @@ export async function GET() {
// AI providers check // AI providers check
try { try {
const { getAIProvider, getChatProvider } = await import('@/lib/ai/providers/registry') const { getSystemConfig } = await import('@/lib/config')
const embeddingProvider = getAIProvider() const config = await getSystemConfig()
const chatProvider = getChatProvider() const { resolveAiRoute } = await import('@/lib/ai/router')
const tagsRoute = resolveAiRoute('tags', config)
const chatRoute = resolveAiRoute('chat', config)
checks.ai = { checks.ai = {
status: 'configured', status: 'configured',
embedding: { provider: embeddingProvider }, embedding: { provider: tagsRoute.providerType, model: tagsRoute.modelName },
chat: { provider: chatProvider }, chat: { provider: chatRoute.providerType, model: chatRoute.modelName },
} }
} catch (e) { } catch (e) {
checks.ai = { status: 'unhealthy', error: e instanceof Error ? e.message : 'Unknown error' } checks.ai = { status: 'unhealthy', error: e instanceof Error ? e.message : 'Unknown error' }

View File

@@ -6,6 +6,7 @@
import { withAiProviderFallback } from '../fallback' import { withAiProviderFallback } from '../fallback'
import { getSystemConfig } from '@/lib/config' import { getSystemConfig } from '@/lib/config'
import { prisma } from '@/lib/prisma'
export interface EmbeddingResult { export interface EmbeddingResult {
embedding: number[] embedding: number[]
@@ -14,7 +15,6 @@ export interface EmbeddingResult {
} }
export class EmbeddingService { export class EmbeddingService {
private readonly EMBEDDING_DIMENSION = 1536
private readonly MAX_CHARS = 15000 private readonly MAX_CHARS = 15000
private truncateForEmbedding(text: string): string { private truncateForEmbedding(text: string): string {
@@ -109,6 +109,38 @@ export class EmbeddingService {
* Check if a note needs embedding regeneration. * Check if a note needs embedding regeneration.
* Uses a content-content comparison (not embedding-content). * Uses a content-content comparison (not embedding-content).
*/ */
async getDbDimension(): Promise<number | null> {
try {
const result: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
)
return result[0]?.dim ?? null
} catch {
return null
}
}
async getModelDimension(): Promise<number | null> {
try {
const { dimension } = await this.generateEmbedding('dimension test')
return dimension
} catch {
return null
}
}
async validateDimension(): Promise<{ dbDimension: number | null; modelDimension: number | null; match: boolean }> {
const [dbDimension, modelDimension] = await Promise.all([
this.getDbDimension(),
this.getModelDimension(),
])
return {
dbDimension,
modelDimension,
match: dbDimension !== null && modelDimension !== null && dbDimension === modelDimension,
}
}
shouldRegenerateEmbedding( shouldRegenerateEmbedding(
noteContent: string, noteContent: string,
_lastEmbeddingContent: string | null, _lastEmbeddingContent: string | null,

View File

@@ -94,14 +94,14 @@ async function sendViaResend(apiKey: string, config: Record<string, string>, { t
}); });
if (error) { if (error) {
// Resend test mode: can only send to the Resend account owner's email const msg = error.message || String(error);
if (error.message?.includes('own email address') || error.name === 'validation_error') { if (msg.includes('own email address')) {
return { return {
success: false, success: false,
error: `Mode test Resend : vous ne pouvez envoyer qu'à l'adresse du compte Resend. Pour envoyer à n'importe qui, vérifiez un domaine sur resend.com/domains et configurez SMTP_FROM avec une adresse de ce domaine.`, error: `Mode test Resend : envoi uniquement vers l'e-mail du compte Resend. Vérifiez le domaine sur resend.com/domains et SMTP_FROM (@domaine vérifié). Détail : ${msg}`,
}; };
} }
return { success: false, error: error.message }; return { success: false, error: msg };
} }
return { success: true, messageId: data?.id }; return { success: true, messageId: data?.id };

View File

@@ -1137,7 +1137,12 @@
"languageDetection": "Language detection", "languageDetection": "Language detection",
"languageDetectionDesc": "Automatically detects the language of each note", "languageDetectionDesc": "Automatically detects the language of each note",
"autoLabeling": "Auto labeling", "autoLabeling": "Auto labeling",
"autoLabelingDesc": "Suggests and applies labels automatically" "autoLabelingDesc": "Suggests and applies labels automatically",
"dimensionMismatch": "Embedding dimension mismatch detected",
"dimensionMismatchDetail": "Database column: vector({{dbDimension}}) — Model outputs: {{modelDimension}} dimensions",
"dimensionMismatchAction": "Embeddings will not work until you migrate the column. This will clear all existing embeddings and re-index them.",
"migrateButton": "Migrate & Re-index",
"migrating": "Migrating..."
}, },
"resend": { "resend": {
"title": "Resend (Recommended)", "title": "Resend (Recommended)",

View File

@@ -1143,7 +1143,12 @@
"languageDetection": "Détection de langue", "languageDetection": "Détection de langue",
"languageDetectionDesc": "Détecte automatiquement la langue de chaque note", "languageDetectionDesc": "Détecte automatiquement la langue de chaque note",
"autoLabeling": "Étiquetage automatique", "autoLabeling": "Étiquetage automatique",
"autoLabelingDesc": "Suggère et applique des étiquettes automatiquement" "autoLabelingDesc": "Suggère et applique des étiquettes automatiquement",
"dimensionMismatch": "Incompatibilité de dimensions d'embedding détectée",
"dimensionMismatchDetail": "Colonne base de données : vector({{dbDimension}}) — Le modèle produit : {{modelDimension}} dimensions",
"dimensionMismatchAction": "Les embeddings ne fonctionneront pas tant que vous ne migrez pas la colonne. Cela effacera tous les embeddings existants et les ré-indexera.",
"migrateButton": "Migrer & Ré-indexer",
"migrating": "Migration en cours..."
}, },
"resend": { "resend": {
"title": "Resend (Recommandé)", "title": "Resend (Recommandé)",