feat: embedding dimension validation + migration system

- Add /api/admin/embeddings/dimension (GET column dim, POST test model dim) - Add /api/admin/embeddings/migrate (alter column, clear, re-index) - Admin form warns on dimension mismatch after save, offers migrate button - Remove hardcoded 1536 from validate endpoint and embedding service - Add validateDimension() utility to EmbeddingService - Fix health route: import prisma correctly, use router instead of missing registry - i18n keys for dimension warning (EN/FR)
2026-05-19 18:45:50 +00:00
parent 28f46860c1
commit 6a8d0eb0a5
10 changed files with 297 additions and 20 deletions
--- a/memento-note/app/(admin)/admin/settings/admin-settings-form.tsx
+++ b/memento-note/app/(admin)/admin/settings/admin-settings-form.tsx
@@ -178,6 +178,9 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
  const [embedFallbackProvider, setEmbedFallbackProvider] = useState<string>(config.AI_PROVIDER_EMBEDDING_FALLBACK || '')
  const [chatFallbackProvider, setChatFallbackProvider] = useState<string>(config.AI_PROVIDER_CHAT_FALLBACK || '')

+  const [dimensionWarning, setDimensionWarning] = useState<{ dbDimension: number; modelDimension: number } | null>(null)
+  const [isMigrating, setIsMigrating] = useState(false)
+
  // Dynamic Models State (for local providers with /api/tags or /v1/models endpoints)
  const [dynamicModels, setDynamicModels] = useState<Record<ModelPurpose, string[]>>({
    tags: [],
@@ -412,6 +415,18 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
        toast.error(t('admin.ai.updateFailed') + ': ' + result.error)
      } else {
        toast.success(t('admin.ai.updateSuccess'))
+
+        setDimensionWarning(null)
+        try {
+          const dimRes = await fetch('/api/admin/embeddings/dimension', { method: 'POST' })
+          const dimData = await dimRes.json()
+          if (dimData.success && !dimData.match) {
+            setDimensionWarning({
+              dbDimension: dimData.dbDimension,
+              modelDimension: dimData.modelDimension,
+            })
+          }
+        } catch {}
      }
    } catch (error: any) {
      setIsSaving(false)
@@ -419,6 +434,24 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
    }
  }

+  const handleMigrateEmbeddings = async () => {
+    setIsMigrating(true)
+    try {
+      const res = await fetch('/api/admin/embeddings/migrate', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })
+      const data = await res.json()
+      if (data.success) {
+        toast.success(data.message)
+        setDimensionWarning(null)
+      } else {
+        toast.error(data.error || 'Migration failed')
+      }
+    } catch (error: any) {
+      toast.error(error.message)
+    } finally {
+      setIsMigrating(false)
+    }
+  }
+
  const handleSaveEmail = async (formData: FormData) => {
    setIsSaving(true)
    const data: Record<string, string> = { EMAIL_PROVIDER: emailProvider }
@@ -426,6 +459,8 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
    if (emailProvider === 'resend') {
      const key = formData.get('RESEND_API_KEY') as string
      if (key) data.RESEND_API_KEY = key
+      const from = formData.get('SMTP_FROM') as string
+      if (from) data.SMTP_FROM = from
    } else {
      data.SMTP_HOST = formData.get('SMTP_HOST') as string
      data.SMTP_PORT = formData.get('SMTP_PORT') as string
@@ -772,6 +807,33 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
              </h3>
              <p className="text-xs text-muted-foreground">{t('admin.ai.embeddingsDescription')}</p>

+              {dimensionWarning && (
+                <div className="rounded-lg border border-amber-500/50 bg-amber-50 dark:bg-amber-950/20 p-3 space-y-2">
+                  <p className="text-sm font-medium text-amber-800 dark:text-amber-200">
+                    {t('admin.ai.dimensionMismatch')}
+                  </p>
+                  <p className="text-xs text-amber-700 dark:text-amber-300">
+                    {t('admin.ai.dimensionMismatchDetail', {
+                      dbDimension: dimensionWarning.dbDimension,
+                      modelDimension: dimensionWarning.modelDimension,
+                    })}
+                  </p>
+                  <p className="text-xs text-amber-600 dark:text-amber-400">
+                    {t('admin.ai.dimensionMismatchAction')}
+                  </p>
+                  <Button
+                    type="button"
+                    size="sm"
+                    variant="outline"
+                    className="border-amber-500 text-amber-800 hover:bg-amber-100 dark:text-amber-200 dark:hover:bg-amber-900"
+                    disabled={isMigrating}
+                    onClick={handleMigrateEmbeddings}
+                  >
+                    {isMigrating ? t('admin.ai.migrating') : t('admin.ai.migrateButton')}
+                  </Button>
+                </div>
+              )}
+
              <div className="space-y-2">
                <Label htmlFor="AI_PROVIDER_EMBEDDING">
                  {t('admin.ai.provider')}
@@ -972,6 +1034,15 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
                  <Input id="RESEND_API_KEY" name="RESEND_API_KEY" type="password" defaultValue={config.RESEND_API_KEY || ''} placeholder="re_..." />
                  <p className="text-xs text-muted-foreground">{t('admin.resend.apiKeyHint')}</p>
                </div>
+                <div className="space-y-2">
+                  <label htmlFor="SMTP_FROM" className="text-sm font-medium">{t('admin.smtp.fromEmail')}</label>
+                  <Input
+                    id="SMTP_FROM"
+                    name="SMTP_FROM"
+                    defaultValue={config.SMTP_FROM || 'noreply@memento-note.com'}
+                    placeholder="noreply@memento-note.com"
+                  />
+                </div>
                {config.RESEND_API_KEY && (
                  <div className="flex items-center gap-2 text-xs text-green-600">
                    <span className="inline-block w-2 h-2 rounded-full bg-green-500" />
--- a/memento-note/app/api/admin/embeddings/dimension/route.ts
+++ b/memento-note/app/api/admin/embeddings/dimension/route.ts
@@ -0,0 +1,59 @@
+import { NextResponse } from 'next/server'
+import { prisma } from '@/lib/prisma'
+import { auth } from '@/auth'
+import { embeddingService } from '@/lib/ai/services/embedding.service'
+
+export async function GET() {
+  try {
+    const session = await auth()
+    if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
+      `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
+    )
+    const dbDimension = colDimResult[0]?.dim ?? null
+
+    const countResult: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
+      `SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
+    )
+    const embeddingCount = Number(countResult[0]?.total ?? 0)
+
+    return NextResponse.json({
+      success: true,
+      dbDimension,
+      embeddingCount,
+    })
+  } catch (error) {
+    console.error('[EMBEDDING_DIMENSION] Error:', error)
+    return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
+  }
+}
+
+export async function POST() {
+  try {
+    const session = await auth()
+    if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    const result = await embeddingService.generateEmbedding('dimension test')
+    const modelDimension = result.dimension
+
+    const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
+      `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
+    )
+    const dbDimension = colDimResult[0]?.dim ?? null
+
+    return NextResponse.json({
+      success: true,
+      modelDimension,
+      dbDimension,
+      match: modelDimension === dbDimension,
+    })
+  } catch (error) {
+    console.error('[EMBEDDING_DIMENSION] Test error:', error)
+    return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
+  }
+}
--- a/memento-note/app/api/admin/embeddings/migrate/route.ts
+++ b/memento-note/app/api/admin/embeddings/migrate/route.ts
@@ -0,0 +1,99 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { prisma } from '@/lib/prisma'
+import { auth } from '@/auth'
+import { embeddingService } from '@/lib/ai/services/embedding.service'
+import { getSystemConfig } from '@/lib/config'
+
+export async function POST(req: NextRequest) {
+  try {
+    const session = await auth()
+    if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
+    }
+
+    const body = await req.json().catch(() => ({}))
+    const targetDimension = body.targetDimension as number | undefined
+
+    const config = await getSystemConfig()
+
+    const testResult = await embeddingService.generateEmbedding('dimension test')
+    const modelDimension = testResult.dimension
+
+    const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
+      `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
+    )
+    const dbDimension = colDimResult[0]?.dim ?? null
+
+    const newDimension = targetDimension || modelDimension
+
+    if (dbDimension === newDimension) {
+      return NextResponse.json({
+        success: true,
+        message: 'Dimensions match, no migration needed',
+        dbDimension,
+        modelDimension,
+      })
+    }
+
+    const existingCount: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
+      `SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
+    )
+    const count = Number(existingCount[0]?.total ?? 0)
+
+    await prisma.$executeRawUnsafe(
+      `DROP INDEX IF EXISTS "NoteEmbedding_embedding_hnsw"`
+    )
+
+    await prisma.$executeRawUnsafe(
+      `TRUNCATE TABLE "NoteEmbedding"`
+    )
+
+    await prisma.$executeRawUnsafe(
+      `ALTER TABLE "NoteEmbedding" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
+    )
+
+    await prisma.$executeRawUnsafe(`
+      CREATE INDEX "NoteEmbedding_embedding_hnsw" ON "NoteEmbedding"
+      USING hnsw ("embedding" vector_cosine_ops)
+      WITH (m = 16, ef_construction = 64)
+    `)
+
+    const docChunkExists: Array<{ exists: boolean }> = await prisma.$queryRawUnsafe(
+      `SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'DocumentChunk' AND column_name = 'embedding') AS exists`
+    )
+
+    if (docChunkExists[0]?.exists) {
+      await prisma.$executeRawUnsafe(
+        `DROP INDEX IF EXISTS "DocumentChunk_embedding_hnsw_idx"`
+      )
+      await prisma.$executeRawUnsafe(
+        `TRUNCATE TABLE "DocumentChunk"`
+      )
+      await prisma.$executeRawUnsafe(
+        `ALTER TABLE "DocumentChunk" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
+      )
+      await prisma.$executeRawUnsafe(`
+        CREATE INDEX "DocumentChunk_embedding_hnsw_idx" ON "DocumentChunk"
+        USING hnsw ("embedding" vector_cosine_ops)
+        WITH (m = 16, ef_construction = 64)
+      `)
+    }
+
+    await prisma.systemConfig.upsert({
+      where: { key: 'AI_EMBEDDING_DIMENSION' },
+      update: { value: String(newDimension) },
+      create: { key: 'AI_EMBEDDING_DIMENSION', value: String(newDimension) },
+    })
+
+    return NextResponse.json({
+      success: true,
+      message: `Migration complete: vector(${dbDimension}) → vector(${newDimension}). ${count} old embeddings cleared. Re-indexing needed.`,
+      previousDimension: dbDimension,
+      newDimension,
+      clearedEmbeddings: count,
+    })
+  } catch (error) {
+    console.error('[EMBEDDING_MIGRATE] Error:', error)
+    return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
+  }
+}
--- a/memento-note/app/api/admin/embeddings/validate/route.ts
+++ b/memento-note/app/api/admin/embeddings/validate/route.ts
@@ -32,10 +32,14 @@ export async function GET() {
    )
    const validCount = Number(withEmbedding[0]?.count ?? 0)

+    const dimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
+      `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
+    )
+    const dbDim = dimResult[0]?.dim ?? 1536
+
    const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
      `SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e
-       WHERE e."embedding" IS NULL
-          OR array_length(string_to_array(replace(replace(e."embedding"::text, '[', ''), ']', ''), ','), 1) != 1536`
+       WHERE e."embedding" IS NULL`
    )
    const invalidCount = Number(invalidResult[0]?.count ?? 0)

@@ -43,6 +47,7 @@ export async function GET() {

    return NextResponse.json({
      success: true,
+      dbDimension: dbDim,
      summary: {
        total,
        valid: validCount - invalidCount,
--- a/memento-note/app/api/admin/health/route.ts
+++ b/memento-note/app/api/admin/health/route.ts
@@ -1,5 +1,5 @@
 import { NextResponse } from 'next/server'
-import { getPrisma } from '@/lib/prisma'
+import { prisma } from '@/lib/prisma'
 import { redis } from '@/lib/redis'

 export const dynamic = 'force-dynamic'
@@ -11,7 +11,6 @@ export async function GET() {
  // Database check
  try {
    const dbStart = Date.now()
-    const prisma = getPrisma()
    const [noteCount, notebookCount, userCount] = await Promise.all([
      prisma.note.count(),
      prisma.notebook.count(),
@@ -47,13 +46,15 @@ export async function GET() {

  // AI providers check
  try {
-    const { getAIProvider, getChatProvider } = await import('@/lib/ai/providers/registry')
-    const embeddingProvider = getAIProvider()
-    const chatProvider = getChatProvider()
+    const { getSystemConfig } = await import('@/lib/config')
+    const config = await getSystemConfig()
+    const { resolveAiRoute } = await import('@/lib/ai/router')
+    const tagsRoute = resolveAiRoute('tags', config)
+    const chatRoute = resolveAiRoute('chat', config)
    checks.ai = {
      status: 'configured',
-      embedding: { provider: embeddingProvider },
-      chat: { provider: chatProvider },
+      embedding: { provider: tagsRoute.providerType, model: tagsRoute.modelName },
+      chat: { provider: chatRoute.providerType, model: chatRoute.modelName },
    }
  } catch (e) {
    checks.ai = { status: 'unhealthy', error: e instanceof Error ? e.message : 'Unknown error' }