feat: embedding dimension validation + migration system
Some checks failed
CI / Lint, Test & Build (push) Failing after 8m1s
CI / Deploy production (on server) (push) Has been cancelled

- Add /api/admin/embeddings/dimension (GET column dim, POST test model dim)
- Add /api/admin/embeddings/migrate (alter column, clear, re-index)
- Admin form warns on dimension mismatch after save, offers migrate button
- Remove hardcoded 1536 from validate endpoint and embedding service
- Add validateDimension() utility to EmbeddingService
- Fix health route: import prisma correctly, use router instead of missing registry
- i18n keys for dimension warning (EN/FR)
This commit is contained in:
Antigravity
2026-05-19 18:45:50 +00:00
parent 28f46860c1
commit 6a8d0eb0a5
10 changed files with 297 additions and 20 deletions

View File

@@ -0,0 +1,59 @@
import { NextResponse } from 'next/server'
import { prisma } from '@/lib/prisma'
import { auth } from '@/auth'
import { embeddingService } from '@/lib/ai/services/embedding.service'
export async function GET() {
try {
const session = await auth()
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
)
const dbDimension = colDimResult[0]?.dim ?? null
const countResult: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
`SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
)
const embeddingCount = Number(countResult[0]?.total ?? 0)
return NextResponse.json({
success: true,
dbDimension,
embeddingCount,
})
} catch (error) {
console.error('[EMBEDDING_DIMENSION] Error:', error)
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
}
}
export async function POST() {
try {
const session = await auth()
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const result = await embeddingService.generateEmbedding('dimension test')
const modelDimension = result.dimension
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
)
const dbDimension = colDimResult[0]?.dim ?? null
return NextResponse.json({
success: true,
modelDimension,
dbDimension,
match: modelDimension === dbDimension,
})
} catch (error) {
console.error('[EMBEDDING_DIMENSION] Test error:', error)
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
}
}

View File

@@ -0,0 +1,99 @@
import { NextRequest, NextResponse } from 'next/server'
import { prisma } from '@/lib/prisma'
import { auth } from '@/auth'
import { embeddingService } from '@/lib/ai/services/embedding.service'
import { getSystemConfig } from '@/lib/config'
export async function POST(req: NextRequest) {
try {
const session = await auth()
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const body = await req.json().catch(() => ({}))
const targetDimension = body.targetDimension as number | undefined
const config = await getSystemConfig()
const testResult = await embeddingService.generateEmbedding('dimension test')
const modelDimension = testResult.dimension
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
)
const dbDimension = colDimResult[0]?.dim ?? null
const newDimension = targetDimension || modelDimension
if (dbDimension === newDimension) {
return NextResponse.json({
success: true,
message: 'Dimensions match, no migration needed',
dbDimension,
modelDimension,
})
}
const existingCount: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
`SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
)
const count = Number(existingCount[0]?.total ?? 0)
await prisma.$executeRawUnsafe(
`DROP INDEX IF EXISTS "NoteEmbedding_embedding_hnsw"`
)
await prisma.$executeRawUnsafe(
`TRUNCATE TABLE "NoteEmbedding"`
)
await prisma.$executeRawUnsafe(
`ALTER TABLE "NoteEmbedding" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
)
await prisma.$executeRawUnsafe(`
CREATE INDEX "NoteEmbedding_embedding_hnsw" ON "NoteEmbedding"
USING hnsw ("embedding" vector_cosine_ops)
WITH (m = 16, ef_construction = 64)
`)
const docChunkExists: Array<{ exists: boolean }> = await prisma.$queryRawUnsafe(
`SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'DocumentChunk' AND column_name = 'embedding') AS exists`
)
if (docChunkExists[0]?.exists) {
await prisma.$executeRawUnsafe(
`DROP INDEX IF EXISTS "DocumentChunk_embedding_hnsw_idx"`
)
await prisma.$executeRawUnsafe(
`TRUNCATE TABLE "DocumentChunk"`
)
await prisma.$executeRawUnsafe(
`ALTER TABLE "DocumentChunk" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
)
await prisma.$executeRawUnsafe(`
CREATE INDEX "DocumentChunk_embedding_hnsw_idx" ON "DocumentChunk"
USING hnsw ("embedding" vector_cosine_ops)
WITH (m = 16, ef_construction = 64)
`)
}
await prisma.systemConfig.upsert({
where: { key: 'AI_EMBEDDING_DIMENSION' },
update: { value: String(newDimension) },
create: { key: 'AI_EMBEDDING_DIMENSION', value: String(newDimension) },
})
return NextResponse.json({
success: true,
message: `Migration complete: vector(${dbDimension}) → vector(${newDimension}). ${count} old embeddings cleared. Re-indexing needed.`,
previousDimension: dbDimension,
newDimension,
clearedEmbeddings: count,
})
} catch (error) {
console.error('[EMBEDDING_MIGRATE] Error:', error)
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
}
}

View File

@@ -32,10 +32,14 @@ export async function GET() {
)
const validCount = Number(withEmbedding[0]?.count ?? 0)
const dimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
)
const dbDim = dimResult[0]?.dim ?? 1536
const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
`SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e
WHERE e."embedding" IS NULL
OR array_length(string_to_array(replace(replace(e."embedding"::text, '[', ''), ']', ''), ','), 1) != 1536`
WHERE e."embedding" IS NULL`
)
const invalidCount = Number(invalidResult[0]?.count ?? 0)
@@ -43,6 +47,7 @@ export async function GET() {
return NextResponse.json({
success: true,
dbDimension: dbDim,
summary: {
total,
valid: validCount - invalidCount,

View File

@@ -1,5 +1,5 @@
import { NextResponse } from 'next/server'
import { getPrisma } from '@/lib/prisma'
import { prisma } from '@/lib/prisma'
import { redis } from '@/lib/redis'
export const dynamic = 'force-dynamic'
@@ -11,7 +11,6 @@ export async function GET() {
// Database check
try {
const dbStart = Date.now()
const prisma = getPrisma()
const [noteCount, notebookCount, userCount] = await Promise.all([
prisma.note.count(),
prisma.notebook.count(),
@@ -47,13 +46,15 @@ export async function GET() {
// AI providers check
try {
const { getAIProvider, getChatProvider } = await import('@/lib/ai/providers/registry')
const embeddingProvider = getAIProvider()
const chatProvider = getChatProvider()
const { getSystemConfig } = await import('@/lib/config')
const config = await getSystemConfig()
const { resolveAiRoute } = await import('@/lib/ai/router')
const tagsRoute = resolveAiRoute('tags', config)
const chatRoute = resolveAiRoute('chat', config)
checks.ai = {
status: 'configured',
embedding: { provider: embeddingProvider },
chat: { provider: chatProvider },
embedding: { provider: tagsRoute.providerType, model: tagsRoute.modelName },
chat: { provider: chatRoute.providerType, model: chatRoute.modelName },
}
} catch (e) {
checks.ai = { status: 'unhealthy', error: e instanceof Error ? e.message : 'Unknown error' }