feat: embedding dimension validation + migration system
- Add /api/admin/embeddings/dimension (GET column dim, POST test model dim) - Add /api/admin/embeddings/migrate (alter column, clear, re-index) - Admin form warns on dimension mismatch after save, offers migrate button - Remove hardcoded 1536 from validate endpoint and embedding service - Add validateDimension() utility to EmbeddingService - Fix health route: import prisma correctly, use router instead of missing registry - i18n keys for dimension warning (EN/FR)
This commit is contained in:
59
memento-note/app/api/admin/embeddings/dimension/route.ts
Normal file
59
memento-note/app/api/admin/embeddings/dimension/route.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import { NextResponse } from 'next/server'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { auth } from '@/auth'
|
||||
import { embeddingService } from '@/lib/ai/services/embedding.service'
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||
)
|
||||
const dbDimension = colDimResult[0]?.dim ?? null
|
||||
|
||||
const countResult: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
|
||||
`SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
|
||||
)
|
||||
const embeddingCount = Number(countResult[0]?.total ?? 0)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
dbDimension,
|
||||
embeddingCount,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('[EMBEDDING_DIMENSION] Error:', error)
|
||||
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST() {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const result = await embeddingService.generateEmbedding('dimension test')
|
||||
const modelDimension = result.dimension
|
||||
|
||||
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||
)
|
||||
const dbDimension = colDimResult[0]?.dim ?? null
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
modelDimension,
|
||||
dbDimension,
|
||||
match: modelDimension === dbDimension,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('[EMBEDDING_DIMENSION] Test error:', error)
|
||||
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
|
||||
}
|
||||
}
|
||||
99
memento-note/app/api/admin/embeddings/migrate/route.ts
Normal file
99
memento-note/app/api/admin/embeddings/migrate/route.ts
Normal file
@@ -0,0 +1,99 @@
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { auth } from '@/auth'
|
||||
import { embeddingService } from '@/lib/ai/services/embedding.service'
|
||||
import { getSystemConfig } from '@/lib/config'
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
const session = await auth()
|
||||
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
|
||||
const body = await req.json().catch(() => ({}))
|
||||
const targetDimension = body.targetDimension as number | undefined
|
||||
|
||||
const config = await getSystemConfig()
|
||||
|
||||
const testResult = await embeddingService.generateEmbedding('dimension test')
|
||||
const modelDimension = testResult.dimension
|
||||
|
||||
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||
)
|
||||
const dbDimension = colDimResult[0]?.dim ?? null
|
||||
|
||||
const newDimension = targetDimension || modelDimension
|
||||
|
||||
if (dbDimension === newDimension) {
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
message: 'Dimensions match, no migration needed',
|
||||
dbDimension,
|
||||
modelDimension,
|
||||
})
|
||||
}
|
||||
|
||||
const existingCount: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
|
||||
`SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
|
||||
)
|
||||
const count = Number(existingCount[0]?.total ?? 0)
|
||||
|
||||
await prisma.$executeRawUnsafe(
|
||||
`DROP INDEX IF EXISTS "NoteEmbedding_embedding_hnsw"`
|
||||
)
|
||||
|
||||
await prisma.$executeRawUnsafe(
|
||||
`TRUNCATE TABLE "NoteEmbedding"`
|
||||
)
|
||||
|
||||
await prisma.$executeRawUnsafe(
|
||||
`ALTER TABLE "NoteEmbedding" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
|
||||
)
|
||||
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE INDEX "NoteEmbedding_embedding_hnsw" ON "NoteEmbedding"
|
||||
USING hnsw ("embedding" vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 64)
|
||||
`)
|
||||
|
||||
const docChunkExists: Array<{ exists: boolean }> = await prisma.$queryRawUnsafe(
|
||||
`SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'DocumentChunk' AND column_name = 'embedding') AS exists`
|
||||
)
|
||||
|
||||
if (docChunkExists[0]?.exists) {
|
||||
await prisma.$executeRawUnsafe(
|
||||
`DROP INDEX IF EXISTS "DocumentChunk_embedding_hnsw_idx"`
|
||||
)
|
||||
await prisma.$executeRawUnsafe(
|
||||
`TRUNCATE TABLE "DocumentChunk"`
|
||||
)
|
||||
await prisma.$executeRawUnsafe(
|
||||
`ALTER TABLE "DocumentChunk" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
|
||||
)
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE INDEX "DocumentChunk_embedding_hnsw_idx" ON "DocumentChunk"
|
||||
USING hnsw ("embedding" vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 64)
|
||||
`)
|
||||
}
|
||||
|
||||
await prisma.systemConfig.upsert({
|
||||
where: { key: 'AI_EMBEDDING_DIMENSION' },
|
||||
update: { value: String(newDimension) },
|
||||
create: { key: 'AI_EMBEDDING_DIMENSION', value: String(newDimension) },
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
message: `Migration complete: vector(${dbDimension}) → vector(${newDimension}). ${count} old embeddings cleared. Re-indexing needed.`,
|
||||
previousDimension: dbDimension,
|
||||
newDimension,
|
||||
clearedEmbeddings: count,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('[EMBEDDING_MIGRATE] Error:', error)
|
||||
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
|
||||
}
|
||||
}
|
||||
@@ -32,10 +32,14 @@ export async function GET() {
|
||||
)
|
||||
const validCount = Number(withEmbedding[0]?.count ?? 0)
|
||||
|
||||
const dimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||
)
|
||||
const dbDim = dimResult[0]?.dim ?? 1536
|
||||
|
||||
const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
|
||||
`SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e
|
||||
WHERE e."embedding" IS NULL
|
||||
OR array_length(string_to_array(replace(replace(e."embedding"::text, '[', ''), ']', ''), ','), 1) != 1536`
|
||||
WHERE e."embedding" IS NULL`
|
||||
)
|
||||
const invalidCount = Number(invalidResult[0]?.count ?? 0)
|
||||
|
||||
@@ -43,6 +47,7 @@ export async function GET() {
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
dbDimension: dbDim,
|
||||
summary: {
|
||||
total,
|
||||
valid: validCount - invalidCount,
|
||||
|
||||
Reference in New Issue
Block a user