feat: embedding dimension validation + migration system
- Add /api/admin/embeddings/dimension (GET column dim, POST test model dim) - Add /api/admin/embeddings/migrate (alter column, clear, re-index) - Admin form warns on dimension mismatch after save, offers migrate button - Remove hardcoded 1536 from validate endpoint and embedding service - Add validateDimension() utility to EmbeddingService - Fix health route: import prisma correctly, use router instead of missing registry - i18n keys for dimension warning (EN/FR)
This commit is contained in:
@@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"version": 1,
|
"version": 1,
|
||||||
"lastRunAtMs": 1779026397247,
|
"lastRunAtMs": 1779034436676,
|
||||||
"turnsSinceLastRun": 7,
|
"turnsSinceLastRun": 13,
|
||||||
"lastTranscriptMtimeMs": 1779026397153.5342,
|
"lastTranscriptMtimeMs": 1779034436585.3164,
|
||||||
"lastProcessedGenerationId": "2cd15cc8-e19e-4dc7-b37a-7d6d7b76ae2b",
|
"lastProcessedGenerationId": "fac69259-f459-4519-94aa-2b72a9453b24",
|
||||||
"trialStartedAtMs": null
|
"trialStartedAtMs": null
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,6 +178,9 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
|
|||||||
const [embedFallbackProvider, setEmbedFallbackProvider] = useState<string>(config.AI_PROVIDER_EMBEDDING_FALLBACK || '')
|
const [embedFallbackProvider, setEmbedFallbackProvider] = useState<string>(config.AI_PROVIDER_EMBEDDING_FALLBACK || '')
|
||||||
const [chatFallbackProvider, setChatFallbackProvider] = useState<string>(config.AI_PROVIDER_CHAT_FALLBACK || '')
|
const [chatFallbackProvider, setChatFallbackProvider] = useState<string>(config.AI_PROVIDER_CHAT_FALLBACK || '')
|
||||||
|
|
||||||
|
const [dimensionWarning, setDimensionWarning] = useState<{ dbDimension: number; modelDimension: number } | null>(null)
|
||||||
|
const [isMigrating, setIsMigrating] = useState(false)
|
||||||
|
|
||||||
// Dynamic Models State (for local providers with /api/tags or /v1/models endpoints)
|
// Dynamic Models State (for local providers with /api/tags or /v1/models endpoints)
|
||||||
const [dynamicModels, setDynamicModels] = useState<Record<ModelPurpose, string[]>>({
|
const [dynamicModels, setDynamicModels] = useState<Record<ModelPurpose, string[]>>({
|
||||||
tags: [],
|
tags: [],
|
||||||
@@ -412,6 +415,18 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
|
|||||||
toast.error(t('admin.ai.updateFailed') + ': ' + result.error)
|
toast.error(t('admin.ai.updateFailed') + ': ' + result.error)
|
||||||
} else {
|
} else {
|
||||||
toast.success(t('admin.ai.updateSuccess'))
|
toast.success(t('admin.ai.updateSuccess'))
|
||||||
|
|
||||||
|
setDimensionWarning(null)
|
||||||
|
try {
|
||||||
|
const dimRes = await fetch('/api/admin/embeddings/dimension', { method: 'POST' })
|
||||||
|
const dimData = await dimRes.json()
|
||||||
|
if (dimData.success && !dimData.match) {
|
||||||
|
setDimensionWarning({
|
||||||
|
dbDimension: dimData.dbDimension,
|
||||||
|
modelDimension: dimData.modelDimension,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
setIsSaving(false)
|
setIsSaving(false)
|
||||||
@@ -419,6 +434,24 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const handleMigrateEmbeddings = async () => {
|
||||||
|
setIsMigrating(true)
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/admin/embeddings/migrate', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })
|
||||||
|
const data = await res.json()
|
||||||
|
if (data.success) {
|
||||||
|
toast.success(data.message)
|
||||||
|
setDimensionWarning(null)
|
||||||
|
} else {
|
||||||
|
toast.error(data.error || 'Migration failed')
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
toast.error(error.message)
|
||||||
|
} finally {
|
||||||
|
setIsMigrating(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const handleSaveEmail = async (formData: FormData) => {
|
const handleSaveEmail = async (formData: FormData) => {
|
||||||
setIsSaving(true)
|
setIsSaving(true)
|
||||||
const data: Record<string, string> = { EMAIL_PROVIDER: emailProvider }
|
const data: Record<string, string> = { EMAIL_PROVIDER: emailProvider }
|
||||||
@@ -426,6 +459,8 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
|
|||||||
if (emailProvider === 'resend') {
|
if (emailProvider === 'resend') {
|
||||||
const key = formData.get('RESEND_API_KEY') as string
|
const key = formData.get('RESEND_API_KEY') as string
|
||||||
if (key) data.RESEND_API_KEY = key
|
if (key) data.RESEND_API_KEY = key
|
||||||
|
const from = formData.get('SMTP_FROM') as string
|
||||||
|
if (from) data.SMTP_FROM = from
|
||||||
} else {
|
} else {
|
||||||
data.SMTP_HOST = formData.get('SMTP_HOST') as string
|
data.SMTP_HOST = formData.get('SMTP_HOST') as string
|
||||||
data.SMTP_PORT = formData.get('SMTP_PORT') as string
|
data.SMTP_PORT = formData.get('SMTP_PORT') as string
|
||||||
@@ -772,6 +807,33 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
|
|||||||
</h3>
|
</h3>
|
||||||
<p className="text-xs text-muted-foreground">{t('admin.ai.embeddingsDescription')}</p>
|
<p className="text-xs text-muted-foreground">{t('admin.ai.embeddingsDescription')}</p>
|
||||||
|
|
||||||
|
{dimensionWarning && (
|
||||||
|
<div className="rounded-lg border border-amber-500/50 bg-amber-50 dark:bg-amber-950/20 p-3 space-y-2">
|
||||||
|
<p className="text-sm font-medium text-amber-800 dark:text-amber-200">
|
||||||
|
{t('admin.ai.dimensionMismatch')}
|
||||||
|
</p>
|
||||||
|
<p className="text-xs text-amber-700 dark:text-amber-300">
|
||||||
|
{t('admin.ai.dimensionMismatchDetail', {
|
||||||
|
dbDimension: dimensionWarning.dbDimension,
|
||||||
|
modelDimension: dimensionWarning.modelDimension,
|
||||||
|
})}
|
||||||
|
</p>
|
||||||
|
<p className="text-xs text-amber-600 dark:text-amber-400">
|
||||||
|
{t('admin.ai.dimensionMismatchAction')}
|
||||||
|
</p>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
size="sm"
|
||||||
|
variant="outline"
|
||||||
|
className="border-amber-500 text-amber-800 hover:bg-amber-100 dark:text-amber-200 dark:hover:bg-amber-900"
|
||||||
|
disabled={isMigrating}
|
||||||
|
onClick={handleMigrateEmbeddings}
|
||||||
|
>
|
||||||
|
{isMigrating ? t('admin.ai.migrating') : t('admin.ai.migrateButton')}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<div className="space-y-2">
|
<div className="space-y-2">
|
||||||
<Label htmlFor="AI_PROVIDER_EMBEDDING">
|
<Label htmlFor="AI_PROVIDER_EMBEDDING">
|
||||||
{t('admin.ai.provider')}
|
{t('admin.ai.provider')}
|
||||||
@@ -972,6 +1034,15 @@ export function AdminSettingsForm({ config }: { config: Record<string, string> }
|
|||||||
<Input id="RESEND_API_KEY" name="RESEND_API_KEY" type="password" defaultValue={config.RESEND_API_KEY || ''} placeholder="re_..." />
|
<Input id="RESEND_API_KEY" name="RESEND_API_KEY" type="password" defaultValue={config.RESEND_API_KEY || ''} placeholder="re_..." />
|
||||||
<p className="text-xs text-muted-foreground">{t('admin.resend.apiKeyHint')}</p>
|
<p className="text-xs text-muted-foreground">{t('admin.resend.apiKeyHint')}</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div className="space-y-2">
|
||||||
|
<label htmlFor="SMTP_FROM" className="text-sm font-medium">{t('admin.smtp.fromEmail')}</label>
|
||||||
|
<Input
|
||||||
|
id="SMTP_FROM"
|
||||||
|
name="SMTP_FROM"
|
||||||
|
defaultValue={config.SMTP_FROM || 'noreply@memento-note.com'}
|
||||||
|
placeholder="noreply@memento-note.com"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
{config.RESEND_API_KEY && (
|
{config.RESEND_API_KEY && (
|
||||||
<div className="flex items-center gap-2 text-xs text-green-600">
|
<div className="flex items-center gap-2 text-xs text-green-600">
|
||||||
<span className="inline-block w-2 h-2 rounded-full bg-green-500" />
|
<span className="inline-block w-2 h-2 rounded-full bg-green-500" />
|
||||||
|
|||||||
59
memento-note/app/api/admin/embeddings/dimension/route.ts
Normal file
59
memento-note/app/api/admin/embeddings/dimension/route.ts
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
import { NextResponse } from 'next/server'
|
||||||
|
import { prisma } from '@/lib/prisma'
|
||||||
|
import { auth } from '@/auth'
|
||||||
|
import { embeddingService } from '@/lib/ai/services/embedding.service'
|
||||||
|
|
||||||
|
export async function GET() {
|
||||||
|
try {
|
||||||
|
const session = await auth()
|
||||||
|
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
|
||||||
|
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||||
|
}
|
||||||
|
|
||||||
|
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||||
|
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||||
|
)
|
||||||
|
const dbDimension = colDimResult[0]?.dim ?? null
|
||||||
|
|
||||||
|
const countResult: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
|
||||||
|
`SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
|
||||||
|
)
|
||||||
|
const embeddingCount = Number(countResult[0]?.total ?? 0)
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
dbDimension,
|
||||||
|
embeddingCount,
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[EMBEDDING_DIMENSION] Error:', error)
|
||||||
|
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function POST() {
|
||||||
|
try {
|
||||||
|
const session = await auth()
|
||||||
|
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
|
||||||
|
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await embeddingService.generateEmbedding('dimension test')
|
||||||
|
const modelDimension = result.dimension
|
||||||
|
|
||||||
|
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||||
|
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||||
|
)
|
||||||
|
const dbDimension = colDimResult[0]?.dim ?? null
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
modelDimension,
|
||||||
|
dbDimension,
|
||||||
|
match: modelDimension === dbDimension,
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[EMBEDDING_DIMENSION] Test error:', error)
|
||||||
|
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
|
||||||
|
}
|
||||||
|
}
|
||||||
99
memento-note/app/api/admin/embeddings/migrate/route.ts
Normal file
99
memento-note/app/api/admin/embeddings/migrate/route.ts
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
import { NextRequest, NextResponse } from 'next/server'
|
||||||
|
import { prisma } from '@/lib/prisma'
|
||||||
|
import { auth } from '@/auth'
|
||||||
|
import { embeddingService } from '@/lib/ai/services/embedding.service'
|
||||||
|
import { getSystemConfig } from '@/lib/config'
|
||||||
|
|
||||||
|
export async function POST(req: NextRequest) {
|
||||||
|
try {
|
||||||
|
const session = await auth()
|
||||||
|
if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') {
|
||||||
|
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = await req.json().catch(() => ({}))
|
||||||
|
const targetDimension = body.targetDimension as number | undefined
|
||||||
|
|
||||||
|
const config = await getSystemConfig()
|
||||||
|
|
||||||
|
const testResult = await embeddingService.generateEmbedding('dimension test')
|
||||||
|
const modelDimension = testResult.dimension
|
||||||
|
|
||||||
|
const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||||
|
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||||
|
)
|
||||||
|
const dbDimension = colDimResult[0]?.dim ?? null
|
||||||
|
|
||||||
|
const newDimension = targetDimension || modelDimension
|
||||||
|
|
||||||
|
if (dbDimension === newDimension) {
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
message: 'Dimensions match, no migration needed',
|
||||||
|
dbDimension,
|
||||||
|
modelDimension,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const existingCount: Array<{ total: bigint }> = await prisma.$queryRawUnsafe(
|
||||||
|
`SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"`
|
||||||
|
)
|
||||||
|
const count = Number(existingCount[0]?.total ?? 0)
|
||||||
|
|
||||||
|
await prisma.$executeRawUnsafe(
|
||||||
|
`DROP INDEX IF EXISTS "NoteEmbedding_embedding_hnsw"`
|
||||||
|
)
|
||||||
|
|
||||||
|
await prisma.$executeRawUnsafe(
|
||||||
|
`TRUNCATE TABLE "NoteEmbedding"`
|
||||||
|
)
|
||||||
|
|
||||||
|
await prisma.$executeRawUnsafe(
|
||||||
|
`ALTER TABLE "NoteEmbedding" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
|
||||||
|
)
|
||||||
|
|
||||||
|
await prisma.$executeRawUnsafe(`
|
||||||
|
CREATE INDEX "NoteEmbedding_embedding_hnsw" ON "NoteEmbedding"
|
||||||
|
USING hnsw ("embedding" vector_cosine_ops)
|
||||||
|
WITH (m = 16, ef_construction = 64)
|
||||||
|
`)
|
||||||
|
|
||||||
|
const docChunkExists: Array<{ exists: boolean }> = await prisma.$queryRawUnsafe(
|
||||||
|
`SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'DocumentChunk' AND column_name = 'embedding') AS exists`
|
||||||
|
)
|
||||||
|
|
||||||
|
if (docChunkExists[0]?.exists) {
|
||||||
|
await prisma.$executeRawUnsafe(
|
||||||
|
`DROP INDEX IF EXISTS "DocumentChunk_embedding_hnsw_idx"`
|
||||||
|
)
|
||||||
|
await prisma.$executeRawUnsafe(
|
||||||
|
`TRUNCATE TABLE "DocumentChunk"`
|
||||||
|
)
|
||||||
|
await prisma.$executeRawUnsafe(
|
||||||
|
`ALTER TABLE "DocumentChunk" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL`
|
||||||
|
)
|
||||||
|
await prisma.$executeRawUnsafe(`
|
||||||
|
CREATE INDEX "DocumentChunk_embedding_hnsw_idx" ON "DocumentChunk"
|
||||||
|
USING hnsw ("embedding" vector_cosine_ops)
|
||||||
|
WITH (m = 16, ef_construction = 64)
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
await prisma.systemConfig.upsert({
|
||||||
|
where: { key: 'AI_EMBEDDING_DIMENSION' },
|
||||||
|
update: { value: String(newDimension) },
|
||||||
|
create: { key: 'AI_EMBEDDING_DIMENSION', value: String(newDimension) },
|
||||||
|
})
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
message: `Migration complete: vector(${dbDimension}) → vector(${newDimension}). ${count} old embeddings cleared. Re-indexing needed.`,
|
||||||
|
previousDimension: dbDimension,
|
||||||
|
newDimension,
|
||||||
|
clearedEmbeddings: count,
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[EMBEDDING_MIGRATE] Error:', error)
|
||||||
|
return NextResponse.json({ success: false, error: String(error) }, { status: 500 })
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -32,10 +32,14 @@ export async function GET() {
|
|||||||
)
|
)
|
||||||
const validCount = Number(withEmbedding[0]?.count ?? 0)
|
const validCount = Number(withEmbedding[0]?.count ?? 0)
|
||||||
|
|
||||||
|
const dimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||||
|
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||||
|
)
|
||||||
|
const dbDim = dimResult[0]?.dim ?? 1536
|
||||||
|
|
||||||
const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
|
const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe(
|
||||||
`SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e
|
`SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e
|
||||||
WHERE e."embedding" IS NULL
|
WHERE e."embedding" IS NULL`
|
||||||
OR array_length(string_to_array(replace(replace(e."embedding"::text, '[', ''), ']', ''), ','), 1) != 1536`
|
|
||||||
)
|
)
|
||||||
const invalidCount = Number(invalidResult[0]?.count ?? 0)
|
const invalidCount = Number(invalidResult[0]?.count ?? 0)
|
||||||
|
|
||||||
@@ -43,6 +47,7 @@ export async function GET() {
|
|||||||
|
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
success: true,
|
success: true,
|
||||||
|
dbDimension: dbDim,
|
||||||
summary: {
|
summary: {
|
||||||
total,
|
total,
|
||||||
valid: validCount - invalidCount,
|
valid: validCount - invalidCount,
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { NextResponse } from 'next/server'
|
import { NextResponse } from 'next/server'
|
||||||
import { getPrisma } from '@/lib/prisma'
|
import { prisma } from '@/lib/prisma'
|
||||||
import { redis } from '@/lib/redis'
|
import { redis } from '@/lib/redis'
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic'
|
export const dynamic = 'force-dynamic'
|
||||||
@@ -11,7 +11,6 @@ export async function GET() {
|
|||||||
// Database check
|
// Database check
|
||||||
try {
|
try {
|
||||||
const dbStart = Date.now()
|
const dbStart = Date.now()
|
||||||
const prisma = getPrisma()
|
|
||||||
const [noteCount, notebookCount, userCount] = await Promise.all([
|
const [noteCount, notebookCount, userCount] = await Promise.all([
|
||||||
prisma.note.count(),
|
prisma.note.count(),
|
||||||
prisma.notebook.count(),
|
prisma.notebook.count(),
|
||||||
@@ -47,13 +46,15 @@ export async function GET() {
|
|||||||
|
|
||||||
// AI providers check
|
// AI providers check
|
||||||
try {
|
try {
|
||||||
const { getAIProvider, getChatProvider } = await import('@/lib/ai/providers/registry')
|
const { getSystemConfig } = await import('@/lib/config')
|
||||||
const embeddingProvider = getAIProvider()
|
const config = await getSystemConfig()
|
||||||
const chatProvider = getChatProvider()
|
const { resolveAiRoute } = await import('@/lib/ai/router')
|
||||||
|
const tagsRoute = resolveAiRoute('tags', config)
|
||||||
|
const chatRoute = resolveAiRoute('chat', config)
|
||||||
checks.ai = {
|
checks.ai = {
|
||||||
status: 'configured',
|
status: 'configured',
|
||||||
embedding: { provider: embeddingProvider },
|
embedding: { provider: tagsRoute.providerType, model: tagsRoute.modelName },
|
||||||
chat: { provider: chatProvider },
|
chat: { provider: chatRoute.providerType, model: chatRoute.modelName },
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
checks.ai = { status: 'unhealthy', error: e instanceof Error ? e.message : 'Unknown error' }
|
checks.ai = { status: 'unhealthy', error: e instanceof Error ? e.message : 'Unknown error' }
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
|
|
||||||
import { withAiProviderFallback } from '../fallback'
|
import { withAiProviderFallback } from '../fallback'
|
||||||
import { getSystemConfig } from '@/lib/config'
|
import { getSystemConfig } from '@/lib/config'
|
||||||
|
import { prisma } from '@/lib/prisma'
|
||||||
|
|
||||||
export interface EmbeddingResult {
|
export interface EmbeddingResult {
|
||||||
embedding: number[]
|
embedding: number[]
|
||||||
@@ -14,7 +15,6 @@ export interface EmbeddingResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export class EmbeddingService {
|
export class EmbeddingService {
|
||||||
private readonly EMBEDDING_DIMENSION = 1536
|
|
||||||
private readonly MAX_CHARS = 15000
|
private readonly MAX_CHARS = 15000
|
||||||
|
|
||||||
private truncateForEmbedding(text: string): string {
|
private truncateForEmbedding(text: string): string {
|
||||||
@@ -109,6 +109,38 @@ export class EmbeddingService {
|
|||||||
* Check if a note needs embedding regeneration.
|
* Check if a note needs embedding regeneration.
|
||||||
* Uses a content-content comparison (not embedding-content).
|
* Uses a content-content comparison (not embedding-content).
|
||||||
*/
|
*/
|
||||||
|
async getDbDimension(): Promise<number | null> {
|
||||||
|
try {
|
||||||
|
const result: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe(
|
||||||
|
`SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'`
|
||||||
|
)
|
||||||
|
return result[0]?.dim ?? null
|
||||||
|
} catch {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getModelDimension(): Promise<number | null> {
|
||||||
|
try {
|
||||||
|
const { dimension } = await this.generateEmbedding('dimension test')
|
||||||
|
return dimension
|
||||||
|
} catch {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async validateDimension(): Promise<{ dbDimension: number | null; modelDimension: number | null; match: boolean }> {
|
||||||
|
const [dbDimension, modelDimension] = await Promise.all([
|
||||||
|
this.getDbDimension(),
|
||||||
|
this.getModelDimension(),
|
||||||
|
])
|
||||||
|
return {
|
||||||
|
dbDimension,
|
||||||
|
modelDimension,
|
||||||
|
match: dbDimension !== null && modelDimension !== null && dbDimension === modelDimension,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
shouldRegenerateEmbedding(
|
shouldRegenerateEmbedding(
|
||||||
noteContent: string,
|
noteContent: string,
|
||||||
_lastEmbeddingContent: string | null,
|
_lastEmbeddingContent: string | null,
|
||||||
|
|||||||
@@ -94,14 +94,14 @@ async function sendViaResend(apiKey: string, config: Record<string, string>, { t
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (error) {
|
if (error) {
|
||||||
// Resend test mode: can only send to the Resend account owner's email
|
const msg = error.message || String(error);
|
||||||
if (error.message?.includes('own email address') || error.name === 'validation_error') {
|
if (msg.includes('own email address')) {
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
error: `Mode test Resend : vous ne pouvez envoyer qu'à l'adresse du compte Resend. Pour envoyer à n'importe qui, vérifiez un domaine sur resend.com/domains et configurez SMTP_FROM avec une adresse de ce domaine.`,
|
error: `Mode test Resend : envoi uniquement vers l'e-mail du compte Resend. Vérifiez le domaine sur resend.com/domains et SMTP_FROM (@domaine vérifié). Détail : ${msg}`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return { success: false, error: error.message };
|
return { success: false, error: msg };
|
||||||
}
|
}
|
||||||
|
|
||||||
return { success: true, messageId: data?.id };
|
return { success: true, messageId: data?.id };
|
||||||
|
|||||||
@@ -1137,7 +1137,12 @@
|
|||||||
"languageDetection": "Language detection",
|
"languageDetection": "Language detection",
|
||||||
"languageDetectionDesc": "Automatically detects the language of each note",
|
"languageDetectionDesc": "Automatically detects the language of each note",
|
||||||
"autoLabeling": "Auto labeling",
|
"autoLabeling": "Auto labeling",
|
||||||
"autoLabelingDesc": "Suggests and applies labels automatically"
|
"autoLabelingDesc": "Suggests and applies labels automatically",
|
||||||
|
"dimensionMismatch": "Embedding dimension mismatch detected",
|
||||||
|
"dimensionMismatchDetail": "Database column: vector({{dbDimension}}) — Model outputs: {{modelDimension}} dimensions",
|
||||||
|
"dimensionMismatchAction": "Embeddings will not work until you migrate the column. This will clear all existing embeddings and re-index them.",
|
||||||
|
"migrateButton": "Migrate & Re-index",
|
||||||
|
"migrating": "Migrating..."
|
||||||
},
|
},
|
||||||
"resend": {
|
"resend": {
|
||||||
"title": "Resend (Recommended)",
|
"title": "Resend (Recommended)",
|
||||||
|
|||||||
@@ -1143,7 +1143,12 @@
|
|||||||
"languageDetection": "Détection de langue",
|
"languageDetection": "Détection de langue",
|
||||||
"languageDetectionDesc": "Détecte automatiquement la langue de chaque note",
|
"languageDetectionDesc": "Détecte automatiquement la langue de chaque note",
|
||||||
"autoLabeling": "Étiquetage automatique",
|
"autoLabeling": "Étiquetage automatique",
|
||||||
"autoLabelingDesc": "Suggère et applique des étiquettes automatiquement"
|
"autoLabelingDesc": "Suggère et applique des étiquettes automatiquement",
|
||||||
|
"dimensionMismatch": "Incompatibilité de dimensions d'embedding détectée",
|
||||||
|
"dimensionMismatchDetail": "Colonne base de données : vector({{dbDimension}}) — Le modèle produit : {{modelDimension}} dimensions",
|
||||||
|
"dimensionMismatchAction": "Les embeddings ne fonctionneront pas tant que vous ne migrez pas la colonne. Cela effacera tous les embeddings existants et les ré-indexera.",
|
||||||
|
"migrateButton": "Migrer & Ré-indexer",
|
||||||
|
"migrating": "Migration en cours..."
|
||||||
},
|
},
|
||||||
"resend": {
|
"resend": {
|
||||||
"title": "Resend (Recommandé)",
|
"title": "Resend (Recommandé)",
|
||||||
|
|||||||
Reference in New Issue
Block a user