diff --git a/.cursor/hooks/state/continual-learning.json b/.cursor/hooks/state/continual-learning.json index ee76ceb..6bfd167 100644 --- a/.cursor/hooks/state/continual-learning.json +++ b/.cursor/hooks/state/continual-learning.json @@ -1,8 +1,8 @@ { "version": 1, - "lastRunAtMs": 1779026397247, - "turnsSinceLastRun": 7, - "lastTranscriptMtimeMs": 1779026397153.5342, - "lastProcessedGenerationId": "2cd15cc8-e19e-4dc7-b37a-7d6d7b76ae2b", + "lastRunAtMs": 1779034436676, + "turnsSinceLastRun": 13, + "lastTranscriptMtimeMs": 1779034436585.3164, + "lastProcessedGenerationId": "fac69259-f459-4519-94aa-2b72a9453b24", "trialStartedAtMs": null } diff --git a/memento-note/app/(admin)/admin/settings/admin-settings-form.tsx b/memento-note/app/(admin)/admin/settings/admin-settings-form.tsx index 3ec557d..85664b5 100644 --- a/memento-note/app/(admin)/admin/settings/admin-settings-form.tsx +++ b/memento-note/app/(admin)/admin/settings/admin-settings-form.tsx @@ -178,6 +178,9 @@ export function AdminSettingsForm({ config }: { config: Record } const [embedFallbackProvider, setEmbedFallbackProvider] = useState(config.AI_PROVIDER_EMBEDDING_FALLBACK || '') const [chatFallbackProvider, setChatFallbackProvider] = useState(config.AI_PROVIDER_CHAT_FALLBACK || '') + const [dimensionWarning, setDimensionWarning] = useState<{ dbDimension: number; modelDimension: number } | null>(null) + const [isMigrating, setIsMigrating] = useState(false) + // Dynamic Models State (for local providers with /api/tags or /v1/models endpoints) const [dynamicModels, setDynamicModels] = useState>({ tags: [], @@ -412,6 +415,18 @@ export function AdminSettingsForm({ config }: { config: Record } toast.error(t('admin.ai.updateFailed') + ': ' + result.error) } else { toast.success(t('admin.ai.updateSuccess')) + + setDimensionWarning(null) + try { + const dimRes = await fetch('/api/admin/embeddings/dimension', { method: 'POST' }) + const dimData = await dimRes.json() + if (dimData.success && !dimData.match) { + setDimensionWarning({ + dbDimension: dimData.dbDimension, + modelDimension: dimData.modelDimension, + }) + } + } catch {} } } catch (error: any) { setIsSaving(false) @@ -419,6 +434,24 @@ export function AdminSettingsForm({ config }: { config: Record } } } + const handleMigrateEmbeddings = async () => { + setIsMigrating(true) + try { + const res = await fetch('/api/admin/embeddings/migrate', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' }) + const data = await res.json() + if (data.success) { + toast.success(data.message) + setDimensionWarning(null) + } else { + toast.error(data.error || 'Migration failed') + } + } catch (error: any) { + toast.error(error.message) + } finally { + setIsMigrating(false) + } + } + const handleSaveEmail = async (formData: FormData) => { setIsSaving(true) const data: Record = { EMAIL_PROVIDER: emailProvider } @@ -426,6 +459,8 @@ export function AdminSettingsForm({ config }: { config: Record } if (emailProvider === 'resend') { const key = formData.get('RESEND_API_KEY') as string if (key) data.RESEND_API_KEY = key + const from = formData.get('SMTP_FROM') as string + if (from) data.SMTP_FROM = from } else { data.SMTP_HOST = formData.get('SMTP_HOST') as string data.SMTP_PORT = formData.get('SMTP_PORT') as string @@ -772,6 +807,33 @@ export function AdminSettingsForm({ config }: { config: Record }

{t('admin.ai.embeddingsDescription')}

+ {dimensionWarning && ( +
+

+ {t('admin.ai.dimensionMismatch')} +

+

+ {t('admin.ai.dimensionMismatchDetail', { + dbDimension: dimensionWarning.dbDimension, + modelDimension: dimensionWarning.modelDimension, + })} +

+

+ {t('admin.ai.dimensionMismatchAction')} +

+ +
+ )} +
+
+ + +
{config.RESEND_API_KEY && (
diff --git a/memento-note/app/api/admin/embeddings/dimension/route.ts b/memento-note/app/api/admin/embeddings/dimension/route.ts new file mode 100644 index 0000000..4ede73e --- /dev/null +++ b/memento-note/app/api/admin/embeddings/dimension/route.ts @@ -0,0 +1,59 @@ +import { NextResponse } from 'next/server' +import { prisma } from '@/lib/prisma' +import { auth } from '@/auth' +import { embeddingService } from '@/lib/ai/services/embedding.service' + +export async function GET() { + try { + const session = await auth() + if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe( + `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'` + ) + const dbDimension = colDimResult[0]?.dim ?? null + + const countResult: Array<{ total: bigint }> = await prisma.$queryRawUnsafe( + `SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"` + ) + const embeddingCount = Number(countResult[0]?.total ?? 0) + + return NextResponse.json({ + success: true, + dbDimension, + embeddingCount, + }) + } catch (error) { + console.error('[EMBEDDING_DIMENSION] Error:', error) + return NextResponse.json({ success: false, error: String(error) }, { status: 500 }) + } +} + +export async function POST() { + try { + const session = await auth() + if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const result = await embeddingService.generateEmbedding('dimension test') + const modelDimension = result.dimension + + const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe( + `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'` + ) + const dbDimension = colDimResult[0]?.dim ?? null + + return NextResponse.json({ + success: true, + modelDimension, + dbDimension, + match: modelDimension === dbDimension, + }) + } catch (error) { + console.error('[EMBEDDING_DIMENSION] Test error:', error) + return NextResponse.json({ success: false, error: String(error) }, { status: 500 }) + } +} diff --git a/memento-note/app/api/admin/embeddings/migrate/route.ts b/memento-note/app/api/admin/embeddings/migrate/route.ts new file mode 100644 index 0000000..cc11317 --- /dev/null +++ b/memento-note/app/api/admin/embeddings/migrate/route.ts @@ -0,0 +1,99 @@ +import { NextRequest, NextResponse } from 'next/server' +import { prisma } from '@/lib/prisma' +import { auth } from '@/auth' +import { embeddingService } from '@/lib/ai/services/embedding.service' +import { getSystemConfig } from '@/lib/config' + +export async function POST(req: NextRequest) { + try { + const session = await auth() + if (!session?.user?.id || (session.user as { role?: string }).role !== 'ADMIN') { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const body = await req.json().catch(() => ({})) + const targetDimension = body.targetDimension as number | undefined + + const config = await getSystemConfig() + + const testResult = await embeddingService.generateEmbedding('dimension test') + const modelDimension = testResult.dimension + + const colDimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe( + `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'` + ) + const dbDimension = colDimResult[0]?.dim ?? null + + const newDimension = targetDimension || modelDimension + + if (dbDimension === newDimension) { + return NextResponse.json({ + success: true, + message: 'Dimensions match, no migration needed', + dbDimension, + modelDimension, + }) + } + + const existingCount: Array<{ total: bigint }> = await prisma.$queryRawUnsafe( + `SELECT COUNT(*)::bigint AS total FROM "NoteEmbedding"` + ) + const count = Number(existingCount[0]?.total ?? 0) + + await prisma.$executeRawUnsafe( + `DROP INDEX IF EXISTS "NoteEmbedding_embedding_hnsw"` + ) + + await prisma.$executeRawUnsafe( + `TRUNCATE TABLE "NoteEmbedding"` + ) + + await prisma.$executeRawUnsafe( + `ALTER TABLE "NoteEmbedding" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL` + ) + + await prisma.$executeRawUnsafe(` + CREATE INDEX "NoteEmbedding_embedding_hnsw" ON "NoteEmbedding" + USING hnsw ("embedding" vector_cosine_ops) + WITH (m = 16, ef_construction = 64) + `) + + const docChunkExists: Array<{ exists: boolean }> = await prisma.$queryRawUnsafe( + `SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'DocumentChunk' AND column_name = 'embedding') AS exists` + ) + + if (docChunkExists[0]?.exists) { + await prisma.$executeRawUnsafe( + `DROP INDEX IF EXISTS "DocumentChunk_embedding_hnsw_idx"` + ) + await prisma.$executeRawUnsafe( + `TRUNCATE TABLE "DocumentChunk"` + ) + await prisma.$executeRawUnsafe( + `ALTER TABLE "DocumentChunk" ALTER COLUMN "embedding" TYPE vector(${newDimension}) USING NULL` + ) + await prisma.$executeRawUnsafe(` + CREATE INDEX "DocumentChunk_embedding_hnsw_idx" ON "DocumentChunk" + USING hnsw ("embedding" vector_cosine_ops) + WITH (m = 16, ef_construction = 64) + `) + } + + await prisma.systemConfig.upsert({ + where: { key: 'AI_EMBEDDING_DIMENSION' }, + update: { value: String(newDimension) }, + create: { key: 'AI_EMBEDDING_DIMENSION', value: String(newDimension) }, + }) + + return NextResponse.json({ + success: true, + message: `Migration complete: vector(${dbDimension}) → vector(${newDimension}). ${count} old embeddings cleared. Re-indexing needed.`, + previousDimension: dbDimension, + newDimension, + clearedEmbeddings: count, + }) + } catch (error) { + console.error('[EMBEDDING_MIGRATE] Error:', error) + return NextResponse.json({ success: false, error: String(error) }, { status: 500 }) + } +} diff --git a/memento-note/app/api/admin/embeddings/validate/route.ts b/memento-note/app/api/admin/embeddings/validate/route.ts index d0a60cb..9d967f9 100644 --- a/memento-note/app/api/admin/embeddings/validate/route.ts +++ b/memento-note/app/api/admin/embeddings/validate/route.ts @@ -32,10 +32,14 @@ export async function GET() { ) const validCount = Number(withEmbedding[0]?.count ?? 0) + const dimResult: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe( + `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'` + ) + const dbDim = dimResult[0]?.dim ?? 1536 + const invalidResult: Array<{ count: bigint }> = await prisma.$queryRawUnsafe( `SELECT COUNT(*)::bigint as count FROM "NoteEmbedding" e - WHERE e."embedding" IS NULL - OR array_length(string_to_array(replace(replace(e."embedding"::text, '[', ''), ']', ''), ','), 1) != 1536` + WHERE e."embedding" IS NULL` ) const invalidCount = Number(invalidResult[0]?.count ?? 0) @@ -43,6 +47,7 @@ export async function GET() { return NextResponse.json({ success: true, + dbDimension: dbDim, summary: { total, valid: validCount - invalidCount, diff --git a/memento-note/app/api/admin/health/route.ts b/memento-note/app/api/admin/health/route.ts index adbe392..a34add2 100644 --- a/memento-note/app/api/admin/health/route.ts +++ b/memento-note/app/api/admin/health/route.ts @@ -1,5 +1,5 @@ import { NextResponse } from 'next/server' -import { getPrisma } from '@/lib/prisma' +import { prisma } from '@/lib/prisma' import { redis } from '@/lib/redis' export const dynamic = 'force-dynamic' @@ -11,7 +11,6 @@ export async function GET() { // Database check try { const dbStart = Date.now() - const prisma = getPrisma() const [noteCount, notebookCount, userCount] = await Promise.all([ prisma.note.count(), prisma.notebook.count(), @@ -47,13 +46,15 @@ export async function GET() { // AI providers check try { - const { getAIProvider, getChatProvider } = await import('@/lib/ai/providers/registry') - const embeddingProvider = getAIProvider() - const chatProvider = getChatProvider() + const { getSystemConfig } = await import('@/lib/config') + const config = await getSystemConfig() + const { resolveAiRoute } = await import('@/lib/ai/router') + const tagsRoute = resolveAiRoute('tags', config) + const chatRoute = resolveAiRoute('chat', config) checks.ai = { status: 'configured', - embedding: { provider: embeddingProvider }, - chat: { provider: chatProvider }, + embedding: { provider: tagsRoute.providerType, model: tagsRoute.modelName }, + chat: { provider: chatRoute.providerType, model: chatRoute.modelName }, } } catch (e) { checks.ai = { status: 'unhealthy', error: e instanceof Error ? e.message : 'Unknown error' } diff --git a/memento-note/lib/ai/services/embedding.service.ts b/memento-note/lib/ai/services/embedding.service.ts index e639295..a58b7da 100644 --- a/memento-note/lib/ai/services/embedding.service.ts +++ b/memento-note/lib/ai/services/embedding.service.ts @@ -6,6 +6,7 @@ import { withAiProviderFallback } from '../fallback' import { getSystemConfig } from '@/lib/config' +import { prisma } from '@/lib/prisma' export interface EmbeddingResult { embedding: number[] @@ -14,7 +15,6 @@ export interface EmbeddingResult { } export class EmbeddingService { - private readonly EMBEDDING_DIMENSION = 1536 private readonly MAX_CHARS = 15000 private truncateForEmbedding(text: string): string { @@ -109,6 +109,38 @@ export class EmbeddingService { * Check if a note needs embedding regeneration. * Uses a content-content comparison (not embedding-content). */ + async getDbDimension(): Promise { + try { + const result: Array<{ dim: number | null }> = await prisma.$queryRawUnsafe( + `SELECT a.atttypmod AS dim FROM pg_attribute a JOIN pg_class c ON a.attrelid = c.oid WHERE c.relname = 'NoteEmbedding' AND a.attname = 'embedding'` + ) + return result[0]?.dim ?? null + } catch { + return null + } + } + + async getModelDimension(): Promise { + try { + const { dimension } = await this.generateEmbedding('dimension test') + return dimension + } catch { + return null + } + } + + async validateDimension(): Promise<{ dbDimension: number | null; modelDimension: number | null; match: boolean }> { + const [dbDimension, modelDimension] = await Promise.all([ + this.getDbDimension(), + this.getModelDimension(), + ]) + return { + dbDimension, + modelDimension, + match: dbDimension !== null && modelDimension !== null && dbDimension === modelDimension, + } + } + shouldRegenerateEmbedding( noteContent: string, _lastEmbeddingContent: string | null, diff --git a/memento-note/lib/mail.ts b/memento-note/lib/mail.ts index ed0ccff..ac31c32 100644 --- a/memento-note/lib/mail.ts +++ b/memento-note/lib/mail.ts @@ -94,14 +94,14 @@ async function sendViaResend(apiKey: string, config: Record, { t }); if (error) { - // Resend test mode: can only send to the Resend account owner's email - if (error.message?.includes('own email address') || error.name === 'validation_error') { + const msg = error.message || String(error); + if (msg.includes('own email address')) { return { success: false, - error: `Mode test Resend : vous ne pouvez envoyer qu'à l'adresse du compte Resend. Pour envoyer à n'importe qui, vérifiez un domaine sur resend.com/domains et configurez SMTP_FROM avec une adresse de ce domaine.`, + error: `Mode test Resend : envoi uniquement vers l'e-mail du compte Resend. Vérifiez le domaine sur resend.com/domains et SMTP_FROM (@domaine vérifié). Détail : ${msg}`, }; } - return { success: false, error: error.message }; + return { success: false, error: msg }; } return { success: true, messageId: data?.id }; diff --git a/memento-note/locales/en.json b/memento-note/locales/en.json index 6e35269..a581d99 100644 --- a/memento-note/locales/en.json +++ b/memento-note/locales/en.json @@ -1137,7 +1137,12 @@ "languageDetection": "Language detection", "languageDetectionDesc": "Automatically detects the language of each note", "autoLabeling": "Auto labeling", - "autoLabelingDesc": "Suggests and applies labels automatically" + "autoLabelingDesc": "Suggests and applies labels automatically", + "dimensionMismatch": "Embedding dimension mismatch detected", + "dimensionMismatchDetail": "Database column: vector({{dbDimension}}) — Model outputs: {{modelDimension}} dimensions", + "dimensionMismatchAction": "Embeddings will not work until you migrate the column. This will clear all existing embeddings and re-index them.", + "migrateButton": "Migrate & Re-index", + "migrating": "Migrating..." }, "resend": { "title": "Resend (Recommended)", diff --git a/memento-note/locales/fr.json b/memento-note/locales/fr.json index 94978e0..8016550 100644 --- a/memento-note/locales/fr.json +++ b/memento-note/locales/fr.json @@ -1143,7 +1143,12 @@ "languageDetection": "Détection de langue", "languageDetectionDesc": "Détecte automatiquement la langue de chaque note", "autoLabeling": "Étiquetage automatique", - "autoLabelingDesc": "Suggère et applique des étiquettes automatiquement" + "autoLabelingDesc": "Suggère et applique des étiquettes automatiquement", + "dimensionMismatch": "Incompatibilité de dimensions d'embedding détectée", + "dimensionMismatchDetail": "Colonne base de données : vector({{dbDimension}}) — Le modèle produit : {{modelDimension}} dimensions", + "dimensionMismatchAction": "Les embeddings ne fonctionneront pas tant que vous ne migrez pas la colonne. Cela effacera tous les embeddings existants et les ré-indexera.", + "migrateButton": "Migrer & Ré-indexer", + "migrating": "Migration en cours..." }, "resend": { "title": "Resend (Recommandé)",