feat: image AI titles (3 suggestions), describe-images action, pin/list fixes, i18n
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 44s
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 44s
- Add image description service + API route for AI-powered image analysis - Image title generation returns 3 selectable suggestions via TitleSuggestions component - Add "Describe images" action in AI assistant (individual + collective) - Fix pin refresh propagation in card and tabs view - Fix note creation refresh in tabs mode, pass all notes to tabs view - Add RTL support (dir="auto") on note content elements - Pass UI language dynamically to AI endpoints instead of hardcoded 'fr' - Add 18 missing i18n keys in both en.json and fr.json - Sparkles button on images for AI title generation (bottom-right, pulse animation) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
151
memento-note/lib/ai/services/image-description.service.ts
Normal file
151
memento-note/lib/ai/services/image-description.service.ts
Normal file
@@ -0,0 +1,151 @@
|
||||
import { generateText } from 'ai'
|
||||
import { readFile } from 'fs/promises'
|
||||
import path from 'path'
|
||||
import { getChatProvider } from '../factory'
|
||||
import { getSystemConfig } from '@/lib/config'
|
||||
|
||||
export interface ImageDescriptionResult {
|
||||
descriptions: Array<{
|
||||
index: number
|
||||
description: string
|
||||
}>
|
||||
suggestions?: Array<{
|
||||
title: string
|
||||
confidence: number
|
||||
reasoning?: string
|
||||
}>
|
||||
combinedSummary?: string
|
||||
}
|
||||
|
||||
const UPLOAD_DIR = path.join(process.cwd(), 'data', 'uploads')
|
||||
|
||||
async function resolveImageAsBase64(imageUrl: string): Promise<string> {
|
||||
const localMatch = imageUrl.match(/\/uploads\/(.+)/)
|
||||
if (localMatch) {
|
||||
const filePath = path.join(UPLOAD_DIR, localMatch[1])
|
||||
const buffer = await readFile(filePath)
|
||||
const ext = path.extname(imageUrl).toLowerCase()
|
||||
const mime = ext === '.png' ? 'image/png' : ext === '.gif' ? 'image/gif' : ext === '.webp' ? 'image/webp' : 'image/jpeg'
|
||||
return `data:${mime};base64,${buffer.toString('base64')}`
|
||||
}
|
||||
|
||||
// Remote URL — fetch and convert
|
||||
const res = await fetch(imageUrl)
|
||||
if (!res.ok) throw new Error(`Failed to fetch image: ${imageUrl}`)
|
||||
const contentType = res.headers.get('content-type') || 'image/jpeg'
|
||||
const arrayBuffer = await res.arrayBuffer()
|
||||
const base64 = Buffer.from(arrayBuffer).toString('base64')
|
||||
return `data:${contentType};base64,${base64}`
|
||||
}
|
||||
|
||||
export async function describeImages(
|
||||
imageUrls: string[],
|
||||
mode: 'description' | 'title',
|
||||
language: string = 'fr'
|
||||
): Promise<ImageDescriptionResult> {
|
||||
const config = await getSystemConfig()
|
||||
const model = getChatProvider(config).getModel()
|
||||
|
||||
const isTitleMode = mode === 'title'
|
||||
const langMap: Record<string, string> = {
|
||||
fr: 'French', en: 'English', fa: 'Persian', ar: 'Arabic',
|
||||
es: 'Spanish', de: 'German', it: 'Italian', pt: 'Portuguese',
|
||||
ru: 'Russian', zh: 'Chinese', ja: 'Japanese', ko: 'Korean',
|
||||
hi: 'Hindi', nl: 'Dutch', pl: 'Polish',
|
||||
}
|
||||
const langName = langMap[language] || 'English'
|
||||
|
||||
// Resolve all images as base64 data URLs (same approach as the chat route)
|
||||
const imageDataUrls = await Promise.all(imageUrls.map(url => resolveImageAsBase64(url)))
|
||||
|
||||
if (isTitleMode) {
|
||||
const prompt = imageUrls.length === 1
|
||||
? `Look carefully at this image and identify every concrete detail you can see: objects, people, animals, text, logos, colors, location/setting, actions, weather, time of day, style (photo/illustration/diagram), and any notable elements.
|
||||
|
||||
Then generate 3 specific, descriptive titles (3-7 words each) in ${langName}. Each title must mention concrete elements actually visible in the image — do NOT use generic or abstract words like "beautiful scene", "interesting image", "visual content". Be precise and factual.
|
||||
|
||||
Good example: "Red bicycle parked near a brick café wall"
|
||||
Bad example: "Beautiful urban scenery"
|
||||
|
||||
Respond ONLY with a JSON array: [{"title": "title1", "confidence": 0.95}, {"title": "title2", "confidence": 0.85}, {"title": "title3", "confidence": 0.75}]`
|
||||
: `Look carefully at these images and identify every concrete detail visible: objects, people, animals, text, logos, colors, locations, actions, weather, styles, and any notable elements across all images.
|
||||
|
||||
Then generate 3 specific, descriptive titles (3-7 words each) in ${langName} that capture what these images collectively show. Each title must mention concrete elements actually visible — do NOT use generic or abstract words like "beautiful scenes", "collection of images". Be precise and factual.
|
||||
|
||||
Good example: "Red bicycle and brick café on a sunny street"
|
||||
Bad example: "Beautiful urban scenery collection"
|
||||
|
||||
Respond ONLY with a JSON array: [{"title": "title1", "confidence": 0.95}, {"title": "title2", "confidence": 0.85}, {"title": "title3", "confidence": 0.75}]`
|
||||
|
||||
const content: any[] = [{ type: 'text', text: prompt }]
|
||||
for (const dataUrl of imageDataUrls) {
|
||||
content.push({ type: 'image', image: dataUrl })
|
||||
}
|
||||
|
||||
const { text } = await generateText({
|
||||
model,
|
||||
messages: [{ role: 'user', content }],
|
||||
})
|
||||
|
||||
// Parse JSON response
|
||||
const jsonMatch = text.match(/\[[\s\S]*\]/)
|
||||
const parsed = jsonMatch ? JSON.parse(jsonMatch[0]) : []
|
||||
|
||||
const suggestions = parsed.map((t: any) => ({
|
||||
title: t.title?.trim().replace(/^["']|["']$/g, '') || '',
|
||||
confidence: Math.round((t.confidence || 0.5) * 100),
|
||||
reasoning: undefined,
|
||||
})).filter((s: any) => s.title)
|
||||
|
||||
return {
|
||||
descriptions: [],
|
||||
suggestions,
|
||||
}
|
||||
}
|
||||
|
||||
// Single image description
|
||||
if (imageUrls.length === 1) {
|
||||
const content: any[] = [
|
||||
{ type: 'text', text: `Describe this image in detail in ${langName}. Be specific about what you see: objects, people, colors, setting, mood, text visible. Keep it under 100 words.` },
|
||||
{ type: 'image', image: imageDataUrls[0] },
|
||||
]
|
||||
|
||||
const { text } = await generateText({
|
||||
model,
|
||||
messages: [{ role: 'user', content }],
|
||||
})
|
||||
|
||||
return {
|
||||
descriptions: [{ index: 0, description: text.trim() }],
|
||||
}
|
||||
}
|
||||
|
||||
// Multiple images: describe each individually
|
||||
const descriptions: Array<{ index: number; description: string }> = []
|
||||
|
||||
for (let i = 0; i < imageDataUrls.length; i++) {
|
||||
const content: any[] = [
|
||||
{ type: 'text', text: `Describe this image (image ${i + 1} of ${imageDataUrls.length}) in ${langName}. Be specific: objects, people, colors, setting, text visible. Under 80 words.` },
|
||||
{ type: 'image', image: imageDataUrls[i] },
|
||||
]
|
||||
|
||||
const { text } = await generateText({
|
||||
model,
|
||||
messages: [{ role: 'user', content }],
|
||||
})
|
||||
|
||||
descriptions.push({ index: i, description: text.trim() })
|
||||
}
|
||||
|
||||
// Combined summary
|
||||
const allDescriptions = descriptions.map(d => d.description).join('\n')
|
||||
const { text: summary } = await generateText({
|
||||
model,
|
||||
prompt: `Based on these individual image descriptions, write a brief (1-2 sentence) overall summary in ${langName} of what these images collectively show:\n\n${allDescriptions}`,
|
||||
})
|
||||
|
||||
return {
|
||||
descriptions,
|
||||
combinedSummary: summary.trim(),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user