feat: brainstorm sessions, PDF document Q&A, embedding fixes, and UI improvements

- Add brainstorm feature with collaborative canvas, AI idea generation, live cursors, playback, and export
- Add PDF upload/extraction/ingestion pipeline with pgvector document search (RAG)
- Add document Q&A overlay with streaming chat and PDF preview
- Add note attachments UI with status polling, grid layout, and auto-scroll
- Add task extraction AI tool and agent executor improvements
- Fix NoteEmbedding missing updatedAt column, re-index 66 notes with 1536-dim embeddings
- Fix brainstorm 'Create Note' button: add success toast and redirect to created note
- Fix memory echo notification infinite polling
- Fix chat route to always include document_search tool
- Add brainstorm i18n keys across all 14 locales
- Add socket server for real-time brainstorm collaboration
- Add hierarchical notebook selector and organize notebook dialog improvements
- Add sidebar brainstorm section with session management
- Update prisma schema with brainstorm tables, attachments, and document chunks

memento-note/lib/ai/services/document-ingestion.service.ts

@@ -0,0 +1,79 @@
+import prisma from '@/lib/prisma'
+import { documentExtractionService } from './document-extraction.service'
+import { documentChunkingService } from './document-chunking.service'
+import { embeddingService } from './embedding.service'
+
+export class DocumentIngestionService {
+  async ingest(attachmentId: string): Promise<void> {
+    const attachment = await prisma.noteAttachment.findUnique({
+      where: { id: attachmentId },
+    })
+    if (!attachment) throw new Error('Attachment not found')
+
+    await prisma.noteAttachment.update({
+      where: { id: attachmentId },
+      data: { status: 'processing' },
+    })
+
+    try {
+      const extracted = await documentExtractionService.extractPdf(attachment.filePath)
+
+      await prisma.noteAttachment.update({
+        where: { id: attachmentId },
+        data: { pageCount: extracted.totalPages },
+      })
+
+      const chunkInputs = extracted.pages.map(p => ({
+        text: p.text,
+        pageNumber: p.pageNumber,
+      }))
+      const chunks = documentChunkingService.chunk(chunkInputs)
+
+      const created = await Promise.all(
+        chunks.map(c =>
+          prisma.documentChunk.create({
+            data: {
+              attachmentId,
+              content: c.content,
+              chunkIndex: c.chunkIndex,
+              pageNumber: c.pageNumber,
+              startChar: c.startChar,
+              endChar: c.endChar,
+              metadata: c.metadata,
+            },
+          })
+        )
+      )
+
+      const BATCH_SIZE = 20
+      for (let i = 0; i < created.length; i += BATCH_SIZE) {
+        const batch = created.slice(i, i + BATCH_SIZE)
+        const texts = batch.map(c => c.content)
+        const embeddings = await embeddingService.generateBatchEmbeddings(texts)
+
+        await Promise.all(
+          batch.map((chunk, idx) =>
+            prisma.$executeRawUnsafe(
+              `UPDATE "DocumentChunk" SET embedding = $1::vector WHERE id = $2`,
+              embeddingService.toVectorString(embeddings[idx].embedding),
+              chunk.id
+            )
+          )
+        )
+      }
+
+      await prisma.noteAttachment.update({
+        where: { id: attachmentId },
+        data: { status: 'ready' },
+      })
+    } catch (error: any) {
+      await prisma.noteAttachment.update({
+        where: { id: attachmentId },
+        data: { status: 'failed', error: error.message?.substring(0, 500) },
+      })
+      throw error
+    }
+  }
+}
+
+export const documentIngestionService = new DocumentIngestionService()