import prisma from '@/lib/prisma' import { documentExtractionService } from './document-extraction.service' import { documentChunkingService } from './document-chunking.service' import { embeddingService } from './embedding.service' export class DocumentIngestionService { async ingest(attachmentId: string): Promise { const attachment = await prisma.noteAttachment.findUnique({ where: { id: attachmentId }, }) if (!attachment) throw new Error('Attachment not found') await prisma.noteAttachment.update({ where: { id: attachmentId }, data: { status: 'processing' }, }) try { const extracted = await documentExtractionService.extractPdf(attachment.filePath) await prisma.noteAttachment.update({ where: { id: attachmentId }, data: { pageCount: extracted.totalPages }, }) const chunkInputs = extracted.pages.map(p => ({ text: p.text, pageNumber: p.pageNumber, })) const chunks = documentChunkingService.chunk(chunkInputs) const created = await Promise.all( chunks.map(c => prisma.documentChunk.create({ data: { attachmentId, content: c.content, chunkIndex: c.chunkIndex, pageNumber: c.pageNumber, startChar: c.startChar, endChar: c.endChar, metadata: c.metadata, }, }) ) ) const BATCH_SIZE = 20 for (let i = 0; i < created.length; i += BATCH_SIZE) { const batch = created.slice(i, i + BATCH_SIZE) const texts = batch.map(c => c.content) const embeddings = await embeddingService.generateBatchEmbeddings(texts) await Promise.all( batch.map((chunk, idx) => prisma.$executeRawUnsafe( `UPDATE "DocumentChunk" SET embedding = $1::vector WHERE id = $2`, embeddingService.toVectorString(embeddings[idx].embedding), chunk.id ) ) ) } await prisma.noteAttachment.update({ where: { id: attachmentId }, data: { status: 'ready' }, }) } catch (error: any) { await prisma.noteAttachment.update({ where: { id: attachmentId }, data: { status: 'failed', error: error.message?.substring(0, 500) }, }) throw error } } } export const documentIngestionService = new DocumentIngestionService()