1fcea6ed7d
All checks were successful
Deploy to Production / Build and Deploy (push) Successful in 7s
- Add brainstorm feature with collaborative canvas, AI idea generation, live cursors, playback, and export - Add PDF upload/extraction/ingestion pipeline with pgvector document search (RAG) - Add document Q&A overlay with streaming chat and PDF preview - Add note attachments UI with status polling, grid layout, and auto-scroll - Add task extraction AI tool and agent executor improvements - Fix NoteEmbedding missing updatedAt column, re-index 66 notes with 1536-dim embeddings - Fix brainstorm 'Create Note' button: add success toast and redirect to created note - Fix memory echo notification infinite polling - Fix chat route to always include document_search tool - Add brainstorm i18n keys across all 14 locales - Add socket server for real-time brainstorm collaboration - Add hierarchical notebook selector and organize notebook dialog improvements - Add sidebar brainstorm section with session management - Update prisma schema with brainstorm tables, attachments, and document chunks
80 lines
2.4 KiB
TypeScript
80 lines
2.4 KiB
TypeScript
import prisma from '@/lib/prisma'
|
|
import { documentExtractionService } from './document-extraction.service'
|
|
import { documentChunkingService } from './document-chunking.service'
|
|
import { embeddingService } from './embedding.service'
|
|
|
|
export class DocumentIngestionService {
|
|
async ingest(attachmentId: string): Promise<void> {
|
|
const attachment = await prisma.noteAttachment.findUnique({
|
|
where: { id: attachmentId },
|
|
})
|
|
if (!attachment) throw new Error('Attachment not found')
|
|
|
|
await prisma.noteAttachment.update({
|
|
where: { id: attachmentId },
|
|
data: { status: 'processing' },
|
|
})
|
|
|
|
try {
|
|
const extracted = await documentExtractionService.extractPdf(attachment.filePath)
|
|
|
|
await prisma.noteAttachment.update({
|
|
where: { id: attachmentId },
|
|
data: { pageCount: extracted.totalPages },
|
|
})
|
|
|
|
const chunkInputs = extracted.pages.map(p => ({
|
|
text: p.text,
|
|
pageNumber: p.pageNumber,
|
|
}))
|
|
const chunks = documentChunkingService.chunk(chunkInputs)
|
|
|
|
const created = await Promise.all(
|
|
chunks.map(c =>
|
|
prisma.documentChunk.create({
|
|
data: {
|
|
attachmentId,
|
|
content: c.content,
|
|
chunkIndex: c.chunkIndex,
|
|
pageNumber: c.pageNumber,
|
|
startChar: c.startChar,
|
|
endChar: c.endChar,
|
|
metadata: c.metadata,
|
|
},
|
|
})
|
|
)
|
|
)
|
|
|
|
const BATCH_SIZE = 20
|
|
for (let i = 0; i < created.length; i += BATCH_SIZE) {
|
|
const batch = created.slice(i, i + BATCH_SIZE)
|
|
const texts = batch.map(c => c.content)
|
|
const embeddings = await embeddingService.generateBatchEmbeddings(texts)
|
|
|
|
await Promise.all(
|
|
batch.map((chunk, idx) =>
|
|
prisma.$executeRawUnsafe(
|
|
`UPDATE "DocumentChunk" SET embedding = $1::vector WHERE id = $2`,
|
|
embeddingService.toVectorString(embeddings[idx].embedding),
|
|
chunk.id
|
|
)
|
|
)
|
|
)
|
|
}
|
|
|
|
await prisma.noteAttachment.update({
|
|
where: { id: attachmentId },
|
|
data: { status: 'ready' },
|
|
})
|
|
} catch (error: any) {
|
|
await prisma.noteAttachment.update({
|
|
where: { id: attachmentId },
|
|
data: { status: 'failed', error: error.message?.substring(0, 500) },
|
|
})
|
|
throw error
|
|
}
|
|
}
|
|
}
|
|
|
|
export const documentIngestionService = new DocumentIngestionService()
|