diff --git a/src/ingestion.ts b/src/ingestion.ts new file mode 100644 index 0000000..8df0bfe --- /dev/null +++ b/src/ingestion.ts @@ -0,0 +1,37 @@ +import { chunkText } from "./chunking.js"; +import { createDocument, storeDocumentChunks } from "./document-repository.js"; +import type { NewDocument } from "./document-repository.js"; +import { generateEmbedding } from "./ollama.js"; + +export const ingestDocument = async ( + document: NewDocument & { content: string }, +): Promise<{ documentId: string; chunkCount: number }> => { + const chunks = chunkText(document.content); + + if (chunks.length === 0) { + throw new Error("Document content cannot be empty"); + } + + const documentId = await createDocument({ + title: document.title, + source: document.source, + }); + + try { + const embeddedChunks = await Promise.all( + chunks.map(async (chunk) => ({ + ...chunk, + embedding: await generateEmbedding(chunk.content), + })), + ); + + await storeDocumentChunks(documentId, embeddedChunks); + + return { + documentId, + chunkCount: embeddedChunks.length, + }; + } catch (error) { + throw error; + } +}; diff --git a/src/server.ts b/src/server.ts index ef170e2..07b96c1 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,12 +1,19 @@ import Fastify, { type FastifyError } from "fastify"; import { z } from "zod"; +import { ingestDocument } from "./ingestion.js"; import { checkOllamaHealth, generateResponse } from "./ollama.js"; const echoSchema = z.object({ message: z.string().min(1).max(500), }); +const documentSchema = z.object({ + title: z.string().trim().min(1).max(200), + source: z.string().trim().max(2_00).optional(), + content: z.string().trim().min(1).max(100_000), +}); + export const buildApp = ( generate = generateResponse, checkHealth = checkOllamaHealth, @@ -95,6 +102,28 @@ export const buildApp = ( } }); + app.post("/documents", async (request, reply) => { + const result = documentSchema.safeParse(request.body); + + if (!result.success) { + return reply.code(400).send({ + error: "Invalid document", + }); + } + + try { + const document = await ingestDocument(result.data); + + return reply.code(201).send(document); + } catch (error) { + request.log.error({ err: error }, "Document ingestion failed"); + + return reply.code(503).send({ + error: "Document ingestion failed", + }); + } + }); + app.get("/test-error", async () => { throw new Error("Sensitive internal detail"); });