feat: add document ingestion endpoint

- Implement document ingestion with text chunking, embedding generation, and database persistence
- Validate document titles, sources, and content before processing
- Add POST /documents returning the document ID and chunk count on success
- Return sanitized validation and ingestion errors
- Log ingestion failures for server-side diagnostics

src/ingestion.ts

@@ -0,0 +1,37 @@
+import { chunkText } from "./chunking.js";
+import { createDocument, storeDocumentChunks } from "./document-repository.js";
+import type { NewDocument } from "./document-repository.js";
+import { generateEmbedding } from "./ollama.js";
+
+export const ingestDocument = async (
+  document: NewDocument & { content: string },
+): Promise<{ documentId: string; chunkCount: number }> => {
+  const chunks = chunkText(document.content);
+
+  if (chunks.length === 0) {
+    throw new Error("Document content cannot be empty");
+  }
+
+  const documentId = await createDocument({
+    title: document.title,
+    source: document.source,
+  });
+
+  try {
+    const embeddedChunks = await Promise.all(
+      chunks.map(async (chunk) => ({
+        ...chunk,
+        embedding: await generateEmbedding(chunk.content),
+      })),
+    );
+
+    await storeDocumentChunks(documentId, embeddedChunks);
+
+    return {
+      documentId,
+      chunkCount: embeddedChunks.length,
+    };
+  } catch (error) {
+    throw error;
+  }
+};

src/server.ts

@@ -1,12 +1,19 @@
 import Fastify, { type FastifyError } from "fastify";
 import { z } from "zod";
 
+import { ingestDocument } from "./ingestion.js";
 import { checkOllamaHealth, generateResponse } from "./ollama.js";
 
 const echoSchema = z.object({
   message: z.string().min(1).max(500),
 });
 
+const documentSchema = z.object({
+  title: z.string().trim().min(1).max(200),
+  source: z.string().trim().max(2_00).optional(),
+  content: z.string().trim().min(1).max(100_000),
+});
+
 export const buildApp = (
   generate = generateResponse,
   checkHealth = checkOllamaHealth,
@@ -95,6 +102,28 @@ export const buildApp = (
     }
   });
 
+  app.post("/documents", async (request, reply) => {
+    const result = documentSchema.safeParse(request.body);
+
+    if (!result.success) {
+      return reply.code(400).send({
+        error: "Invalid document",
+      });
+    }
+
+    try {
+      const document = await ingestDocument(result.data);
+
+      return reply.code(201).send(document);
+    } catch (error) {
+      request.log.error({ err: error }, "Document ingestion failed");
+
+      return reply.code(503).send({
+        error: "Document ingestion failed",
+      });
+    }
+  });
+
   app.get("/test-error", async () => {
     throw new Error("Sensitive internal detail");
   });