c2b47e7645
- Add document ingestion with text chunking, embedding generation, and PostgreSQL persistence - Configure PostgreSQL with pgvector, document schemas, and database migrations - Implement semantic search using Ollama embeddings and pgvector cosine similarity - Add POST /documents, POST /search, and POST /ask endpoints with validation and sanitized error handling - Generate context-grounded answers with source attribution - Filter retrieved chunks using a minimum similarity threshold of 0.5 and return fallback responses when no relevant context is available - Add unit and integration tests for ingestion, search, question answering, validation, and failure handling - Update README with Phase 3 capabilities, setup instructions, and API documentation
37 lines
994 B
TypeScript
37 lines
994 B
TypeScript
import { chunkText } from "./chunking.js";
|
|
import { createDocument, storeDocumentChunks } from "./document-repository.js";
|
|
import type { NewDocument } from "./document-repository.js";
|
|
import { generateEmbedding } from "./ollama.js";
|
|
|
|
export const ingestDocument = async (
|
|
document: NewDocument & { content: string },
|
|
): Promise<{ documentId: string; chunkCount: number }> => {
|
|
const chunks = chunkText(document.content);
|
|
|
|
if (chunks.length === 0) {
|
|
throw new Error("Document content cannot be empty");
|
|
}
|
|
|
|
const documentId = await createDocument({
|
|
title: document.title,
|
|
source: document.source,
|
|
});
|
|
|
|
try {
|
|
const embeddedChunks = await Promise.all(
|
|
chunks.map(async (chunk) => ({
|
|
...chunk,
|
|
embedding: await generateEmbedding(chunk.content),
|
|
})),
|
|
);
|
|
|
|
await storeDocumentChunks(documentId, embeddedChunks);
|
|
|
|
return {
|
|
documentId,
|
|
chunkCount: embeddedChunks.length,
|
|
};
|
|
} catch (error) {
|
|
throw error;
|
|
}
|
|
};
|