c2b47e7645
- Add document ingestion with text chunking, embedding generation, and PostgreSQL persistence - Configure PostgreSQL with pgvector, document schemas, and database migrations - Implement semantic search using Ollama embeddings and pgvector cosine similarity - Add POST /documents, POST /search, and POST /ask endpoints with validation and sanitized error handling - Generate context-grounded answers with source attribution - Filter retrieved chunks using a minimum similarity threshold of 0.5 and return fallback responses when no relevant context is available - Add unit and integration tests for ingestion, search, question answering, validation, and failure handling - Update README with Phase 3 capabilities, setup instructions, and API documentation
74 lines
1.5 KiB
TypeScript
74 lines
1.5 KiB
TypeScript
import { pool } from "./db.js";
|
|
import type { TextChunk } from "./chunking.js";
|
|
|
|
export interface NewDocument {
|
|
title: string;
|
|
source?: string;
|
|
}
|
|
|
|
export interface StoredChunk extends TextChunk {
|
|
embedding: number[];
|
|
}
|
|
|
|
export const createDocument = async (
|
|
document: NewDocument,
|
|
): Promise<string> => {
|
|
const result = await pool.query<{ id: string }>(
|
|
`INSERT INTO documents (title, source, status)
|
|
VALUES ($1, $2, 'processing')
|
|
RETURNING id`,
|
|
[document.title, document.source ?? null],
|
|
);
|
|
|
|
const id = result.rows[0]?.id;
|
|
|
|
if (!id) {
|
|
throw new Error("Failed to create document");
|
|
}
|
|
|
|
return id;
|
|
};
|
|
|
|
export const storeDocumentChunks = async (
|
|
documentId: string,
|
|
chunks: StoredChunk[],
|
|
): Promise<void> => {
|
|
const client = await pool.connect();
|
|
|
|
try {
|
|
await client.query("BEGIN");
|
|
|
|
for (const chunk of chunks) {
|
|
await client.query(
|
|
`INSERT INTO document_chunks
|
|
(document_id, chunk_index, content, embedding)
|
|
VALUES ($1, $2, $3, $4::vector)`,
|
|
[
|
|
documentId,
|
|
chunk.chunkIndex,
|
|
chunk.content,
|
|
`[${chunk.embedding.join(",")}]`,
|
|
],
|
|
);
|
|
}
|
|
|
|
await client.query(
|
|
`UPDATE documents
|
|
SET status = 'completed'
|
|
WHERE id = $1`,
|
|
[documentId],
|
|
);
|
|
|
|
await client.query("COMMIT");
|
|
} catch (error) {
|
|
try {
|
|
await client.query("ROLLBACK");
|
|
} catch {
|
|
// Preserve the original error.
|
|
}
|
|
|
|
throw error;
|
|
} finally {
|
|
client.release();
|
|
}
|
|
};
|