c2b47e7645
- Add document ingestion with text chunking, embedding generation, and PostgreSQL persistence - Configure PostgreSQL with pgvector, document schemas, and database migrations - Implement semantic search using Ollama embeddings and pgvector cosine similarity - Add POST /documents, POST /search, and POST /ask endpoints with validation and sanitized error handling - Generate context-grounded answers with source attribution - Filter retrieved chunks using a minimum similarity threshold of 0.5 and return fallback responses when no relevant context is available - Add unit and integration tests for ingestion, search, question answering, validation, and failure handling - Update README with Phase 3 capabilities, setup instructions, and API documentation
23 lines
797 B
SQL
23 lines
797 B
SQL
CREATE EXTENSION IF NOT EXISTS vector;
|
|
|
|
CREATE TABLE documents (
|
|
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
title TEXT NOT NULL,
|
|
source TEXT,
|
|
status TEXT NOT NULL DEFAULT 'pending'
|
|
CHECK (status IN ('pending', 'processing', 'completed', 'failed')),
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
);
|
|
|
|
CREATE TABLE document_chunks (
|
|
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
|
chunk_index INTEGER NOT NULL CHECK (chunk_index >= 0),
|
|
content TEXT NOT NULL,
|
|
embedding vector(768) NOT NULL,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
UNIQUE (document_id, chunk_index)
|
|
);
|
|
|
|
CREATE INDEX document_chunks_document_id_idx
|
|
ON document_chunks (document_id);
|