c2b47e7645
- Add document ingestion with text chunking, embedding generation, and PostgreSQL persistence - Configure PostgreSQL with pgvector, document schemas, and database migrations - Implement semantic search using Ollama embeddings and pgvector cosine similarity - Add POST /documents, POST /search, and POST /ask endpoints with validation and sanitized error handling - Generate context-grounded answers with source attribution - Filter retrieved chunks using a minimum similarity threshold of 0.5 and return fallback responses when no relevant context is available - Add unit and integration tests for ingestion, search, question answering, validation, and failure handling - Update README with Phase 3 capabilities, setup instructions, and API documentation
48 lines
1.3 KiB
TypeScript
48 lines
1.3 KiB
TypeScript
import { describe, it } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
|
|
import { chunkText } from "./chunking.js";
|
|
|
|
describe("chunkText", () => {
|
|
it("returns no chunks for empty or whitespace-only text", () => {
|
|
assert.deepEqual(chunkText(""), []);
|
|
assert.deepEqual(chunkText(" \n\t "), []);
|
|
});
|
|
|
|
it("returns one chunk when text fits within the chunk size", () => {
|
|
assert.deepEqual(chunkText("Hello world"), [
|
|
{ chunkIndex: 0, content: "Hello world" },
|
|
]);
|
|
});
|
|
|
|
it("splits text into chunks with the configured overlap", () => {
|
|
const chunks = chunkText("abcdefghijkl", {
|
|
chunkSize: 6,
|
|
overlap: 2,
|
|
});
|
|
|
|
assert.deepEqual(chunks, [
|
|
{ chunkIndex: 0, content: "abcdef" },
|
|
{ chunkIndex: 1, content: "efghij" },
|
|
{ chunkIndex: 2, content: "ijkl" },
|
|
]);
|
|
});
|
|
|
|
it("assigns sequential chunk indices", () => {
|
|
const chunks = chunkText("abcdefghijkl", {
|
|
chunkSize: 4,
|
|
overlap: 1,
|
|
});
|
|
|
|
assert.deepEqual(
|
|
chunks.map((chunk) => chunk.chunkIndex),
|
|
[0, 1, 2, 3],
|
|
);
|
|
});
|
|
|
|
it("rejects invalid chunk sizes", () => {
|
|
assert.throws(() => chunkText("text", { chunkSize: 0 }));
|
|
assert.throws(() => chunkText("text", { chunkSize: -1 }));
|
|
assert.throws(() => chunkText("text", { chunkSize: 1.5 }));
|
|
});
|
|
});
|