dae610d2b2
- Implement chunkText with configurable chunk size and overlap defaults - Normalize input text and return no chunks for empty or whitespace-only content - Assign sequential indices to chunks while preserving configured overlap - Validate chunk size and overlap constraints - Add tests for empty input, chunk boundaries, overlap, indexing, and invalid sizes
48 lines
1.3 KiB
TypeScript
48 lines
1.3 KiB
TypeScript
import { describe, it } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
|
|
import { chunkText } from "./chunking.js";
|
|
|
|
describe("chunkText", () => {
|
|
it("returns no chunks for empty or whitespace-only text", () => {
|
|
assert.deepEqual(chunkText(""), []);
|
|
assert.deepEqual(chunkText(" \n\t "), []);
|
|
});
|
|
|
|
it("returns one chunk when text fits within the chunk size", () => {
|
|
assert.deepEqual(chunkText("Hello world"), [
|
|
{ chunkIndex: 0, content: "Hello world" },
|
|
]);
|
|
});
|
|
|
|
it("splits text into chunks with the configured overlap", () => {
|
|
const chunks = chunkText("abcdefghijkl", {
|
|
chunkSize: 6,
|
|
overlap: 2,
|
|
});
|
|
|
|
assert.deepEqual(chunks, [
|
|
{ chunkIndex: 0, content: "abcdef" },
|
|
{ chunkIndex: 1, content: "efghij" },
|
|
{ chunkIndex: 2, content: "ijkl" },
|
|
]);
|
|
});
|
|
|
|
it("assigns sequential chunk indices", () => {
|
|
const chunks = chunkText("abcdefghijkl", {
|
|
chunkSize: 4,
|
|
overlap: 1,
|
|
});
|
|
|
|
assert.deepEqual(
|
|
chunks.map((chunk) => chunk.chunkIndex),
|
|
[0, 1, 2, 3],
|
|
);
|
|
});
|
|
|
|
it("rejects invalid chunk sizes", () => {
|
|
assert.throws(() => chunkText("text", { chunkSize: 0 }));
|
|
assert.throws(() => chunkText("text", { chunkSize: -1 }));
|
|
assert.throws(() => chunkText("text", { chunkSize: 1.5 }));
|
|
});
|
|
});
|