feat: add configurable text chunking with overlap

- Implement chunkText with configurable chunk size and overlap defaults
- Normalize input text and return no chunks for empty or whitespace-only content
- Assign sequential indices to chunks while preserving configured overlap
- Validate chunk size and overlap constraints
- Add tests for empty input, chunk boundaries, overlap, indexing, and invalid sizes

src/chunking.test.ts

@@ -0,0 +1,48 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+
+import { chunkText } from "./chunking.js";
+
+describe("chunkText", () => {
+  it("returns no chunks for empty or whitespace-only text", () => {
+    assert.deepEqual(chunkText(""), []);
+    assert.deepEqual(chunkText("   \n\t "), []);
+  });
+
+  it("returns one chunk when text fits within the chunk size", () => {
+    assert.deepEqual(chunkText("Hello world"), [
+      { chunkIndex: 0, content: "Hello world" },
+    ]);
+  });
+
+  it("splits text into chunks with the configured overlap", () => {
+    const chunks = chunkText("abcdefghijkl", {
+      chunkSize: 6,
+      overlap: 2,
+    });
+
+    assert.deepEqual(chunks, [
+      { chunkIndex: 0, content: "abcdef" },
+      { chunkIndex: 1, content: "efghij" },
+      { chunkIndex: 2, content: "ijkl" },
+    ]);
+  });
+
+  it("assigns sequential chunk indices", () => {
+    const chunks = chunkText("abcdefghijkl", {
+      chunkSize: 4,
+      overlap: 1,
+    });
+
+    assert.deepEqual(
+      chunks.map((chunk) => chunk.chunkIndex),
+      [0, 1, 2, 3],
+    );
+  });
+
+  it("rejects invalid chunk sizes", () => {
+    assert.throws(() => chunkText("text", { chunkSize: 0 }));
+    assert.throws(() => chunkText("text", { chunkSize: -1 }));
+    assert.throws(() => chunkText("text", { chunkSize: 1.5 }));
+  });
+});

src/chunking.ts

@@ -0,0 +1,54 @@
+import { error } from "node:console";
+
+export interface TextChunk {
+  chunkIndex: number;
+  content: string;
+}
+
+interface ChunkingOptions {
+  chunkSize?: number;
+  overlap?: number;
+}
+
+const DEFAULT_CHUNK_SIZE = 1_000;
+const DEFAULT_OVERLAP = 200;
+
+export const chunkText = (
+  text: string,
+  options: ChunkingOptions = {},
+): TextChunk[] => {
+  const chunkSize = options.chunkSize ?? DEFAULT_CHUNK_SIZE;
+  const overlap = options.overlap ?? DEFAULT_OVERLAP;
+
+  if (!Number.isInteger(chunkSize) || chunkSize <= 0) {
+    throw new Error("Chunk size must be a positive integer");
+  }
+
+  if (!Number.isInteger(overlap) || overlap < 0 || overlap >= chunkSize) {
+    throw new Error(
+      "Overlap must be a non-negative integer smaller than chunk size",
+    );
+  }
+
+  const normalizedText = text.trim();
+
+  if (!normalizedText) {
+    return [];
+  }
+
+  const chunks: TextChunk[] = [];
+  const step = chunkSize - overlap;
+
+  for (let start = 0; start < normalizedText.length; start += step) {
+    const content = normalizedText.slice(start, start + chunkSize).trim();
+
+    if (content) {
+      chunks.push({
+        chunkIndex: chunks.length,
+        content,
+      });
+    }
+  }
+
+  return chunks;
+};