From dae610d2b2e70f79899618fe4d63d89d700602f9 Mon Sep 17 00:00:00 2001
From: Brandon Wolfgang <developer@brandonwolfgang.com>
Date: Thu, 22 Oct 2026 10:01:12 -0500
Subject: [PATCH] feat: add configurable text chunking with overlap - Implement
 chunkText with configurable chunk size and overlap defaults - Normalize input
 text and return no chunks for empty or whitespace-only content - Assign
 sequential indices to chunks while preserving configured overlap - Validate
 chunk size and overlap constraints - Add tests for empty input, chunk
 boundaries, overlap, indexing, and invalid sizes

---
 src/chunking.test.ts | 48 +++++++++++++++++++++++++++++++++++++++
 src/chunking.ts      | 54 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 src/chunking.test.ts
 create mode 100644 src/chunking.ts

diff --git a/src/chunking.test.ts b/src/chunking.test.ts
new file mode 100644
index 0000000..75eb7cb
--- /dev/null
+++ b/src/chunking.test.ts
@@ -0,0 +1,48 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+
+import { chunkText } from "./chunking.js";
+
+describe("chunkText", () => {
+  it("returns no chunks for empty or whitespace-only text", () => {
+    assert.deepEqual(chunkText(""), []);
+    assert.deepEqual(chunkText("   \n\t "), []);
+  });
+
+  it("returns one chunk when text fits within the chunk size", () => {
+    assert.deepEqual(chunkText("Hello world"), [
+      { chunkIndex: 0, content: "Hello world" },
+    ]);
+  });
+
+  it("splits text into chunks with the configured overlap", () => {
+    const chunks = chunkText("abcdefghijkl", {
+      chunkSize: 6,
+      overlap: 2,
+    });
+
+    assert.deepEqual(chunks, [
+      { chunkIndex: 0, content: "abcdef" },
+      { chunkIndex: 1, content: "efghij" },
+      { chunkIndex: 2, content: "ijkl" },
+    ]);
+  });
+
+  it("assigns sequential chunk indices", () => {
+    const chunks = chunkText("abcdefghijkl", {
+      chunkSize: 4,
+      overlap: 1,
+    });
+
+    assert.deepEqual(
+      chunks.map((chunk) => chunk.chunkIndex),
+      [0, 1, 2, 3],
+    );
+  });
+
+  it("rejects invalid chunk sizes", () => {
+    assert.throws(() => chunkText("text", { chunkSize: 0 }));
+    assert.throws(() => chunkText("text", { chunkSize: -1 }));
+    assert.throws(() => chunkText("text", { chunkSize: 1.5 }));
+  });
+});
diff --git a/src/chunking.ts b/src/chunking.ts
new file mode 100644
index 0000000..dbed64d
--- /dev/null
+++ b/src/chunking.ts
@@ -0,0 +1,54 @@
+import { error } from "node:console";
+
+export interface TextChunk {
+  chunkIndex: number;
+  content: string;
+}
+
+interface ChunkingOptions {
+  chunkSize?: number;
+  overlap?: number;
+}
+
+const DEFAULT_CHUNK_SIZE = 1_000;
+const DEFAULT_OVERLAP = 200;
+
+export const chunkText = (
+  text: string,
+  options: ChunkingOptions = {},
+): TextChunk[] => {
+  const chunkSize = options.chunkSize ?? DEFAULT_CHUNK_SIZE;
+  const overlap = options.overlap ?? DEFAULT_OVERLAP;
+
+  if (!Number.isInteger(chunkSize) || chunkSize <= 0) {
+    throw new Error("Chunk size must be a positive integer");
+  }
+
+  if (!Number.isInteger(overlap) || overlap < 0 || overlap >= chunkSize) {
+    throw new Error(
+      "Overlap must be a non-negative integer smaller than chunk size",
+    );
+  }
+
+  const normalizedText = text.trim();
+
+  if (!normalizedText) {
+    return [];
+  }
+
+  const chunks: TextChunk[] = [];
+  const step = chunkSize - overlap;
+
+  for (let start = 0; start < normalizedText.length; start += step) {
+    const content = normalizedText.slice(start, start + chunkSize).trim();
+
+    if (content) {
+      chunks.push({
+        chunkIndex: chunks.length,
+        content,
+      });
+    }
+  }
+
+  return chunks;
+};