add chunkText helper (#35)

Author: ogroppo

CHANGELOG.md

@@ -1,5 +1,11 @@
 # deverything
 
+## 4.10.0
+
+### Minor Changes
+
+- chunkText
+
 ## 4.9.0
 
 ### Minor Changes

README.md

@@ -96,6 +96,7 @@ Contributions always welcome!
   - `arrayIntersection()` get the intersection of two arrays
 - `capitalize()` word => Word
 - `chunkArray()` split array into chunks
+- `chunkText()` split text into chunks of a given character size, optionally preserving sentence boundaries via `Intl.Segmenter`
 - `chunkedAll()` process all items in chunks
 - `chunkedAsync()` process async operations in chunks
 - `chunkedDynamic()` process with dynamic chunk sizes

package.json

@@ -1,6 +1,6 @@
 {
   "name": "deverything",
-  "version": "4.9.0",
+  "version": "4.10.0",
   "description": "Everything you need for Dev",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",

src/helpers/chunkText.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, test } from "vitest";
+import { chunkText } from "./chunkText";
+
+describe("chunkText", () => {
+  test("splits text into chunks of the given size", () => {
+    expect(chunkText("abcdefgh", 3)).toEqual(["abc", "def", "gh"]);
+    expect(chunkText("hello", 5)).toEqual(["hello"]);
+    expect(chunkText("hello", 10)).toEqual(["hello"]);
+    expect(chunkText("", 3)).toEqual([]);
+  });
+
+  const text = "Hello world. How are you? I am fine.";
+
+  test("preserves sentence shorter than chunk size", () => {
+    const chunks = chunkText(
+      text,
+      5, // shorter than sentence
+      { preserveOnBreak: "sentence" }
+    );
+    expect(chunks).toEqual([
+      "Hello",
+      " worl",
+      "d. ",
+      "How a",
+      "re yo",
+      "u? ",
+      "I am ",
+      "fine.",
+    ]);
+    expect(chunks.some((chunk) => chunk.length > 5)).toBe(false);
+  });
+
+  test("preserves sentence shorter than chunk size", () => {
+    const chunks = chunkText(text, 25, { preserveOnBreak: "sentence" });
+    expect(chunks).toEqual(["Hello world. ", "How are you? I am fine."]);
+    expect(chunks.some((chunk) => chunk.length > 25)).toBe(false);
+  });
+
+  test("preserves words shorter than chunk size", () => {
+    const chunks = chunkText(text, 25, { preserveOnBreak: "word" });
+    expect(chunks).toEqual(["Hello world. How are you?", " I am fine."]);
+    expect(chunks.some((chunk) => chunk.length > 25)).toBe(false);
+  });
+});

src/helpers/chunkText.ts

@@ -0,0 +1,42 @@
+import { chunkArray } from "./chunkArray";
+
+type ChunkTextOptions = {
+  preserveOnBreak?: "sentence" | "word";
+};
+
+export const chunkText = (
+  text: string,
+  chunkMaxSize: number,
+  options?: ChunkTextOptions
+): string[] => {
+  const segmenter = new Intl.Segmenter(undefined, {
+    granularity: options?.preserveOnBreak,
+  });
+  const segments = [...segmenter.segment(text)].map((s) => s.segment);
+
+  const chunks: string[] = [];
+  let current = "";
+
+  for (const segment of segments) {
+    if (segment.length > chunkMaxSize) {
+      if (current) {
+        chunks.push(current);
+        current = "";
+      }
+      chunks.push(
+        ...chunkArray([...segment], chunkMaxSize).map((chars) => chars.join(""))
+      );
+    } else if (current.length + segment.length > chunkMaxSize) {
+      chunks.push(current);
+      current = segment;
+    } else {
+      current += segment;
+    }
+  }
+
+  if (current) {
+    chunks.push(current);
+  }
+
+  return chunks;
+};

src/helpers/index.ts

@@ -3,6 +3,7 @@ export * from "./arrayDiff";
 export * from "./arrayIntersection";
 export * from "./capitalize";
 export * from "./chunkArray";
+export * from "./chunkText";
 export * from "./chunkedAll";
 export * from "./chunkedAsync";
 export * from "./chunkedDynamic";