From c1b243ea85ebb0dcedf6658fd86ee7aadc40824c Mon Sep 17 00:00:00 2001 From: Timur Rakhmatullin <174210871+TimurRakhmatullin86@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:32:08 -0700 Subject: [PATCH 1/4] Do not count whitespace runs as words in RecursiveDocumentSplitter fallback The word-mode fixed-size fallback split words with re.findall(r"\S+|\s+", text), which also yields multi-character whitespace tokens (" ", "\t", "\f"). The loop counted every token that was not exactly a single space toward the chunk length, so a multi-space or page-break token was counted as a word - producing chunks shorter than split_length and, for a whitespace-only document, a whitespace-only chunk. Count a token only when it contains a real word (word.strip()). --- .../preprocessors/recursive_splitter.py | 4 ++- ...hitespace-word-count-a22d97f7deb28dfa.yaml | 7 +++++ .../preprocessors/test_recursive_splitter.py | 26 ++++++++++++------- 3 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/fix-recursive-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index 129ec133091..e81d9902f4c 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -380,7 +380,9 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", " current_length = 0 for word in words: - if word != " ": + # re.findall above also yields multi-character whitespace tokens (" ", "\t"). + # Only count real words toward the length; any whitespace run is a separator. + if word.strip(): current_chunk.append(word) current_length += 1 if current_length == self.split_length and current_chunk: diff --git a/releasenotes/notes/fix-recursive-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml b/releasenotes/notes/fix-recursive-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml new file mode 100644 index 00000000000..1f98c51c811 --- /dev/null +++ b/releasenotes/notes/fix-recursive-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + ``RecursiveDocumentSplitter``'s word-mode fixed-size fallback no longer counts a run of + whitespace (e.g. a double space, tab, or page break) as a word. Previously such a run was + counted toward the chunk length, producing chunks smaller than ``split_length`` and, for a + document made only of whitespace, a whitespace-only chunk. diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index d334af5b8e2..dd1350422cd 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -502,7 +502,7 @@ def test_run_split_by_dot_count_page_breaks_word_unit() -> None: documents = document_splitter.run(documents=[Document(content=text)])["documents"] - assert len(documents) == 8 + assert len(documents) == 7 assert documents[0].content == "Sentence on page 1." assert documents[0].meta["page_number"] == 1 assert documents[0].meta["split_id"] == 0 @@ -533,16 +533,11 @@ def test_run_split_by_dot_count_page_breaks_word_unit() -> None: assert documents[5].meta["split_id"] == 5 assert documents[5].meta["split_idx_start"] == text.index(documents[5].content) - assert documents[6].content == "\f\f Sentence on page" + assert documents[6].content == "\f\f Sentence on page 5." assert documents[6].meta["page_number"] == 5 assert documents[6].meta["split_id"] == 6 assert documents[6].meta["split_idx_start"] == text.index(documents[6].content) - assert documents[7].content == " 5." - assert documents[7].meta["page_number"] == 5 - assert documents[7].meta["split_id"] == 7 - assert documents[7].meta["split_idx_start"] == text.index(documents[7].content) - def test_run_split_by_word_count_page_breaks_word_unit(): splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=0, separators=[" "], split_unit="word") @@ -899,9 +894,10 @@ def test_run_custom_split_by_dot_and_overlap_3_char_unit(): document_splitter = RecursiveDocumentSplitter(separators=["."], split_length=4, split_overlap=0, split_unit="word") text = "\x0c\x0c Sentence on page 5." chunks = document_splitter._fall_back_to_fixed_chunking(text, split_units="word") - assert len(chunks) == 2 - assert chunks[0] == "\x0c\x0c Sentence on page" - assert chunks[1] == " 5." + # The leading page breaks are whitespace, not words, so the four real words fit one chunk + # instead of being split mid-sentence. + assert len(chunks) == 1 + assert chunks[0] == "\x0c\x0c Sentence on page 5." def test_run_serialization_in_pipeline(): @@ -1136,3 +1132,13 @@ def test_fallback_overlap_token_unit(): assert len(result) > 1 for chunk in result: assert splitter._chunk_length(chunk.content) <= 4 + + +def test_word_fallback_does_not_count_multichar_whitespace_as_words(): + # The word-mode fixed-size fallback (used when no configured separator matches) must treat any + # run of whitespace as a separator; " " or "\t" must not be counted as a word. + splitter = RecursiveDocumentSplitter(split_length=1, split_overlap=0, split_unit="word", separators=["\n\n"]) + chunks = splitter.run([Document(content="hello world")])["documents"] + # Exactly one real word per chunk and no whitespace-only chunk. + assert [chunk.content.strip() for chunk in chunks] == ["hello", "world"] + assert all(chunk.content.strip() for chunk in chunks) From 949454567d642b72c7210adfa1fe82719e793b64 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 30 Jul 2026 17:06:00 +0200 Subject: [PATCH 2/4] updating fix for edge case + updating release notes --- haystack/components/preprocessors/recursive_splitter.py | 9 ++++++++- test/components/preprocessors/test_recursive_splitter.py | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index e81d9902f4c..ba34ebea9f2 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -392,8 +392,15 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", " else: current_chunk.append(word) - if current_chunk: + if current_length > 0: chunks.append("".join(current_chunk)) + elif current_chunk: + # Only whitespace is left over (e.g. trailing whitespace): attach it to the + # previous chunk instead of emitting a whitespace-only chunk on its own. + if chunks: + chunks[-1] += "".join(current_chunk) + else: + chunks.append("".join(current_chunk)) elif split_units == "char": for i in range(0, self._chunk_length(text), self.split_length): chunks.append(text[i : i + self.split_length]) diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index dd1350422cd..3a1b8449a70 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -1142,3 +1142,11 @@ def test_word_fallback_does_not_count_multichar_whitespace_as_words(): # Exactly one real word per chunk and no whitespace-only chunk. assert [chunk.content.strip() for chunk in chunks] == ["hello", "world"] assert all(chunk.content.strip() for chunk in chunks) + + +def test_fallback_word_unit_no_trailing_whitespace_only_chunk(): + """Trailing whitespace after the last real word must not be emitted as its own whitespace-only chunk.""" + splitter = RecursiveDocumentSplitter(split_length=1, split_overlap=0, split_unit="word", separators=["\n\n"]) + result = splitter.run([Document(content="hello world ")])["documents"] + + assert all(doc.content.strip() for doc in result) From 8762f5cb796ebb5b75cc7515f183a4f065a9997e Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 30 Jul 2026 17:06:17 +0200 Subject: [PATCH 3/4] updating fix for edge case + updating release notes --- ...-trailing-whitespace-chunk-d04e66f19a7a49d6.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 releasenotes/notes/fix-recursive-splitter-trailing-whitespace-chunk-d04e66f19a7a49d6.yaml diff --git a/releasenotes/notes/fix-recursive-splitter-trailing-whitespace-chunk-d04e66f19a7a49d6.yaml b/releasenotes/notes/fix-recursive-splitter-trailing-whitespace-chunk-d04e66f19a7a49d6.yaml new file mode 100644 index 00000000000..c5632dcd963 --- /dev/null +++ b/releasenotes/notes/fix-recursive-splitter-trailing-whitespace-chunk-d04e66f19a7a49d6.yaml @@ -0,0 +1,13 @@ +--- +fixes: + - | + ``RecursiveDocumentSplitter``'s word-mode fixed-size fallback no longer counts a run of + whitespace (e.g. a double space, tab, or page break) as a word, so it no longer produces + chunks smaller than ``split_length``. It also no longer emits a whitespace-only chunk when + the text ends in whitespace right after a chunk boundary; that trailing whitespace is now + attached to the previous chunk instead. + + This changes the exact chunk boundaries and chunk count produced by the word-unit fallback + for any text containing such whitespace runs. Documents already split and indexed under the + old behavior will produce different chunks if re-split after upgrading, so re-index any + document store that relies on stable chunk boundaries from this fallback path. From 26e51dda975edf5bac7145007ac7820a3b2fac83 Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Thu, 30 Jul 2026 17:08:32 +0200 Subject: [PATCH 4/4] cleaning up --- ...ve-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 releasenotes/notes/fix-recursive-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml diff --git a/releasenotes/notes/fix-recursive-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml b/releasenotes/notes/fix-recursive-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml deleted file mode 100644 index 1f98c51c811..00000000000 --- a/releasenotes/notes/fix-recursive-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -fixes: - - | - ``RecursiveDocumentSplitter``'s word-mode fixed-size fallback no longer counts a run of - whitespace (e.g. a double space, tab, or page break) as a word. Previously such a run was - counted toward the chunk length, producing chunks smaller than ``split_length`` and, for a - document made only of whitespace, a whitespace-only chunk.