Do not count whitespace runs as words in RecursiveDocumentSplitter fallback

The word-mode fixed-size fallback split words with re.findall(r"\S+|\s+", text),
which also yields multi-character whitespace tokens ("  ", "\t", "\f"). The loop
counted every token that was not exactly a single space toward the chunk length,
so a multi-space or page-break token was counted as a word - producing chunks
shorter than split_length and, for a whitespace-only document, a whitespace-only
chunk. Count a token only when it contains a real word (word.strip()).

Author: TimurRakhmatullin86

haystack/components/preprocessors/recursive_splitter.py

@@ -380,7 +380,9 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "
             current_length = 0
 
             for word in words:
-                if word != " ":
+                # re.findall above also yields multi-character whitespace tokens ("  ", "\t").
+                # Only count real words toward the length; any whitespace run is a separator.
+                if word.strip():
                     current_chunk.append(word)
                     current_length += 1
                     if current_length == self.split_length and current_chunk:

releasenotes/notes/fix-recursive-splitter-whitespace-word-count-a22d97f7deb28dfa.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    `RecursiveDocumentSplitter`'s word-mode fixed-size fallback no longer counts a run of
+    whitespace (e.g. a double space, tab, or page break) as a word. Previously such a run was
+    counted toward the chunk length, producing chunks smaller than `split_length` and, for a
+    document made only of whitespace, a whitespace-only chunk.

test/components/preprocessors/test_recursive_splitter.py

@@ -502,7 +502,7 @@ def test_run_split_by_dot_count_page_breaks_word_unit() -> None:
 
     documents = document_splitter.run(documents=[Document(content=text)])["documents"]
 
-    assert len(documents) == 8
+    assert len(documents) == 7
     assert documents[0].content == "Sentence on page 1."
     assert documents[0].meta["page_number"] == 1
     assert documents[0].meta["split_id"] == 0
@@ -533,16 +533,11 @@ def test_run_split_by_dot_count_page_breaks_word_unit() -> None:
     assert documents[5].meta["split_id"] == 5
     assert documents[5].meta["split_idx_start"] == text.index(documents[5].content)
 
-    assert documents[6].content == "\f\f Sentence on page"
+    assert documents[6].content == "\f\f Sentence on page 5."
     assert documents[6].meta["page_number"] == 5
     assert documents[6].meta["split_id"] == 6
     assert documents[6].meta["split_idx_start"] == text.index(documents[6].content)
 
-    assert documents[7].content == " 5."
-    assert documents[7].meta["page_number"] == 5
-    assert documents[7].meta["split_id"] == 7
-    assert documents[7].meta["split_idx_start"] == text.index(documents[7].content)
-
 
 def test_run_split_by_word_count_page_breaks_word_unit():
     splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=0, separators=[" "], split_unit="word")
@@ -899,9 +894,10 @@ def test_run_custom_split_by_dot_and_overlap_3_char_unit():
     document_splitter = RecursiveDocumentSplitter(separators=["."], split_length=4, split_overlap=0, split_unit="word")
     text = "\x0c\x0c Sentence on page 5."
     chunks = document_splitter._fall_back_to_fixed_chunking(text, split_units="word")
-    assert len(chunks) == 2
-    assert chunks[0] == "\x0c\x0c Sentence on page"
-    assert chunks[1] == " 5."
+    # The leading page breaks are whitespace, not words, so the four real words fit one chunk
+    # instead of being split mid-sentence.
+    assert len(chunks) == 1
+    assert chunks[0] == "\x0c\x0c Sentence on page 5."
 
 
 def test_run_serialization_in_pipeline():
@@ -1136,3 +1132,13 @@ def test_fallback_overlap_token_unit():
     assert len(result) > 1
     for chunk in result:
         assert splitter._chunk_length(chunk.content) <= 4
+
+
+def test_word_fallback_does_not_count_multichar_whitespace_as_words():
+    # The word-mode fixed-size fallback (used when no configured separator matches) must treat any
+    # run of whitespace as a separator; "  " or "\t" must not be counted as a word.
+    splitter = RecursiveDocumentSplitter(split_length=1, split_overlap=0, split_unit="word", separators=["\n\n"])
+    chunks = splitter.run([Document(content="hello  world")])["documents"]
+    # Exactly one real word per chunk and no whitespace-only chunk.
+    assert [chunk.content.strip() for chunk in chunks] == ["hello", "world"]
+    assert all(chunk.content.strip() for chunk in chunks)