Do not count whitespace runs as words in RecursiveDocumentSplitter fallback

haystack/components/preprocessors/recursive_splitter.py

@@ -381,7 +381,9 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "
             current_length = 0
 
             for word in words:
-                if word != " ":
+                # re.findall above also yields multi-character whitespace tokens ("  ", "\t").
+                # Only count real words toward the length; any whitespace run is a separator.
+                if word.strip():
                     current_chunk.append(word)
                     current_length += 1
                     if current_length == self.split_length and current_chunk:
@@ -391,8 +393,15 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "
                 else:
                     current_chunk.append(word)
 
-            if current_chunk:
+            if current_length > 0:
                 chunks.append("".join(current_chunk))
+            elif current_chunk:
+                # Only whitespace is left over (e.g. trailing whitespace): attach it to the
+                # previous chunk instead of emitting a whitespace-only chunk on its own.
+                if chunks:
+                    chunks[-1] += "".join(current_chunk)
+                else:
+                    chunks.append("".join(current_chunk))
         elif split_units == "char":
             for i in range(0, self._chunk_length(text), self.split_length):
                 chunks.append(text[i : i + self.split_length])

releasenotes/notes/fix-recursive-splitter-trailing-whitespace-chunk-d04e66f19a7a49d6.yaml

@@ -0,0 +1,13 @@
+---
+fixes:
+  - |
+    ``RecursiveDocumentSplitter``'s word-mode fixed-size fallback no longer counts a run of
+    whitespace (e.g. a double space, tab, or page break) as a word, so it no longer produces
+    chunks smaller than ``split_length``. It also no longer emits a whitespace-only chunk when
+    the text ends in whitespace right after a chunk boundary; that trailing whitespace is now
+    attached to the previous chunk instead.
+
+    This changes the exact chunk boundaries and chunk count produced by the word-unit fallback
+    for any text containing such whitespace runs. Documents already split and indexed under the
+    old behavior will produce different chunks if re-split after upgrading, so re-index any
+    document store that relies on stable chunk boundaries from this fallback path.

test/components/preprocessors/test_recursive_splitter.py

@@ -504,7 +504,7 @@ def test_run_split_by_dot_count_page_breaks_word_unit() -> None:
 
     documents = document_splitter.run(documents=[Document(content=text)])["documents"]
 
-    assert len(documents) == 8
+    assert len(documents) == 7
     assert documents[0].content == "Sentence on page 1."
     assert documents[0].meta["page_number"] == 1
     assert documents[0].meta["split_id"] == 0
@@ -535,16 +535,11 @@ def test_run_split_by_dot_count_page_breaks_word_unit() -> None:
     assert documents[5].meta["split_id"] == 5
     assert documents[5].meta["split_idx_start"] == text.index(documents[5].content)
 
-    assert documents[6].content == "\f\f Sentence on page"
+    assert documents[6].content == "\f\f Sentence on page 5."
     assert documents[6].meta["page_number"] == 5
     assert documents[6].meta["split_id"] == 6
     assert documents[6].meta["split_idx_start"] == text.index(documents[6].content)
 
-    assert documents[7].content == " 5."
-    assert documents[7].meta["page_number"] == 5
-    assert documents[7].meta["split_id"] == 7
-    assert documents[7].meta["split_idx_start"] == text.index(documents[7].content)
-
 
 def test_run_split_by_word_count_page_breaks_word_unit():
     splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=0, separators=[" "], split_unit="word")
@@ -901,9 +896,10 @@ def test_run_custom_split_by_dot_and_overlap_3_char_unit():
     document_splitter = RecursiveDocumentSplitter(separators=["."], split_length=4, split_overlap=0, split_unit="word")
     text = "\x0c\x0c Sentence on page 5."
     chunks = document_splitter._fall_back_to_fixed_chunking(text, split_units="word")
-    assert len(chunks) == 2
-    assert chunks[0] == "\x0c\x0c Sentence on page"
-    assert chunks[1] == " 5."
+    # The leading page breaks are whitespace, not words, so the four real words fit one chunk
+    # instead of being split mid-sentence.
+    assert len(chunks) == 1
+    assert chunks[0] == "\x0c\x0c Sentence on page 5."
 
 
 def test_run_serialization_in_pipeline():
@@ -1156,3 +1152,21 @@ def test_fallback_overlap_token_unit():
     assert len(result) > 1
     for chunk in result:
         assert splitter._chunk_length(chunk.content) <= 4
+
+
+def test_word_fallback_does_not_count_multichar_whitespace_as_words():
+    # The word-mode fixed-size fallback (used when no configured separator matches) must treat any
+    # run of whitespace as a separator; "  " or "\t" must not be counted as a word.
+    splitter = RecursiveDocumentSplitter(split_length=1, split_overlap=0, split_unit="word", separators=["\n\n"])
+    chunks = splitter.run([Document(content="hello  world")])["documents"]
+    # Exactly one real word per chunk and no whitespace-only chunk.
+    assert [chunk.content.strip() for chunk in chunks] == ["hello", "world"]
+    assert all(chunk.content.strip() for chunk in chunks)
+
+
+def test_fallback_word_unit_no_trailing_whitespace_only_chunk():
+    """Trailing whitespace after the last real word must not be emitted as its own whitespace-only chunk."""
+    splitter = RecursiveDocumentSplitter(split_length=1, split_overlap=0, split_unit="word", separators=["\n\n"])
+    result = splitter.run([Document(content="hello world ")])["documents"]
+
+    assert all(doc.content.strip() for doc in result)