diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index 0ddef2aac2..30de09b2ca 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -381,7 +381,9 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", " current_length = 0 for word in words: - if word != " ": + # re.findall above also yields multi-character whitespace tokens (" ", "\t"). + # Only count real words toward the length; any whitespace run is a separator. + if word.strip(): current_chunk.append(word) current_length += 1 if current_length == self.split_length and current_chunk: @@ -391,8 +393,15 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", " else: current_chunk.append(word) - if current_chunk: + if current_length > 0: chunks.append("".join(current_chunk)) + elif current_chunk: + # Only whitespace is left over (e.g. trailing whitespace): attach it to the + # previous chunk instead of emitting a whitespace-only chunk on its own. + if chunks: + chunks[-1] += "".join(current_chunk) + else: + chunks.append("".join(current_chunk)) elif split_units == "char": for i in range(0, self._chunk_length(text), self.split_length): chunks.append(text[i : i + self.split_length]) diff --git a/releasenotes/notes/fix-recursive-splitter-trailing-whitespace-chunk-d04e66f19a7a49d6.yaml b/releasenotes/notes/fix-recursive-splitter-trailing-whitespace-chunk-d04e66f19a7a49d6.yaml new file mode 100644 index 0000000000..c5632dcd96 --- /dev/null +++ b/releasenotes/notes/fix-recursive-splitter-trailing-whitespace-chunk-d04e66f19a7a49d6.yaml @@ -0,0 +1,13 @@ +--- +fixes: + - | + ``RecursiveDocumentSplitter``'s word-mode fixed-size fallback no longer counts a run of + whitespace (e.g. a double space, tab, or page break) as a word, so it no longer produces + chunks smaller than ``split_length``. It also no longer emits a whitespace-only chunk when + the text ends in whitespace right after a chunk boundary; that trailing whitespace is now + attached to the previous chunk instead. + + This changes the exact chunk boundaries and chunk count produced by the word-unit fallback + for any text containing such whitespace runs. Documents already split and indexed under the + old behavior will produce different chunks if re-split after upgrading, so re-index any + document store that relies on stable chunk boundaries from this fallback path. diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index 7be763018b..4e9c64915f 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -504,7 +504,7 @@ def test_run_split_by_dot_count_page_breaks_word_unit() -> None: documents = document_splitter.run(documents=[Document(content=text)])["documents"] - assert len(documents) == 8 + assert len(documents) == 7 assert documents[0].content == "Sentence on page 1." assert documents[0].meta["page_number"] == 1 assert documents[0].meta["split_id"] == 0 @@ -535,16 +535,11 @@ def test_run_split_by_dot_count_page_breaks_word_unit() -> None: assert documents[5].meta["split_id"] == 5 assert documents[5].meta["split_idx_start"] == text.index(documents[5].content) - assert documents[6].content == "\f\f Sentence on page" + assert documents[6].content == "\f\f Sentence on page 5." assert documents[6].meta["page_number"] == 5 assert documents[6].meta["split_id"] == 6 assert documents[6].meta["split_idx_start"] == text.index(documents[6].content) - assert documents[7].content == " 5." - assert documents[7].meta["page_number"] == 5 - assert documents[7].meta["split_id"] == 7 - assert documents[7].meta["split_idx_start"] == text.index(documents[7].content) - def test_run_split_by_word_count_page_breaks_word_unit(): splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=0, separators=[" "], split_unit="word") @@ -901,9 +896,10 @@ def test_run_custom_split_by_dot_and_overlap_3_char_unit(): document_splitter = RecursiveDocumentSplitter(separators=["."], split_length=4, split_overlap=0, split_unit="word") text = "\x0c\x0c Sentence on page 5." chunks = document_splitter._fall_back_to_fixed_chunking(text, split_units="word") - assert len(chunks) == 2 - assert chunks[0] == "\x0c\x0c Sentence on page" - assert chunks[1] == " 5." + # The leading page breaks are whitespace, not words, so the four real words fit one chunk + # instead of being split mid-sentence. + assert len(chunks) == 1 + assert chunks[0] == "\x0c\x0c Sentence on page 5." def test_run_serialization_in_pipeline(): @@ -1156,3 +1152,21 @@ def test_fallback_overlap_token_unit(): assert len(result) > 1 for chunk in result: assert splitter._chunk_length(chunk.content) <= 4 + + +def test_word_fallback_does_not_count_multichar_whitespace_as_words(): + # The word-mode fixed-size fallback (used when no configured separator matches) must treat any + # run of whitespace as a separator; " " or "\t" must not be counted as a word. + splitter = RecursiveDocumentSplitter(split_length=1, split_overlap=0, split_unit="word", separators=["\n\n"]) + chunks = splitter.run([Document(content="hello world")])["documents"] + # Exactly one real word per chunk and no whitespace-only chunk. + assert [chunk.content.strip() for chunk in chunks] == ["hello", "world"] + assert all(chunk.content.strip() for chunk in chunks) + + +def test_fallback_word_unit_no_trailing_whitespace_only_chunk(): + """Trailing whitespace after the last real word must not be emitted as its own whitespace-only chunk.""" + splitter = RecursiveDocumentSplitter(split_length=1, split_overlap=0, split_unit="word", separators=["\n\n"]) + result = splitter.run([Document(content="hello world ")])["documents"] + + assert all(doc.content.strip() for doc in result)