From 45ea0d165150d3d888279893969d4d9ed8724498 Mon Sep 17 00:00:00 2001 From: Osamaali313 <86572800+Osamaali313@users.noreply.github.com> Date: Mon, 29 Jun 2026 22:08:40 +0300 Subject: [PATCH 1/4] fix: populate _split_overlap metadata for word/token units in RecursiveDocumentSplitter _add_overlap_info computed the overlap length with self._chunk_length(prev_doc.content), which returns a word/token count for word/token split units, while curr_pos and split_idx_start are character offsets. Mixing units made overlap_length negative, so the guard never fired and the _split_overlap metadata was silently left empty for word/token splitting with overlap (char units were unaffected). Measure the overlap and ranges in characters via len(prev_doc.content). Adds a regression test and a release note. --- .../preprocessors/recursive_splitter.py | 12 +++++++++--- ...ken-overlap-metadata-9f3c1a7be0d24856.yaml | 9 +++++++++ .../preprocessors/test_recursive_splitter.py | 19 +++++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index 90232701ca9..5a8ef819a41 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -403,15 +403,21 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", " def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: list[Document]) -> None: prev_doc = new_docs[-1] - overlap_length = self._chunk_length(prev_doc.content) - (curr_pos - prev_doc.meta["split_idx_start"]) # type: ignore + # curr_pos and split_idx_start are character offsets, so the overlap and + # range must be measured in characters too. Using self._chunk_length() + # here would mix units for word/token splitting (it returns a word/token + # count), making overlap_length negative and silently dropping the + # _split_overlap metadata. + prev_doc_length = len(prev_doc.content) # type: ignore + overlap_length = prev_doc_length - (curr_pos - prev_doc.meta["split_idx_start"]) if overlap_length > 0: prev_doc.meta["_split_overlap"].append({"doc_id": new_doc.id, "range": (0, overlap_length)}) new_doc.meta["_split_overlap"].append( { "doc_id": prev_doc.id, "range": ( - self._chunk_length(prev_doc.content) - overlap_length, # type: ignore - self._chunk_length(prev_doc.content), # type: ignore + prev_doc_length - overlap_length, + prev_doc_length, ), } ) diff --git a/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml b/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml new file mode 100644 index 00000000000..2dc152d70a5 --- /dev/null +++ b/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixed `RecursiveDocumentSplitter` not populating the `_split_overlap` + metadata when `split_unit` is `"word"` or `"token"` and `split_overlap` + is greater than 0. The overlap length was computed with a word/token count + while the offsets it is compared against are character positions, so it + became negative and the overlap metadata was silently dropped. The overlap + is now measured in characters, matching the ranges it records. diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index 7022e645f3d..923e618ed66 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -750,6 +750,25 @@ def test_run_split_by_dot_and_overlap_1_word_unit_split_idx_start(): ) +def test_run_split_by_dot_and_overlap_1_word_unit_split_overlap_metadata(): + """ + _split_overlap must be populated when split_unit="word" and split_overlap > 0. + + Regression: the overlap length was computed with a word/token count while + curr_pos/split_idx_start are character offsets, so it went negative and the + _split_overlap metadata was silently left empty for word/token units. + """ + splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, separators=["."], split_unit="word") + text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four." + chunks = splitter.run([Document(content=text)])["documents"] + assert len(chunks) == 5 + # First chunk overlaps with the next, last with the previous, middle with both. + assert any(o["doc_id"] == chunks[1].id for o in chunks[0].meta["_split_overlap"]) + assert any(o["doc_id"] == chunks[3].id for o in chunks[4].meta["_split_overlap"]) + for i in (1, 2, 3): + assert chunks[i].meta["_split_overlap"], f"chunk {i} has empty _split_overlap" + + def test_run_trigger_dealing_with_remaining_word_larger_than_split_length(): splitter = RecursiveDocumentSplitter(split_length=3, split_overlap=2, separators=["."], split_unit="word") text = """A simple sentence1. A bright sentence2. A clever sentence3""" From 1b5b15f43403e97a175d5cb31ea4c3a7e4868942 Mon Sep 17 00:00:00 2001 From: Osamaali313 Date: Mon, 6 Jul 2026 09:55:51 +0300 Subject: [PATCH 2/4] Address review feedback - Shorten the _add_overlap_info comment (per review). - Rename test to test_word_unit_split_populates_split_overlap_metadata and drop the docstring paragraph referencing the old behavior. - Use double backticks for inline code in the release note (reST) so the release-notes RST check passes. - Run ruff format (collapse the _split_overlap dict literal) to fix the format check. --- .../preprocessors/recursive_splitter.py | 15 +++------------ ...d-token-overlap-metadata-9f3c1a7be0d24856.yaml | 4 ++-- .../preprocessors/test_recursive_splitter.py | 6 +----- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index 5a8ef819a41..254650bdc55 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -403,23 +403,14 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", " def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: list[Document]) -> None: prev_doc = new_docs[-1] - # curr_pos and split_idx_start are character offsets, so the overlap and - # range must be measured in characters too. Using self._chunk_length() - # here would mix units for word/token splitting (it returns a word/token - # count), making overlap_length negative and silently dropping the - # _split_overlap metadata. + # curr_pos and split_idx_start are character offsets, so measure the + # overlap and range in characters too (not via _chunk_length, which returns a word/token count). prev_doc_length = len(prev_doc.content) # type: ignore overlap_length = prev_doc_length - (curr_pos - prev_doc.meta["split_idx_start"]) if overlap_length > 0: prev_doc.meta["_split_overlap"].append({"doc_id": new_doc.id, "range": (0, overlap_length)}) new_doc.meta["_split_overlap"].append( - { - "doc_id": prev_doc.id, - "range": ( - prev_doc_length - overlap_length, - prev_doc_length, - ), - } + {"doc_id": prev_doc.id, "range": (prev_doc_length - overlap_length, prev_doc_length)} ) def _run_one(self, doc: Document) -> list[Document]: diff --git a/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml b/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml index 2dc152d70a5..24d9622419e 100644 --- a/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml +++ b/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml @@ -1,8 +1,8 @@ --- fixes: - | - Fixed `RecursiveDocumentSplitter` not populating the `_split_overlap` - metadata when `split_unit` is `"word"` or `"token"` and `split_overlap` + Fixed ``RecursiveDocumentSplitter`` not populating the ``_split_overlap`` + metadata when ``split_unit`` is ``"word"`` or ``"token"`` and ``split_overlap`` is greater than 0. The overlap length was computed with a word/token count while the offsets it is compared against are character positions, so it became negative and the overlap metadata was silently dropped. The overlap diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index 923e618ed66..86b236d81eb 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -750,13 +750,9 @@ def test_run_split_by_dot_and_overlap_1_word_unit_split_idx_start(): ) -def test_run_split_by_dot_and_overlap_1_word_unit_split_overlap_metadata(): +def test_word_unit_split_populates_split_overlap_metadata(): """ _split_overlap must be populated when split_unit="word" and split_overlap > 0. - - Regression: the overlap length was computed with a word/token count while - curr_pos/split_idx_start are character offsets, so it went negative and the - _split_overlap metadata was silently left empty for word/token units. """ splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, separators=["."], split_unit="word") text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four." From 89bd1c8c79321bd968e1f3b8885081852bbb5544 Mon Sep 17 00:00:00 2001 From: Osamaali313 Date: Mon, 6 Jul 2026 11:39:16 +0300 Subject: [PATCH 3/4] test: assert exact _split_overlap ranges + add token-unit test Per review: replace the word-unit test's loose assertions with direct asserts on the exact chunk contents and _split_overlap ranges, and add a matching token-unit test. Values verified against the splitter output. --- .../preprocessors/test_recursive_splitter.py | 50 ++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index 86b236d81eb..112bdabd155 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -752,17 +752,55 @@ def test_run_split_by_dot_and_overlap_1_word_unit_split_idx_start(): def test_word_unit_split_populates_split_overlap_metadata(): """ - _split_overlap must be populated when split_unit="word" and split_overlap > 0. + _split_overlap ranges must be character offsets into the referenced chunk when + split_unit="word" and split_overlap > 0 """ splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, separators=["."], split_unit="word") text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four." chunks = splitter.run([Document(content=text)])["documents"] assert len(chunks) == 5 - # First chunk overlaps with the next, last with the previous, middle with both. - assert any(o["doc_id"] == chunks[1].id for o in chunks[0].meta["_split_overlap"]) - assert any(o["doc_id"] == chunks[3].id for o in chunks[4].meta["_split_overlap"]) - for i in (1, 2, 3): - assert chunks[i].meta["_split_overlap"], f"chunk {i} has empty _split_overlap" + + assert chunks[0].content == "This is sentence one." + assert chunks[0].meta["_split_overlap"] == [{"doc_id": chunks[1].id, "range": (0, 4)}] # "one." + + assert chunks[1].content == "one. This is sentence" + assert chunks[1].meta["_split_overlap"] == [ + {"doc_id": chunks[0].id, "range": (17, 21)}, # "one." + {"doc_id": chunks[2].id, "range": (0, 8)}, # "sentence" + ] + + assert chunks[2].content == "sentence two. This is" + assert chunks[2].meta["_split_overlap"] == [ + {"doc_id": chunks[1].id, "range": (13, 21)}, # "sentence" + {"doc_id": chunks[3].id, "range": (0, 2)}, # "is" + ] + + assert chunks[3].content == "is sentence three. This" + assert chunks[3].meta["_split_overlap"] == [ + {"doc_id": chunks[2].id, "range": (19, 21)}, # "is" + {"doc_id": chunks[4].id, "range": (0, 4)}, # "This" + ] + + assert chunks[4].content == "This is sentence four." + assert chunks[4].meta["_split_overlap"] == [{"doc_id": chunks[3].id, "range": (19, 23)}] # "This" + + +def test_token_unit_split_populates_split_overlap_metadata(): + """ + _split_overlap ranges must be character offsets into the referenced chunk when + split_unit="token" and split_overlap > 0 + """ + splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, separators=["."], split_unit="token") + text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four." + chunks = splitter.run([Document(content=text)])["documents"] + assert len(chunks) == 8 + + assert chunks[0].meta["_split_overlap"] == [{"doc_id": chunks[1].id, "range": (0, 4)}] + assert chunks[1].meta["_split_overlap"] == [ + {"doc_id": chunks[0].id, "range": (16, 20)}, + {"doc_id": chunks[2].id, "range": (0, 1)}, + ] + assert chunks[7].meta["_split_overlap"] == [{"doc_id": chunks[6].id, "range": (9, 18)}] def test_run_trigger_dealing_with_remaining_word_larger_than_split_length(): From ec66eb1b06233e33472736e3acbfe1742a9e14ea Mon Sep 17 00:00:00 2001 From: Sebastian Husch Lee <10526848+sjrl@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:00:41 +0200 Subject: [PATCH 4/4] Update test/components/preprocessors/test_recursive_splitter.py --- test/components/preprocessors/test_recursive_splitter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index 112bdabd155..40e9b671530 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -785,6 +785,7 @@ def test_word_unit_split_populates_split_overlap_metadata(): assert chunks[4].meta["_split_overlap"] == [{"doc_id": chunks[3].id, "range": (19, 23)}] # "This" +@pytest.mark.integration def test_token_unit_split_populates_split_overlap_metadata(): """ _split_overlap ranges must be character offsets into the referenced chunk when