diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index 90232701ca9..254650bdc55 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -403,17 +403,14 @@ def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", " def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: list[Document]) -> None: prev_doc = new_docs[-1] - overlap_length = self._chunk_length(prev_doc.content) - (curr_pos - prev_doc.meta["split_idx_start"]) # type: ignore + # curr_pos and split_idx_start are character offsets, so measure the + # overlap and range in characters too (not via _chunk_length, which returns a word/token count). + prev_doc_length = len(prev_doc.content) # type: ignore + overlap_length = prev_doc_length - (curr_pos - prev_doc.meta["split_idx_start"]) if overlap_length > 0: prev_doc.meta["_split_overlap"].append({"doc_id": new_doc.id, "range": (0, overlap_length)}) new_doc.meta["_split_overlap"].append( - { - "doc_id": prev_doc.id, - "range": ( - self._chunk_length(prev_doc.content) - overlap_length, # type: ignore - self._chunk_length(prev_doc.content), # type: ignore - ), - } + {"doc_id": prev_doc.id, "range": (prev_doc_length - overlap_length, prev_doc_length)} ) def _run_one(self, doc: Document) -> list[Document]: diff --git a/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml b/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml new file mode 100644 index 00000000000..24d9622419e --- /dev/null +++ b/releasenotes/notes/fix-recursive-splitter-word-token-overlap-metadata-9f3c1a7be0d24856.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixed ``RecursiveDocumentSplitter`` not populating the ``_split_overlap`` + metadata when ``split_unit`` is ``"word"`` or ``"token"`` and ``split_overlap`` + is greater than 0. The overlap length was computed with a word/token count + while the offsets it is compared against are character positions, so it + became negative and the overlap metadata was silently dropped. The overlap + is now measured in characters, matching the ranges it records. diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index 7022e645f3d..40e9b671530 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -750,6 +750,60 @@ def test_run_split_by_dot_and_overlap_1_word_unit_split_idx_start(): ) +def test_word_unit_split_populates_split_overlap_metadata(): + """ + _split_overlap ranges must be character offsets into the referenced chunk when + split_unit="word" and split_overlap > 0 + """ + splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, separators=["."], split_unit="word") + text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four." + chunks = splitter.run([Document(content=text)])["documents"] + assert len(chunks) == 5 + + assert chunks[0].content == "This is sentence one." + assert chunks[0].meta["_split_overlap"] == [{"doc_id": chunks[1].id, "range": (0, 4)}] # "one." + + assert chunks[1].content == "one. This is sentence" + assert chunks[1].meta["_split_overlap"] == [ + {"doc_id": chunks[0].id, "range": (17, 21)}, # "one." + {"doc_id": chunks[2].id, "range": (0, 8)}, # "sentence" + ] + + assert chunks[2].content == "sentence two. This is" + assert chunks[2].meta["_split_overlap"] == [ + {"doc_id": chunks[1].id, "range": (13, 21)}, # "sentence" + {"doc_id": chunks[3].id, "range": (0, 2)}, # "is" + ] + + assert chunks[3].content == "is sentence three. This" + assert chunks[3].meta["_split_overlap"] == [ + {"doc_id": chunks[2].id, "range": (19, 21)}, # "is" + {"doc_id": chunks[4].id, "range": (0, 4)}, # "This" + ] + + assert chunks[4].content == "This is sentence four." + assert chunks[4].meta["_split_overlap"] == [{"doc_id": chunks[3].id, "range": (19, 23)}] # "This" + + +@pytest.mark.integration +def test_token_unit_split_populates_split_overlap_metadata(): + """ + _split_overlap ranges must be character offsets into the referenced chunk when + split_unit="token" and split_overlap > 0 + """ + splitter = RecursiveDocumentSplitter(split_length=4, split_overlap=1, separators=["."], split_unit="token") + text = "This is sentence one. This is sentence two. This is sentence three. This is sentence four." + chunks = splitter.run([Document(content=text)])["documents"] + assert len(chunks) == 8 + + assert chunks[0].meta["_split_overlap"] == [{"doc_id": chunks[1].id, "range": (0, 4)}] + assert chunks[1].meta["_split_overlap"] == [ + {"doc_id": chunks[0].id, "range": (16, 20)}, + {"doc_id": chunks[2].id, "range": (0, 1)}, + ] + assert chunks[7].meta["_split_overlap"] == [{"doc_id": chunks[6].id, "range": (9, 18)}] + + def test_run_trigger_dealing_with_remaining_word_larger_than_split_length(): splitter = RecursiveDocumentSplitter(split_length=3, split_overlap=2, separators=["."], split_unit="word") text = """A simple sentence1. A bright sentence2. A clever sentence3"""