From d0e2fca7c52e50a8113fe8b33eeba0859628d0a4 Mon Sep 17 00:00:00 2001 From: onatozmenn Date: Sat, 25 Jul 2026 11:58:53 +0300 Subject: [PATCH 1/2] fix: set source_id on RecursiveDocumentSplitter chunks RecursiveDocumentSplitter wrote only `parent_id` on the chunks it produced. Every other splitter in the library writes `source_id`, and that is the key the rest of Haystack reads to tie a chunk back to the document it came from. The practical effect is that its output does not compose with SentenceWindowRetriever, which reads `source_id` by default and raises when it is missing. The same pipeline works with DocumentSplitter and fails with RecursiveDocumentSplitter: DocumentSplitter -> OK, 3 context documents RecursiveDocumentSplitter -> ValueError: The retrieved documents must have 'source_id' in their metadata. Chunks now carry `source_id` as well as `parent_id`, which keeps its previous value so callers already reading it are unaffected. While here, the class docstring example claimed a meta key named `original_id`, which the code has never written. Its output block also did not match what the example produces: it was generated with a different `split_length` than the one shown, and reported `_split_overlap` as `[]` where a run with `split_overlap=0` gives `None`. The example now reproduces the output it claims. --- .../preprocessors/recursive_splitter.py | 15 ++++++--- ...e-splitter-source-id-4efff9b9344f57e4.yaml | 12 +++++++ .../preprocessors/test_recursive_splitter.py | 32 +++++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/recursive-splitter-source-id-4efff9b9344f57e4.yaml diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index 129ec133091..21002477e71 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -38,7 +38,7 @@ class RecursiveDocumentSplitter: from haystack import Document from haystack.components.preprocessors import RecursiveDocumentSplitter - chunker = RecursiveDocumentSplitter(split_length=260, split_overlap=0, separators=["\\n\\n", "\\n", ".", " "]) + chunker = RecursiveDocumentSplitter(split_length=15, split_overlap=0, separators=["\\n\\n", "\\n", ".", " "]) text = ('''Artificial intelligence (AI) - Introduction AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems. @@ -47,10 +47,11 @@ class RecursiveDocumentSplitter: doc_chunks = chunker.run([doc]) print(doc_chunks["documents"]) # [ - # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\\n\\n', meta: {'original_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': []}) - # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\\n', meta: {'original_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': []}) - # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'original_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': []}) - # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'original_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': []}) + # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\\n\\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': None, 'page_number': 1}) + # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': None, 'page_number': 1}) + # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': None, 'page_number': 1}) + # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': None, 'page_number': 1}) + # Document(id=..., content: 'vehicles; generative and creative tools; and superhuman play and analysis in strategy games.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 4, 'split_idx_start': 350, '_split_overlap': None, 'page_number': 1}) # ] ``` """ # noqa: E501 @@ -425,6 +426,10 @@ def _run_one(self, doc: Document) -> list[Document]: for split_nr, chunk in enumerate(chunks): meta = deepcopy(doc.meta) + # `source_id` is the key every other splitter writes, and the one components + # like SentenceWindowRetriever look for by default. `parent_id` carries the + # same value and is kept for callers already reading it. + meta["source_id"] = doc.id meta["parent_id"] = doc.id meta["split_id"] = split_nr meta["split_idx_start"] = current_position diff --git a/releasenotes/notes/recursive-splitter-source-id-4efff9b9344f57e4.yaml b/releasenotes/notes/recursive-splitter-source-id-4efff9b9344f57e4.yaml new file mode 100644 index 00000000000..f0eed7e2705 --- /dev/null +++ b/releasenotes/notes/recursive-splitter-source-id-4efff9b9344f57e4.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + Fixed ``RecursiveDocumentSplitter`` not setting the ``source_id`` meta field on the chunks + it produces. It wrote only ``parent_id``, while every other splitter in the library + (``DocumentSplitter``, ``CSVDocumentSplitter``, ``EmbeddingBasedDocumentSplitter``, + ``HierarchicalDocumentSplitter``, ``MarkdownHeaderSplitter`` and ``PythonCodeSplitter``) + writes ``source_id``. Components that follow that convention therefore rejected its output: + ``SentenceWindowRetriever`` reads ``source_id`` by default and raises when it is absent, so + it failed with "The retrieved documents must have 'source_id' in their metadata." on a + pipeline that worked with any other splitter. Chunks now carry ``source_id`` as well as + ``parent_id``, which keeps its previous value for callers already reading it. diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index d334af5b8e2..3e05518e091 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -11,6 +11,8 @@ from haystack import Document, Pipeline from haystack.components.preprocessors.recursive_splitter import RecursiveDocumentSplitter from haystack.components.preprocessors.sentence_tokenizer import SentenceSplitter +from haystack.components.retrievers.sentence_window_retriever import SentenceWindowRetriever +from haystack.document_stores.in_memory import InMemoryDocumentStore def test_get_custom_sentence_tokenizer_success(): @@ -1055,6 +1057,36 @@ def test_recursive_splitter_generates_unique_ids_and_correct_meta(): assert chunk.meta["split_id"] == idx +def test_recursive_splitter_sets_source_id(): + """`source_id` is the key the rest of the library uses to tie a chunk back to its document.""" + source_doc = Document(content="Haystack is awesome. " * 5) + + chunks = RecursiveDocumentSplitter(split_length=3).run([source_doc])["documents"] + + assert chunks + for chunk in chunks: + assert chunk.meta["source_id"] == source_doc.id + # parent_id carried the same value before source_id existed, so it stays. + assert chunk.meta["parent_id"] == source_doc.id + + +def test_recursive_splitter_output_works_with_sentence_window_retriever(): + """SentenceWindowRetriever looks up `source_id` by default and raises when it is + absent, so chunks from this splitter used to be unusable with it.""" + source_doc = Document(content="Haystack is awesome. " * 10) + chunks = RecursiveDocumentSplitter(split_length=3).run([source_doc])["documents"] + assert len(chunks) > 2 + + store = InMemoryDocumentStore() + store.write_documents(chunks) + + result = SentenceWindowRetriever(document_store=store, window_size=1).run(retrieved_documents=[chunks[1]]) + + # The middle chunk plus one neighbour on each side. + assert len(result["context_documents"]) == 3 + assert result["context_windows"] + + def test_warm_up_is_idempotent_sentence(monkeypatch): splitter = RecursiveDocumentSplitter(separators=["sentence", " "]) From f332ee48fe271edb46d61fc1d8a6692e751f801e Mon Sep 17 00:00:00 2001 From: onatozmenn Date: Mon, 27 Jul 2026 19:50:55 +0300 Subject: [PATCH 2/2] Address review: drop cross-component comments, fold source_id into the meta test Signed-off-by: onatozmenn --- .../preprocessors/recursive_splitter.py | 3 --- .../preprocessors/test_recursive_splitter.py | 18 +++--------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/haystack/components/preprocessors/recursive_splitter.py b/haystack/components/preprocessors/recursive_splitter.py index 21002477e71..0ddef2aac24 100644 --- a/haystack/components/preprocessors/recursive_splitter.py +++ b/haystack/components/preprocessors/recursive_splitter.py @@ -426,9 +426,6 @@ def _run_one(self, doc: Document) -> list[Document]: for split_nr, chunk in enumerate(chunks): meta = deepcopy(doc.meta) - # `source_id` is the key every other splitter writes, and the one components - # like SentenceWindowRetriever look for by default. `parent_id` carries the - # same value and is kept for callers already reading it. meta["source_id"] = doc.id meta["parent_id"] = doc.id meta["split_id"] = split_nr diff --git a/test/components/preprocessors/test_recursive_splitter.py b/test/components/preprocessors/test_recursive_splitter.py index 3e05518e091..7be763018be 100644 --- a/test/components/preprocessors/test_recursive_splitter.py +++ b/test/components/preprocessors/test_recursive_splitter.py @@ -1051,28 +1051,16 @@ def test_recursive_splitter_generates_unique_ids_and_correct_meta(): # IDs must be unique assert len({c.id for c in chunks}) == len(chunks) - # parent_id and split_id checks + # source_id, parent_id and split_id checks for idx, chunk in enumerate(chunks): - assert chunk.meta["parent_id"] == source_doc.id - assert chunk.meta["split_id"] == idx - - -def test_recursive_splitter_sets_source_id(): - """`source_id` is the key the rest of the library uses to tie a chunk back to its document.""" - source_doc = Document(content="Haystack is awesome. " * 5) - - chunks = RecursiveDocumentSplitter(split_length=3).run([source_doc])["documents"] - - assert chunks - for chunk in chunks: assert chunk.meta["source_id"] == source_doc.id - # parent_id carried the same value before source_id existed, so it stays. assert chunk.meta["parent_id"] == source_doc.id + assert chunk.meta["split_id"] == idx def test_recursive_splitter_output_works_with_sentence_window_retriever(): """SentenceWindowRetriever looks up `source_id` by default and raises when it is - absent, so chunks from this splitter used to be unusable with it.""" + absent""" source_doc = Document(content="Haystack is awesome. " * 10) chunks = RecursiveDocumentSplitter(split_length=3).run([source_doc])["documents"] assert len(chunks) > 2