fix: set source_id on RecursiveDocumentSplitter chunks

RecursiveDocumentSplitter wrote only `parent_id` on the chunks it produced.
Every other splitter in the library writes `source_id`, and that is the key the
rest of Haystack reads to tie a chunk back to the document it came from.

The practical effect is that its output does not compose with
SentenceWindowRetriever, which reads `source_id` by default and raises when it
is missing. The same pipeline works with DocumentSplitter and fails with
RecursiveDocumentSplitter:

    DocumentSplitter          -> OK, 3 context documents
    RecursiveDocumentSplitter -> ValueError: The retrieved documents must have
                                 'source_id' in their metadata.

Chunks now carry `source_id` as well as `parent_id`, which keeps its previous
value so callers already reading it are unaffected.

While here, the class docstring example claimed a meta key named `original_id`,
which the code has never written. Its output block also did not match what the
example produces: it was generated with a different `split_length` than the one
shown, and reported `_split_overlap` as `[]` where a run with `split_overlap=0`
gives `None`. The example now reproduces the output it claims.

Author: onatozmenn

haystack/components/preprocessors/recursive_splitter.py

@@ -38,7 +38,7 @@ class RecursiveDocumentSplitter:
     from haystack import Document
     from haystack.components.preprocessors import RecursiveDocumentSplitter
 
-    chunker = RecursiveDocumentSplitter(split_length=260, split_overlap=0, separators=["\\n\\n", "\\n", ".", " "])
+    chunker = RecursiveDocumentSplitter(split_length=15, split_overlap=0, separators=["\\n\\n", "\\n", ".", " "])
     text = ('''Artificial intelligence (AI) - Introduction
 
     AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.
@@ -47,10 +47,11 @@ class RecursiveDocumentSplitter:
     doc_chunks = chunker.run([doc])
     print(doc_chunks["documents"])
     # [
-    # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\\n\\n', meta: {'original_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': []})
-    # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\\n', meta: {'original_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': []})
-    # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'original_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': []})
-    # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'original_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': []})
+    # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\\n\\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': None, 'page_number': 1})
+    # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': None, 'page_number': 1})
+    # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': None, 'page_number': 1})
+    # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': None, 'page_number': 1})
+    # Document(id=..., content: 'vehicles; generative and creative tools; and superhuman play and analysis in strategy games.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 4, 'split_idx_start': 350, '_split_overlap': None, 'page_number': 1})
     # ]
     ```
     """  # noqa: E501
@@ -425,6 +426,10 @@ def _run_one(self, doc: Document) -> list[Document]:
 
         for split_nr, chunk in enumerate(chunks):
             meta = deepcopy(doc.meta)
+            # `source_id` is the key every other splitter writes, and the one components
+            # like SentenceWindowRetriever look for by default. `parent_id` carries the
+            # same value and is kept for callers already reading it.
+            meta["source_id"] = doc.id
             meta["parent_id"] = doc.id
             meta["split_id"] = split_nr
             meta["split_idx_start"] = current_position

releasenotes/notes/recursive-splitter-source-id-4efff9b9344f57e4.yaml

@@ -0,0 +1,12 @@
+---
+fixes:
+  - |
+    Fixed ``RecursiveDocumentSplitter`` not setting the ``source_id`` meta field on the chunks
+    it produces. It wrote only ``parent_id``, while every other splitter in the library
+    (``DocumentSplitter``, ``CSVDocumentSplitter``, ``EmbeddingBasedDocumentSplitter``,
+    ``HierarchicalDocumentSplitter``, ``MarkdownHeaderSplitter`` and ``PythonCodeSplitter``)
+    writes ``source_id``. Components that follow that convention therefore rejected its output:
+    ``SentenceWindowRetriever`` reads ``source_id`` by default and raises when it is absent, so
+    it failed with "The retrieved documents must have 'source_id' in their metadata." on a
+    pipeline that worked with any other splitter. Chunks now carry ``source_id`` as well as
+    ``parent_id``, which keeps its previous value for callers already reading it.

test/components/preprocessors/test_recursive_splitter.py

@@ -11,6 +11,8 @@
 from haystack import Document, Pipeline
 from haystack.components.preprocessors.recursive_splitter import RecursiveDocumentSplitter
 from haystack.components.preprocessors.sentence_tokenizer import SentenceSplitter
+from haystack.components.retrievers.sentence_window_retriever import SentenceWindowRetriever
+from haystack.document_stores.in_memory import InMemoryDocumentStore
 
 
 def test_get_custom_sentence_tokenizer_success():
@@ -1055,6 +1057,36 @@ def test_recursive_splitter_generates_unique_ids_and_correct_meta():
         assert chunk.meta["split_id"] == idx
 
 
+def test_recursive_splitter_sets_source_id():
+    """`source_id` is the key the rest of the library uses to tie a chunk back to its document."""
+    source_doc = Document(content="Haystack is awesome. " * 5)
+
+    chunks = RecursiveDocumentSplitter(split_length=3).run([source_doc])["documents"]
+
+    assert chunks
+    for chunk in chunks:
+        assert chunk.meta["source_id"] == source_doc.id
+        # parent_id carried the same value before source_id existed, so it stays.
+        assert chunk.meta["parent_id"] == source_doc.id
+
+
+def test_recursive_splitter_output_works_with_sentence_window_retriever():
+    """SentenceWindowRetriever looks up `source_id` by default and raises when it is
+    absent, so chunks from this splitter used to be unusable with it."""
+    source_doc = Document(content="Haystack is awesome. " * 10)
+    chunks = RecursiveDocumentSplitter(split_length=3).run([source_doc])["documents"]
+    assert len(chunks) > 2
+
+    store = InMemoryDocumentStore()
+    store.write_documents(chunks)
+
+    result = SentenceWindowRetriever(document_store=store, window_size=1).run(retrieved_documents=[chunks[1]])
+
+    # The middle chunk plus one neighbour on each side.
+    assert len(result["context_documents"]) == 3
+    assert result["context_windows"]
+
+
 def test_warm_up_is_idempotent_sentence(monkeypatch):
     splitter = RecursiveDocumentSplitter(separators=["sentence", " "])