fix: set source_id on RecursiveDocumentSplitter chunks (#12155)

Signed-off-by: onatozmenn <onatozmen44@gmail.com>

Author: onatozmenn

haystack/components/preprocessors/recursive_splitter.py

@@ -38,7 +38,7 @@ class RecursiveDocumentSplitter:
     from haystack import Document
     from haystack.components.preprocessors import RecursiveDocumentSplitter
 
-    chunker = RecursiveDocumentSplitter(split_length=260, split_overlap=0, separators=["\\n\\n", "\\n", ".", " "])
+    chunker = RecursiveDocumentSplitter(split_length=15, split_overlap=0, separators=["\\n\\n", "\\n", ".", " "])
     text = ('''Artificial intelligence (AI) - Introduction
 
     AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.
@@ -47,10 +47,11 @@ class RecursiveDocumentSplitter:
     doc_chunks = chunker.run([doc])
     print(doc_chunks["documents"])
     # [
-    # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\\n\\n', meta: {'original_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': []})
-    # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\\n', meta: {'original_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': []})
-    # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'original_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': []})
-    # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'original_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': []})
+    # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\\n\\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': None, 'page_number': 1})
+    # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': None, 'page_number': 1})
+    # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': None, 'page_number': 1})
+    # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': None, 'page_number': 1})
+    # Document(id=..., content: 'vehicles; generative and creative tools; and superhuman play and analysis in strategy games.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 4, 'split_idx_start': 350, '_split_overlap': None, 'page_number': 1})
     # ]
     ```
     """  # noqa: E501
@@ -425,6 +426,7 @@ def _run_one(self, doc: Document) -> list[Document]:
 
         for split_nr, chunk in enumerate(chunks):
             meta = deepcopy(doc.meta)
+            meta["source_id"] = doc.id
             meta["parent_id"] = doc.id
             meta["split_id"] = split_nr
             meta["split_idx_start"] = current_position

releasenotes/notes/recursive-splitter-source-id-4efff9b9344f57e4.yaml

@@ -0,0 +1,12 @@
+---
+fixes:
+  - |
+    Fixed ``RecursiveDocumentSplitter`` not setting the ``source_id`` meta field on the chunks
+    it produces. It wrote only ``parent_id``, while every other splitter in the library
+    (``DocumentSplitter``, ``CSVDocumentSplitter``, ``EmbeddingBasedDocumentSplitter``,
+    ``HierarchicalDocumentSplitter``, ``MarkdownHeaderSplitter`` and ``PythonCodeSplitter``)
+    writes ``source_id``. Components that follow that convention therefore rejected its output:
+    ``SentenceWindowRetriever`` reads ``source_id`` by default and raises when it is absent, so
+    it failed with "The retrieved documents must have 'source_id' in their metadata." on a
+    pipeline that worked with any other splitter. Chunks now carry ``source_id`` as well as
+    ``parent_id``, which keeps its previous value for callers already reading it.

test/components/preprocessors/test_recursive_splitter.py

@@ -11,6 +11,8 @@
 from haystack import Document, Pipeline
 from haystack.components.preprocessors.recursive_splitter import RecursiveDocumentSplitter
 from haystack.components.preprocessors.sentence_tokenizer import SentenceSplitter
+from haystack.components.retrievers.sentence_window_retriever import SentenceWindowRetriever
+from haystack.document_stores.in_memory import InMemoryDocumentStore
 
 
 def test_get_custom_sentence_tokenizer_success():
@@ -1049,12 +1051,30 @@ def test_recursive_splitter_generates_unique_ids_and_correct_meta():
     # IDs must be unique
     assert len({c.id for c in chunks}) == len(chunks)
 
-    # parent_id and split_id checks
+    # source_id, parent_id and split_id checks
     for idx, chunk in enumerate(chunks):
+        assert chunk.meta["source_id"] == source_doc.id
         assert chunk.meta["parent_id"] == source_doc.id
         assert chunk.meta["split_id"] == idx
 
 
+def test_recursive_splitter_output_works_with_sentence_window_retriever():
+    """SentenceWindowRetriever looks up `source_id` by default and raises when it is
+    absent"""
+    source_doc = Document(content="Haystack is awesome. " * 10)
+    chunks = RecursiveDocumentSplitter(split_length=3).run([source_doc])["documents"]
+    assert len(chunks) > 2
+
+    store = InMemoryDocumentStore()
+    store.write_documents(chunks)
+
+    result = SentenceWindowRetriever(document_store=store, window_size=1).run(retrieved_documents=[chunks[1]])
+
+    # The middle chunk plus one neighbour on each side.
+    assert len(result["context_documents"]) == 3
+    assert result["context_windows"]
+
+
 def test_warm_up_is_idempotent_sentence(monkeypatch):
     splitter = RecursiveDocumentSplitter(separators=["sentence", " "])