fix: MarkdownHeaderSplitter silently drops a trailing header with no body text (#12064)

Co-authored-by: Loi Nguyen <1948922+lntutor@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Julian Risch <julian.risch@deepset.ai>

Author: 3 people

haystack/components/preprocessors/markdown_header_splitter.py

@@ -147,13 +147,15 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]:
         # process headers and build chunks
         chunks: list[dict] = []
         header_stack: list[str | None] = [None] * 6
-        pending_headers: list[str] = []  # store empty headers to prepend to next content
+        pending_start: int | None = None  # start offset in text of the first buffered empty header
+        pending_header_text: str | None = None  # text of the last buffered empty header
+        pending_level = 0  # level of the last buffered empty header
         has_content = False  # flag to track if any header has content
 
         for i, match in enumerate(matches):
             # extract header info
             header_prefix = match.group(1)
-            header_text = match.group(2)
+            header_text = match.group(2).strip()  # keep surrounding whitespace out of the metadata
             level = len(header_prefix)
 
             # get content
@@ -169,8 +171,12 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]:
             # skip splits w/o content
             if not content.strip():  # this strip is needed to avoid counting whitespace as content
                 if self.keep_headers:
-                    header_line = f"{header_prefix} {header_text}"
-                    pending_headers.append(header_line)
+                    # buffer this header so it gets prepended to the next contentful chunk; only the
+                    # offset of the first header in the run is needed since the chunk is sliced from text
+                    if pending_start is None:
+                        pending_start = match.start()
+                    pending_header_text = header_text
+                    pending_level = level
                 continue
 
             has_content = True  # at least one header has content
@@ -183,16 +189,13 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]:
             )
 
             if self.keep_headers:
-                header_line = f"{header_prefix} {header_text}"
-                # add pending & current header to content
-                chunk_content = ""
-                if pending_headers:
-                    chunk_content += "\n".join(pending_headers) + "\n"
-                chunk_content += f"{header_line}{content}"
+                # Slice from the first buffered empty header (if any) so the buffered header lines and
+                # the whitespace between them are preserved byte-exactly.
+                chunk_content = text[(pending_start if pending_start is not None else match.start()) : end]
                 chunks.append(
                     {"content": chunk_content, "meta": {"header": header_text, "parent_headers": parent_headers}}
                 )
-                pending_headers = []  # reset pending headers
+                pending_start = None  # reset buffered headers
             else:
                 chunks.append({"content": content, "meta": {"header": header_text, "parent_headers": parent_headers}})
 
@@ -203,6 +206,18 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]:
             )
             return [{"content": text, "meta": {}}]
 
+        # Flush any trailing headers that had no body text of their own. A header at the very end of the
+        # document (or a run of such headers) never gets a following content chunk to be prepended to, so
+        # without this it would be silently dropped. Slicing the original text keeps reconstruction exact.
+        if pending_start is not None:
+            parent_headers = [h for h in header_stack[: pending_level - 1] if h is not None]
+            chunks.append(
+                {
+                    "content": text[pending_start:],
+                    "meta": {"header": pending_header_text, "parent_headers": parent_headers},
+                }
+            )
+
         return chunks
 
     def _apply_secondary_splitting(self, documents: list[Document]) -> list[Document]:

releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml

@@ -0,0 +1,19 @@
+---
+fixes:
+  - |
+    Fixed ``MarkdownHeaderSplitter`` silently dropping a trailing header that has
+    no body text. With ``keep_headers=True`` (the default), a header at the end of
+    the document whose only content is whitespace was buffered to prepend to the
+    next chunk, but with no following chunk it was never emitted, so the split
+    documents no longer reconstructed the original text. Such trailing headers
+    are now emitted as a final chunk.
+  - |
+    Fixed ``MarkdownHeaderSplitter`` collapsing blank lines that follow a header
+    with no body text. With ``keep_headers=True``, such headers were re-joined with
+    a single newline when prepended to the next chunk, so the split documents did
+    not reconstruct the original text. Chunk content is now sliced from the
+    original text and is byte-exact.
+  - |
+    Fixed ``MarkdownHeaderSplitter`` including surrounding whitespace in the
+    ``header`` and ``parent_headers`` metadata fields. The header text is now
+    stripped; chunk content still keeps the header line's original whitespace.

test/components/preprocessors/test_markdown_header_splitter.py

@@ -838,3 +838,35 @@ def test_page_break_handling_with_multiple_headers(sample_text_with_page_breaks)
     # reconstruct original
     reconstructed_text = "".join(doc.content for doc in split_docs)
     assert reconstructed_text == sample_text_with_page_breaks
+
+
+def test_trailing_header_without_content_is_not_dropped():
+    text = "# Header 1\nContent 1.\n# Header 2\n"
+    docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"]
+    assert "".join(doc.content for doc in docs) == text
+    assert docs[-1].content == "# Header 2\n"
+    assert docs[-1].meta["header"] == "Header 2"
+    assert docs[-1].meta["parent_headers"] == []
+
+
+def test_middle_header_without_content_preserves_blank_lines():
+    # buffered headers are sliced from the original text, so blank lines between a header with no
+    # body and the next contentful header are preserved instead of collapsed to a single newline
+    text = "# Header 1\n\n\n# Header 2\nContent.\n"
+    docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"]
+    assert "".join(doc.content for doc in docs) == text
+
+
+def test_header_metadata_is_stripped_but_content_is_byte_exact():
+    text = "# Header 1   \nContent.\n"
+    docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"]
+    assert docs[0].meta["header"] == "Header 1"
+    # the chunk content keeps the header line's original trailing whitespace
+    assert "".join(doc.content for doc in docs) == text
+
+
+def test_whitespace_only_trailing_header_has_empty_header_metadata():
+    text = "# Header 1\nContent.\n#   \n"
+    docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"]
+    assert "".join(doc.content for doc in docs) == text
+    assert docs[-1].meta["header"] == ""