fix: MarkdownHeaderSplitter dropped content before an embedded header

With keep_headers=False the secondary split re-stripped a leading header using a
non-anchored re.search, which matched the first '#' line anywhere in the body (an
embedded lower-level header, or a '#' comment inside a fenced code block) and
discarded everything before it. Anchor the match to the start of the content with
re.match so only a genuine leading header is stripped.

Author: Otis0408

haystack/components/preprocessors/markdown_header_splitter.py

@@ -223,7 +223,7 @@ def _apply_secondary_splitting(self, documents: list[Document]) -> list[Document
 
             if not self.keep_headers:  # skip header extraction if keep_headers
                 # extract header information
-                header_match = re.search(self._header_pattern, doc.content)
+                header_match = re.match(self._header_pattern, doc.content)
                 if header_match:
                     content_for_splitting = doc.content[header_match.end() :]
 

releasenotes/notes/fix-markdown-splitter-content-loss-3e4f5a6b7c8d9012.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    Fixed ``MarkdownHeaderSplitter`` dropping content when ``keep_headers=False``.
+    The secondary split re-stripped a leading header with a non-anchored search,
+    which matched the first ``#`` line anywhere in the body (an embedded
+    lower-level header, or a ``#`` comment inside a fenced code block) and
+    discarded everything before it. The match is now anchored to the start of the
+    content, so only a genuine leading header is stripped.

test/components/preprocessors/test_markdown_header_splitter.py

@@ -250,6 +250,29 @@ def test_preserve_document_metadata():
     assert split_docs[0].content == "\nContent"
 
 
+def test_secondary_split_keeps_content_before_embedded_header():
+    """With keep_headers=False, prose before an embedded lower-level header must
+    not be dropped during the secondary split."""
+    splitter = MarkdownHeaderSplitter(
+        keep_headers=False, header_split_levels=[1], secondary_split="word", split_length=100
+    )
+    docs = splitter.run(documents=[Document(content="# Main\nintro paragraph text\n## Sub\nmore text\n")])["documents"]
+    combined = "".join(doc.content or "" for doc in docs)
+    assert "intro paragraph text" in combined
+
+
+def test_secondary_split_keeps_content_before_code_fence_comment():
+    """With keep_headers=False, prose before a '#' comment inside a fenced code block must
+    not be dropped during the secondary split (the comment is not a real header)."""
+    splitter = MarkdownHeaderSplitter(
+        keep_headers=False, header_split_levels=[1], secondary_split="word", split_length=100
+    )
+    content = "# Main\nsome intro text\n```python\n# a comment in code\n```\nmore text\n"
+    docs = splitter.run(documents=[Document(content=content)])["documents"]
+    combined = "".join(doc.content or "" for doc in docs)
+    assert "some intro text" in combined
+
+
 # Error and edge case handling
 def test_non_text_document():
     """Test that the component correctly handles non-text documents."""