diff --git a/haystack/components/preprocessors/markdown_header_splitter.py b/haystack/components/preprocessors/markdown_header_splitter.py index 890fc1f279a..f3ebf1adbeb 100644 --- a/haystack/components/preprocessors/markdown_header_splitter.py +++ b/haystack/components/preprocessors/markdown_header_splitter.py @@ -223,7 +223,7 @@ def _apply_secondary_splitting(self, documents: list[Document]) -> list[Document if not self.keep_headers: # skip header extraction if keep_headers # extract header information - header_match = re.search(self._header_pattern, doc.content) + header_match = re.match(self._header_pattern, doc.content) if header_match: content_for_splitting = doc.content[header_match.end() :] diff --git a/releasenotes/notes/fix-markdown-splitter-content-loss-3e4f5a6b7c8d9012.yaml b/releasenotes/notes/fix-markdown-splitter-content-loss-3e4f5a6b7c8d9012.yaml new file mode 100644 index 00000000000..4adb8793ed9 --- /dev/null +++ b/releasenotes/notes/fix-markdown-splitter-content-loss-3e4f5a6b7c8d9012.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixed ``MarkdownHeaderSplitter`` dropping content when ``keep_headers=False``. + The secondary split re-stripped a leading header with a non-anchored search, + which matched the first ``#`` line anywhere in the body (an embedded + lower-level header, or a ``#`` comment inside a fenced code block) and + discarded everything before it. The match is now anchored to the start of the + content, so only a genuine leading header is stripped. diff --git a/test/components/preprocessors/test_markdown_header_splitter.py b/test/components/preprocessors/test_markdown_header_splitter.py index cdbb4989e35..8f5f3c43eba 100644 --- a/test/components/preprocessors/test_markdown_header_splitter.py +++ b/test/components/preprocessors/test_markdown_header_splitter.py @@ -250,6 +250,29 @@ def test_preserve_document_metadata(): assert split_docs[0].content == "\nContent" +def test_secondary_split_keeps_content_before_embedded_header(): + """With keep_headers=False, prose before an embedded lower-level header must + not be dropped during the secondary split.""" + splitter = MarkdownHeaderSplitter( + keep_headers=False, header_split_levels=[1], secondary_split="word", split_length=100 + ) + docs = splitter.run(documents=[Document(content="# Main\nintro paragraph text\n## Sub\nmore text\n")])["documents"] + combined = "".join(doc.content or "" for doc in docs) + assert "intro paragraph text" in combined + + +def test_secondary_split_keeps_content_before_code_fence_comment(): + """With keep_headers=False, prose before a '#' comment inside a fenced code block must + not be dropped during the secondary split (the comment is not a real header).""" + splitter = MarkdownHeaderSplitter( + keep_headers=False, header_split_levels=[1], secondary_split="word", split_length=100 + ) + content = "# Main\nsome intro text\n```python\n# a comment in code\n```\nmore text\n" + docs = splitter.run(documents=[Document(content=content)])["documents"] + combined = "".join(doc.content or "" for doc in docs) + assert "some intro text" in combined + + # Error and edge case handling def test_non_text_document(): """Test that the component correctly handles non-text documents."""