From 5ef2337f1aa1d9f3f9fce1a518ad8656939aadd3 Mon Sep 17 00:00:00 2001 From: Loi Nguyen <1948922+lntutor@users.noreply.github.com> Date: Sun, 19 Jul 2026 20:13:13 +0700 Subject: [PATCH 1/2] fix: MarkdownHeaderSplitter drops a trailing header with no body text With keep_headers=True (the default), a header whose only content is whitespace is buffered into pending_headers to be prepended to the next content chunk. A header (or run of headers) at the very end of the document has no following chunk, so the buffer was never flushed and the header text was dropped -- breaking the reconstruction invariant that the split contents rejoin to the original, and inconsistent with a middle empty header, which is preserved. Flush any trailing buffered headers as a final chunk, sliced from the original text so reconstruction stays exact. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N6RtoHuxrDqTUo9Mw9h4Cv --- .../preprocessors/markdown_header_splitter.py | 20 +++++++++++++++++++ ...tter-trailing-header-9c1a2b3d4e5f6a7b.yaml | 9 +++++++++ .../test_markdown_header_splitter.py | 9 +++++++++ 3 files changed, 38 insertions(+) create mode 100644 releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml diff --git a/haystack/components/preprocessors/markdown_header_splitter.py b/haystack/components/preprocessors/markdown_header_splitter.py index f3ebf1adbeb..59bc0dab10b 100644 --- a/haystack/components/preprocessors/markdown_header_splitter.py +++ b/haystack/components/preprocessors/markdown_header_splitter.py @@ -148,6 +148,9 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: chunks: list[dict] = [] header_stack: list[str | None] = [None] * 6 pending_headers: list[str] = [] # store empty headers to prepend to next content + pending_start: int | None = None # start offset in text of the first buffered empty header + pending_header_text: str | None = None # text of the last buffered empty header + pending_level = 0 # level of the last buffered empty header has_content = False # flag to track if any header has content for i, match in enumerate(matches): @@ -170,7 +173,11 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: if not content.strip(): # this strip is needed to avoid counting whitespace as content if self.keep_headers: header_line = f"{header_prefix} {header_text}" + if not pending_headers: + pending_start = match.start() pending_headers.append(header_line) + pending_header_text = header_text + pending_level = level continue has_content = True # at least one header has content @@ -193,6 +200,7 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: {"content": chunk_content, "meta": {"header": header_text, "parent_headers": parent_headers}} ) pending_headers = [] # reset pending headers + pending_start = None else: chunks.append({"content": content, "meta": {"header": header_text, "parent_headers": parent_headers}}) @@ -203,6 +211,18 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: ) return [{"content": text, "meta": {}}] + # Flush any trailing headers that had no body text of their own. A header at the very end of the + # document (or a run of such headers) never gets a following content chunk to be prepended to, so + # without this it would be silently dropped. Slicing the original text keeps reconstruction exact. + if pending_headers and pending_start is not None: + parent_headers = [h for h in header_stack[: pending_level - 1] if h is not None] + chunks.append( + { + "content": text[pending_start:], + "meta": {"header": pending_header_text, "parent_headers": parent_headers}, + } + ) + return chunks def _apply_secondary_splitting(self, documents: list[Document]) -> list[Document]: diff --git a/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml b/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml new file mode 100644 index 00000000000..8a24b95c4ed --- /dev/null +++ b/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixed `MarkdownHeaderSplitter` silently dropping a trailing header that has + no body text. With `keep_headers=True` (the default), a header at the end of + the document whose only content is whitespace was buffered to prepend to the + next chunk, but with no following chunk it was never emitted, so the split + documents no longer reconstructed the original text. Such trailing headers + are now emitted as a final chunk. diff --git a/test/components/preprocessors/test_markdown_header_splitter.py b/test/components/preprocessors/test_markdown_header_splitter.py index 8f5f3c43eba..7f89282d443 100644 --- a/test/components/preprocessors/test_markdown_header_splitter.py +++ b/test/components/preprocessors/test_markdown_header_splitter.py @@ -838,3 +838,12 @@ def test_page_break_handling_with_multiple_headers(sample_text_with_page_breaks) # reconstruct original reconstructed_text = "".join(doc.content for doc in split_docs) assert reconstructed_text == sample_text_with_page_breaks + + +def test_trailing_header_without_content_is_not_dropped(): + text = "# Header 1\nContent 1.\n# Header 2\n" + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + assert "".join(doc.content for doc in docs) == text + assert docs[-1].content == "# Header 2\n" + assert docs[-1].meta["header"] == "Header 2" + assert docs[-1].meta["parent_headers"] == [] From 250c6c932cc4d09bf1f2411475dd3d50b027ab95 Mon Sep 17 00:00:00 2001 From: Julian Risch Date: Wed, 29 Jul 2026 20:01:44 +0200 Subject: [PATCH 2/2] fix: make MarkdownHeaderSplitter chunk content byte-exact and strip header metadata Follow-up to the trailing-header fix in this PR, addressing two coupled review findings. The trailing-header flush already sliced the original text, but the pre-existing path for a *middle* header with no body still rebuilt content by joining buffered header lines with "\n". That normalized whitespace, so blank lines between such a header and the next contentful one were collapsed and reconstruction stayed lossy: "# H1\n\n\n# H2\nContent.\n" -> "# H1\n# H2\nContent.\n" Both paths now slice from the original text, so the two agree on fidelity and keep_headers=True reconstruction is byte-exact in every buffered-header case. Because content is no longer rebuilt from header_prefix/header_text, the pending_headers list is only ever read as a boolean -- "pending_start is not None" already carries that signal -- so it and both header_line constructions are removed. Net less state in the loop than before. That in turn makes it safe to strip the captured header text, which the regex ^(#{1,6}) (.+)$ had always let leak into metadata ("# Header 1 " gave header == "Header 1 ", and a trailing "# " gave header == " "). Stripping now only affects header / parent_headers; chunk content keeps the header line's original whitespace. Done before this change it would have silently mutated emitted content. Adds three regression tests, each failing before this commit. Also applies the release-note double-backtick convention that pre-commit enforces. Co-Authored-By: Claude Opus 5 (1M context) --- .../preprocessors/markdown_header_splitter.py | 23 ++++++++----------- ...tter-trailing-header-9c1a2b3d4e5f6a7b.yaml | 14 +++++++++-- .../test_markdown_header_splitter.py | 23 +++++++++++++++++++ 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/haystack/components/preprocessors/markdown_header_splitter.py b/haystack/components/preprocessors/markdown_header_splitter.py index 59bc0dab10b..72d0b571334 100644 --- a/haystack/components/preprocessors/markdown_header_splitter.py +++ b/haystack/components/preprocessors/markdown_header_splitter.py @@ -147,7 +147,6 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: # process headers and build chunks chunks: list[dict] = [] header_stack: list[str | None] = [None] * 6 - pending_headers: list[str] = [] # store empty headers to prepend to next content pending_start: int | None = None # start offset in text of the first buffered empty header pending_header_text: str | None = None # text of the last buffered empty header pending_level = 0 # level of the last buffered empty header @@ -156,7 +155,7 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: for i, match in enumerate(matches): # extract header info header_prefix = match.group(1) - header_text = match.group(2) + header_text = match.group(2).strip() # keep surrounding whitespace out of the metadata level = len(header_prefix) # get content @@ -172,10 +171,10 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: # skip splits w/o content if not content.strip(): # this strip is needed to avoid counting whitespace as content if self.keep_headers: - header_line = f"{header_prefix} {header_text}" - if not pending_headers: + # buffer this header so it gets prepended to the next contentful chunk; only the + # offset of the first header in the run is needed since the chunk is sliced from text + if pending_start is None: pending_start = match.start() - pending_headers.append(header_line) pending_header_text = header_text pending_level = level continue @@ -190,17 +189,13 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: ) if self.keep_headers: - header_line = f"{header_prefix} {header_text}" - # add pending & current header to content - chunk_content = "" - if pending_headers: - chunk_content += "\n".join(pending_headers) + "\n" - chunk_content += f"{header_line}{content}" + # Slice from the first buffered empty header (if any) so the buffered header lines and + # the whitespace between them are preserved byte-exactly. + chunk_content = text[(pending_start if pending_start is not None else match.start()) : end] chunks.append( {"content": chunk_content, "meta": {"header": header_text, "parent_headers": parent_headers}} ) - pending_headers = [] # reset pending headers - pending_start = None + pending_start = None # reset buffered headers else: chunks.append({"content": content, "meta": {"header": header_text, "parent_headers": parent_headers}}) @@ -214,7 +209,7 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: # Flush any trailing headers that had no body text of their own. A header at the very end of the # document (or a run of such headers) never gets a following content chunk to be prepended to, so # without this it would be silently dropped. Slicing the original text keeps reconstruction exact. - if pending_headers and pending_start is not None: + if pending_start is not None: parent_headers = [h for h in header_stack[: pending_level - 1] if h is not None] chunks.append( { diff --git a/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml b/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml index 8a24b95c4ed..b76ecd34dd7 100644 --- a/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml +++ b/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml @@ -1,9 +1,19 @@ --- fixes: - | - Fixed `MarkdownHeaderSplitter` silently dropping a trailing header that has - no body text. With `keep_headers=True` (the default), a header at the end of + Fixed ``MarkdownHeaderSplitter`` silently dropping a trailing header that has + no body text. With ``keep_headers=True`` (the default), a header at the end of the document whose only content is whitespace was buffered to prepend to the next chunk, but with no following chunk it was never emitted, so the split documents no longer reconstructed the original text. Such trailing headers are now emitted as a final chunk. + - | + Fixed ``MarkdownHeaderSplitter`` collapsing blank lines that follow a header + with no body text. With ``keep_headers=True``, such headers were re-joined with + a single newline when prepended to the next chunk, so the split documents did + not reconstruct the original text. Chunk content is now sliced from the + original text and is byte-exact. + - | + Fixed ``MarkdownHeaderSplitter`` including surrounding whitespace in the + ``header`` and ``parent_headers`` metadata fields. The header text is now + stripped; chunk content still keeps the header line's original whitespace. diff --git a/test/components/preprocessors/test_markdown_header_splitter.py b/test/components/preprocessors/test_markdown_header_splitter.py index 7f89282d443..ee7e0571580 100644 --- a/test/components/preprocessors/test_markdown_header_splitter.py +++ b/test/components/preprocessors/test_markdown_header_splitter.py @@ -847,3 +847,26 @@ def test_trailing_header_without_content_is_not_dropped(): assert docs[-1].content == "# Header 2\n" assert docs[-1].meta["header"] == "Header 2" assert docs[-1].meta["parent_headers"] == [] + + +def test_middle_header_without_content_preserves_blank_lines(): + # buffered headers are sliced from the original text, so blank lines between a header with no + # body and the next contentful header are preserved instead of collapsed to a single newline + text = "# Header 1\n\n\n# Header 2\nContent.\n" + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + assert "".join(doc.content for doc in docs) == text + + +def test_header_metadata_is_stripped_but_content_is_byte_exact(): + text = "# Header 1 \nContent.\n" + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + assert docs[0].meta["header"] == "Header 1" + # the chunk content keeps the header line's original trailing whitespace + assert "".join(doc.content for doc in docs) == text + + +def test_whitespace_only_trailing_header_has_empty_header_metadata(): + text = "# Header 1\nContent.\n# \n" + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + assert "".join(doc.content for doc in docs) == text + assert docs[-1].meta["header"] == ""