diff --git a/haystack/components/preprocessors/markdown_header_splitter.py b/haystack/components/preprocessors/markdown_header_splitter.py index f3ebf1adbeb..72d0b571334 100644 --- a/haystack/components/preprocessors/markdown_header_splitter.py +++ b/haystack/components/preprocessors/markdown_header_splitter.py @@ -147,13 +147,15 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: # process headers and build chunks chunks: list[dict] = [] header_stack: list[str | None] = [None] * 6 - pending_headers: list[str] = [] # store empty headers to prepend to next content + pending_start: int | None = None # start offset in text of the first buffered empty header + pending_header_text: str | None = None # text of the last buffered empty header + pending_level = 0 # level of the last buffered empty header has_content = False # flag to track if any header has content for i, match in enumerate(matches): # extract header info header_prefix = match.group(1) - header_text = match.group(2) + header_text = match.group(2).strip() # keep surrounding whitespace out of the metadata level = len(header_prefix) # get content @@ -169,8 +171,12 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: # skip splits w/o content if not content.strip(): # this strip is needed to avoid counting whitespace as content if self.keep_headers: - header_line = f"{header_prefix} {header_text}" - pending_headers.append(header_line) + # buffer this header so it gets prepended to the next contentful chunk; only the + # offset of the first header in the run is needed since the chunk is sliced from text + if pending_start is None: + pending_start = match.start() + pending_header_text = header_text + pending_level = level continue has_content = True # at least one header has content @@ -183,16 +189,13 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: ) if self.keep_headers: - header_line = f"{header_prefix} {header_text}" - # add pending & current header to content - chunk_content = "" - if pending_headers: - chunk_content += "\n".join(pending_headers) + "\n" - chunk_content += f"{header_line}{content}" + # Slice from the first buffered empty header (if any) so the buffered header lines and + # the whitespace between them are preserved byte-exactly. + chunk_content = text[(pending_start if pending_start is not None else match.start()) : end] chunks.append( {"content": chunk_content, "meta": {"header": header_text, "parent_headers": parent_headers}} ) - pending_headers = [] # reset pending headers + pending_start = None # reset buffered headers else: chunks.append({"content": content, "meta": {"header": header_text, "parent_headers": parent_headers}}) @@ -203,6 +206,18 @@ def _split_text_by_markdown_headers(self, text: str, doc_id: str) -> list[dict]: ) return [{"content": text, "meta": {}}] + # Flush any trailing headers that had no body text of their own. A header at the very end of the + # document (or a run of such headers) never gets a following content chunk to be prepended to, so + # without this it would be silently dropped. Slicing the original text keeps reconstruction exact. + if pending_start is not None: + parent_headers = [h for h in header_stack[: pending_level - 1] if h is not None] + chunks.append( + { + "content": text[pending_start:], + "meta": {"header": pending_header_text, "parent_headers": parent_headers}, + } + ) + return chunks def _apply_secondary_splitting(self, documents: list[Document]) -> list[Document]: diff --git a/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml b/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml new file mode 100644 index 00000000000..b76ecd34dd7 --- /dev/null +++ b/releasenotes/notes/fix-markdown-splitter-trailing-header-9c1a2b3d4e5f6a7b.yaml @@ -0,0 +1,19 @@ +--- +fixes: + - | + Fixed ``MarkdownHeaderSplitter`` silently dropping a trailing header that has + no body text. With ``keep_headers=True`` (the default), a header at the end of + the document whose only content is whitespace was buffered to prepend to the + next chunk, but with no following chunk it was never emitted, so the split + documents no longer reconstructed the original text. Such trailing headers + are now emitted as a final chunk. + - | + Fixed ``MarkdownHeaderSplitter`` collapsing blank lines that follow a header + with no body text. With ``keep_headers=True``, such headers were re-joined with + a single newline when prepended to the next chunk, so the split documents did + not reconstruct the original text. Chunk content is now sliced from the + original text and is byte-exact. + - | + Fixed ``MarkdownHeaderSplitter`` including surrounding whitespace in the + ``header`` and ``parent_headers`` metadata fields. The header text is now + stripped; chunk content still keeps the header line's original whitespace. diff --git a/test/components/preprocessors/test_markdown_header_splitter.py b/test/components/preprocessors/test_markdown_header_splitter.py index 8f5f3c43eba..ee7e0571580 100644 --- a/test/components/preprocessors/test_markdown_header_splitter.py +++ b/test/components/preprocessors/test_markdown_header_splitter.py @@ -838,3 +838,35 @@ def test_page_break_handling_with_multiple_headers(sample_text_with_page_breaks) # reconstruct original reconstructed_text = "".join(doc.content for doc in split_docs) assert reconstructed_text == sample_text_with_page_breaks + + +def test_trailing_header_without_content_is_not_dropped(): + text = "# Header 1\nContent 1.\n# Header 2\n" + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + assert "".join(doc.content for doc in docs) == text + assert docs[-1].content == "# Header 2\n" + assert docs[-1].meta["header"] == "Header 2" + assert docs[-1].meta["parent_headers"] == [] + + +def test_middle_header_without_content_preserves_blank_lines(): + # buffered headers are sliced from the original text, so blank lines between a header with no + # body and the next contentful header are preserved instead of collapsed to a single newline + text = "# Header 1\n\n\n# Header 2\nContent.\n" + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + assert "".join(doc.content for doc in docs) == text + + +def test_header_metadata_is_stripped_but_content_is_byte_exact(): + text = "# Header 1 \nContent.\n" + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + assert docs[0].meta["header"] == "Header 1" + # the chunk content keeps the header line's original trailing whitespace + assert "".join(doc.content for doc in docs) == text + + +def test_whitespace_only_trailing_header_has_empty_header_metadata(): + text = "# Header 1\nContent.\n# \n" + docs = MarkdownHeaderSplitter().run(documents=[Document(content=text)])["documents"] + assert "".join(doc.content for doc in docs) == text + assert docs[-1].meta["header"] == ""