diff --git a/haystack/components/preprocessors/document_cleaner.py b/haystack/components/preprocessors/document_cleaner.py index 33a3d5e73c7..f186e75fcf9 100644 --- a/haystack/components/preprocessors/document_cleaner.py +++ b/haystack/components/preprocessors/document_cleaner.py @@ -341,7 +341,9 @@ def _find_longest_common_ngram(self, sequences: list[str], min_ngram: int = 3, m :returns: The longest ngram that all sequences have in common. """ sequences = [s for s in sequences if s] # filter empty sequences - if not sequences: + if len(sequences) < 2: + # a single sequence has no ngram "in common" with any other; treating + # its own longest ngram as a repeated header/footer would wipe it return "" seqs_ngrams = map(partial(self._allngram, min_ngram=min_ngram, max_ngram=max_ngram), sequences) intersection = reduce(set.intersection, seqs_ngrams) diff --git a/releasenotes/notes/fix-document-cleaner-3-page-wipe-1c2d3e4f5a6b7089.yaml b/releasenotes/notes/fix-document-cleaner-3-page-wipe-1c2d3e4f5a6b7089.yaml new file mode 100644 index 00000000000..e0087d2f87d --- /dev/null +++ b/releasenotes/notes/fix-document-cleaner-3-page-wipe-1c2d3e4f5a6b7089.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixed ``DocumentCleaner`` wiping the middle page of a 3-page document when + ``remove_repeated_substrings=True``. Header/footer detection compares the + pages between the first and last, which is a single page for a 3-page + document; its own longest ngram was then treated as a repeated header/footer + and removed. Detection now requires at least two pages to compare, so unique + content is preserved while genuine repeated headers/footers are still removed. diff --git a/test/components/preprocessors/test_document_cleaner.py b/test/components/preprocessors/test_document_cleaner.py index 31567563b24..9e834acf112 100644 --- a/test/components/preprocessors/test_document_cleaner.py +++ b/test/components/preprocessors/test_document_cleaner.py @@ -121,6 +121,17 @@ def test_remove_repeated_substrings(self): result = cleaner.run(documents=[Document(content=text)]) assert result["documents"][0].content == expected_text + def test_remove_repeated_substrings_preserves_unique_middle_page(self): + cleaner = DocumentCleaner( + remove_empty_lines=False, remove_extra_whitespaces=False, remove_repeated_substrings=True + ) + text = "PAGE ONE\fThe quick brown fox jumps high\fPAGE THREE" + result = cleaner.run(documents=[Document(content=text)])["documents"][0] + assert result.content.split("\f")[1] == "The quick brown fox jumps high" + # With no genuine repeated header/footer, all three pages must round-trip unchanged and in order. + assert result.content.split("\f") == ["PAGE ONE", "The quick brown fox jumps high", "PAGE THREE"] + assert result.content == text + def test_copy_metadata(self): cleaner = DocumentCleaner() documents = [