fix: DocumentCleaner wiped the middle page of a 3-page document

With remove_repeated_substrings=True, header/footer detection compares the pages
between the first and last. For a 3-page document that slice is a single page,
and _find_longest_common_ngram reduced set.intersection over that one page's
ngrams to the page's own longest ngram, which was then removed from every page,
wiping the unique middle page. Require at least two sequences before deriving a
common ngram.

Author: Otis0408

haystack/components/preprocessors/document_cleaner.py

@@ -341,7 +341,9 @@ def _find_longest_common_ngram(self, sequences: list[str], min_ngram: int = 3, m
         :returns: The longest ngram that all sequences have in common.
         """
         sequences = [s for s in sequences if s]  # filter empty sequences
-        if not sequences:
+        if len(sequences) < 2:
+            # a single sequence has no ngram "in common" with any other; treating
+            # its own longest ngram as a repeated header/footer would wipe it
             return ""
         seqs_ngrams = map(partial(self._allngram, min_ngram=min_ngram, max_ngram=max_ngram), sequences)
         intersection = reduce(set.intersection, seqs_ngrams)

releasenotes/notes/fix-document-cleaner-3-page-wipe-1c2d3e4f5a6b7089.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    Fixed ``DocumentCleaner`` wiping the middle page of a 3-page document when
+    ``remove_repeated_substrings=True``. Header/footer detection compares the
+    pages between the first and last, which is a single page for a 3-page
+    document; its own longest ngram was then treated as a repeated header/footer
+    and removed. Detection now requires at least two pages to compare, so unique
+    content is preserved while genuine repeated headers/footers are still removed.

test/components/preprocessors/test_document_cleaner.py

@@ -121,6 +121,17 @@ def test_remove_repeated_substrings(self):
         result = cleaner.run(documents=[Document(content=text)])
         assert result["documents"][0].content == expected_text
 
+    def test_remove_repeated_substrings_preserves_unique_middle_page(self):
+        cleaner = DocumentCleaner(
+            remove_empty_lines=False, remove_extra_whitespaces=False, remove_repeated_substrings=True
+        )
+        text = "PAGE ONE\fThe quick brown fox jumps high\fPAGE THREE"
+        result = cleaner.run(documents=[Document(content=text)])["documents"][0]
+        assert result.content.split("\f")[1] == "The quick brown fox jumps high"
+        # With no genuine repeated header/footer, all three pages must round-trip unchanged and in order.
+        assert result.content.split("\f") == ["PAGE ONE", "The quick brown fox jumps high", "PAGE THREE"]
+        assert result.content == text
+
     def test_copy_metadata(self):
         cleaner = DocumentCleaner()
         documents = [