fix: DocumentCleaner wiped the middle page of a 3-page document

haystack/components/preprocessors/document_cleaner.py

@@ -341,7 +341,9 @@ def _find_longest_common_ngram(self, sequences: list[str], min_ngram: int = 3, m
         :returns: The longest ngram that all sequences have in common.
         """
         sequences = [s for s in sequences if s]  # filter empty sequences
-        if not sequences:
+        if len(sequences) < 2:
+            # a single sequence has no ngram "in common" with any other; treating
+            # its own longest ngram as a repeated header/footer would wipe it
             return ""
         seqs_ngrams = map(partial(self._allngram, min_ngram=min_ngram, max_ngram=max_ngram), sequences)
         intersection = reduce(set.intersection, seqs_ngrams)

releasenotes/notes/fix-document-cleaner-3-page-wipe-1c2d3e4f5a6b7089.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    Fixed ``DocumentCleaner`` wiping the middle page of a 3-page document when
+    ``remove_repeated_substrings=True``. Header/footer detection compares the
+    pages between the first and last, which is a single page for a 3-page
+    document; its own longest ngram was then treated as a repeated header/footer
+    and removed. Detection now requires at least two pages to compare, so unique
+    content is preserved while genuine repeated headers/footers are still removed.

test/components/preprocessors/test_document_cleaner.py

@@ -121,6 +121,17 @@ def test_remove_repeated_substrings(self):
         result = cleaner.run(documents=[Document(content=text)])
         assert result["documents"][0].content == expected_text
 
+    def test_remove_repeated_substrings_preserves_unique_middle_page(self):
+        cleaner = DocumentCleaner(
+            remove_empty_lines=False, remove_extra_whitespaces=False, remove_repeated_substrings=True
+        )
+        text = "PAGE ONE\fThe quick brown fox jumps high\fPAGE THREE"
+        result = cleaner.run(documents=[Document(content=text)])["documents"][0]
+        assert result.content.split("\f")[1] == "The quick brown fox jumps high"
+        # With no genuine repeated header/footer, all three pages must round-trip unchanged and in order.
+        assert result.content.split("\f") == ["PAGE ONE", "The quick brown fox jumps high", "PAGE THREE"]
+        assert result.content == text
+
     def test_copy_metadata(self):
         cleaner = DocumentCleaner()
         documents = [