fix: suppress table caption artifacts across page boundaries

StevenVincentOne · StevenVincentOne · commit cf8829ea69c8 · 2026-03-19T15:21:12.000-07:00
diff --git a/java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/markdown/MarkdownGenerator.java b/java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/markdown/MarkdownGenerator.java
@@ -65,6 +65,10 @@ public class MarkdownGenerator implements Closeable {
         "^(pass@1|cons@\\d+|rating)(?:\\s+(pass@1|cons@\\d+|rating))+\\s*$",
         Pattern.CASE_INSENSITIVE
     );
+    private static final Pattern FOOTNOTE_PLACEHOLDER_PATTERN = Pattern.compile(
+        "^(?:\\d+https?://\\S+)(?:\\s+\\d+https?://\\S+)*$",
+        Pattern.CASE_INSENSITIVE
+    );
     private static final Pattern BENCHMARK_PATTERN = Pattern.compile(
         "(AIME 2024|MATH-500|CNMO 2024|GPQA(?: Diamond)?|LiveCodeBench|Codeforces|SWE Verified|Aider-Polyglot|MMLU(?:-Redux|-Pro)?|DROP|IF-Eval|SimpleQA|FRAMES|AlpacaEval2\\.0|ArenaHard|CLUEWSC|C-Eval|C-SimpleQA)",
         Pattern.CASE_INSENSITIVE
@@ -88,10 +92,15 @@ public class MarkdownGenerator implements Closeable {
 
     public void writeToMarkdown(List<List<IObject>> contents) {
         try {
+            List<Set<Integer>> pageSkipIndices = new java.util.ArrayList<>(contents.size());
+            for (List<IObject> pageContents : contents) {
+                pageSkipIndices.add(collectTableArtifactIndices(pageContents));
+            }
+            extendCrossPageTableArtifactSkips(contents, pageSkipIndices);
             for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
                 writePageSeparator(pageNumber);
                 List<IObject> pageContents = contents.get(pageNumber);
-                Set<Integer> skipIndices = collectTableArtifactIndices(pageContents);
+                Set<Integer> skipIndices = pageSkipIndices.get(pageNumber);
                 for (int contentIndex = 0; contentIndex < pageContents.size(); contentIndex++) {
                     if (skipIndices.contains(contentIndex)) {
                         continue;
@@ -360,6 +369,26 @@ protected boolean shouldWriteTableBody() {
         return Config.MARKDOWN_TABLE_OUTPUT_FULL.equals(markdownTableOutput);
     }
 
+    protected void extendCrossPageTableArtifactSkips(List<List<IObject>> contents, List<Set<Integer>> pageSkipIndices) {
+        if (Config.MARKDOWN_TABLE_OUTPUT_FULL.equals(markdownTableOutput)) {
+            return;
+        }
+        for (int pageNumber = 0; pageNumber < contents.size(); pageNumber++) {
+            List<IObject> pageContents = contents.get(pageNumber);
+            Set<Integer> pageSkips = pageSkipIndices.get(pageNumber);
+
+            int firstMeaningful = findFirstMeaningfulContentIndex(pageContents, pageSkips);
+            if (firstMeaningful >= 0 && isTableCaptionText(normalizeContentText(pageContents.get(firstMeaningful))) && pageNumber > 0) {
+                walkTableArtifactRange(contents.get(pageNumber - 1), pageSkipIndices.get(pageNumber - 1), contents.get(pageNumber - 1).size(), -1);
+            }
+
+            int lastMeaningful = findLastMeaningfulContentIndex(pageContents, pageSkips);
+            if (lastMeaningful >= 0 && isTableCaptionText(normalizeContentText(pageContents.get(lastMeaningful))) && pageNumber + 1 < contents.size()) {
+                walkTableArtifactRange(contents.get(pageNumber + 1), pageSkipIndices.get(pageNumber + 1), -1, 1);
+            }
+        }
+    }
+
     protected Set<Integer> collectTableArtifactIndices(List<IObject> pageContents) {
         Set<Integer> skip = new HashSet<>();
         if (Config.MARKDOWN_TABLE_OUTPUT_FULL.equals(markdownTableOutput)) {
@@ -411,6 +440,11 @@ protected void walkTableArtifactRange(List<IObject> pageContents, Set<Integer> s
                 continue;
             }
             if (looksNarrativeText(text)) {
+                if (direction > 0 && shouldSkipDanglingNarrativeFragment(pageContents, index, text)) {
+                    skip.add(index);
+                    index += direction;
+                    continue;
+                }
                 break;
             }
             break;
@@ -421,11 +455,51 @@ protected boolean isHeadingContent(IObject content) {
         return content instanceof SemanticHeading;
     }
 
+    protected int findFirstMeaningfulContentIndex(List<IObject> pageContents, Set<Integer> skip) {
+        for (int index = 0; index < pageContents.size(); index++) {
+            if (skip.contains(index)) {
+                continue;
+            }
+            String text = normalizeContentText(pageContents.get(index));
+            if (!text.isEmpty() || pageContents.get(index) instanceof TableBorder) {
+                return index;
+            }
+        }
+        return -1;
+    }
+
+    protected int findLastMeaningfulContentIndex(List<IObject> pageContents, Set<Integer> skip) {
+        for (int index = pageContents.size() - 1; index >= 0; index--) {
+            if (skip.contains(index)) {
+                continue;
+            }
+            String text = normalizeContentText(pageContents.get(index));
+            if (!text.isEmpty() || pageContents.get(index) instanceof TableBorder) {
+                return index;
+            }
+        }
+        return -1;
+    }
+
     protected boolean isTableOutputOff() {
         return Config.MARKDOWN_TABLE_OUTPUT_OFF.equals(markdownTableOutput);
     }
 
     protected String normalizeContentText(IObject content) {
+        if (content instanceof PDFList) {
+            StringBuilder builder = new StringBuilder();
+            for (ListItem item : ((PDFList) content).getListItems()) {
+                String value = String.valueOf(item).replaceAll("\\s+", " ").trim();
+                if (value.isEmpty()) {
+                    continue;
+                }
+                if (builder.length() > 0) {
+                    builder.append(' ');
+                }
+                builder.append(value);
+            }
+            return builder.toString();
+        }
         if (!(content instanceof SemanticTextNode)) {
             return "";
         }
@@ -466,6 +540,9 @@ protected boolean looksTableArtifactText(String text) {
         if (NUMERIC_ONLY_PATTERN.matcher(text).matches()) {
             return true;
         }
+        if (FOOTNOTE_PLACEHOLDER_PATTERN.matcher(text).matches()) {
+            return true;
+        }
         if (TABLE_HEADER_TEXT_PATTERN.matcher(text).matches()) {
             return true;
         }
@@ -507,6 +584,26 @@ protected boolean looksTableArtifactText(String text) {
         return false;
     }
 
+    protected boolean shouldSkipDanglingNarrativeFragment(List<IObject> pageContents, int index, String text) {
+        if (text.isEmpty() || !Character.isLowerCase(text.charAt(0))) {
+            return false;
+        }
+        for (int nextIndex = index + 1; nextIndex < pageContents.size(); nextIndex++) {
+            IObject next = pageContents.get(nextIndex);
+            String nextText = normalizeContentText(next);
+            if (nextText.isEmpty()) {
+                continue;
+            }
+            if (isHeadingContent(next) || isTableCaptionText(nextText)) {
+                return true;
+            }
+            if (looksNarrativeText(nextText) && !looksTableArtifactText(nextText)) {
+                return false;
+            }
+        }
+        return false;
+    }
+
     protected String getLineBreak() {
         if (isInsideTable()) {
             return MarkdownSyntax.HTML_LINE_BREAK_TAG;
diff --git a/java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/markdown/MarkdownGeneratorTest.java b/java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/markdown/MarkdownGeneratorTest.java
@@ -10,13 +10,16 @@
 import org.junit.jupiter.api.io.TempDir;
 import org.opendataloader.pdf.api.Config;
 import org.verapdf.wcag.algorithms.entities.IObject;
+import org.verapdf.wcag.algorithms.entities.SemanticHeading;
 import org.verapdf.wcag.algorithms.entities.SemanticParagraph;
 import org.verapdf.wcag.algorithms.entities.content.TextChunk;
 import org.verapdf.wcag.algorithms.entities.content.TextLine;
 import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
 import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
 import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
 import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
+import org.verapdf.wcag.algorithms.entities.lists.ListItem;
+import org.verapdf.wcag.algorithms.entities.lists.PDFList;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
@@ -152,6 +155,30 @@ void testCaptionOnlySkipsFlattenedTableArtifactsAroundCaption() throws IOExcepti
         assertTrue(markdown.contains("This paragraph should remain after the caption because it is narrative."));
     }
 
+    @Test
+    void testCaptionOnlySkipsFootnotePlaceholderListsAndDanglingLowercaseFragments() throws IOException {
+        Config config = new Config();
+        config.setOutputFolder(tempDir.toString());
+        config.setMarkdownTableOutput(Config.MARKDOWN_TABLE_OUTPUT_CAPTION_ONLY);
+        File inputPdf = tempDir.resolve("dangling.pdf").toFile();
+        Files.writeString(inputPdf.toPath(), "");
+
+        try (TestMarkdownGenerator generator = new TestMarkdownGenerator(inputPdf, config)) {
+            generator.writePage(List.of(
+                createFootnoteList("1https://example.com", "2https://example.com"),
+                createCaption("Table 4 | Comparison between models."),
+                createCaption("performance of the model will improve soon."),
+                createHeading("3.2. Distilled Model Evaluation", 5)
+            ));
+        }
+
+        String markdown = Files.readString(tempDir.resolve("dangling.md"));
+        assertFalse(markdown.contains("1https://example.com"));
+        assertFalse(markdown.contains("performance of the model will improve soon."));
+        assertTrue(markdown.contains("Table 4 | Comparison between models."));
+        assertTrue(markdown.contains("##### 3.2. Distilled Model Evaluation"));
+    }
+
     /**
      * Helper method that mirrors the heading prefix generation logic in
      * MarkdownGenerator.writeHeading().
@@ -229,6 +256,33 @@ private TableBorderCell createCell(int row, int col, String text) {
         return cell;
     }
 
+    private SemanticHeading createHeading(String text, int level) {
+        SemanticHeading heading = new SemanticHeading(createCaption(text));
+        heading.setHeadingLevel(level);
+        return heading;
+    }
+
+    private PDFList createFootnoteList(String... items) {
+        PDFList list = new PDFList();
+        for (String itemText : items) {
+            ListItem item = new ListItem(new BoundingBox(), null);
+            item.add(new TextLine(new TextChunk(
+                new BoundingBox(0, 10.0, 10.0, 20.0, 20.0),
+                itemText,
+                "Font1",
+                10,
+                700,
+                0,
+                20.0,
+                new double[]{0.0},
+                null,
+                0
+            )));
+            list.add(item);
+        }
+        return list;
+    }
+
     private static class TestMarkdownGenerator extends MarkdownGenerator {
         TestMarkdownGenerator(File inputPdf, Config config) throws IOException {
             super(inputPdf, config);
