@ (#55)

Drop rotated/vertical text runs from PDF extraction

Academic PDFs with a rotated left-margin stamp (arXiv:
"arXiv:1706.03762v7 [cs.CL] 2 Aug 2023") had that vertical text read in
reverse and it survived as junk headings/titles — the document title
became "3202 guA" (reversed "Aug 2023") and sections like "]LC.sc["
(reversed "[cs.CL]") and "7v26730.6071:viXra".

pdftable already classifies each word run: extractPDFRows now skips runs
that are not Upright or whose reading Direction is vertical ("ttb"/"btt")
via isRotatedRun. This removes margin stamps, page-edge watermarks, and
rotated figure labels — page furniture, never body prose — while keeping
upright LTR and RTL text untouched.

Verified on the arXiv "Attention Is All You Need" PDF: title now
"Attention Is All You Need", zero reversed-stamp sections, clean outline
(Abstract / 1 Introduction / 2 Background / 3 Model Architecture).
@

Author: hallelx2

pkg/parser/pdf.go

@@ -1152,6 +1152,17 @@ func extractPDFRows(doc pdftable.Document) ([]pdfRow, error) {
 			return b
 		}
 		for _, wd := range words {
+			// Drop rotated / vertical text runs — arXiv left-margin
+			// stamps, page-edge watermarks, rotated figure labels. These
+			// are page furniture, not body prose, and pdftable reads a
+			// rotated run in an order that surfaces as reversed text
+			// ("3202 guA" for a rotated "Aug 2023", "]LC.sc[" for
+			// "[cs.CL]"), which then pollutes the outline with junk
+			// headings/titles. Upright left-to-right / right-to-left
+			// prose (including real RTL languages) is kept.
+			if isRotatedRun(wd.Upright, wd.Direction) {
+				continue
+			}
 			b := find(wd.Y1)
 			b.words = append(b.words, wd)
 			if wd.FontSize > b.maxFS {
@@ -1531,6 +1542,24 @@ func looksLikeHeading(s string) bool {
 	return true
 }
 
+// isRotatedRun reports whether a pdftable word run is rotated / vertical
+// text — page furniture (margin stamps, watermarks, rotated labels)
+// rather than body prose. pdftable marks such a run as not Upright and/or
+// with a vertical reading Direction ("ttb"/"btt"); it reads them in an
+// order that surfaces as reversed text, so dropping them keeps the outline
+// clean. Upright horizontal prose in either LTR or RTL scripts is body
+// text and is never dropped.
+func isRotatedRun(upright bool, direction string) bool {
+	if !upright {
+		return true
+	}
+	switch strings.ToLower(direction) {
+	case "ttb", "btt":
+		return true
+	}
+	return false
+}
+
 // isBoldFont reports whether a PDF font name denotes a bold weight. SEC filing
 // section headings are typically bold at body font size (not larger), so this is
 // how we recover them — a size-only heuristic misses them entirely.

pkg/parser/rotated_text_test.go

@@ -0,0 +1,31 @@
+package parser
+
+import "testing"
+
+// TestIsRotatedRun locks the orientation filter that keeps rotated /
+// vertical page furniture (arXiv margin stamps, watermarks) out of the
+// extracted row stream while preserving upright prose in any script.
+func TestIsRotatedRun(t *testing.T) {
+	cases := []struct {
+		name      string
+		upright   bool
+		direction string
+		want      bool
+	}{
+		{"upright ltr prose is kept", true, "ltr", false},
+		{"upright rtl prose is kept", true, "rtl", false},
+		{"upright with empty direction is kept", true, "", false},
+		{"not upright is dropped", false, "ltr", true},
+		{"vertical top-to-bottom is dropped", true, "ttb", true},
+		{"vertical bottom-to-top is dropped", true, "btt", true},
+		{"vertical, case-insensitive", true, "TTB", true},
+		{"not upright AND vertical is dropped", false, "btt", true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := isRotatedRun(tc.upright, tc.direction); got != tc.want {
+				t.Errorf("isRotatedRun(%v, %q) = %v, want %v", tc.upright, tc.direction, got, tc.want)
+			}
+		})
+	}
+}