fix(parser): drop publisher/license boilerplate rows

arXiv (and similar) print a rotated license stamp down the page
margin: "Provided proper attribution is provided, Google hereby
grants permission to reproduce the tables and figures ... solely for
use in journalistic or scholarly works." Split across rows, each
fragment passed looksLikeHeading and became a junk top-level section
(and sometimes the document title).

isBoilerplateLine drops rows matching a curated signature list
(license/copyright/preprint phrases) plus the bare arXiv id stamp,
before they reach structure or content extraction. Signatures are
deliberately specific — verified no false positives on real prose
incl. sentences that merely contain "permission" or "rights".

Author: hallelx2

pkg/parser/boilerplate_test.go

@@ -0,0 +1,33 @@
+package parser
+
+import "testing"
+
+func TestIsBoilerplateLine(t *testing.T) {
+	boiler := []string{
+		"Provided proper attribution is provided, Google hereby grants permission to",
+		"reproduce the tables and figures in this paper solely for use in journalistic or",
+		"All Rights Reserved.",
+		"This work is licensed under a Creative Commons Attribution 4.0 License.",
+		"arXiv:1706.03762v7 [cs.CL] 2 Aug 2023",
+		"Preprint. Under review.",
+	}
+	for _, s := range boiler {
+		if !isBoilerplateLine(s) {
+			t.Errorf("expected boilerplate: %q", s)
+		}
+	}
+	content := []string{
+		"Attention Is All You Need",
+		"3 Model Architecture",
+		"The dominant sequence transduction models are based on attention.",
+		"We grant the model permission to attend to all positions.", // 'permission' but not boilerplate
+		"Results",
+		"References",
+		"Recommendation: initiate low-dose therapy (evidence grade A).",
+	}
+	for _, s := range content {
+		if isBoilerplateLine(s) {
+			t.Errorf("false positive on content: %q", s)
+		}
+	}
+}

pkg/parser/pdf.go

@@ -279,6 +279,13 @@ func extractPDFRows(reader *pdflib.Reader) ([]pdfRow, error) {
 			if text == "" {
 				continue
 			}
+			// Drop publisher/preprint boilerplate (e.g. the rotated
+			// arXiv license stamp in the left margin). Left in, it
+			// pollutes the structure with junk top-level "headings"
+			// and the document title.
+			if isBoilerplateLine(text) {
+				continue
+			}
 			out = append(out, pdfRow{
 				page:     pageNum,
 				fontSize: b.maxFS,
@@ -452,6 +459,43 @@ func normalizeForMatch(s string) string {
 	return strings.TrimSpace(b.String())
 }
 
+// boilerplateSignatures are substrings (lower-cased) that mark a line
+// as publisher / preprint / license boilerplate rather than document
+// content. Kept deliberately specific so we never drop real prose —
+// each phrase is something that essentially only appears in a
+// copyright/license stamp.
+var boilerplateSignatures = []string{
+	"hereby grants permission",
+	"provided proper attribution is provided",
+	"solely for use in journalistic",
+	"all rights reserved",
+	"is licensed under a creative commons",
+	"licensed under cc by",
+	"this work is licensed under",
+	"permission to make digital or hard copies",
+	"copyright held by the owner",
+	"preprint. under review",
+	"preprint submitted to",
+}
+
+// isBoilerplateLine reports whether a row is publisher/license noise.
+// Matches the curated signature list, plus the bare arXiv id stamp
+// ("arXiv:2401.01234v2 [cs.CL] 5 Jan 2024") that arXiv prints down
+// the page margin.
+func isBoilerplateLine(s string) bool {
+	low := strings.ToLower(strings.TrimSpace(s))
+	for _, sig := range boilerplateSignatures {
+		if strings.Contains(low, sig) {
+			return true
+		}
+	}
+	// arXiv margin stamp: starts with "arxiv:" followed by a digit.
+	if strings.HasPrefix(low, "arxiv:") && len(low) > 6 && low[6] >= '0' && low[6] <= '9' {
+		return true
+	}
+	return false
+}
+
 func looksLikeHeading(s string) bool {
 	// Headings are rarely > 14 words and never end with sentence punctuation
 	// from the middle of a paragraph.