fix(parser): also drop short license-tail fragments

The license sentence splits across rows; the final 'scholarly works.'
fragment slipped through. Drop short (<=4 word) rows that contain a
license-tail fragment, guarded by length so real prose using the
phrase in a full sentence is kept.

Author: hallelx2

pkg/parser/boilerplate_test.go

@@ -10,6 +10,7 @@ func TestIsBoilerplateLine(t *testing.T) {
 		"This work is licensed under a Creative Commons Attribution 4.0 License.",
 		"arXiv:1706.03762v7 [cs.CL] 2 Aug 2023",
 		"Preprint. Under review.",
+		"scholarly works.", // short license-tail fragment
 	}
 	for _, s := range boiler {
 		if !isBoilerplateLine(s) {
@@ -24,6 +25,7 @@ func TestIsBoilerplateLine(t *testing.T) {
 		"Results",
 		"References",
 		"Recommendation: initiate low-dose therapy (evidence grade A).",
+		"This study was published in several scholarly works over the past decade.", // 'scholarly works' inside real prose
 	}
 	for _, s := range content {
 		if isBoilerplateLine(s) {

pkg/parser/pdf.go

@@ -478,10 +478,22 @@ var boilerplateSignatures = []string{
 	"preprint submitted to",
 }
 
+// boilerplateFragments are short tails of a license sentence that the
+// PDF splits onto their own row (e.g. "...journalistic or scholarly
+// works." → a lone "scholarly works." row). These are too generic to
+// match anywhere in a line, so we only drop them when the whole row
+// is a short fragment (≤ 4 words) — real prose using the phrase lives
+// inside a longer sentence.
+var boilerplateFragments = []string{
+	"scholarly works",
+	"journalistic or",
+	"or scholarly",
+}
+
 // isBoilerplateLine reports whether a row is publisher/license noise.
-// Matches the curated signature list, plus the bare arXiv id stamp
-// ("arXiv:2401.01234v2 [cs.CL] 5 Jan 2024") that arXiv prints down
-// the page margin.
+// Matches the curated signature list, the bare arXiv id stamp
+// ("arXiv:2401.01234v2 [cs.CL] 5 Jan 2024"), and short license-tail
+// fragments.
 func isBoilerplateLine(s string) bool {
 	low := strings.ToLower(strings.TrimSpace(s))
 	for _, sig := range boilerplateSignatures {
@@ -493,6 +505,14 @@ func isBoilerplateLine(s string) bool {
 	if strings.HasPrefix(low, "arxiv:") && len(low) > 6 && low[6] >= '0' && low[6] <= '9' {
 		return true
 	}
+	// Short license-tail fragments.
+	if len(strings.Fields(low)) <= 4 {
+		for _, frag := range boilerplateFragments {
+			if strings.Contains(low, frag) {
+				return true
+			}
+		}
+	}
 	return false
 }