fix(parser): PDF text via pdftable v0.4.0 (fixes reversed/garbled extraction) (#51)

* fix(parser): extract PDF text via pdftable v0.4.0, not ledongthuc

PDF section titles came out reversed and full of U+FFFD (e.g.
'Retrieval'->'laveirteR', plus 654 spurious sections) because text rows
were sourced from ledongthuc/pdf, whose weak font-encoding + ordering
mangles real design/academic PDFs. pdftable (our pdfplumber port) reads
them correctly; v0.4.0 also fixes its word spacing (size-relative
tolerance + explicit-space boundaries).

Switch extractPDFRows to pdftable's Words() with those options, reusing
the existing row bucketing / bold / letter-spacing-collapse logic.
ledongthuc/pdf is now used ONLY for /Outlines and its failure is
non-fatal (text no longer depends on it).

Verified: the Vectorless whitepaper now parses to 6 clean sections with
correct titles and zero replacement chars (was 654 reversed).

* chore(deps): go mod tidy after pdftable v0.4.0 bump

* fix(parser): serialize pdftable Words() to avoid data race

pdftable's Words() mutates the same package-level state as OpenBytes and
is not concurrent-safe; parallel ingest parses tripped the race detector.
Hold pdftableOpenMu during row extraction, matching the existing guard on
OpenBytes.

Author: hallelx2

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/go-chi/chi/v5 v5.2.5
 	github.com/google/uuid v1.6.0
 	github.com/hallelx2/llmgate v0.3.0
-	github.com/hallelx2/pdftable v0.3.1
+	github.com/hallelx2/pdftable v0.4.0
 	github.com/hibiken/asynq v0.26.0
 	github.com/jackc/pgx/v5 v5.9.2
 	github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728

go.sum

@@ -134,8 +134,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+u
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
 github.com/hallelx2/llmgate v0.3.0 h1:OLm2L41WFAqE2ka64RUdLirXmVRbvbY8zu7N3NSkfS8=
 github.com/hallelx2/llmgate v0.3.0/go.mod h1:MK2Ol/5CIweTQ2/9eSiTJ5g/KSSuobNZL9TD3s57JxY=
-github.com/hallelx2/pdftable v0.3.1 h1:Uqe+9G8s9jrGYwxk8dEMXBCB+SlzvWPmW0Ze5863W1I=
-github.com/hallelx2/pdftable v0.3.1/go.mod h1:pxNlc4D43wjzis7M6EfgQZvHOsQ4okggm+xqUu+OokI=
+github.com/hallelx2/pdftable v0.4.0 h1:ldF8qQrUbejWsbB/JSIqliEXF2jF8z8kjEsbKS8r2BU=
+github.com/hallelx2/pdftable v0.4.0/go.mod h1:pxNlc4D43wjzis7M6EfgQZvHOsQ4okggm+xqUu+OokI=
 github.com/hhrutter/lzw v1.0.0 h1:laL89Llp86W3rRs83LvKbwYRx6INE8gDn0XNb1oXtm0=
 github.com/hhrutter/lzw v1.0.0/go.mod h1:2HC6DJSn/n6iAZfgM3Pg+cP1KxeWc3ezG8bBqW5+WEo=
 github.com/hhrutter/pkcs7 v0.2.2 h1:xMoifoVWah1LNym3C0pomEiLmyJyVIBXt/8oTPyPz+8=

pkg/parser/pdf.go

@@ -307,19 +307,16 @@ func (p *PDF) parseDoc(_ context.Context, buf []byte) (*ParsedDoc, error) {
 	}
 	defer func() { _ = pdoc.Close() }() // best-effort close
 
-	reader, err := pdflib.NewReader(bytes.NewReader(docBytes), int64(len(docBytes)))
-	if err != nil {
-		// ledongthuc/pdf failed on a PDF pdftable accepted (some
-		// xref-stream variants are pdftable-only). Without ledongthuc
-		// we lose both outline access AND the primitive layer the
-		// row extractor uses — so we bail with a clear message rather
-		// than emit garbled text from pdftable.Words() (its word
-		// grouping concatenates words on standard-14 fonts without
-		// AFM metrics; see v0.4.x followup).
-		return nil, fmt.Errorf("pdf: open: ledongthuc/pdf backend rejected the document: %w", err)
+	// ledongthuc/pdf is now used ONLY for /Outlines (bookmarks). Text and
+	// rows come from pdftable's Words() (v0.4.0: correct order + spacing).
+	// A reader failure is therefore non-fatal — we just lose the outline
+	// hint and fall back to the font-size heuristic on the pdftable rows.
+	reader, rerr := pdflib.NewReader(bytes.NewReader(docBytes), int64(len(docBytes)))
+	if rerr != nil {
+		reader = nil
 	}
 
-	rows, err := extractPDFRows(reader)
+	rows, err := extractPDFRows(pdoc)
 	if err != nil {
 		return nil, err
 	}
@@ -964,24 +961,48 @@ type pdfRow struct {
 //
 // Once pdftable bundles standard-14 AFM metrics (v0.4.x goal) we can
 // swap back to its Words() output.
-func extractPDFRows(reader *pdflib.Reader) ([]pdfRow, error) {
-	numPages := reader.NumPage()
+func extractPDFRows(doc pdftable.Document) ([]pdfRow, error) {
+	// pdftable's Words() mutates the same package-level state as OpenBytes
+	// (see pdftableOpenMu) — it is not safe for concurrent callers, and
+	// ingest workers parse documents in parallel. Serialize it too until
+	// pdftable itself is made concurrency-safe. (Proper fix tracked in the
+	// Foundational Libraries project.)
+	pdftableOpenMu.Lock()
+	defer pdftableOpenMu.Unlock()
+
+	numPages := doc.NumPages()
 	var out []pdfRow
 
+	// pdftable v0.4.0 word extraction: a size-relative gap tolerance plus
+	// explicit-space boundaries give correct reading order and word
+	// spacing on real design/academic PDFs — replacing the ledongthuc
+	// glyph path that produced reversed, replacement-char-laden text.
+	wopts := pdftable.WordOpts{
+		XTolerance:        3,
+		XToleranceRatio:   0.15,
+		YTolerance:        3,
+		HorizontalLTR:     true,
+		VerticalTTB:       true,
+		Expand:            true,
+		UseExplicitSpaces: true,
+	}
+
 	for pageNum := 1; pageNum <= numPages; pageNum++ {
-		page := reader.Page(pageNum)
-		if page.V.IsNull() {
+		page, perr := doc.Page(pageNum)
+		if perr != nil {
+			continue
+		}
+		words, werr := page.Words(wopts)
+		if werr != nil {
 			continue
 		}
-		content := page.Content()
 
-		// Group letters by (approximate) baseline Y. Values within 2pt
-		// are considered the same row — PDFs frequently jitter Y by a
-		// fraction.
+		// Bucket words into rows by visual top (Y1), within 2pt — PDFs
+		// frequently jitter Y by a fraction.
 		type rowBucket struct {
 			y     float64
 			maxFS float64
-			chars []pdflib.Text
+			words []pdftable.Word
 		}
 		var buckets []*rowBucket
 		find := func(y float64) *rowBucket {
@@ -994,37 +1015,30 @@ func extractPDFRows(reader *pdflib.Reader) ([]pdfRow, error) {
 			buckets = append(buckets, b)
 			return b
 		}
-		for _, t := range content.Text {
-			b := find(t.Y)
-			b.chars = append(b.chars, t)
-			if t.FontSize > b.maxFS {
-				b.maxFS = t.FontSize
+		for _, wd := range words {
+			b := find(wd.Y1)
+			b.words = append(b.words, wd)
+			if wd.FontSize > b.maxFS {
+				b.maxFS = wd.FontSize
 			}
 		}
 		sort.Slice(buckets, func(i, j int) bool { return buckets[i].y > buckets[j].y })
 
 		for _, b := range buckets {
-			sort.Slice(b.chars, func(i, j int) bool { return b.chars[i].X < b.chars[j].X })
+			// Left-to-right. pdftable already resolved word boundaries +
+			// spacing, so we just order the words and join them.
+			sort.Slice(b.words, func(i, j int) bool { return b.words[i].X0 < b.words[j].X0 })
 			var sb strings.Builder
-			var lastX float64
 			boldGlyphs, totalGlyphs := 0, 0
-			for i, ch := range b.chars {
-				// Insert a space when the gap between the previous
-				// glyph's end and this glyph's start exceeds 0.20 of
-				// the font size. Tuned against real PDFs (arXiv +
-				// SEC 10-Ks): word-boundary gaps land around
-				// 0.20-0.30·fontSize; intra-word kerning stays well
-				// below 0.10.
-				if i > 0 && ch.X-lastX > ch.FontSize*0.20 {
+			for i, wd := range b.words {
+				if i > 0 {
 					sb.WriteString(" ")
 				}
-				sb.WriteString(ch.S)
-				lastX = ch.X + ch.W
-				if strings.TrimSpace(ch.S) != "" {
-					totalGlyphs++
-					if isBoldFont(ch.Font) {
-						boldGlyphs++
-					}
+				sb.WriteString(wd.Text)
+				n := len([]rune(strings.TrimSpace(wd.Text)))
+				totalGlyphs += n
+				if isBoldFont(wd.FontName) {
+					boldGlyphs += n
 				}
 			}
 			// Wide letter-tracking — common on filing cover pages and