fix(ingest): strip control bytes and fold ligatures in extracted PDF text

Author: codebybilal18

app/ingest/parse.py

@@ -17,6 +17,7 @@
 import io
 import logging
 import re
+import unicodedata
 from dataclasses import dataclass, field
 
 from app.config import settings
@@ -38,6 +39,30 @@
 # large that the request cost doubles for no accuracy gain.
 RENDER_DPI = 150
 
+# Control characters Postgres will not store in a text column, minus the
+# whitespace we want to keep. A single stray NUL in a PDF's text stream aborts
+# the whole insert with
+#   invalid byte sequence for encoding "UTF8": 0x00
+# and takes the entire document's ingestion down with it. Five of the thirteen
+# papers in the benchmark corpus contain at least one.
+_CONTROL_CHARS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
+
+
+def clean_extracted_text(text: str) -> str:
+    """Make PDF text storable and searchable.
+
+    Two problems, both invisible until they bite:
+
+    * **NUL and control bytes** — Postgres rejects them outright.
+    * **Typographic ligatures** — PDF text streams encode "fi" as U+FB01 and
+      "ff" as U+FB00, so "inverted file" extracts as "inverted file" and
+      "effectiveness" as "effectiveness". Those are *different strings*: a
+      keyword search for "file" misses them, and the tokens an embedding model
+      sees are not the ones the reader typed. NFKC folds them back, along with
+      other compatibility forms such as non-breaking spaces.
+    """
+    return _CONTROL_CHARS.sub("", unicodedata.normalize("NFKC", text))
+
 
 @dataclass
 class Heading:
@@ -80,7 +105,7 @@ def _extract_headings_native(page) -> tuple[str, list[Heading]]:
     try:
         blocks = page.get_text("dict")["blocks"]
     except Exception:  # pragma: no cover - malformed PDFs
-        return page.get_text("text"), []
+        return clean_extracted_text(page.get_text("text")), []
 
     lines: list[tuple[str, float, bool]] = []
     sizes: list[float] = []
@@ -98,7 +123,7 @@ def _extract_headings_native(page) -> tuple[str, list[Heading]]:
             sizes.append(size)
 
     if not lines:
-        return page.get_text("text"), []
+        return clean_extracted_text(page.get_text("text")), []
 
     sorted_sizes = sorted(sizes)
     body_size = sorted_sizes[len(sorted_sizes) // 2]  # median == body text
@@ -109,7 +134,8 @@ def _extract_headings_native(page) -> tuple[str, list[Heading]]:
     out_parts: list[str] = []
     headings: list[Heading] = []
     offset = 0
-    for text, size, bold in lines:
+    for raw_text, size, bold in lines:
+        text = clean_extracted_text(raw_text)
         level = level_of.get(round(size, 1))
         # A short, bold, title-cased line is a heading even at body size —
         # common in filings and specs where headings aren't set larger.
@@ -178,7 +204,7 @@ async def parse_page_with_vision(data: bytes, page_number: int, meter: UsageMete
         cached_input_tokens=result.cached_input_tokens,
         label="vision_ocr",
     )
-    text = (result.text or "").strip()
+    text = clean_extracted_text(result.text or "").strip()
     if text == "[BLANK PAGE]":
         text = ""
     return ParsedPage(

app/ingest/pipeline.py

@@ -567,12 +567,29 @@ async def ingest_to_completion(
     Used by the CLI, the eval corpus loader, and local development — anywhere a
     caller can afford to block. On serverless the client polls instead.
     """
+    # Counters on TickResult are per-tick. Returning the last one reports the
+    # terminal INDEXING->READY step, which did no work: a multi-tick document
+    # would log "ready — 0 pages, 0 chunks" despite ingesting perfectly.
+    totals = {"pages": 0, "written": 0, "embedded": 0, "dupes": 0, "cost": 0.0}
     result: TickResult | None = None
+
     for _ in range(max_ticks):
         result = await ingest_tick(session, document_id, config)
         await session.commit()
+        totals["pages"] += result.pages_parsed
+        totals["written"] += result.chunks_written
+        totals["embedded"] += result.chunks_embedded
+        totals["dupes"] += result.duplicates_skipped
+        totals["cost"] += result.cost_usd
         if result.done:
-            return result
+            break
+
     if result is None:
         raise IngestError("no ticks executed")
+
+    result.pages_parsed = totals["pages"]
+    result.chunks_written = totals["written"]
+    result.chunks_embedded = totals["embedded"]
+    result.duplicates_skipped = totals["dupes"]
+    result.cost_usd = totals["cost"]
     return result

evals/golden_set.jsonl

@@ -19,7 +19,7 @@
 {"id":"q01","question":"What are the two RAG model variants introduced in the RAG paper?","expected":"RAG-Sequence, which uses the same retrieved document to generate the whole sequence, and RAG-Token, which can use a different document for each generated token.","expected_chunks":["rag.pdf#~RAG-Sequence","rag.pdf#~RAG-Token"],"category":"factual","difficulty":"easy"}
 {"id":"q02","question":"Which generator model does the RAG paper use?","expected":"BART-large, a pre-trained sequence-to-sequence transformer, is used as the parametric generator.","expected_chunks":["rag.pdf#~BART"],"category":"factual","difficulty":"easy"}
 {"id":"q03","question":"What does the parameter M control in the HNSW index?","expected":"M is the number of established connections (neighbours) per element per layer — the graph's out-degree, which trades index size and build time against search quality.","expected_chunks":["hnsw.pdf#~number of established connections"],"category":"factual","difficulty":"medium"}
-{"id":"q04","question":"What does ef_construction control when building an HNSW index?","expected":"It is the size of the dynamic candidate list used during index construction; larger values produce a higher-quality graph at the cost of longer build time.","expected_chunks":["hnsw.pdf#~ef_construction"],"category":"factual","difficulty":"medium"}
+{"id":"q04","question":"What does ef_construction control when building an HNSW index?","expected":"It is the size of the dynamic candidate list used during index construction; larger values produce a higher-quality graph at the cost of longer build time.","expected_chunks":["hnsw.pdf#~efconstruction"],"category":"factual","difficulty":"medium"}
 {"id":"q05","question":"Which pooling strategy did Sentence-BERT find worked best?","expected":"Mean pooling over the token output vectors (the MEAN strategy) outperformed both the CLS token and max pooling.","expected_chunks":["sentence-bert.pdf#~pooling"],"category":"factual","difficulty":"medium"}
 {"id":"q06","question":"What is the central finding of the Lost in the Middle paper?","expected":"Model performance is highest when the relevant information appears at the very beginning or the very end of the input context and degrades substantially when it is in the middle, producing a U-shaped performance curve.","expected_chunks":["lost-in-the-middle.pdf#~middle"],"category":"factual","difficulty":"easy"}
 {"id":"q07","question":"Which metrics does the RAGAS framework propose?","expected":"Faithfulness, answer relevance, and context relevance — evaluated without human-annotated ground-truth references.","expected_chunks":["ragas.pdf#~faithfulness"],"category":"factual","difficulty":"easy"}

tests/test_chunking.py

@@ -172,3 +172,35 @@ def test_configs_a_and_b_land_in_different_spaces():
     assert space_for(get_config("B")) == space_for(get_config("C")) == space_for(get_config("D"))
     # E uses a different embedder and therefore a different vector dimension.
     assert space_for(get_config("E")) != space_for(get_config("D"))
+
+
+# --- extracted-text cleaning ------------------------------------------------
+
+
+def test_nul_bytes_are_stripped():
+    """Postgres rejects NUL in a text column outright, aborting the insert with
+    `invalid byte sequence for encoding "UTF8": 0x00` — which takes the whole
+    document's ingestion down. Five of the thirteen benchmark PDFs contain one.
+    """
+    from app.ingest.parse import clean_extracted_text
+
+    assert clean_extracted_text("before\x00after") == "beforeafter"
+    assert "\x00" not in clean_extracted_text("a\x00b\x01c\x1fd\x7fe")
+
+
+def test_whitespace_is_preserved():
+    from app.ingest.parse import clean_extracted_text
+
+    assert clean_extracted_text("a\nb\tc\rd") == "a\nb\tc\rd"
+
+
+def test_ligatures_are_folded_to_ascii():
+    """PDF text streams encode "fi" as U+FB01 and "ff" as U+FB00, so a keyword
+    search for "file" misses "inverted file" entirely and the embedding model
+    sees tokens the reader never typed."""
+    from app.ingest.parse import clean_extracted_text
+
+    assert clean_extracted_text("inverted file") == "inverted file"
+    assert clean_extracted_text("effectiveness") == "effectiveness"
+    assert clean_extracted_text("difficult") == "difficult"
+    assert clean_extracted_text("flow") == "flow"