feat: implement static PDF cascade with Tesseract OCR, PyMuPDF, and LLM context capping

Author: utkarshqz

api/routes/templates.py

@@ -71,23 +71,17 @@ def get_template_by_id(template_id: int, db: Session = Depends(get_db)):
     return tpl
 
 
-# ── PyMuPDF-based deterministic field finder ───────────────────────────
+# ── PyMuPDF-based deterministic field finder ───────────────────────
 #
-#  Approach (for static non-fillable PDFs with a text layer):
+#  PATH 1 — Text-layer PDFs  (typed Word docs, digital PDFs)
+#  PATH 2 — Image-based PDFs (screenshots pasted into Word, scanned docs)
+#         → Tesseract OCR on the rendered page image
+#  PATH 3 — Final fallback   (Gemma Vision, for exotic/complex cases)
 #
-#  1.  PyMuPDF page.get_text("words") → word-level bboxes (x0,y0,x1,y1,text)
-#  2.  Group words into visual lines by Y coordinate (3pt tolerance)
-#  3.  Lines containing ":" or "____" are field lines
-#  4.  Blank zone = underscore position OR right-edge of colon word + small gap
-#  5.  Gemma (text-only) cleans raw label into snake_case — no vision needed
-#  6.  Coordinates stored as percentages of page size in the DB
-#      → filler converts back to absolute pts at render time
-#
-#  PyMuPDF is used instead of pdfplumber because it is already installed
-#  as part of the core stack (fitz) and avoids an optional dependency.
-#
-#  Gemma Vision is ONLY used as a fallback when the PDF has NO text layer
-#  (i.e. a flat image scan with no searchable text).
+#  Gemma is used ONLY for:
+#    a) Cleaning raw label text to snake_case (both paths)
+#    b) Final fallback vision scan (path 3 only)
+#  Gemma NEVER estimates coordinates.
 
 def _scan_fields_with_pymupdf(pdf_path: str) -> list[dict]:
     """
@@ -115,6 +109,191 @@ def _scan_fields_with_pymupdf(pdf_path: str) -> list[dict]:
     return found_fields
 
 
+def _scan_fields_with_tesseract(pdf_path: str, dpi: int = 200) -> list[dict]:
+    """
+    PATH 2: Deterministic field detection for image-based static PDFs.
+
+    Renders each page to a raster image (via PyMuPDF) then runs Tesseract OCR
+    with pytesseract.image_to_data() to get word-level bounding boxes.
+
+    Coordinate system:
+      Tesseract returns pixel coordinates relative to the rendered image.
+      We convert: px → PDF points via  pt = px * (72 / dpi)
+      Then store as percentages of page size (same as PyMuPDF path).
+
+    Uses the same colon / underscore field detection logic as the text-layer path,
+    so a single downstream pipeline handles both PDF types identically.
+
+    Requires: pytesseract and Tesseract binary (v5+ recommended).
+    """
+    import fitz
+    import pytesseract
+    from PIL import Image
+    import io
+
+    # Point pytesseract at the Tesseract binary (Windows path)
+    pytesseract.pytesseract.tesseract_cmd = r"C:\Program Files\Tesseract-OCR\tesseract.exe"
+
+    found_fields = []
+    doc = fitz.open(pdf_path)
+    print(f"[TESSERACT] Opened '{pdf_path}' — {len(doc)} page(s) at {dpi} DPI")
+
+    for page_num in range(len(doc)):
+        page = doc[page_num]
+        pw_pt = page.rect.width    # page width  in PDF points
+        ph_pt = page.rect.height   # page height in PDF points
+
+        # Render page to image
+        mat = fitz.Matrix(dpi / 72, dpi / 72)
+        pix = page.get_pixmap(matrix=mat, alpha=False)
+        img = Image.open(io.BytesIO(pix.tobytes("png")))
+        img_w, img_h = img.size
+        print(f"[TESSERACT] Page {page_num}: rendered {img_w}x{img_h}px (PDF: {pw_pt:.0f}x{ph_pt:.0f}pt)")
+
+        # Preprocess image for better OCR on low-quality/scanned PDFs
+        # Grayscale + contrast enhancement helps Tesseract significantly
+        from PIL import ImageOps, ImageFilter
+        img_gray = img.convert("L")                   # grayscale
+        img_enhanced = ImageOps.autocontrast(img_gray, cutoff=2)  # boost contrast
+
+        # Run Tesseract OCR — returns word-level data
+        ocr_data = pytesseract.image_to_data(
+            img_enhanced,
+            output_type=pytesseract.Output.DICT,
+            config="--psm 6"   # assume uniform block of text
+        )
+
+        # Build word dicts, filtering out empty/low-confidence tokens
+        # Tesseract bbox: left, top, width, height (all in pixels)
+        words = []
+        n = len(ocr_data["text"])
+        for i in range(n):
+            txt  = ocr_data["text"][i].strip()
+            conf = int(ocr_data["conf"][i])
+            # conf == -1 → non-word token (block/para/line level), skip
+            # conf >= 0  → actual OCR word (0=very uncertain, 100=certain)
+            if not txt or conf < 0:
+                continue
+            px_left = ocr_data["left"][i]
+            px_top  = ocr_data["top"][i]
+            px_w    = ocr_data["width"][i]
+            px_h    = ocr_data["height"][i]
+
+            # Convert pixel coords → PDF points
+            scale = 72.0 / dpi
+            x0 = px_left * scale
+            y0 = px_top  * scale
+            x1 = (px_left + px_w) * scale
+            y1 = (px_top  + px_h) * scale
+
+            words.append({"x0": x0, "y0": y0, "x1": x1, "y1": y1, "text": txt})
+
+        print(f"[TESSERACT] Page {page_num}: {len(words)} confident words")
+        for w in words[:12]:
+            print(f"  word '{w['text']:20s}' x0={w['x0']:6.1f} y0={w['y0']:6.1f}")
+        if len(words) > 12:
+            print(f"  ... and {len(words) - 12} more")
+
+        # ── Group words into visual lines (same logic as PyMuPDF path) ────
+        Y_TOL = 4   # slightly larger tolerance for OCR jitter
+        sorted_words = sorted(words, key=lambda w: (w["y0"], w["x0"]))
+
+        lines: list[list[dict]] = []
+        current_line: list[dict] = []
+        current_y: float | None = None
+
+        for w in sorted_words:
+            if current_y is None or abs(w["y0"] - current_y) <= Y_TOL:
+                current_line.append(w)
+                if current_y is None:
+                    current_y = w["y0"]
+            else:
+                if current_line:
+                    lines.append(current_line)
+                current_line = [w]
+                current_y = w["y0"]
+        if current_line:
+            lines.append(current_line)
+
+        print(f"[TESSERACT] Grouped into {len(lines)} visual lines")
+        for idx, lw in enumerate(lines):
+            lt = " ".join(w["text"] for w in lw)
+            print(f"  line {idx:02d}: '{lt[:90]}'")
+
+        # ── Identical field detection logic as PyMuPDF path ─────────────
+        for line_words in lines:
+            line_words = sorted(line_words, key=lambda w: w["x0"])
+            full_text = " ".join(w["text"] for w in line_words)
+
+            has_colon       = ":" in full_text
+            has_underscores = "_" * 4 in full_text or full_text.count("_") >= 4
+
+            if not (has_colon or has_underscores):
+                continue
+
+            # Skip header lines with content after colon
+            if has_colon and not has_underscores:
+                colon_pos = full_text.find(":")
+                after_colon = full_text[colon_pos + 1:].strip()
+                if after_colon and not after_colon.startswith("_"):
+                    label_part = full_text[:colon_pos].strip().lower()
+                    HEADER_WORDS = {"title", "form", "section", "page", "date", "version", "revision"}
+                    if any(hw in label_part for hw in HEADER_WORDS):
+                        continue
+
+            label_text    = full_text.strip()
+            answer_x: float | None = None
+            answer_y      = min(w["y0"] for w in line_words)
+            answer_bottom = max(w["y1"] for w in line_words)
+
+            if has_underscores:
+                for w in line_words:
+                    if "_" * 4 in w["text"]:
+                        answer_x = w["x0"]
+                        answer_bottom = w["y1"]
+                        label_parts = [lw["text"] for lw in line_words if lw["x0"] < w["x0"]]
+                        if label_parts:
+                            label_text = " ".join(label_parts)
+                        break
+
+            if answer_x is None and has_colon:
+                for w in line_words:
+                    if ":" in w["text"]:
+                        answer_x = w["x1"] + 2
+                        label_parts = []
+                        for lw in line_words:
+                            label_parts.append(lw["text"])
+                            if ":" in lw["text"]:
+                                break
+                        label_text = " ".join(label_parts)
+                        break
+
+            if answer_x is None:
+                answer_x = line_words[-1]["x1"] + 2
+
+            clean_label = re.sub(r'^[\d\.\)\s]+', '', label_text)
+            clean_label = clean_label.rstrip(': _\n').strip()
+
+            if not clean_label:
+                continue
+
+            found_fields.append({
+                "raw_label":     clean_label,
+                "answer_x":      round(answer_x, 2),
+                "answer_y":      round(answer_y, 2),
+                "answer_bottom": round(answer_bottom, 2),
+                "page_num":      page_num,
+                "page_w":        round(pw_pt, 2),
+                "page_h":        round(ph_pt, 2),
+                "line_text":     full_text.strip(),
+            })
+            print(f"  [FIELD] '{clean_label:25s}' answer_x={answer_x:.1f}pt  y0={answer_y:.1f}pt")
+
+    doc.close()
+    print(f"[TESSERACT] Done — {len(found_fields)} field(s) found")
+    return found_fields
+
+
 async def _clean_labels_with_gemma(raw_labels: list[str], image_bytes: bytes) -> list[str]:
     """
     Optional: Ask Gemma to convert raw label strings to clean snake_case names.
@@ -195,7 +374,7 @@ async def scan_static_template(template_id: int, db: Session = Depends(get_db)):
     Scan Once → Fill Forever.
 
     For PDFs WITH a text layer:
-      1. pdfplumber extracts word-level coordinates (deterministic)
+      1. PyMuPDF extracts word-level coordinates (deterministic)
       2. We find field lines (colon/underscore patterns)
       3. Gemma cleans up the label names (semantic)
       4. Absolute point coordinates are saved to FormFieldCoordinates
@@ -238,67 +417,73 @@ async def scan_static_template(template_id: int, db: Session = Depends(get_db)):
     has_text_layer = word_count > 5
     print(f"[SCAN] {page_w:.0f}×{page_h:.0f}pt | words={word_count} | text_layer={has_text_layer}")
 
+    raw_fields: list[dict] = []
     found_fields: list[dict] = []
     scan_mode = "unknown"
 
-    # ── PRIMARY PATH: PyMuPDF deterministic scan ─────────────────────────
+    # ── PATH 1: PyMuPDF — text-layer PDFs ────────────────────────────────
     if has_text_layer:
         scan_mode = "pymupdf"
-        print("[SCAN] Using PyMuPDF deterministic word-level scan...")
-
+        print("[SCAN] PATH 1: PyMuPDF deterministic word-level scan...")
         try:
             raw_fields = _scan_fields_with_pymupdf(tpl.pdf_path)
         except Exception as e:
-            print(f"[SCAN] ⚠️ PyMuPDF scan crashed: {e}")
-            import traceback
-            traceback.print_exc()
+            print(f"[SCAN] ⚠️ PyMuPDF crashed: {e}")
+            import traceback; traceback.print_exc()
+            raw_fields = []
+
+    # ── PATH 2: Tesseract OCR — image-based PDFs ─────────────────────────
+    if not raw_fields:
+        reason = "no text layer" if not has_text_layer else "PyMuPDF found 0 fields"
+        print(f"[SCAN] PATH 2: Tesseract OCR ({reason})...")
+        scan_mode = "tesseract"
+        try:
+            raw_fields = _scan_fields_with_tesseract(tpl.pdf_path)
+        except Exception as e:
+            print(f"[SCAN] ⚠️ Tesseract crashed: {e}")
+            import traceback; traceback.print_exc()
             raw_fields = []
 
-        if raw_fields:
-            print(f"[SCAN] Found {len(raw_fields)} field lines:")
-            for f in raw_fields:
-                print(f"   '{f['raw_label']:30s}' → answer_x={f['answer_x']:.1f}pt  y={f['answer_y']:.1f}pt")
-
-            # Ask Gemma to clean up label names (semantic intelligence)
-            raw_labels = [f["raw_label"] for f in raw_fields]
-            clean_labels = await _clean_labels_with_gemma(raw_labels, img_bytes)
-
-            for i, rf in enumerate(raw_fields):
-                snake_label = clean_labels[i] if i < len(clean_labels) else "unknown_field"
-                found_fields.append({
-                    "label": snake_label,
-                    "display": rf["raw_label"],
-                    "x": rf["answer_x"],       # absolute PDF points
-                    "y": rf["answer_y"],        # absolute PDF points
-                    "w": rf["page_w"] * 0.9 - rf["answer_x"],  # remaining width
-                    "h": rf["answer_bottom"] - rf["answer_y"],  # line height
-                    "page_num": rf["page_num"],
-                    "page_w": rf["page_w"],
-                    "page_h": rf["page_h"],
-                    "type": "text",
-                    "coord_unit": "pt",         # MARKER: absolute points, not %
-                })
-
-            print(f"\n[SCAN] ✅ pdfplumber matched {len(found_fields)} fields")
-
-    # ── FALLBACK PATH: Gemma Vision for image-only PDFs ──────────────────
+    # ── Shared: clean labels with Gemma (text→text only, no vision) ───────
+    if raw_fields:
+        print(f"[SCAN] {len(raw_fields)} raw field(s) — asking Gemma to clean labels...")
+        raw_labels  = [f["raw_label"] for f in raw_fields]
+        clean_labels = await _clean_labels_with_gemma(raw_labels, img_bytes)
+
+        for i, rf in enumerate(raw_fields):
+            snake_label = clean_labels[i] if i < len(clean_labels) else "unknown_field"
+            found_fields.append({
+                "label":      snake_label,
+                "display":    rf["raw_label"],
+                "x":          rf["answer_x"],
+                "y":          rf["answer_y"],
+                "w":          rf["page_w"] * 0.9 - rf["answer_x"],
+                "h":          rf["answer_bottom"] - rf["answer_y"],
+                "page_num":   rf["page_num"],
+                "page_w":     rf["page_w"],
+                "page_h":     rf["page_h"],
+                "type":       "text",
+                "coord_unit": "pt",
+            })
+        print(f"[SCAN] ✅ {scan_mode} → {len(found_fields)} fields with exact coords")
+
+    # ── PATH 3: Gemma Vision — last resort only ───────────────────────────
     if not found_fields:
-        reason = "no text layer" if not has_text_layer else "pdfplumber found 0 fields"
-        print(f"[SCAN] Fallback → Gemma Vision ({reason})...")
+        print("[SCAN] PATH 3: Gemma Vision (last resort — both deterministic paths failed)...")
         scan_mode = "vision"
         llm = LLM()
         try:
             found_fields = await llm.async_vision_scan_fields(img_bytes)
-            # Vision fields come as percentages — mark them accordingly
             for vf in found_fields:
-                vf["coord_unit"] = "pct"
+                vf["coord_unit"] = "pct"  # Gemma returns guessed percentages
             print(f"[SCAN] Gemma vision: {len(found_fields)} fields")
         except Exception as e:
-            raise AppError(f"Vision scan failed: {e}", status_code=500)
+            raise AppError(f"All scan paths failed: {e}", status_code=500)
 
     if not found_fields:
         return {"status": "no_fields_found", "message": "No fields detected."}
 
+
     # ── Save to Data Lake (FormFieldCoordinates) ─────────────────────────
     stored_coords, semantic_fields = [], {}
     for vf in found_fields:

frontend/index.html

@@ -1245,7 +1245,7 @@ <h1 class="hero-h1">REPORT<br /><span class="outline">ONCE.</span></h1>
         <div class="result">
           <div class="result-loading" id="resLoading">
             <div class="spinner"></div>
-            <div class="loading-msg"><b>Mistral</b> is extracting data and filling your form...</div>
+            <div class="loading-msg"><b>FireForm AI</b> is extracting data and filling your form...</div>
           </div>
           <div class="result-success" id="resSuccess">
             <div class="success-header">
@@ -1589,8 +1589,8 @@ <h1 class="hero-h1">REPORT<br /><span class="outline">ONCE.</span></h1>
       document.getElementById('btnFill').disabled = true;
       const lm = document.querySelector('.loading-msg');
       lm.innerHTML = (mode === 'batch' && selectedIds.size > 1)
-        ? `<b>Mistral</b> is filling ${selectedIds.size} forms in one pass...`
-        : `<b>Mistral</b> is extracting data and filling your form...`;
+        ? `<b>FireForm AI</b> is filling ${selectedIds.size} forms in one pass...`
+        : `<b>FireForm AI</b> is extracting data and filling your form...`;
       document.getElementById('resLoading').classList.add('show');
       try {
         if (mode === 'batch' && selectedIds.size > 1) await fillBatch(text);

requirements.txt

@@ -15,3 +15,5 @@ python-multipart
 pymupdf
 pdfplumber
 reportlab
+pytesseract
+Pillow