feat: vision pipeline — Gemma 3 scan, static PDF overlay filler, Data Lake UI, semantic mapper fix

Author: utkarshqz

api/main.py

@@ -2,13 +2,20 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
+from sqlmodel import SQLModel
+from api.db.database import engine
+from api.db import models  # Ensures models are registered
 from api.routes import templates, forms, transcribe, incidents
 from api.errors.base import AppError
 from typing import Union
 import os
 
 app = FastAPI()
 
+@app.on_event("startup")
+def on_startup():
+    SQLModel.metadata.create_all(engine)
+
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],

api/routes/incidents.py

@@ -133,7 +133,20 @@ async def generate_pdf_from_lake(
     print(f"[DATA LAKE] Generating '{template.name}' from incident {incident_id} via Semantic Mapper")
 
     master_data = json.loads(incident.master_json)
-    tpl_fields = list(template.fields.keys()) if isinstance(template.fields, dict) else template.fields
+
+    # FIX #2 & #4: Determine target fields based on whether this is a static or AcroForm PDF.
+    # For static PDFs (scanned with vision), use the coordinate labels as targets.
+    # For regular AcroForms, use the stored template field names.
+    from api.db.repositories import get_template_coordinates
+    coords = get_template_coordinates(db, template_id)
+    
+    if coords:
+        # Static PDF path: use the field labels from the scanned coordinates
+        tpl_fields = [c.field_label for c in coords]
+        print(f"[DATA LAKE] Static PDF detected — using {len(tpl_fields)} scanned coordinate labels as targets")
+    else:
+        # AcroForm path: use stored field names from template
+        tpl_fields = list(template.fields.keys()) if isinstance(template.fields, dict) else template.fields
 
     # --- THE MAGIC BRIDGE: AI Semantic Mapper ---
     from src.llm import LLM
@@ -153,10 +166,19 @@ async def generate_pdf_from_lake(
     # Fill PDF
     filler = Filler()
     try:
-        output_path = filler.fill_form_with_data(
-            pdf_form=template.pdf_path,
-            data=mapped_data
-        )
+        if coords:
+            print(f"[DATA LAKE] Found {len(coords)} coordinates, using static PDF filler.")
+            output_path = filler.fill_static_pdf(
+                pdf_form=template.pdf_path,
+                coordinates=coords,
+                data=mapped_data
+            )
+        else:
+            print("[DATA LAKE] No coordinates found, using dynamic AcroForm filler.")
+            output_path = filler.fill_form_with_data(
+                pdf_form=template.pdf_path,
+                data=mapped_data
+            )
     except Exception as e:
         raise AppError(f"PDF generation failed: {str(e)}", status_code=500)
 

api/routes/templates.py

@@ -40,22 +40,18 @@ async def create(
         from pypdf import PdfReader
 
         # Read real field names directly from original PDF
-        # Use /T (internal name) as both key and label
-        # Real names like "JobTitle", "Phone Number" are already human-readable
         reader = PdfReader(save_path)
         raw_fields = reader.get_fields() or {}
 
         fields = {}
         for internal_name, field_data in raw_fields.items():
-            # Use /TU tooltip if available, otherwise prettify /T name
             label = None
             if isinstance(field_data, dict):
                 label = field_data.get("/TU")
             if not label:
-                # Prettify: "JobTitle" → "Job Title", "DATE7_af_date" → "Date"
                 import re
                 label = re.sub(r'([a-z])([A-Z])', r'\1 \2', internal_name)
-                label = re.sub(r'_af_.*$', '', label)  # strip "_af_date" suffix
+                label = re.sub(r'_af_.*$', '', label)
                 label = label.replace('_', ' ').strip().title()
             fields[internal_name] = label
 
@@ -86,4 +82,88 @@ def get_template_by_id(
     tpl = get_template(db, template_id)
     if not tpl:
         raise AppError("Template not found", status_code=404)
-    return tpl
+    return tpl
+
+
+@router.post("/{template_id}/scan")
+async def scan_static_template(
+    template_id: int,
+    db: Session = Depends(get_db)
+):
+    """
+    Uses Gemma 3 Vision model to scan a static/non-fillable PDF once,
+    detect all blank fields by visual analysis, and store their coordinates
+    in the Data Lake. Scan Once → Fill Forever.
+    """
+    print(f"\n\n🚨 [SCAN] Vision scan requested for template {template_id}\n")
+    from api.db.repositories import get_template, get_template_coordinates, create_field_coordinates
+    from api.db.models import FormFieldCoordinates
+
+    tpl = get_template(db, template_id)
+    if not tpl:
+        raise AppError("Template not found", status_code=404)
+
+    if not os.path.exists(tpl.pdf_path):
+        raise AppError("Template PDF not found on disk", status_code=404)
+
+    existing_coords = get_template_coordinates(db, template_id)
+    if existing_coords:
+        return {
+            "status": "already_scanned",
+            "message": "Template already has coordinate data",
+            "fields_found": len(existing_coords)
+        }
+
+    import fitz
+    from src.llm import LLM
+
+    doc = fitz.open(tpl.pdf_path)
+    if len(doc) == 0:
+        raise AppError("PDF has no pages", status_code=400)
+
+    # Render page 1 as a high-res image for the vision model
+    page = doc[0]
+    pix = page.get_pixmap(dpi=150)
+    img_bytes = pix.tobytes("png")
+    doc.close()
+
+    llm = LLM()
+    try:
+        vision_fields = await llm.async_vision_scan_fields(img_bytes)
+        print(f"[VISION] Found {len(vision_fields)} fields on static template.")
+    except Exception as e:
+        raise AppError(f"Vision scan failed: {e}", status_code=500)
+
+    if not vision_fields:
+        return {"status": "no_fields_found", "message": "Vision model found no fields."}
+
+    # Save coordinates into DB
+    stored_coords = []
+    semantic_fields = {}
+
+    for vf in vision_fields:
+        c = FormFieldCoordinates(
+            template_id=template_id,
+            field_label=vf.get("label", "unknown_field"),
+            page_number=0,
+            x=float(vf.get("x", 0)),
+            y=float(vf.get("y", 0)),
+            width=float(vf.get("w", vf.get("width", 20))),
+            height=float(vf.get("h", vf.get("height", 5))),
+            field_type=vf.get("type", "text")
+        )
+        stored_coords.append(c)
+        semantic_fields[c.field_label] = c.field_label.replace("_", " ").title()
+
+    create_field_coordinates(db, stored_coords)
+
+    if not tpl.fields or len(tpl.fields) == 0:
+        tpl.fields = semantic_fields
+        db.add(tpl)
+        db.commit()
+
+    return {
+        "status": "success",
+        "message": f"Vision scan complete — {len(stored_coords)} fields mapped.",
+        "fields": semantic_fields
+    }

frontend/index.html

@@ -659,6 +659,26 @@
       margin-bottom: 24px;
     }
 
+    .btn-scan {
+      display: inline-flex;
+      align-items: center;
+      gap: 8px;
+      padding: 6px 14px;
+      border-radius: 6px;
+      font-family: var(--mono);
+      font-size: 0.68rem;
+      margin-bottom: 24px;
+      margin-left: 10px;
+      background: rgba(255, 69, 0, 0.1);
+      border: 1px solid rgba(255, 69, 0, 0.4);
+      color: var(--fire);
+      cursor: pointer;
+      transition: all 0.2s;
+    }
+    .btn-scan:hover {
+      background: rgba(255, 69, 0, 0.2);
+    }
+
     .selected-tpl.none {
       background: rgba(255, 255, 255, 0.02);
       border: 1px solid var(--border);
@@ -1186,7 +1206,10 @@ <h1 class="hero-h1">REPORT<br /><span class="outline">ONCE.</span></h1>
       </div>
 
       <div class="form-area">
-        <div class="selected-tpl none" id="tplBadge">← Select a template from the sidebar</div>
+        <div style="display:flex;">
+          <div class="selected-tpl none" id="tplBadge">← Select a template from the sidebar</div>
+          <button id="btnScan" class="btn-scan" style="display:none;" onclick="scanTemplate()">👁️ AI Layout Scan</button>
+        </div>
         <div class="field-label">
           <span>Incident Description <span class="req">*</span></span>
           <span class="char-count" id="charCount">0 chars</span>
@@ -1204,6 +1227,14 @@ <h1 class="hero-h1">REPORT<br /><span class="outline">ONCE.</span></h1>
         <input class="input-field" id="incidentIdInput" type="text"
           placeholder="Incident ID (Optional - To append to an existing Data Lake record)" style="margin-bottom:14px;"
           oninput="updateFillBtn()" />
+          
+        <div class="upload-zone" id="photoZone" style="margin-bottom:14px; padding: 15px;">
+          <input type="file" id="photoInput" accept="image/*" capture="environment" onchange="onPhotoChosen(this)" />
+          <span class="upload-zone-icon" style="font-size: 1.5rem; margin-bottom: 5px;">📸</span>
+          <div class="upload-zone-text"><b>Add Scene Photo</b><br/>(AI will dynamically analyze details into lake)</div>
+          <div class="file-chosen" id="photoChosen"></div>
+        </div>
+        
         <textarea id="incidentText"
           placeholder="Officer Hernandez responding to a structure fire at 742 Evergreen Terrace. Two occupants evacuated safely. Minor smoke inhalation treated on scene by EMS. Unit 7 on scene 14:32, cleared 16:45. Handed off to Deputy Martinez..."
           oninput="onTextInput(this)"></textarea>
@@ -1314,6 +1345,10 @@ <h1 class="hero-h1">REPORT<br /><span class="outline">ONCE.</span></h1>
       const b = document.getElementById('tplBadge');
       b.className = 'selected-tpl set'; b.textContent = `\ud83d\udccb ${name}`;
       document.getElementById('step2').classList.add('done');
+      document.getElementById('btnScan').style.display = 'inline-flex';
+      document.getElementById('btnScan').textContent = '👁️ AI Layout Scan';
+      document.getElementById('btnScan').style.color = 'var(--fire)';
+      document.getElementById('btnScan').style.borderColor = 'rgba(255, 69, 0, 0.4)';
       updateFillBtn();
     }
 
@@ -1325,6 +1360,7 @@ <h1 class="hero-h1">REPORT<br /><span class="outline">ONCE.</span></h1>
       const b = document.getElementById('tplBadge');
       if (count === 0) { b.className = 'selected-tpl none'; b.textContent = '\u2190 Check multiple templates for batch fill'; }
       else { b.className = 'selected-tpl set'; b.textContent = `\ud83d\udccb ${count} template${count > 1 ? 's' : ''} selected for batch`; }
+      document.getElementById('btnScan').style.display = 'none';
       if (count > 0) document.getElementById('step2').classList.add('done');
       updateFillBtn(); loadTemplates();
     }
@@ -1351,6 +1387,63 @@ <h1 class="hero-h1">REPORT<br /><span class="outline">ONCE.</span></h1>
       }
     });
 
+    async function scanTemplate() {
+      if (!selectedTplId) return;
+      const btn = document.getElementById('btnScan');
+      btn.disabled = true;
+      btn.textContent = 'Scanning...';
+      try {
+        const r = await fetch(`${API}/templates/${selectedTplId}/scan`, { method: 'POST' });
+        const data = await r.json();
+        if (r.ok || data.status === 'already_scanned') {
+          btn.textContent = '✓ Scanned';
+          btn.style.color = 'var(--green)';
+          btn.style.borderColor = 'var(--green)';
+        } else {
+          btn.textContent = '✗ Scan Failed';
+        }
+      } catch {
+        btn.textContent = '✗ Scan Failed';
+      }
+      btn.disabled = false;
+    }
+
+    async function onPhotoChosen(input) {
+      if (input.files.length > 0) {
+        const file = input.files[0];
+        const ch = document.getElementById('photoChosen');
+        ch.textContent = '⏳ Analyzing scene details...';
+        ch.style.display = 'block';
+        ch.style.color = 'var(--ember)';
+        
+        let incidentId = document.getElementById('incidentIdInput').value.trim();
+        if (!incidentId) {
+          const d = new Date();
+          incidentId = `INC-${d.getFullYear()}-${String(d.getMonth()+1).padStart(2,'0')}${String(d.getDate()).padStart(2,'0')}-${String(d.getHours()).padStart(2,'0')}${String(d.getMinutes()).padStart(2,'0')}`;
+          document.getElementById('incidentIdInput').value = incidentId;
+        }
+        
+        const fd = new FormData();
+        fd.append('file', file);
+        
+        try {
+          const r = await fetch(`${API}/incidents/${incidentId}/scene-photo`, { method: 'POST', body: fd});
+          const data = await r.json();
+          if (r.ok || r.status === 200) {
+             ch.textContent = '✓ Scene visually analyzed & logged to Data Lake!';
+             ch.style.color = 'var(--green)';
+             updateFillBtn();
+          } else {
+             ch.textContent = '✗ Analysis failed';
+             ch.style.color = 'var(--red)';
+          }
+        } catch {
+             ch.textContent = '✗ Analysis failed';
+             ch.style.color = 'var(--red)';
+        }
+      }
+    }
+
     async function saveTemplate() {
       const file = document.getElementById('pdfInput').files[0];
       const name = document.getElementById('tplName').value.trim() || file?.name.replace('.pdf', '') || 'Untitled';

src/filler.py

@@ -323,8 +323,15 @@ def fill_static_pdf(self, pdf_form: str, coordinates: list, data: dict) -> str:
                 rect = fitz.Rect(x_pts, y_pts, x_pts + w_pts, y_pts + h_pts)
 
                 if coord.field_type == "text":
-                    # fitz.TEXT_ALIGN_LEFT = 0
-                    page.insert_textbox(rect, val_str, fontsize=10, fontname="helv", color=(0, 0, 0), align=0)
+                    font_size = 9
+                    # For very flat rects (drawn lines have h < 3pt),
+                    # expand the rect upward so text is visually on top of the line.
+                    if h_pts < font_size:
+                        fill_rect = fitz.Rect(x_pts, y_pts - font_size - 2, x_pts + w_pts, y_pts + 1)
+                    else:
+                        fill_rect = rect
+                    page.insert_textbox(fill_rect, val_str, fontsize=font_size,
+                                        fontname="helv", color=(0, 0, 0), align=0)
                 elif coord.field_type == "checkbox":
                     page.draw_rect(rect, color=(0,0,0), width=1)
                     if val_str.lower() in TRUTHY_VALUES: