feat: Implement Data Lake and AI Semantic Mapper for seamless Multi-Form Generation (ICS-209 & ICS-214)

Author: utkarshqz

api/routes/incidents.py

@@ -57,11 +57,37 @@ async def extract_to_data_lake(
     merged_fields = {}
 
     if all_templates:
-        # Build superset from all known templates
-        for tpl in all_templates:
-            if isinstance(tpl.fields, dict):
-                merged_fields.update(tpl.fields)
-        print(f"[DATA LAKE] Base schema: {len(merged_fields)} template fields across {len(all_templates)} templates")
+        # Instead of pulling 1000+ unreadable AcroForm fields from all templates,
+        # we provide a core Universal ICS Schema to guide the LLM, and let its dynamic 
+        # schema-less extraction catch everything else. This prevents 503 timeouts!
+        merged_fields = {
+            "incident_name": "Incident Name",
+            "incident_number": "Incident Number",
+            "date_time_of_report": "Date and Time of Report",
+            "report_version": "Report Version",
+            "incident_commander": "Incident Commander",
+            "incident_management_organization": "Incident Management Organization",
+            "incident_description": "Incident Description / Sub-type",
+            "location": "Location",
+            "current_incident_size_acres": "Current Incident Size (Acres)",
+            "percent_contained": "Percent Contained",
+            "estimated_containment_date": "Estimated Containment Date",
+            "cause_of_incident": "Cause of Incident",
+            "weather_conditions": "Weather Conditions (Current)",
+            "significant_events": "Significant Events / Critical Activity",
+            "planned_actions": "Planned Actions for Next 24 Hours",
+            "operational_period_from": "Operational Period From",
+            "operational_period_to": "Operational Period To",
+            "from": "From Time",
+            "to": "To Time",
+            "unit_name": "Unit Name / Designators",
+            "unit_leader": "Unit Leader (Name and ICS Position)",
+            "personnel_assigned": "Personnel Assigned",
+            "activity_log": "Activity Log",
+            "prepared_by": "Prepared By",
+            "preparation_date_time": "Date and Time of Preparation"
+        }
+        print(f"[DATA LAKE] Using Universal ICS Schema ({len(merged_fields)} fields) to prevent LLM timeout.")
 
     try:
         llm = LLM(transcript_text=input_text, target_fields=merged_fields)
@@ -134,9 +160,7 @@ async def generate_pdf_from_lake(
 
     master_data = json.loads(incident.master_json)
 
-    # FIX #2 & #4: Determine target fields based on whether this is a static or AcroForm PDF.
-    # For static PDFs (scanned with vision), use the coordinate labels as targets.
-    # For regular AcroForms, use the stored template field names.
+    # Determine target fields based on whether this is a static or AcroForm PDF.
     from api.db.repositories import get_template_coordinates
     coords = get_template_coordinates(db, template_id)
 
@@ -146,7 +170,19 @@ async def generate_pdf_from_lake(
         print(f"[DATA LAKE] Static PDF detected — using {len(tpl_fields)} scanned coordinate labels as targets")
     else:
         # AcroForm path: use stored field names from template
-        tpl_fields = list(template.fields.keys()) if isinstance(template.fields, dict) else template.fields
+        raw_fields = list(template.fields.keys()) if isinstance(template.fields, dict) else template.fields
+        
+        # --- JUNK FILTER ---
+        import re
+        tpl_fields = []
+        for f in raw_fields:
+            if not f: continue
+            basename = str(f).split('.')[-1].split('[')[0].lower()
+            if re.match(r'^(textfield|text|checkbox|check|button|btn|radio|listbox|combo|rectangle|line)\d*$', basename):
+                continue
+            tpl_fields.append(f)
+            
+        print(f"[DATA LAKE] Junk Filter: compressed {len(raw_fields)} raw structural fields down to {len(tpl_fields)} meaningful targets.")
 
     # --- THE MAGIC BRIDGE: AI Semantic Mapper ---
     from src.llm import LLM
@@ -297,64 +333,4 @@ def generate_narrative(incident_id: str, db: Session = Depends(get_db)):
         "narrative": narrative,
         "format": "markdown",
         "generated_at": datetime.utcnow().isoformat()
-    }
-
-
-# ── Vision Model Endpoints ──────────────────────────────
-
-from fastapi import UploadFile, File
-
-@router.post("/{incident_id}/scene-photo")
-async def add_scene_photo(
-    incident_id: str,
-    file: UploadFile = File(...),
-    db: Session = Depends(get_db)
-):
-    """
-    Multimodal Vision Integration.
-    Upload a scene photo to the incident. Uses Gemma 3 vision model to generate
-    a professional narrative of the scene and merges it into the Data Lake Master JSON.
-    """
-    if not incident_id:
-        raise AppError("Missing incident_id", status_code=400)
-        
-    incident = get_incident(db, incident_id)
-    if not incident:
-        # Auto-create if it doesn't exist
-        incident = IncidentMasterData(
-            incident_id=incident_id,
-            master_json="{}",
-            transcript_text="[VISION] Image submitted."
-        )
-        create_incident(db, incident)
-
-    import base64
-    image_bytes = await file.read()
-    
-    # Run the vision model (async)
-    from src.llm import LLM
-    llm = LLM()
-    try:
-        scene_narrative = await llm.async_vision_describe_scene(image_bytes)
-        print(f"[VISION] Scene Narrative generated: {scene_narrative[:50]}...")
-    except Exception as e:
-        raise AppError(f"Vision model failed: {e}", status_code=500)
-    
-    # Update the data lake using the smart consensus merge
-    new_data = {
-        "scene_visual_analysis": scene_narrative
-    }
-    
-    update_incident_json(
-        db, 
-        incident_id, 
-        new_data, 
-        new_transcript=f"[VISUAL OBSERVATION]: {scene_narrative}"
-    )
-
-    return {
-        "incident_id": incident_id,
-        "status": "photo_analyzed_and_merged",
-        "narrative_preview": scene_narrative[:100] + "...",
-        "message": "Scene analysis merged into master data lake."
     }

api/routes/templates.py

@@ -220,74 +220,105 @@ def _scan_fields_with_tesseract(pdf_path: str, dpi: int = 200) -> list[dict]:
             lt = " ".join(w["text"] for w in lw)
             print(f"  line {idx:02d}: '{lt[:90]}'")
 
-        # ── Identical field detection logic as PyMuPDF path ─────────────
+        # ── Enhanced field detection: multi-colon + numbered patterns ───────
+        # Handles forms like ICS-214 where:
+        #   - Multiple "Label: [blank] Label: [blank]" appear on one line
+        #   - Numbered fields "1. Incident Name" have no colon but ARE fillable
         for line_words in lines:
             line_words = sorted(line_words, key=lambda w: w["x0"])
-            full_text = " ".join(w["text"] for w in line_words)
+            full_text  = " ".join(w["text"] for w in line_words)
 
             has_colon       = ":" in full_text
             has_underscores = "_" * 4 in full_text or full_text.count("_") >= 4
 
-            if not (has_colon or has_underscores):
-                continue
-
-            # Skip header lines with content after colon
-            if has_colon and not has_underscores:
-                colon_pos = full_text.find(":")
-                after_colon = full_text[colon_pos + 1:].strip()
-                if after_colon and not after_colon.startswith("_"):
-                    label_part = full_text[:colon_pos].strip().lower()
-                    HEADER_WORDS = {"title", "form", "section", "page", "date", "version", "revision"}
-                    if any(hw in label_part for hw in HEADER_WORDS):
-                        continue
-
-            label_text    = full_text.strip()
-            answer_x: float | None = None
             answer_y      = min(w["y0"] for w in line_words)
             answer_bottom = max(w["y1"] for w in line_words)
 
+            # ── Strategy A: UNDERSCORE blanks ───────────────────────────
             if has_underscores:
                 for w in line_words:
                     if "_" * 4 in w["text"]:
-                        answer_x = w["x0"]
-                        answer_bottom = w["y1"]
+                        answer_x   = w["x0"]
                         label_parts = [lw["text"] for lw in line_words if lw["x0"] < w["x0"]]
-                        if label_parts:
-                            label_text = " ".join(label_parts)
-                        break
-
-            if answer_x is None and has_colon:
-                for w in line_words:
-                    if ":" in w["text"]:
-                        answer_x = w["x1"] + 2
-                        label_parts = []
-                        for lw in line_words:
-                            label_parts.append(lw["text"])
-                            if ":" in lw["text"]:
-                                break
-                        label_text = " ".join(label_parts)
-                        break
-
-            if answer_x is None:
-                answer_x = line_words[-1]["x1"] + 2
-
-            clean_label = re.sub(r'^[\d\.\)\s]+', '', label_text)
-            clean_label = clean_label.rstrip(': _\n').strip()
-
-            if not clean_label:
-                continue
+                        label_text  = " ".join(label_parts) if label_parts else full_text
+                        clean_label = re.sub(r'^[\d\.\)\s]+', '', label_text).rstrip(': _\n').strip()
+                        if clean_label:
+                            found_fields.append({
+                                "raw_label": clean_label, "answer_x": round(answer_x, 2),
+                                "answer_y": round(answer_y, 2), "answer_bottom": round(w["y1"], 2),
+                                "page_num": page_num, "page_w": round(pw_pt, 2), "page_h": round(ph_pt, 2),
+                                "line_text": full_text.strip(),
+                            })
+                            print(f"  [FIELD] '{clean_label:25s}' answer_x={answer_x:.1f}pt  y0={answer_y:.1f}pt")
+                continue   # underscore lines done
+
+            # ── Strategy B: ALL colons in one line → one field each ──────
+            if has_colon:
+                HEADER_WORDS = {"title", "form", "section", "page", "version", "revision", "ics", "noaa"}
+                # Find every word that contains ":" and treat each as a field label endpoint
+                colon_words = [i for i, w in enumerate(line_words) if ":" in w["text"]]
+                for ci, colon_idx in enumerate(colon_words):
+                    # Label = words from previous colon_word+1 up to this colon_word (inclusive)
+                    prev_end = colon_words[ci - 1] + 1 if ci > 0 else 0
+                    label_parts = [line_words[j]["text"] for j in range(prev_end, colon_idx + 1)]
+                    label_text  = " ".join(label_parts)
+                    clean_label = re.sub(r'^[\d\.\)\s/]+', '', label_text).rstrip(': _\n').strip()
+
+                    if not clean_label or len(clean_label) < 2:
+                        continue
+                    if any(hw in clean_label.lower() for hw in HEADER_WORDS):
+                        continue
 
-            found_fields.append({
-                "raw_label":     clean_label,
-                "answer_x":      round(answer_x, 2),
-                "answer_y":      round(answer_y, 2),
-                "answer_bottom": round(answer_bottom, 2),
-                "page_num":      page_num,
-                "page_w":        round(pw_pt, 2),
-                "page_h":        round(ph_pt, 2),
-                "line_text":     full_text.strip(),
-            })
-            print(f"  [FIELD] '{clean_label:25s}' answer_x={answer_x:.1f}pt  y0={answer_y:.1f}pt")
+                    # answer_x = right edge of the colon word + small padding
+                    answer_x = line_words[colon_idx]["x1"] + 2
+
+                    # answer width = distance to next colon word (or page edge)
+                    if ci + 1 < len(colon_words):
+                        next_label_start = line_words[colon_words[ci + 1] - 1]["x0"] if colon_words[ci + 1] > 0 else pw_pt
+                        field_w = max(next_label_start - answer_x - 4, 20)
+                    else:
+                        field_w = pw_pt * 0.9 - answer_x
+
+                    found_fields.append({
+                        "raw_label": clean_label, "answer_x": round(answer_x, 2),
+                        "answer_y": round(answer_y, 2), "answer_bottom": round(answer_bottom, 2),
+                        "page_num": page_num, "page_w": round(pw_pt, 2), "page_h": round(ph_pt, 2),
+                        "line_text": full_text.strip(),
+                    })
+                    print(f"  [FIELD] '{clean_label:25s}' answer_x={answer_x:.1f}pt  y0={answer_y:.1f}pt")
+                continue   # colon lines done — all colons processed
+
+            # ── Strategy C: Numbered fields "1. Incident Name" (no colon) ──
+            # ICS-214 has headers like "1. Incident Name 2. Operational Period"
+            # Find all numbered tokens, split the line at each, create one field per segment
+            numbered_positions = [(m.start(), m.group()) for m in re.finditer(r'(?<!\d)\d+\.(?!\d)', full_text)]
+            if len(numbered_positions) >= 1:
+                # Split full_text into segments between numbered markers
+                segments = []
+                for si, (start, _) in enumerate(numbered_positions):
+                    end = numbered_positions[si + 1][0] if si + 1 < len(numbered_positions) else len(full_text)
+                    segments.append(full_text[start:end].strip())
+
+                for seg in segments:
+                    # The answer area is to the RIGHT of this label segment's last word in the line
+                    seg_clean = re.sub(r'^\d+\.\s*', '', seg).strip().rstrip(': _').strip()
+                    if not seg_clean or len(seg_clean) < 2:
+                        continue
+                    # Find x position of the last word in this segment
+                    seg_words = [w for w in line_words if w["text"] in seg.split()]
+                    if seg_words:
+                        last_seg_word = max(seg_words, key=lambda w: w["x1"])
+                        answer_x = last_seg_word["x1"] + 4
+                    else:
+                        answer_x = line_words[-1]["x1"] + 4
+
+                    found_fields.append({
+                        "raw_label": seg_clean, "answer_x": round(answer_x, 2),
+                        "answer_y": round(answer_y, 2), "answer_bottom": round(answer_bottom, 2),
+                        "page_num": page_num, "page_w": round(pw_pt, 2), "page_h": round(ph_pt, 2),
+                        "line_text": full_text.strip(),
+                    })
+                    print(f"  [FIELD] '{seg_clean:25s}' answer_x={answer_x:.1f}pt  y0={answer_y:.1f}pt")
 
     doc.close()
     print(f"[TESSERACT] Done — {len(found_fields)} field(s) found")
@@ -467,9 +498,14 @@ async def scan_static_template(template_id: int, db: Session = Depends(get_db)):
             })
         print(f"[SCAN] ✅ {scan_mode} → {len(found_fields)} fields with exact coords")
 
-    # ── PATH 3: Gemma Vision — last resort only ───────────────────────────
-    if not found_fields:
-        print("[SCAN] PATH 3: Gemma Vision (last resort — both deterministic paths failed)...")
+    # ── PATH 3: Gemma Vision — true last resort only ──────────────────────
+    # Only fire if BOTH deterministic paths found absolutely nothing.
+    # Gemma Vision is slow (~5-10 min) and coordinates can be imprecise
+    # (off-page for complex table forms). Use only when Tesseract gets 0 fields.
+    MIN_FIELDS_THRESHOLD = 0
+    if len(found_fields) <= MIN_FIELDS_THRESHOLD:
+        reason = "0 fields found" if not found_fields else f"only {len(found_fields)} fields found (table form needs Gemma vision)"
+        print(f"[SCAN] PATH 3: Gemma Vision ({reason})...")
         scan_mode = "vision"
         llm = LLM()
         try:

src/filler.py

@@ -349,24 +349,35 @@ def fill_static_pdf(self, pdf_form: str, coordinates: list, data: dict) -> str:
                 # Convert percentages (0-100) back to absolute PDF points
                 x_pts = (coord.x / 100.0) * page_w
                 y_pts_from_top = (coord.y / 100.0) * page_h
+                w_pts = (coord.width / 100.0) * page_w
 
                 # CRITICAL: reportlab uses BOTTOM-LEFT origin
-                # pdfplumber/our DB uses TOP-LEFT origin
-                # So we need to flip the Y axis
                 y_reportlab = page_h - y_pts_from_top
 
-                # Draw the text slightly above the baseline
-                # (y_reportlab points to the TOP of the line; text needs
-                #  to sit on the baseline which is near the bottom of the line)
-                font_size = 10
-                h_pts = (coord.height / 100.0) * page_h
-                text_y = y_reportlab - h_pts + 2  # near bottom of field area
-
-                c.setFontSize(font_size)
-                c.drawString(x_pts, text_y, val_str)
+                # Use Paragraph to support word-wrapping and newlines natively
+                from reportlab.platypus import Paragraph
+                from reportlab.lib.styles import getSampleStyleSheet
+                style = getSampleStyleSheet()["Normal"]
+                style.fontName = "Helvetica"
+                style.fontSize = 10
+                style.leading = 12
+                
+                # If width is 0 or extremely small due to edge scanning, give it a default reasonable width
+                avail_w = w_pts if w_pts > 40 else 250
+
+                html_val = val_str.replace("\n", "<br/>")
+                p = Paragraph(html_val, style)
+                
+                # Wrap computes how much actual box height (bh) the text needs
+                bw, bh = p.wrap(avail_w, page_h)  
+                
+                # drawOn places the very bottom of the paragraph. 
+                # Since y_reportlab is the top of our field boundary, we subtract bh to anchor it below.
+                draw_y = y_reportlab - bh
+                p.drawOn(c, x_pts, draw_y)
 
                 fields_filled += 1
-                print(f"  [OVERLAY] '{coord.field_label}' → '{val_str}' at ({x_pts:.1f}, {text_y:.1f})")
+                print(f"  [OVERLAY] '{coord.field_label}' (WRAP w={avail_w:.0f}pt) → {val_str[:30]}... at ({x_pts:.1f}, {draw_y:.1f})")
 
             c.save()
             packet.seek(0)