Add figure quality gate and reject bad crops

Author: 917Dhj

references/figure-placement.md

@@ -45,6 +45,21 @@ Do not let scripts make the final semantic choice; scripts should only prepare c
 - Do not produce a text-only note first and then ask the user in a follow-up whether figures should be inserted.
 - If no figure can be confidently replaced, finish the note with placeholders and explain that outcome in the final response.
 
+## Visual Quality Gate
+
+Figure/table insertion has two separate gates:
+- identity match: the candidate label, caption, and local context match the planned figure/table
+- visual usability: the crop actually contains the visual body needed by the reader
+
+A label or caption match is not insertion approval.
+Fail closed when visual usability is weak: keep the placeholder instead of inserting the candidate.
+
+Reject candidates that are:
+- caption-only crops
+- tables with no visible table body
+- large text, title-page, or abstract crops masquerading as figures
+- crops where the visual body is tiny relative to the crop
+
 ## Placeholder Requirements
 
 Each placeholder should include:

references/final-writing.md

@@ -209,6 +209,10 @@ The note should preserve the full figure/table structure even when image extract
 If the bundle contains candidate figure pages or candidate image files:
 - use them as evidence for semantic matching
 - prefer the candidate with the strongest caption/page-context agreement
+- treat identity match and visual usability as separate gates
+- never treat a matching label or caption as sufficient approval to insert an image
+- reject caption-only crops, missing table bodies, large text/title/abstract crops, and crops with very low visual body ratio
+- if visual quality is missing, ambiguous, or failed, keep the placeholder
 - still make the final decision yourself rather than trusting the candidate ranking blindly
 
 Final-note figure rules:

references/model-synthesis.md

@@ -34,7 +34,10 @@ The rule is simple:
    If `pdf_assets` is present in the bundle, use it for semantic figure selection:
    - inspect page-level image metadata
    - inspect figure candidate pages and candidate images from `figure_plan`
+   - inspect `figure_assets[].quality_signals` when available
    - match likely figures by page proximity, caption context, and candidate snippets
+   - require both identity match and visual usability before insertion
+   - fail closed on caption-only crops, missing table bodies, large text/title/abstract crops, or very low visual body ratio
    - keep the final semantic matching decision on the model side
    Build the note in placeholder-first order:
    - plan placeholders for all major figures/tables that matter to the note

scripts/build_synthesis_bundle.py

@@ -133,6 +133,29 @@ def sanitize_page_assets(assets_wrapper: dict, *, limit: int = 24) -> list[dict]
     return sanitized
 
 
+def sanitize_figure_assets(assets_wrapper: dict, *, limit: int = 48) -> list[dict]:
+    sanitized: list[dict] = []
+    for item in (assets_wrapper.get("figure_assets", []) or [])[:limit]:
+        if not isinstance(item, dict):
+            continue
+        record = {
+            "filename": item.get("filename", ""),
+            "path": item.get("path", ""),
+            "page_number": item.get("page_number", 0),
+            "label": item.get("label", ""),
+            "kind": item.get("kind", ""),
+            "caption_text": normalize_whitespace(str(item.get("caption_text", ""))),
+            "width": item.get("width", 0),
+            "height": item.get("height", 0),
+            "size_bytes": item.get("size_bytes", 0),
+            "extraction_level": item.get("extraction_level", ""),
+        }
+        if isinstance(item.get("quality_signals"), dict):
+            record["quality_signals"] = item.get("quality_signals")
+        sanitized.append(record)
+    return sanitized
+
+
 def bundle(metadata: dict, evidence_wrapper: dict, figures_wrapper: dict, assets_wrapper: dict) -> dict:
     evidence_pack = evidence_wrapper.get("evidence_pack", {}) if isinstance(evidence_wrapper.get("evidence_pack"), dict) else {}
     figure_plan = figures_wrapper.get("figure_plan", {}) if isinstance(figures_wrapper.get("figure_plan"), dict) else {}
@@ -177,6 +200,7 @@ def bundle(metadata: dict, evidence_wrapper: dict, figures_wrapper: dict, assets
             "images_dir": assets_wrapper.get("images_dir", ""),
             "page_assets": sanitize_page_assets(assets_wrapper),
             "image_assets": assets_wrapper.get("image_assets", []),
+            "figure_assets": sanitize_figure_assets(assets_wrapper),
             "ocr_available": assets_wrapper.get("ocr_available", False),
         },
         "summary": evidence_wrapper.get("summary", {}),
@@ -340,6 +364,9 @@ def bundle(metadata: dict, evidence_wrapper: dict, figures_wrapper: dict, assets
             },
             "figure_rules": [
                 "先规划主要图表的占位标签，再决定哪些可以替换成真实图片",
+                "label/caption match is not insertion approval; it only proves identity, and visual usability is a separate hard gate.",
+                "Reject caption-only crops, missing table body crops, large text/title/abstract crops, and low visual body ratio crops.",
+                "If visual quality is missing, ambiguous, or marked review/reject, fail closed: keep the placeholder instead of inserting the image.",
                 "如果没有高置信度图像匹配，不要删除占位标签",
                 "图注必须保留论文原始编号，例如 Fig. 1、Table 2",
                 "如果插入的是局部子图或不完整裁剪，必须明确说明",

scripts/contracts.py

@@ -45,6 +45,29 @@ class EquationCandidate(TypedDict, total=False):
     kind_hint: str
 
 
+class FigureQualitySignals(TypedDict, total=False):
+    visual_quality_status: str
+    quality_reason_codes: list[str]
+    page_coverage_ratio: float
+    visual_rect_count: int
+    visual_body_ratio: float
+    paragraph_text_chars: int
+    table_body_rows: int
+    caption_text_chars: int
+
+
+class FigureAssetCandidate(TypedDict, total=False):
+    filename: str
+    path: str
+    width: int
+    height: int
+    size_bytes: int
+    label: str
+    extraction_level: str
+    quality_signals: FigureQualitySignals
+    candidate_status: str
+
+
 class EvidencePack(TypedDict, total=False):
     paper_id: str
     problem_evidence: list[EvidenceItem]
@@ -75,8 +98,9 @@ class FigurePlanItem(TypedDict, total=False):
     priority: int
     anchor_text: str
     insert_mode: str
-    figure_asset_candidate: dict[str, Any]
+    figure_asset_candidate: FigureAssetCandidate
     candidate_pages: list[dict[str, Any]]
+    candidate_status: str
     matching_strategy: str
 
 

scripts/extract_pdf_assets.py

@@ -60,6 +60,80 @@ def _looks_like_data_row(text: str) -> bool:
     return numeric_tokens >= max(2, len(tokens) // 2)
 
 
+def _rect_area(bbox: tuple[float, float, float, float]) -> float:
+    return max(0.0, bbox[2] - bbox[0]) * max(0.0, bbox[3] - bbox[1])
+
+
+def _intersection_area(
+    a: tuple[float, float, float, float],
+    b: tuple[float, float, float, float],
+) -> float:
+    x0 = max(a[0], b[0])
+    y0 = max(a[1], b[1])
+    x1 = min(a[2], b[2])
+    y1 = min(a[3], b[3])
+    return _rect_area((x0, y0, x1, y1))
+
+
+def _rects_intersect(
+    a: tuple[float, float, float, float],
+    b: tuple[float, float, float, float],
+) -> bool:
+    return _intersection_area(a, b) > 0
+
+
+def _classify_visual_quality(
+    *,
+    kind: str,
+    page_coverage_ratio: float,
+    visual_rect_count: int,
+    visual_body_ratio: float,
+    paragraph_text_chars: int,
+    table_body_rows: int,
+    caption_text_chars: int,
+) -> dict:
+    """Classify whether a caption-matched crop is visually usable.
+
+    This is intentionally conservative. A label/caption match proves identity,
+    but not that the rendered crop contains the figure or table body.
+    """
+    normalized_kind = kind.strip().lower()
+    reasons: list[str] = []
+
+    if normalized_kind == "table":
+        if table_body_rows <= 0:
+            reasons.append("table_body_missing")
+        if table_body_rows <= 1 and visual_body_ratio < 0.03 and caption_text_chars >= 40:
+            reasons.append("caption_only_suspected")
+        status = "reject" if reasons else "usable"
+    else:
+        if paragraph_text_chars >= 450:
+            reasons.append("large_text_block_suspected")
+        if page_coverage_ratio >= 0.70 and paragraph_text_chars >= 250:
+            reasons.append("oversized_page_crop")
+        if visual_rect_count <= 1 and visual_body_ratio < 0.03:
+            reasons.append("low_visual_body_ratio")
+        if any(code in reasons for code in ("large_text_block_suspected", "oversized_page_crop", "low_visual_body_ratio")):
+            status = "reject"
+        elif visual_rect_count == 0 or visual_body_ratio < 0.08:
+            if "low_visual_body_ratio" not in reasons:
+                reasons.append("low_visual_body_ratio")
+            status = "review"
+        else:
+            status = "usable"
+
+    return {
+        "visual_quality_status": status,
+        "quality_reason_codes": reasons,
+        "page_coverage_ratio": round(page_coverage_ratio, 6),
+        "visual_rect_count": int(visual_rect_count),
+        "visual_body_ratio": round(visual_body_ratio, 6),
+        "paragraph_text_chars": int(paragraph_text_chars),
+        "table_body_rows": int(table_body_rows),
+        "caption_text_chars": int(caption_text_chars),
+    }
+
+
 def _classify_caption_kind(label: str) -> str:
     """Return 'table' if the caption label starts with 'Table', else 'figure'."""
     return "table" if label.strip().lower().startswith("table") else "figure"
@@ -246,6 +320,23 @@ def _collect_drawing_rects(page) -> list[tuple[float, float, float, float]]:
     return rects
 
 
+def _visual_signal_for_bbox(page, bbox: tuple[float, float, float, float]) -> tuple[int, float]:
+    """Return visual rect count and visual-area ratio inside a crop."""
+    crop_area = _rect_area(bbox)
+    if crop_area <= 0:
+        return 0, 0.0
+    rects = _collect_xref_rects(page) + _collect_drawing_rects(page)
+    count = 0
+    visual_area = 0.0
+    for rect in rects:
+        area = _intersection_area(rect, bbox)
+        if area <= 0:
+            continue
+        count += 1
+        visual_area += area
+    return count, min(1.0, visual_area / crop_area)
+
+
 def _find_body_text_blocks(page) -> list[tuple[float, float, float, float, str]]:
     """Return bounding boxes of body-text blocks (non-caption) sorted top-to-bottom."""
     results: list[tuple[float, float, float, float, str]] = []
@@ -308,6 +399,67 @@ def _find_paragraph_blocks(page, *, min_chars: int = 200) -> list[tuple[float, f
     return results
 
 
+def _count_paragraph_text_chars_in_bbox(
+    page,
+    bbox: tuple[float, float, float, float],
+    caption_bbox: tuple[float, float, float, float],
+) -> int:
+    """Count prose-like text intersecting a crop, excluding the caption area."""
+    chars = 0
+    blocks = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE)["blocks"]
+    for block in blocks:
+        if block.get("type") != 0:
+            continue
+        bb = tuple(block["bbox"])
+        if not _rects_intersect(bb, bbox):
+            continue
+        if _intersection_area(bb, caption_bbox) / max(_rect_area(bb), 1.0) > 0.6:
+            continue
+
+        lines = block.get("lines", [])
+        line_texts = [
+            "".join(s.get("text", "") for s in line.get("spans", [])).strip()
+            for line in lines
+        ]
+        line_texts = [text for text in line_texts if text]
+        full_text = normalize_whitespace(" ".join(line_texts))
+        if len(full_text) < 80:
+            continue
+        if CAPTION_RE.match(full_text):
+            continue
+        numeric_rows = sum(1 for text in line_texts if _looks_like_data_row(text))
+        if line_texts and numeric_rows > len(line_texts) * 0.4:
+            continue
+        chars += len(full_text)
+    return chars
+
+
+def _quality_signals_for_crop(
+    page,
+    kind: str,
+    bbox: tuple[float, float, float, float],
+    caption_anchor: dict,
+    page_rect,
+    *,
+    table_body_rows: int,
+) -> dict:
+    page_area = _rect_area((page_rect.x0, page_rect.y0, page_rect.x1, page_rect.y1))
+    page_coverage_ratio = _rect_area(bbox) / page_area if page_area > 0 else 0.0
+    visual_rect_count, visual_body_ratio = _visual_signal_for_bbox(page, bbox)
+    caption_bbox = tuple(caption_anchor["bbox"])
+    paragraph_text_chars = _count_paragraph_text_chars_in_bbox(page, bbox, caption_bbox)
+    caption_text_chars = len(normalize_whitespace(str(caption_anchor.get("line_text", ""))))
+    return _classify_visual_quality(
+        kind=kind,
+        page_coverage_ratio=page_coverage_ratio,
+        visual_rect_count=visual_rect_count,
+        visual_body_ratio=visual_body_ratio,
+        paragraph_text_chars=paragraph_text_chars,
+        table_body_rows=table_body_rows,
+        caption_text_chars=caption_text_chars,
+    )
+
+
 def _clip_to_page(
     bbox: tuple[float, float, float, float],
     page_rect,
@@ -594,6 +746,17 @@ def _estimate_table_bbox(
     next_anchor: dict | None,
     page_rect,
 ) -> tuple[float, float, float, float] | None:
+    result = _estimate_table_bbox_with_rows(page, caption_anchor, prev_anchor, next_anchor, page_rect)
+    return result[0] if result is not None else None
+
+
+def _estimate_table_bbox_with_rows(
+    page,
+    caption_anchor: dict,
+    prev_anchor: dict | None,
+    next_anchor: dict | None,
+    page_rect,
+) -> tuple[tuple[float, float, float, float], int] | None:
     r"""Estimate the bounding box of a table.
 
     Tables in academic papers come in two layouts:
@@ -651,12 +814,18 @@ def _estimate_table_bbox(
         return None
 
     chosen: list[tuple[float, float, float, float]]
+    chosen_data_rows: int
     if up_data > down_data:
         chosen = up_lines
+        chosen_data_rows = up_data
     else:
         chosen = down_lines
+        chosen_data_rows = down_data
 
-    return _finalize_table_bbox(page, caption_anchor, chosen, page_rect)
+    bbox = _finalize_table_bbox(page, caption_anchor, chosen, page_rect)
+    if bbox is None:
+        return None
+    return bbox, chosen_data_rows
 
 
 def _render_crop(page, bbox: tuple[float, float, float, float], dpi: int) -> bytes:
@@ -688,8 +857,13 @@ def extract_figure_regions(
         kind = anchor.get("kind", "figure")
 
         bbox: tuple[float, float, float, float] | None
+        table_body_rows = 0
         if kind == "table":
-            bbox = _estimate_table_bbox(page, anchor, prev_anchor, next_anchor, page_rect)
+            table_result = _estimate_table_bbox_with_rows(page, anchor, prev_anchor, next_anchor, page_rect)
+            if table_result is not None:
+                bbox, table_body_rows = table_result
+            else:
+                bbox = None
             if bbox is None:
                 # Fall back to the figure-shape estimator in case the table is
                 # actually rendered as an embedded image.
@@ -716,6 +890,14 @@ def extract_figure_regions(
 
         width_px = int((bbox[2] - bbox[0]) * dpi / 72.0)
         height_px = int((bbox[3] - bbox[1]) * dpi / 72.0)
+        quality_signals = _quality_signals_for_crop(
+            page,
+            kind,
+            bbox,
+            anchor,
+            page_rect,
+            table_body_rows=table_body_rows,
+        )
 
         assets.append(
             {
@@ -731,6 +913,7 @@ def extract_figure_regions(
                 "bbox_pt": list(bbox),
                 "size_bytes": len(png_bytes),
                 "extraction_level": "figure",
+                "quality_signals": quality_signals,
             }
         )