fix(evals): report citation support as unmeasured rather than zero when verification is off

Author: codebybilal18

app/evals/graders/deterministic.py

@@ -65,7 +65,7 @@ def grade_refusal(question: GoldenQuestion, response: dict) -> RefusalGrade:
 @dataclass
 class CitationGrade:
     validity: float
-    support: float
+    support: float | None
     citation_count: int
     invalid_count: int
     unsupported_count: int
@@ -99,7 +99,14 @@ def grade_citations(response: dict) -> CitationGrade:
         # where refusals are excluded from this metric.
         validity = 1.0 if verification.get("is_refusal") else 0.0
 
-    support = float(verification.get("citation_coverage", 0.0))
+    # `citation_coverage` counts an unverified claim as unsupported, which is
+    # correct at runtime but wrong as a *measurement* when verification was
+    # switched off for the config: every claim comes back UNVERIFIED and the
+    # metric reads 0.000, indistinguishable from "nothing was supported". None
+    # means not measured, and the aggregate skips it.
+    support = (
+        float(verification.get("citation_coverage", 0.0)) if verification.get("verified") else None
+    )
     unsupported = len(verification.get("unsupported_claims", []))
 
     return CitationGrade(

tests/test_retrieval_logic.py

@@ -204,3 +204,60 @@ def test_meter_breaks_cost_down_by_stage():
     by_stage = meter.cost_by_label()
     assert set(by_stage) == {"generation", "embed"}
     assert meter.total_cost_usd == pytest.approx(0.40 + 0.02)
+
+
+# --- unmeasured vs zero -----------------------------------------------------
+
+
+def test_citation_support_is_none_when_verification_was_disabled():
+    """Configs A-C run with verification off. `citation_coverage` counts every
+    unverified claim as unsupported, so the metric came out 0.000 — visually
+    identical to "none of the citations were supported", when the truth is that
+    nothing was measured. A published table cannot carry that ambiguity.
+    """
+    from app.evals.graders.deterministic import grade_citations
+
+    unverified = grade_citations(
+        {
+            "verification": {
+                "verified": False,
+                "citations": [{"label": "C1", "resolved": True}],
+                "invalid_labels": [],
+                "claims": [{"claim": "x", "verdict": "unverified"}],
+                "citation_coverage": 0.0,
+            }
+        }
+    )
+    assert unverified.support is None
+    assert unverified.validity == 1.0  # resolution still measured; it is free
+
+    verified = grade_citations(
+        {
+            "verification": {
+                "verified": True,
+                "citations": [{"label": "C1", "resolved": True}],
+                "invalid_labels": [],
+                "claims": [{"claim": "x", "verdict": "yes"}],
+                "citation_coverage": 1.0,
+            }
+        }
+    )
+    assert verified.support == 1.0
+
+
+def test_a_genuinely_unsupported_answer_still_scores_zero():
+    """The None must mean 'not measured', never 'measured as bad'."""
+    from app.evals.graders.deterministic import grade_citations
+
+    graded = grade_citations(
+        {
+            "verification": {
+                "verified": True,
+                "citations": [{"label": "C1", "resolved": True}],
+                "invalid_labels": [],
+                "claims": [{"claim": "x", "verdict": "no"}],
+                "citation_coverage": 0.0,
+            }
+        }
+    )
+    assert graded.support == 0.0