fix(eval): fail closed on incomplete metric evidence

Author: neutronstar238

examples/optimization/eval_optimize_loop/run_pipeline.py

@@ -1931,10 +1931,16 @@ def summarize_evaluate_result(result: Any, evalset_payload: dict[str, Any]) -> d
         actual_text, expected_text = _extract_actual_expected(runs[0], case_by_id[eval_id])
         error_message = None
         for run in runs:
-            run_passed = run_passed and _is_passed_status(run.final_eval_status)
+            run_metrics = list(run.overall_eval_metric_results or [])
+            run_passed = (
+                run_passed
+                and _is_passed_status(run.final_eval_status)
+                and bool(run_metrics)
+                and all(_is_passed_status(metric.eval_status) for metric in run_metrics)
+            )
             if run.error_message and error_message is None:
                 error_message = sanitize_report_text(run.error_message)
-            for metric in run.overall_eval_metric_results:
+            for metric in run_metrics:
                 score = metric.score
                 metric_passed = _is_passed_status(metric.eval_status)
                 details = getattr(metric, "details", None)

tests/evaluation/test_eval_optimize_loop_example.py

@@ -829,6 +829,48 @@ def test_summary_omits_thoughts_and_redacts_provider_credentials_from_report_tex
     assert secret not in serialized_summary
 
 
+def test_summary_fails_closed_when_a_reported_metric_was_not_evaluated():
+    module = load_pipeline_module()
+    payload = load_report(EXAMPLE_DIR / "val.evalset.json")
+    case = payload["eval_cases"][0]
+    run = SimpleNamespace(
+        eval_metric_result_per_invocation=[],
+        final_eval_status="passed",
+        error_message=None,
+        overall_eval_metric_results=[
+            SimpleNamespace(
+                metric_name=ROUTE_TOOL_ARGS_METRIC,
+                score=None,
+                eval_status="not_evaluated",
+                details=None,
+                threshold=1.0,
+            ),
+            SimpleNamespace(
+                metric_name="llm_rubric_response",
+                score=1.0,
+                eval_status="passed",
+                details=None,
+                threshold=0.66,
+            ),
+        ],
+    )
+    result = SimpleNamespace(
+        results_by_eval_set_id={
+            payload["eval_set_id"]: SimpleNamespace(
+                eval_results_by_eval_id={case["eval_id"]: [run]},
+            )
+        }
+    )
+
+    summary = module.summarize_evaluate_result(result, payload)
+    case_result = summary["case_results"][0]
+
+    assert case_result["passed"] is False
+    assert case_result["root_cause"] in module.TAXONOMY
+    assert case_result["reasons"]
+    assert case["eval_id"] in summary["failed_case_ids"]
+
+
 @pytest.mark.parametrize(
     "sensitive_text",
     [