fix(evaluation): 收紧报告脱敏与遥测完整性

Author: guocfu

examples/optimization/eval_optimize_loop/report_builder.py

@@ -24,13 +24,15 @@
 _MISSING_TOKEN_REASON = (
     "Reflection LM calls were observed but optimizer token usage was not reported."
 )
+_INVALID_TOKEN_REASON = "Optimizer token usage was malformed or inconsistent."
 _REDACTED = "[REDACTED]"
 _SENSITIVE_ENV_NAMES = ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL")
 _SENSITIVE_KEY_VALUE = re.compile(
-    r"(?P<prefix>[\"']?(?:api_?key|base_?url|authorization)[\"']?\s*[:=]\s*)"
+    r"(?P<prefix>[\"']?(?:api[_-]?key|base[_-]?url|authorization)[\"']?\s*[:=]\s*)"
     r"(?P<value>[\"'][^\"']*[\"']|(?:(?:bearer|basic|token)\s+)?[^\s,;}\]]+)",
     re.IGNORECASE,
 )
+_HTTP_URL = re.compile(r"https?://[^\s,;}\]<>\"']+", re.IGNORECASE)
 _BEARER_VALUE = re.compile(
     r"\bbearer(?:\s+|\s*[:=]\s*)[\"']?[^\s,;}\]\"']+[\"']?",
     re.IGNORECASE,
@@ -60,7 +62,19 @@ def _redact_error_message(error: Exception) -> str:
         lambda match: f"{match.group('prefix')}{_REDACTED}",
         message,
     )
-    return _BEARER_VALUE.sub(f"Bearer {_REDACTED}", message)
+    message = _BEARER_VALUE.sub(f"Bearer {_REDACTED}", message)
+    return _HTTP_URL.sub(_REDACTED, message)
+
+
+def _is_complete_token_usage(value: object) -> bool:
+    if not isinstance(value, dict):
+        return False
+    required = ("prompt", "completion", "total")
+    if not all(key in value for key in required):
+        return False
+    if not all(type(value[key]) is int and value[key] >= 0 for key in required):
+        return False
+    return value["total"] == value["prompt"] + value["completion"]
 
 
 def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObservation:
@@ -76,8 +90,12 @@ def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObserv
     native = result.optimize_result
     reflection_calls = native.total_reflection_lm_calls
     cost_missing = reflection_calls > 0 and native.total_llm_cost <= 0
-    token_usage = dict(native.total_token_usage)
-    tokens_missing = reflection_calls > 0 and token_usage.get("total", 0) <= 0
+    token_usage = native.total_token_usage
+    token_usage_valid = _is_complete_token_usage(token_usage)
+    tokens_missing = (
+        not token_usage_valid
+        or (reflection_calls > 0 and token_usage["total"] <= 0)
+    )
     return OptimizerResourceObservation(
         scope_note=_OPTIMIZER_SCOPE,
         total_rounds=OptimizerResourceValue[int](
@@ -96,7 +114,11 @@ def _optimizer_resources(result: PipelineStageResult) -> OptimizerResourceObserv
             status="unavailable" if tokens_missing else "available",
             value=None if tokens_missing else token_usage,
             unit="tokens",
-            reason=_MISSING_TOKEN_REASON if tokens_missing else None,
+            reason=(
+                _INVALID_TOKEN_REASON
+                if tokens_missing and not token_usage_valid
+                else _MISSING_TOKEN_REASON if tokens_missing else None
+            ),
         ),
         duration_seconds=OptimizerResourceValue[float](
             status="available", value=native.duration_seconds, unit="seconds",

tests/evaluation/test_eval_optimize_loop_stage5_report_builder.py

@@ -227,6 +227,100 @@ async def test_failure_report_redacts_environment_values_and_common_secret_forms
     assert "retryable=true" in report.error_message
 
 
+def test_failure_report_redacts_hyphenated_keys_and_bare_urls(tmp_path):
+    root = _copy_example(tmp_path)
+    prepared = prepare_run(root / "pipeline.json", run_id="report_failure_hyphenated")
+    error = RuntimeError(
+        "X-API-Key: hyphen-api-secret x-api-key=lower-hyphen-secret; "
+        "endpoint https://private.example/v1?token=secret and http://10.0.0.8:8080/run"
+    )
+
+    report = build_failure_report(
+        prepared, progress=_progress(), error=error,
+        source_prompt_hashes={}, existing_artifacts=[],
+        generated_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc),
+    )
+
+    assert "hyphen-api-secret" not in report.error_message
+    assert "lower-hyphen-secret" not in report.error_message
+    assert "https://private.example/v1?token=secret" not in report.error_message
+    assert "http://10.0.0.8:8080/run" not in report.error_message
+    assert "endpoint [REDACTED]" in report.error_message
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "token_usage",
+    [
+        {},
+        {"prompt": 1, "completion": 2},
+        {"prompt": 1, "completion": "2", "total": 3},
+        {"prompt": True, "completion": 2, "total": 3},
+        {"prompt": -1, "completion": 2, "total": 1},
+        {"prompt": 1, "completion": 2, "total": 4},
+        ["not", "a", "dict"],
+    ],
+)
+async def test_real_report_marks_malformed_token_usage_unavailable(tmp_path, token_usage):
+    root = _copy_example(tmp_path)
+    prepared = prepare_run(root / "pipeline.json", run_id="report_real_bad_tokens")
+    fake_result = await run_fake_stage(prepared, scenario="improve")
+    candidate = OptimizerCandidateProposal.model_validate({
+        **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer",
+        "optimizer_status": "SUCCEEDED", "finish_reason": "completed",
+        "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0,
+        "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}",
+    })
+    native = _optimize_result(
+        await prepared.source_target.read_all(), candidate.prompts,
+    ).model_copy(update={"total_token_usage": token_usage})
+    result = RealStageResult(
+        **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate,
+        optimize_result=native,
+    )
+
+    report = build_optimization_report(
+        prepared, result, progress=_progress(),
+        finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc),
+    )
+
+    assert report.optimizer_resources.token_usage.status == "unavailable"
+    assert report.optimizer_resources.token_usage.value is None
+
+
+@pytest.mark.asyncio
+async def test_real_report_allows_consistent_zero_token_usage_without_reflection_calls(tmp_path):
+    root = _copy_example(tmp_path)
+    prepared = prepare_run(root / "pipeline.json", run_id="report_real_zero_tokens")
+    fake_result = await run_fake_stage(prepared, scenario="improve")
+    candidate = OptimizerCandidateProposal.model_validate({
+        **fake_result.candidate.model_dump(exclude={"scenario", "seed"}), "provider": "agent_optimizer",
+        "optimizer_status": "SUCCEEDED", "finish_reason": "completed",
+        "baseline_pass_rate": 1 / 3, "best_pass_rate": 1.0,
+        "candidate_id": f"optimizer-{fake_result.candidate.candidate_id[-12:]}",
+    })
+    native = _optimize_result(
+        await prepared.source_target.read_all(), candidate.prompts,
+    ).model_copy(update={
+        "total_reflection_lm_calls": 0,
+        "total_token_usage": {"prompt": 0, "completion": 0, "total": 0, "cached": 7},
+    })
+    result = RealStageResult(
+        **fake_result.model_dump(exclude={"scenario", "candidate"}), candidate=candidate,
+        optimize_result=native,
+    )
+
+    report = build_optimization_report(
+        prepared, result, progress=_progress(),
+        finished_at=datetime(2026, 7, 18, 0, 1, tzinfo=timezone.utc),
+    )
+
+    assert report.optimizer_resources.token_usage.status == "available"
+    assert report.optimizer_resources.token_usage.value == {
+        "prompt": 0, "completion": 0, "total": 0, "cached": 7,
+    }
+
+
 def test_report_dto_invariants_reject_inconsistent_artifact_states():
     with pytest.raises(ValueError, match="available artifacts require"):
         ArtifactReference(