fix(bench): break circular import in predictor/__init__.py (Tracer-Cloud#2802)

* fix(bench): break circular import in predictor/__init__.py
* refactor(bench): extract taxonomy + vocabulary from predictor; add L0 report panel

Author: YauhenBichel

tests/benchmarks/_framework/reporting.py

@@ -226,6 +226,16 @@
 # opensre-only instrumentation. Useful as internal diagnostics, but NOT present
 # in the paper, so they are reported in a separate panel to avoid implying a
 # comparison that doesn't exist.
+# L0 investigation-native metrics (opensre prose, keyword parser). Distinct from
+# L1 ``a1`` which scores the predictor's rank-1 formalization.
+_L0_INVESTIGATION_METRICS = [
+    "investigation_a1",
+    "investigation_partial_a1",
+    "investigation_object_a1",
+    "translation_loss",
+]
+_L0_CI_METRICS = frozenset({"investigation_a1", "investigation_object_a1"})
+
 _OPENSRE_ONLY_METRICS = [
     "partial_a1",
     "partial_a3",
@@ -696,6 +706,92 @@ def _render_decomposition_html(
     return parts
 
 
+def _render_l0_investigation_markdown(
+    by_lm: dict[str, dict[str, list[dict[str, Any]]]],
+) -> list[str]:
+    """L0 panel: investigation-native metrics from opensre prose (not predictor)."""
+    if not by_lm:
+        return []
+    flat = [c for modes in by_lm.values() for cs in modes.values() for c in cs]
+    present = [m for m in _L0_INVESTIGATION_METRICS if _per_cell_metric(flat, m)]
+    if not present:
+        return []
+
+    lines: list[str] = []
+    lines.append("### Investigation quality — L0 (opensre prose, not paper-comparable)")
+    lines.append("")
+    lines.append(
+        "_L0 scores a keyword-parsed triple from opensre's investigation prose "
+        "(``report`` / ``root_cause`` / causal chain). L1 ``a1`` in the headline "
+        "scores the predictor's rank-1 formalization. The gap "
+        "``a1 − investigation_a1`` is translation loss; "
+        "``translation_loss`` flags cases where L0 is right but L1 is wrong._"
+    )
+    lines.append("")
+    header = "| LLM | variant | " + " | ".join(present) + " |"
+    sep = "|" + "|".join(["---"] * (len(present) + 2)) + "|"
+    lines.append(header)
+    lines.append(sep)
+    for llm in sorted(by_lm.keys()):
+        for mode in sorted(by_lm[llm].keys()):
+            mode_cells = by_lm[llm][mode]
+            row = [f"`{llm}`", mode]
+            for metric in present:
+                scen_vals = _scenario_means(mode_cells, metric)
+                mean, lo, hi, n = _mean_with_ci(scen_vals)
+                if metric in _L0_CI_METRICS and len(scen_vals) >= 2:
+                    row.append(f"{mean:.2f} [{lo:.2f}–{hi:.2f}]")
+                elif n:
+                    row.append(f"{mean:.2f}")
+                else:
+                    row.append("—")
+            lines.append("| " + " | ".join(row) + " |")
+    lines.append("")
+    return lines
+
+
+def _render_l0_investigation_html(
+    by_lm: dict[str, dict[str, list[dict[str, Any]]]],
+    esc: Any,
+) -> list[str]:
+    """HTML mirror of :func:`_render_l0_investigation_markdown`."""
+    if not by_lm:
+        return []
+    flat = [c for modes in by_lm.values() for cs in modes.values() for c in cs]
+    present = [m for m in _L0_INVESTIGATION_METRICS if _per_cell_metric(flat, m)]
+    if not present:
+        return []
+
+    parts: list[str] = []
+    parts.append("<h3>Investigation quality — L0 (opensre prose, not paper-comparable)</h3>")
+    parts.append(
+        "<p><small>L0 scores a keyword-parsed triple from opensre's investigation "
+        "prose. L1 <code>a1</code> scores the predictor's rank-1 formalization. "
+        "The gap <code>a1 − investigation_a1</code> is translation loss.</small></p>"
+    )
+    parts.append("<table><thead><tr><th>LLM</th><th>variant</th>")
+    for m in present:
+        parts.append(f"<th>{esc(m)}</th>")
+    parts.append("</tr></thead><tbody>")
+    for llm in sorted(by_lm.keys()):
+        for mode in sorted(by_lm[llm].keys()):
+            mode_cells = by_lm[llm][mode]
+            parts.append(f"<tr><td><code>{esc(llm)}</code></td><td>{esc(mode)}</td>")
+            for metric in present:
+                scen_vals = _scenario_means(mode_cells, metric)
+                mean, lo, hi, n = _mean_with_ci(scen_vals)
+                if metric in _L0_CI_METRICS and len(scen_vals) >= 2:
+                    cell_txt = f"{mean:.2f}<br><small>[{lo:.2f}–{hi:.2f}]</small>"
+                elif n:
+                    cell_txt = f"{mean:.2f}"
+                else:
+                    cell_txt = "—"
+                parts.append(f'<td class="metric">{cell_txt}</td>')
+            parts.append("</tr>")
+    parts.append("</tbody></table>")
+    return parts
+
+
 # --------------------------------------------------------------------------- #
 # Markdown rendering                                                          #
 # --------------------------------------------------------------------------- #
@@ -806,6 +902,9 @@ def _render_markdown(
     # --- Decomposition: where the accuracy goes (Track 2) ---
     lines.extend(_render_decomposition_markdown(cells, by_lm))
 
+    # --- L0 investigation quality (opensre prose — not paper-comparable) ---
+    lines.extend(_render_l0_investigation_markdown(by_lm))
+
     # --- opensre-only diagnostics (NOT in the paper, NOT comparable) ---
     if by_lm:
         flat = [c for modes in by_lm.values() for cs in modes.values() for c in cs]
@@ -1110,6 +1209,9 @@ def esc(s: Any) -> str:
         # Decomposition (Track 2)
         parts.extend(_render_decomposition_html(cells, by_lm, esc))
 
+        # L0 investigation quality
+        parts.extend(_render_l0_investigation_html(by_lm, esc))
+
         # opensre-only diagnostics (segregated — not paper-comparable)
         flat = [c for modes in by_lm.values() for cs in modes.values() for c in cs]
         present = [m for m in _OPENSRE_ONLY_METRICS if _per_cell_metric(flat, m)]

tests/benchmarks/_framework/tests/test_reporting.py

@@ -183,6 +183,31 @@ def test_html_includes_inline_style_no_external_deps(tmp_path: Path) -> None:
     assert "<script src=" not in html
 
 
+def test_markdown_includes_l0_investigation_panel_when_metrics_present(tmp_path: Path) -> None:
+    """L0 investigation metrics surface in report.md when cells record them."""
+    _write_report_json(tmp_path)
+    cases_dir = tmp_path / "cases"
+    cell = {
+        "case": {"case_id": "case-001", "metadata": {"fault_category": "Runtime"}},
+        "run": {"mode": "opensre+llm", "llm": "claude_sonnet"},
+        "score": {
+            "metrics": {
+                "a1": 0.0,
+                "investigation_a1": 0.75,
+                "investigation_object_a1": 0.80,
+                "translation_loss": 0.25,
+            }
+        },
+        "ok": True,
+    }
+    (cases_dir / "case-001__opensre+llm__claude_sonnet__0.json").write_text(json.dumps(cell))
+    out = render_report_dir(tmp_path, formats=["markdown"])
+    md = out["markdown"].read_text()
+    assert "Investigation quality — L0" in md
+    assert "investigation_a1" in md
+    assert "translation_loss" in md
+
+
 def test_render_works_without_cases_directory(tmp_path: Path) -> None:
     """report.json is the source of truth; cases/ is optional."""
     _write_report_json(tmp_path)

tests/benchmarks/cloudopsbench/closed_vocabulary.py

@@ -0,0 +1,141 @@
+"""Closed-vocabulary constants for CloudOpsBench scoring and prediction.
+
+Package-level module (not under ``predictor/``) so ``taxonomy`` and
+``scoring`` can import fault-object lists without loading
+``predictor/__init__.py``. The predictor subpackage re-exports these
+from ``predictor.vocabulary`` for backward compatibility.
+"""
+
+from __future__ import annotations
+
+# Declaring ``__all__`` tells CodeQL that these names are the module's
+# public surface — they are not "unused globals" but intentional
+# re-exports consumed by ``taxonomy``, ``scoring``, and
+# ``predictor.vocabulary`` (which re-exports them for legacy callers).
+__all__ = [
+    "_FAULT_OBJECT_NAMESPACES",
+    "_FAULT_OBJECT_NODES",
+    "_FAULT_OBJECT_SERVICES",
+    "_ROOT_CAUSES",
+    "_TAXONOMY_CATEGORIES",
+]
+
+_TAXONOMY_CATEGORIES: tuple[str, ...] = (
+    "Admission_Fault",
+    "Scheduling_Fault",
+    "Infrastructure_Fault",
+    "Startup_Fault",
+    "Runtime_Fault",
+    "Service_Routing_Fault",
+    "Performance_Fault",
+)
+
+_ROOT_CAUSES: tuple[str, ...] = (
+    # Scheduling
+    "missing_service_account",
+    "node_cordon_mismatch",
+    "node_affinity_mismatch",
+    "node_selector_mismatch",
+    "pod_anti_affinity_conflict",
+    "taint_toleration_mismatch",
+    "cpu_capacity_mismatch",
+    "memory_capacity_mismatch",
+    # Infrastructure
+    "node_network_delay",
+    "node_network_packet_loss",
+    "containerd_unavailable",
+    "kubelet_unavailable",
+    "kube_proxy_unavailable",
+    "kube_scheduler_unavailable",
+    # Startup
+    "image_registry_dns_failure",
+    "incorrect_image_reference",
+    "missing_image_pull_secret",
+    "pvc_selector_mismatch",
+    "pvc_storage_class_mismatch",
+    "pvc_access_mode_mismatch",
+    "pvc_capacity_mismatch",
+    "pv_binding_occupied",
+    "volume_mount_permission_denied",
+    # Runtime
+    "oom_killed",
+    "liveness_probe_incorrect_protocol",
+    "liveness_probe_incorrect_port",
+    "liveness_probe_incorrect_timing",
+    "readiness_probe_incorrect_protocol",
+    "readiness_probe_incorrect_port",
+    "mysql_invalid_credentials",
+    "mysql_invalid_port",
+    "missing_secret_binding",
+    "db_connection_exhaustion",
+    "db_readonly_mode",
+    "gateway_misrouted",
+    "deployment_zero_replicas",
+    # Service routing
+    "service_selector_mismatch",
+    "service_port_mapping_mismatch",
+    "service_protocol_mismatch",
+    "service_env_var_address_mismatch",
+    "service_sidecar_port_conflict",
+    "service_dns_resolution_failure",
+    # Performance
+    "pod_network_delay",
+    "pod_cpu_overload",
+    # Admission
+    "namespace_cpu_quota_exceeded",
+    "namespace_memory_quota_exceeded",
+    "namespace_pod_quota_exceeded",
+    "namespace_service_quota_exceeded",
+    "namespace_storage_quota_exceeded",
+)
+
+_FAULT_OBJECT_SERVICES: tuple[str, ...] = (
+    # online-boutique
+    "adservice",
+    "cartservice",
+    "checkoutservice",
+    "currencyservice",
+    "emailservice",
+    "frontend",
+    "paymentservice",
+    "productcatalogservice",
+    "recommendationservice",
+    "redis-cart",
+    "shippingservice",
+    # train-ticket
+    "ts-gateway-service",
+    "ts-order-service",
+    "ts-payment-service",
+    "ts-travel-service",
+    "ts-user-service",
+    "ts-auth-service",
+    "ts-route-service",
+    "ts-ticket-office-service",
+    "ts-assurance-service",
+    "ts-basic-service",
+    "ts-cancel-service",
+    "ts-config-service",
+    "ts-consign-service",
+    "ts-contacts-service",
+    "ts-delivery-service",
+    "ts-food-delivery-service",
+    "ts-food-service",
+    "ts-inside-payment-service",
+    "ts-notification-service",
+    "ts-order-other-service",
+    "ts-preserve-service",
+    "ts-price-service",
+    "ts-seat-service",
+    "ts-security-service",
+    "ts-station-food-service",
+    "ts-station-service",
+    "ts-train-food-service",
+    "ts-train-service",
+    "ts-travel2-service",
+    "ts-voucher-service",
+    "ts-wait-order-service",
+    "tsdb-mysql",
+)
+
+_FAULT_OBJECT_NODES: tuple[str, ...] = ("master", "worker-01", "worker-02", "worker-03")
+_FAULT_OBJECT_NAMESPACES: tuple[str, ...] = ("boutique", "train-ticket")

tests/benchmarks/cloudopsbench/predictor/__init__.py

@@ -29,10 +29,8 @@
 
 from __future__ import annotations
 
-from tests.benchmarks.cloudopsbench.predictor.investigation_handoff import (
-    align_predictions_to_investigation,
-    apply_investigation_handoff,
-)
+from typing import Any
+
 from tests.benchmarks.cloudopsbench.predictor.llm_call import (
     _FENCED_JSON,
     _build_system_prompt,
@@ -102,3 +100,23 @@
     # llm_call_structured_openai
     "emit_paper_predictions_structured",
 ]
+
+_INVESTIGATION_HANDOFF_EXPORTS = frozenset(
+    {"align_predictions_to_investigation", "apply_investigation_handoff"}
+)
+
+
+def __getattr__(name: str) -> Any:
+    """Lazy-load investigation handoff so importing ``vocabulary`` from this
+    package does not pull in handoff (and its scoring dependencies) at init."""
+    if name in _INVESTIGATION_HANDOFF_EXPORTS:
+        from tests.benchmarks.cloudopsbench.predictor.investigation_handoff import (
+            align_predictions_to_investigation,
+            apply_investigation_handoff,
+        )
+
+        return {
+            "align_predictions_to_investigation": align_predictions_to_investigation,
+            "apply_investigation_handoff": apply_investigation_handoff,
+        }[name]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

tests/benchmarks/cloudopsbench/predictor/investigation_handoff.py

@@ -31,7 +31,7 @@
 import re
 from typing import Any
 
-from tests.benchmarks.cloudopsbench.scoring import _taxonomy_for_root_cause
+from tests.benchmarks.cloudopsbench.taxonomy import taxonomy_for_root_cause
 
 logger = logging.getLogger(__name__)
 
@@ -231,7 +231,7 @@ def align_predictions_to_investigation(
         {
             **prediction,
             "rank": new_rank + 1,
-            "fault_taxonomy": _taxonomy_for_root_cause(str(prediction.get("root_cause") or "")),
+            "fault_taxonomy": taxonomy_for_root_cause(str(prediction.get("root_cause") or "")),
         }
         for new_rank, prediction in enumerate(new_order)
     ]

tests/benchmarks/cloudopsbench/predictor/llm_call.py

@@ -41,7 +41,7 @@
     _ROOT_CAUSES,
     _TAXONOMY_CATEGORIES,
 )
-from tests.benchmarks.cloudopsbench.scoring import _taxonomy_for_root_cause
+from tests.benchmarks.cloudopsbench.taxonomy import taxonomy_for_root_cause
 
 logger = logging.getLogger(__name__)
 
@@ -312,7 +312,7 @@ def _parse_predictions(text: str) -> dict[str, Any] | None:
         # Lever A: snap onto the dataset's closed vocabulary before scoring so
         # near-miss tokens don't auto-fail the exact-match scorer.
         normalized_root_cause = _snap_root_cause(root_cause)
-        derived_taxonomy = _taxonomy_for_root_cause(normalized_root_cause)
+        derived_taxonomy = taxonomy_for_root_cause(normalized_root_cause)
         llm_taxonomy = (prediction.get("fault_taxonomy") or "").strip()
         if llm_taxonomy and llm_taxonomy != derived_taxonomy:
             logger.info(

tests/benchmarks/cloudopsbench/predictor/llm_call_structured_openai.py

@@ -71,7 +71,7 @@
     _ROOT_CAUSES,
     _TAXONOMY_CATEGORIES,
 )
-from tests.benchmarks.cloudopsbench.scoring import _taxonomy_for_root_cause
+from tests.benchmarks.cloudopsbench.taxonomy import taxonomy_for_root_cause
 
 logger = logging.getLogger(__name__)
 
@@ -209,7 +209,7 @@ def emit_paper_predictions_structured(
         # though the schema constrains the LLM's emit, the scorer's mapping
         # is the canonical ground truth and may differ from what the LLM
         # picked for the same root_cause.
-        derived_taxonomy = _taxonomy_for_root_cause(prediction.root_cause)
+        derived_taxonomy = taxonomy_for_root_cause(prediction.root_cause)
         cleaned.append(
             {
                 "rank": prediction.rank,