refactor(bench): _framework decoupling — 5 phases + workflow/UX cleanup (Tracer-Cloud#2801)

* refactoring
* fixing naming convention

Author: YauhenBichel

.github/workflows/benchmark-image.yml

@@ -1,4 +1,10 @@
-name: Benchmark image — build + push to ECR
+name: Benchmark image — build + push to ECR (any adapter)
+
+# Adapter-agnostic image build. The bench image carries the full
+# ``tests/benchmarks/`` tree, so every registered adapter ships in the
+# same image. ``benchmark-run.yml`` selects which adapter actually runs
+# via its ``config`` input. See ``tests/benchmarks/_framework/registry.py``
+# for the registration contract.
 
 # Builds infra/bench/Dockerfile.bench and pushes the resulting image to the
 # opensre-bench ECR repository. The bench container is what

.github/workflows/benchmark-promote-image.yml

@@ -1,5 +1,9 @@
-name: Benchmark image — promote tag to task definition
+name: Benchmark image — promote tag to task definition (any adapter)
 
+# Adapter-agnostic image promotion. Rebinds the ECS task definition to
+# a specific image tag. The image carries every registered adapter; the
+# config supplied to ``benchmark-run.yml`` picks which one runs.
+#
 # Manually-triggered workflow that runs `terraform apply -var=image_tag=<TAG>`
 # in infra/bench/ to register a new ECS task definition revision pointing at
 # the chosen ECR image. This is the privileged "deploy" step that comes
@@ -18,7 +22,7 @@ name: Benchmark image — promote tag to task definition
 #     lock-table read/write, iam:PassRole) are granted by the github_actions_run_bench
 #     inline policy in infra/bench/iam_oidc.tf — any apply of that module attaches them.
 #   - Repo secrets seeded into AWS Secrets Manager
-#   - Repo vars set (AWS_ACCOUNT_ID etc., see tests/benchmarks/cloudopsbench/AWS_BENCH_SETUP.md step 4)
+#   - Repo vars set (AWS_ACCOUNT_ID etc., see infra/bench/AWS_BENCH_SETUP.md step 4)
 
 on:
   workflow_dispatch:
@@ -51,7 +55,7 @@ jobs:
           AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }}
         run: |
           if [ -z "${AWS_ACCOUNT_ID:-}" ]; then
-            echo "::error::Missing repo variable AWS_ACCOUNT_ID. See tests/benchmarks/cloudopsbench/AWS_BENCH_SETUP.md."
+            echo "::error::Missing repo variable AWS_ACCOUNT_ID. See infra/bench/AWS_BENCH_SETUP.md."
             exit 1
           fi
 

.github/workflows/benchmark-run.yml

@@ -1,13 +1,20 @@
-name: Benchmark — run on Fargate
+name: Benchmark — run on Fargate (any adapter)
 
 # Manually-triggered benchmark run on AWS Fargate.
 #
-# Why ECS RunTask instead of running on a GitHub-hosted ubuntu runner:
-# a full Cloud-OpsBench grid runs for hours and writes hundreds of MB of
-# artifacts — Fargate handles it cleanly, has the AWS Secrets Manager
-# entries already wired in via infra/bench/, and writes to the
-# per-run S3 bucket. The workflow's job is just to launch the task and
-# print where to watch.
+# Adapter-agnostic. This workflow does NOT know or care which benchmark
+# is running. The ``config`` input names a YAML file; the YAML names an
+# adapter (``benchmark: <name>``); the framework's registry resolves it
+# to a registered ``BenchmarkAdapter`` subclass. The same workflow runs
+# any adapter packaged into the bench image — CloudOpsBench today,
+# OpenRCA / ToolCallBench / future adapters tomorrow, with zero changes
+# here. See ``tests/benchmarks/_framework/registry.py``.
+#
+# Why ECS RunTask instead of a GitHub-hosted ubuntu runner: a full bench
+# grid runs for hours and writes hundreds of MB of artifacts — Fargate
+# handles it cleanly, has AWS Secrets Manager wired in via infra/bench/,
+# and writes to the per-run S3 bucket. The workflow's job is just to
+# launch the task and print where to watch.
 #
 # Trigger from the GitHub UI:
 #   Actions → "Benchmark run (manual)" → Run workflow → fill inputs
@@ -24,9 +31,14 @@ on:
   workflow_dispatch:
     inputs:
       config:
-        description: 'Path to YAML config inside the container'
+        # No adapter-specific default — the operator must point at a
+        # specific config. Leaving the field empty surfaces the choice
+        # rather than silently dispatching the CloudOpsBench smoke. The
+        # path is relative to the container's repo root. Examples:
+        #   tests/benchmarks/cloudopsbench/configs/cloudopsbench_smoke.yml
+        #   tests/benchmarks/<adapter>/configs/<config>.yml
+        description: 'Path to YAML config inside the container (e.g. tests/benchmarks/<adapter>/configs/<config>.yml)'
         required: true
-        default: 'tests/benchmarks/cloudopsbench/configs/cloudopsbench_smoke.yml'
       dev_mode:
         description: 'Dev mode (skip integrity gates, no pre-reg needed)'
         type: boolean
@@ -64,7 +76,7 @@ jobs:
         # Fail loudly BEFORE the AWS auth step if any required repo variable
         # is missing. Without this, an unset var would surface downstream as
         # an opaque error like "Task Definition can not be blank" from
-        # describe-task-definition. See tests/benchmarks/cloudopsbench/AWS_BENCH_SETUP.md step 4 for how
+        # describe-task-definition. See infra/bench/AWS_BENCH_SETUP.md step 4 for how
         # to set these.
         env:
           AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }}
@@ -83,7 +95,7 @@ jobs:
           if [ "${#missing[@]}" -gt 0 ]; then
             echo "::error::Missing repo variable(s): ${missing[*]}"
             echo "::error::Set them under Settings → Secrets and variables → Actions → Variables."
-            echo "::error::Values come from \`cd infra/bench && terraform output\` — see tests/benchmarks/cloudopsbench/AWS_BENCH_SETUP.md step 4."
+            echo "::error::Values come from \`cd infra/bench && terraform output\` — see infra/bench/AWS_BENCH_SETUP.md step 4."
             exit 1
           fi
           echo "All 5 required repo variables are set."

…chmarks/cloudopsbench/AWS_BENCH_SETUP.md infra/bench/AWS_BENCH_SETUP.mdtests/benchmarks/cloudopsbench/AWS_BENCH_SETUP.md renamed to infra/bench/AWS_BENCH_SETUP.md tests/benchmarks/cloudopsbench/AWS_BENCH_SETUP.md renamed to infra/bench/AWS_BENCH_SETUP.md

@@ -90,6 +90,47 @@ class MyAdapter(BenchmarkAdapter):
     """
 
 
+# --------------------------------------------------------------------------- #
+# Overfit-dimensions schema                                                   #
+# --------------------------------------------------------------------------- #
+
+
+class OverfitDimensions(BaseModel):
+    """Metadata keys that overfit guards use to extract case attributes.
+
+    The framework's overfit module (``_framework/overfit.py``) runs
+    five guards that attribute lift across (a) the corpus's "system"
+    dimension, (b) the corpus's "stratum" / fault-category dimension,
+    and (c) per-case ground-truth objects. Different adapters emit
+    cells with different metadata layouts — CloudOpsBench uses
+    ``metadata["system"]`` / ``metadata["fault_category"]`` /
+    ``metadata["ground_truth"]["fault_object"]``; another adapter may
+    use different key names.
+
+    This model lets each adapter declare its key names without
+    requiring the framework to know about them. The defaults match
+    CloudOpsBench's schema so existing call sites continue to work;
+    other adapters override ``BenchmarkAdapter.overfit_dimensions``
+    to point at their own keys.
+
+    Phase 3 of the framework decoupling: the previous overfit guards
+    hardcoded ``c["case"]["metadata"]["system"]`` etc. inline, which
+    silently coupled the framework to CloudOpsBench's metadata shape.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    system_key: str = "system"
+    """``case.metadata[<key>]`` — the corpus's "system" attribute."""
+
+    stratum_key: str = "fault_category"
+    """``case.metadata[<key>]`` — the corpus's stratum / category attribute."""
+
+    gt_object_key: str = "fault_object"
+    """``case.metadata["ground_truth"][<key>]`` — the GT object name used
+    by Guard C's cluster fingerprinting."""
+
+
 # --------------------------------------------------------------------------- #
 # The adapter interface                                                       #
 # --------------------------------------------------------------------------- #
@@ -137,6 +178,47 @@ def apply_config_overrides(self, config: Any) -> None:  # noqa: ARG002 — defau
         """
         return None
 
+    def overfit_dimensions(self) -> OverfitDimensions:
+        """Metadata keys the overfit guards should consult for this adapter.
+
+        Default returns ``OverfitDimensions()`` which matches the
+        CloudOpsBench schema (``system``, ``fault_category``,
+        ``ground_truth.fault_object``). Adapters with a different
+        metadata layout override this to point the guards at their own
+        keys.
+
+        Phase 3 of the framework decoupling: previously
+        ``_framework/overfit.py`` indexed into these keys inline,
+        coupling the framework to one adapter's metadata shape. This
+        hook moves the schema declaration to the adapter that owns it.
+        """
+        return OverfitDimensions()
+
+    def extend_provenance(self, provenance: dict[str, Any]) -> dict[str, Any]:
+        """Optional: inject adapter-specific fields into the provenance dict.
+
+        Called by ``_framework/provenance.py::capture_provenance`` after
+        the framework finishes assembling its standard sections
+        (``code``, ``config``, ``pre_registration``, ``models``,
+        ``environment``, ``dataset``, ``run_inputs``). Adapters can:
+          - add a new top-level key (e.g. an adapter-specific run note)
+          - extend an existing section (e.g. add ``min_tool_calls`` to
+            ``run_inputs``)
+          - return the dict unchanged
+
+        Default is identity. The hook exists so the framework's
+        provenance module stays adapter-agnostic — it does NOT import or
+        reach into any adapter's internals to build the artifact. Before
+        Phase 4 the framework imported ``cloudopsbench.bench_agent``
+        directly to read ``min_tool_calls``; that import is gone and
+        CloudOpsBench now overrides this hook instead.
+
+        Implementations should mutate-and-return for performance, or
+        return a fresh dict if a non-destructive transform is needed —
+        the framework respects the return value either way.
+        """
+        return provenance
+
     @abstractmethod
     def load_cases(self, filters: CaseFilters) -> Iterator[BenchmarkCase]:
         """Stream cases matching the filter. Seeded random selection is the

tests/benchmarks/_framework/adapter_base.py

@@ -19,7 +19,11 @@
 
 from __future__ import annotations
 
-from tests.benchmarks._framework.adapter_base import AdapterCapabilities, BenchmarkAdapter
+from tests.benchmarks._framework.adapter_base import (
+    AdapterCapabilities,
+    BenchmarkAdapter,
+    OverfitDimensions,
+)
 from tests.benchmarks._framework.registry import (
     build_adapter,
     capabilities_for,
@@ -47,6 +51,7 @@
     "CaseScore",
     "MetricSchema",
     "Mode",
+    "OverfitDimensions",
     "RunContext",
     "RunResult",
     "build_adapter",

tests/benchmarks/_framework/adapters.py

@@ -34,8 +34,8 @@ def _default_report_formats() -> list[Literal["json", "markdown", "html"]]:
     return ["json", "markdown"]
 
 
-def _capabilities_for_lint(benchmark_name: str) -> AdapterCapabilities:
-    """Look up an adapter's capabilities for use during config lint.
+def _resolve_capabilities_or_default(benchmark_name: str) -> AdapterCapabilities:
+    """Look up an adapter's capabilities; fall back to the all-False default.
 
     Unknown adapters return the all-False default — every gated feature
     is refused. This is intentional: a typo in ``config.benchmark`` (e.g.
@@ -245,7 +245,7 @@ def lint(self) -> list[str]:
         # ``benchmark == "cloudopsbench"``) means a new adapter that opts
         # in to ``supports_agent_variant=True`` is automatically accepted
         # by the framework without changes here.
-        adapter_caps = _capabilities_for_lint(self.benchmark)
+        adapter_caps = _resolve_capabilities_or_default(self.benchmark)
         if self.agent_variant != "default" and not adapter_caps.supports_agent_variant:
             errors.append(
                 f"agent_variant={self.agent_variant!r} requires the "