feat(evals): allow AgentX framework override

Author: adibarra

.github/workflows/benchmark-tmpl.yml

@@ -121,6 +121,11 @@ on:
         required: false
         type: string
         default: ""
+      eval-framework:
+        description: "Explicit eval framework override"
+        required: false
+        type: string
+        default: ""
       swebench-gen-mode:
         description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
         required: false
@@ -162,6 +167,7 @@ env:
   TOTAL_CPU_DRAM_GB: ${{ inputs.total-cpu-dram-gb }}
   DURATION: ${{ inputs.duration }}
   EVAL_LIMIT: ${{ inputs.eval-limit }}
+  EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
   SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }}
   AIPERF_FAILED_REQUEST_THRESHOLD: '0.10'
   RESULT_DIR: /workspace/results

.github/workflows/e2e-tests.yml

@@ -26,6 +26,11 @@ on:
                 required: false
                 type: string
                 default: ""
+            eval-framework:
+                description: "Explicit eval framework override (for example, lm-eval on an agentic recipe)"
+                required: false
+                type: string
+                default: ""
             swebench-gen-mode:
                 description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
                 required: false
@@ -55,6 +60,11 @@ on:
                 required: false
                 type: string
                 default: ""
+            eval-framework:
+                description: "Explicit eval framework override (for example, lm-eval on an agentic recipe)"
+                required: false
+                type: string
+                default: ""
             swebench-gen-mode:
                 description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
                 required: false
@@ -280,6 +290,7 @@ jobs:
             run-eval: true
             eval-only: true
             eval-limit: ${{ inputs.eval-limit }}
+            eval-framework: ${{ inputs.eval-framework }}
             swebench-gen-mode: ${{ inputs.swebench-gen-mode }}
             scenario-type: agentic-coding
             ref: ${{ inputs.ref }}

utils/evals/test_run_eval_dispatch.py

@@ -342,6 +342,17 @@ def test_default_graded_eval_suite_is_registered():
     )
 
 
+def test_agentic_eval_workflow_forwards_framework_override():
+    repo_root = BENCHMARK_LIB.parents[1]
+    e2e_workflow = (repo_root / ".github/workflows/e2e-tests.yml").read_text()
+    benchmark_workflow = (
+        repo_root / ".github/workflows/benchmark-tmpl.yml"
+    ).read_text()
+
+    assert "eval-framework: ${{ inputs.eval-framework }}" in e2e_workflow
+    assert "EVAL_FRAMEWORK: ${{ inputs.eval-framework }}" in benchmark_workflow
+
+
 def test_swebench_single_shot_registers_task_yaml():
     script = r'''
 source "$BENCHMARK_LIB"