fix: dispatch agentic SWE-bench eval rows via a dedicated bucket and sweep job

Oseltamivir · Oseltamivir · commit d374605e25a0 · 2026-07-18T13:31:54.000+08:00
Since #1947, generate_sweep_configs --evals-only marks the top-concurrency agentic arm of each config as a SWE-bench eval row, but two consumers were never taught about agentic eval rows, so any PR touching a single-node agentic-coding config failed its sweep: 1. Schema: ChangelogMatrixEntry.evals only accepted SingleNodeMatrixEntry (fixed-seq-len), so process_changelog.py's final model_validate rejected the agentic row and check-changelog failed before any GPU work. 2. Dispatch: even with the schema widened, run-sweep.yml's sweep-evals job dispatches every eval row with fixed-seq-len inputs; an agentic row gets empty ISL/OSL/MAX_MODEL_LEN and scenario-type defaults to fixed-seq-len, so the launcher runs the fixed_seq_len benchmark script, which exits on check_env_vars. Fix: route agentic eval rows into a new agentic_evals bucket (mirroring the single_node['agentic'] design) dispatched by a new sweep-agentic-evals job cloned from sweep-agentic's input block plus run-eval/eval-only, so an agentic row can never reach the fixed-seq-len eval dispatch. The agentic model accepts optional run-eval/eval-only carried by eval rows; exclude_none keeps benchmark row output byte-identical. 中文：自 #1947 起，generate_sweep_configs --evals-only 会将每个配置中并发度最高的 agentic 搜索空间条目标记为 SWE-bench 评估行，但下游两个消费方并不认识 agentic 评估行，导致任何涉及单节点 agentic-coding 配置的 PR 扫描失败：其一，schema 层 ChangelogMatrixEntry.evals 仅接受固定序列长度条目，check-changelog 直接报错； 其二，调度层 sweep-evals 以固定序列长度的输入派发评估行，ISL/OSL/MAX_MODEL_LEN 为空且 scenario-type 回落为 fixed-seq-len，启动器因此运行错误的基准测试脚本并 在 check_env_vars 处退出。本修复新增 agentic_evals 独立分桶（与 single_node['agentic'] 设计一致），由新的 sweep-agentic-evals 任务按 sweep-agentic 的输入派发并附加 run-eval/eval-only，使 agentic 评估行不可能进入 固定序列长度的评估调度路径；基准测试行的输出保持逐字节不变。
diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml
@@ -628,6 +628,58 @@ jobs:
             run-eval: true
             eval-only: true
 
+    # Agentic (SWE-bench) eval rows carry the agentic input shape, so they are
+    # dispatched with sweep-agentic's inputs rather than sweep-evals' fixed-seq-len
+    # inputs (isl/osl/max-model-len, which agentic rows don't have).
+    sweep-agentic-evals:
+        needs: [setup, canary-select, canary-sweep]
+        if: >-
+            ${{
+              !cancelled() &&
+              needs.setup.result == 'success' &&
+              needs.setup.outputs.reuse-enabled != 'true' &&
+              (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') &&
+              toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != '[]' &&
+              toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != 'null'
+            }}
+        uses: ./.github/workflows/benchmark-tmpl.yml
+        name: agentic eval /
+        strategy:
+            fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast-no-canary') }}
+            matrix:
+                config: ${{ fromJson(needs.setup.outputs.search-space-config).agentic_evals }}
+        secrets: inherit
+        with:
+            exp-name: ${{ matrix.config.exp-name }}
+            runner: ${{ matrix.config.runner }}
+            image: ${{ matrix.config.image }}
+            model: ${{ matrix.config.model }}
+            model-prefix: ${{ matrix.config.model-prefix }}
+            framework: ${{ matrix.config.framework }}
+            precision: ${{ matrix.config.precision }}
+            router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }}
+            kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }}
+            tp: ${{ matrix.config.tp }}
+            pp: ${{ matrix.config.pp }}
+            dcp-size: ${{ matrix.config.dcp-size }}
+            pcp-size: ${{ matrix.config.pcp-size }}
+            ep: ${{ matrix.config.ep }}
+            dp-attn: ${{ matrix.config.dp-attn }}
+            conc: ${{ matrix.config.conc }}
+            kv-offloading: ${{ matrix.config.kv-offloading }}
+            kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }}
+            kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }}
+            total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }}
+            duration: ${{ matrix.config.duration }}
+            isl: '0'
+            osl: '0'
+            max-model-len: '0'
+            spec-decoding: ${{ matrix.config.spec-decoding }}
+            disagg: ${{ 'false' }}
+            run-eval: true
+            eval-only: true
+            scenario-type: agentic-coding
+
     sweep-multi-node-evals:
         needs: [setup, canary-select, canary-sweep]
         if: >-
@@ -714,8 +766,8 @@ jobs:
             result-prefix: "bmk"
 
     collect-evals:
-        needs: [sweep-evals, sweep-multi-node-evals, setup]
-        if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }}
+        needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, setup]
+        if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }}
         uses: ./.github/workflows/collect-evals.yml
         secrets: inherit
 
diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py
@@ -18,6 +18,7 @@
     SingleNodeMasterConfigEntry,
     MultiNodeMasterConfigEntry,
     ChangelogEntry,
+    ChangelogMatrixEntry,
     validate_matrix_entry,
     validate_master_config,
     validate_runner_config,
@@ -1400,6 +1401,113 @@ def test_scenario_type_must_be_nonempty_and_supported(self, scenario_type):
             })
 
 
+# =============================================================================
+# Test ChangelogMatrixEntry
+# =============================================================================
+
+@pytest.fixture
+def valid_agentic_eval_row():
+    """Agentic SWE-bench eval row as emitted by generate_sweep_configs --evals-only."""
+    return {
+        "image": "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec",
+        "model": "deepseek-ai/DeepSeek-V4-Pro",
+        "model-prefix": "dsv4",
+        "precision": "fp4",
+        "framework": "vllm",
+        "runner": "cluster:b300-nv",
+        "tp": 8,
+        "pp": 1,
+        "dcp-size": 1,
+        "pcp-size": 1,
+        "ep": 8,
+        "dp-attn": True,
+        "spec-decoding": "mtp",
+        "conc": 224,
+        "kv-offloading": "none",
+        "router": {"name": "vllm-router", "version": "0.1.14"},
+        "total-cpu-dram-gb": 0,
+        "duration": 3600,
+        "exp-name": "dsv4_tp8_conc224_kvnone_spec-mtp",
+        "scenario-type": "agentic-coding",
+        "run-eval": True,
+        "eval-only": True,
+    }
+
+
+@pytest.fixture
+def valid_changelog_metadata():
+    return {
+        "base_ref": "base",
+        "head_ref": "head",
+        "entries": [{
+            "config-keys": ["test-config"],
+            "description": ["Test entry"],
+            "pr-link": "https://github.com/SemiAnalysisAI/InferenceX/pull/1",
+        }],
+    }
+
+
+class TestChangelogMatrixEntry:
+    """Tests for the final search-space contract consumed by run-sweep.yml."""
+
+    def test_agentic_eval_rows_validate_in_agentic_evals(
+        self, valid_agentic_eval_row, valid_changelog_metadata,
+    ):
+        entry = ChangelogMatrixEntry.model_validate({
+            "agentic_evals": [valid_agentic_eval_row],
+            "changelog_metadata": valid_changelog_metadata,
+        })
+
+        assert entry.agentic_evals[0].scenario_type == "agentic-coding"
+        assert entry.agentic_evals[0].run_eval is True
+        assert entry.agentic_evals[0].eval_only is True
+
+    def test_evals_bucket_rejects_agentic_rows(
+        self, valid_agentic_eval_row, valid_changelog_metadata,
+    ):
+        """The `evals` bucket is dispatched with fixed-seq-len inputs
+        (isl/osl/max-model-len), so agentic rows must never validate there."""
+        with pytest.raises(ValueError):
+            ChangelogMatrixEntry.model_validate({
+                "evals": [valid_agentic_eval_row],
+                "changelog_metadata": valid_changelog_metadata,
+            })
+
+    def test_evals_bucket_accepts_fixed_seq_len_rows(
+        self, valid_single_node_matrix_entry, valid_changelog_metadata,
+    ):
+        eval_row = {
+            **valid_single_node_matrix_entry,
+            "run-eval": True,
+            "eval-only": True,
+        }
+        entry = ChangelogMatrixEntry.model_validate({
+            "evals": [eval_row],
+            "changelog_metadata": valid_changelog_metadata,
+        })
+
+        assert entry.evals[0].run_eval is True
+        assert entry.evals[0].eval_only is True
+
+    def test_agentic_benchmark_rows_dump_without_eval_flags(
+        self, valid_agentic_eval_row, valid_changelog_metadata,
+    ):
+        """Benchmark rows omit run-eval/eval-only, and exclude_none must keep
+        them out of the dump so the dispatched row shape is unchanged."""
+        benchmark_row = {
+            k: v for k, v in valid_agentic_eval_row.items()
+            if k not in ("run-eval", "eval-only")
+        }
+        entry = ChangelogMatrixEntry.model_validate({
+            "single_node": {"agentic": [benchmark_row]},
+            "changelog_metadata": valid_changelog_metadata,
+        })
+        dumped = entry.model_dump(by_alias=True, exclude_none=True)
+
+        assert "run-eval" not in dumped["single_node"]["agentic"][0]
+        assert "eval-only" not in dumped["single_node"]["agentic"][0]
+
+
 # =============================================================================
 # Test load_config_files
 # =============================================================================
diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py
@@ -291,6 +291,10 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
     duration: int = Field(alias=Fields.DURATION.value)
     exp_name: str = Field(alias=Fields.EXP_NAME.value)
     scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value)
+    # Agentic eval rows (SWE-bench) carry run-eval/eval-only; benchmark rows
+    # omit them, and exclude_none keeps them out of dumped benchmark output.
+    run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value)
+    eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value)
 
     @model_validator(mode='after')
     def validate_kv_offload_fields(self):
@@ -890,6 +894,13 @@ class ChangelogMatrixEntry(BaseModel):
     multi_node: dict[str, list[Union[MultiNodeMatrixEntry, MultiNodeAgenticMatrixEntry]]
                      ] = Field(default_factory=dict)
     evals: list[SingleNodeMatrixEntry] = Field(default_factory=list)
+    # Agentic (SWE-bench) eval rows live in their own bucket rather than a
+    # union inside `evals`: each bucket maps 1:1 to a run-sweep.yml job with a
+    # static input block, so an agentic row can never reach the fixed-seq-len
+    # eval dispatch (which reads isl/osl/max-model-len and would launch the
+    # wrong benchmark script).
+    agentic_evals: list[SingleNodeAgenticMatrixEntry] = Field(
+        default_factory=list)
     multinode_evals: list[MultiNodeMatrixEntry] = Field(default_factory=list)
     changelog_metadata: ChangelogMetadata
 
diff --git a/utils/process_changelog.py b/utils/process_changelog.py
@@ -137,6 +137,7 @@ def main():
         "single_node": defaultdict(list),
         "multi_node": defaultdict(list),
         "evals": [],
+        "agentic_evals": [],
         "multinode_evals": [],
         "changelog_metadata": {
             "base_ref": args.base_ref,
@@ -271,7 +272,18 @@ def main():
             seq_len_str = seq_len_to_str(result["isl"], result["osl"])
             final_results["single_node"][seq_len_str].append(result)
 
-    final_results["evals"] = [e for e in all_eval_results if e.get("prefill") is None]
+    # Agentic eval rows go to their own bucket so run-sweep.yml can dispatch
+    # them with agentic inputs (scenario-type, kv-offloading, ...) instead of
+    # the fixed-seq-len inputs (isl/osl/max-model-len) they don't have.
+    single_node_evals = [e for e in all_eval_results if e.get("prefill") is None]
+    final_results["evals"] = [
+        e for e in single_node_evals
+        if e.get("scenario-type") != "agentic-coding"
+    ]
+    final_results["agentic_evals"] = [
+        e for e in single_node_evals
+        if e.get("scenario-type") == "agentic-coding"
+    ]
     final_results["multinode_evals"] = [e for e in all_eval_results if e.get("prefill") is not None]
 
     # Validate final results structure
diff --git a/utils/recover_failed_ingest.py b/utils/recover_failed_ingest.py
@@ -492,6 +492,7 @@ def build_config(
         )
         + len(config.get("multi_node", {}).get("agentic", []) or []),
         "eval_jobs": len(config.get("evals", []) or [])
+        + len(config.get("agentic_evals", []) or [])
         + len(config.get("multinode_evals", []) or []),
     }
 
diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py
@@ -583,3 +583,107 @@ def fake_run(command, **kwargs):
     assert "--all-evals" not in commands[2]
     assert _scenario_values(commands[2]) == ["fixed-seq-len"]
     json.loads(capsys.readouterr().out)
+
+
+def test_eval_rows_split_into_fixed_and_agentic_buckets(
+    monkeypatch,
+    capsys,
+):
+    """Realistic eval rows must pass final validation and land in the bucket
+    matching their dispatch job: fixed-seq-len rows in `evals`, agentic
+    (SWE-bench) rows in `agentic_evals`."""
+    added_yaml = """
+- config-keys:
+    - test-config
+  description:
+    - Mixed fixed-seq-len and agentic eval selection
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1
+"""
+
+    fixed_eval_row = {
+        "image": "vllm/vllm-openai:v0.11.0",
+        "model": "deepseek-ai/DeepSeek-V4-Pro",
+        "model-prefix": "dsv4",
+        "precision": "fp4",
+        "framework": "vllm",
+        "spec-decoding": "mtp",
+        "runner": "cluster:b300-nv",
+        "isl": 8192,
+        "osl": 1024,
+        "tp": 8,
+        "pp": 1,
+        "dcp-size": 1,
+        "pcp-size": 1,
+        "ep": 8,
+        "dp-attn": True,
+        "conc": 224,
+        "max-model-len": 10240,
+        "exp-name": "dsv4_8k1k_fixed_eval",
+        "disagg": False,
+        "run-eval": True,
+        "eval-only": True,
+    }
+    agentic_eval_row = {
+        "image": "vllm/vllm-openai:v0.11.0",
+        "model": "deepseek-ai/DeepSeek-V4-Pro",
+        "model-prefix": "dsv4",
+        "precision": "fp4",
+        "framework": "vllm",
+        "runner": "cluster:b300-nv",
+        "tp": 8,
+        "pp": 1,
+        "dcp-size": 1,
+        "pcp-size": 1,
+        "ep": 8,
+        "dp-attn": True,
+        "spec-decoding": "mtp",
+        "conc": 224,
+        "kv-offloading": "none",
+        "router": {"name": "vllm-router", "version": "0.1.14"},
+        "total-cpu-dram-gb": 0,
+        "duration": 3600,
+        "exp-name": "dsv4_agentic_eval",
+        "scenario-type": "agentic-coding",
+        "run-eval": True,
+        "eval-only": True,
+    }
+
+    monkeypatch.setattr(
+        process_changelog,
+        "get_added_lines",
+        lambda *_: added_yaml,
+    )
+    monkeypatch.setattr(
+        process_changelog,
+        "load_config_files",
+        lambda _: {"test-config": {}},
+    )
+
+    def fake_run(command, **kwargs):
+        if "--evals-only" in command:
+            stdout = json.dumps([fixed_eval_row, agentic_eval_row])
+        else:
+            stdout = "[]"
+        return SimpleNamespace(stdout=stdout)
+
+    monkeypatch.setattr(subprocess, "run", fake_run)
+    monkeypatch.setattr(sys, "argv", [
+        "process_changelog.py",
+        "--base-ref", "base",
+        "--head-ref", "head",
+        "--changelog-file", "perf-changelog.yaml",
+    ])
+
+    process_changelog.main()
+
+    output = json.loads(capsys.readouterr().out)
+    assert [row["exp-name"] for row in output["evals"]] == [
+        "dsv4_8k1k_fixed_eval"
+    ]
+    assert [row["exp-name"] for row in output["agentic_evals"]] == [
+        "dsv4_agentic_eval"
+    ]
+    assert output["multinode_evals"] == []
+    assert output["agentic_evals"][0]["scenario-type"] == "agentic-coding"
+    assert output["agentic_evals"][0]["run-eval"] is True
+    assert output["agentic_evals"][0]["eval-only"] is True

Original file line number	Diff line number	Diff line change
`@@ -492,6 +492,7 @@ def build_config(`
`492`	`492`	`)`
`493`	`493`	`+ len(config.get("multi_node", {}).get("agentic", []) or []),`
`494`	`494`	`"eval_jobs": len(config.get("evals", []) or [])`
	`495`	`+ + len(config.get("agentic_evals", []) or [])`
`495`	`496`	`+ len(config.get("multinode_evals", []) or []),`
`496`	`497`	`}`
`497`	`498`