From d374605e25a07d5a6906f8ccf076f2e5e9ea60c2 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:31:35 +0800 Subject: [PATCH 1/4] fix: dispatch agentic SWE-bench eval rows via a dedicated bucket and sweep job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since #1947, generate_sweep_configs --evals-only marks the top-concurrency agentic arm of each config as a SWE-bench eval row, but two consumers were never taught about agentic eval rows, so any PR touching a single-node agentic-coding config failed its sweep: 1. Schema: ChangelogMatrixEntry.evals only accepted SingleNodeMatrixEntry (fixed-seq-len), so process_changelog.py's final model_validate rejected the agentic row and check-changelog failed before any GPU work. 2. Dispatch: even with the schema widened, run-sweep.yml's sweep-evals job dispatches every eval row with fixed-seq-len inputs; an agentic row gets empty ISL/OSL/MAX_MODEL_LEN and scenario-type defaults to fixed-seq-len, so the launcher runs the fixed_seq_len benchmark script, which exits on check_env_vars. Fix: route agentic eval rows into a new agentic_evals bucket (mirroring the single_node['agentic'] design) dispatched by a new sweep-agentic-evals job cloned from sweep-agentic's input block plus run-eval/eval-only, so an agentic row can never reach the fixed-seq-len eval dispatch. The agentic model accepts optional run-eval/eval-only carried by eval rows; exclude_none keeps benchmark row output byte-identical. 中文：自 #1947 起，generate_sweep_configs --evals-only 会将每个配置中并发度最高的 agentic 搜索空间条目标记为 SWE-bench 评估行，但下游两个消费方并不认识 agentic 评估行，导致任何涉及单节点 agentic-coding 配置的 PR 扫描失败：其一，schema 层 ChangelogMatrixEntry.evals 仅接受固定序列长度条目，check-changelog 直接报错； 其二，调度层 sweep-evals 以固定序列长度的输入派发评估行，ISL/OSL/MAX_MODEL_LEN 为空且 scenario-type 回落为 fixed-seq-len，启动器因此运行错误的基准测试脚本并 在 check_env_vars 处退出。本修复新增 agentic_evals 独立分桶（与 single_node['agentic'] 设计一致），由新的 sweep-agentic-evals 任务按 sweep-agentic 的输入派发并附加 run-eval/eval-only，使 agentic 评估行不可能进入 固定序列长度的评估调度路径；基准测试行的输出保持逐字节不变。 --- .github/workflows/run-sweep.yml | 56 ++++++++++++- utils/matrix_logic/test_validation.py | 108 ++++++++++++++++++++++++++ utils/matrix_logic/validation.py | 11 +++ utils/process_changelog.py | 14 +++- utils/recover_failed_ingest.py | 1 + utils/test_process_changelog.py | 104 +++++++++++++++++++++++++ 6 files changed, 291 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 011353c68a..041bc6c165 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -628,6 +628,58 @@ jobs: run-eval: true eval-only: true + # Agentic (SWE-bench) eval rows carry the agentic input shape, so they are + # dispatched with sweep-agentic's inputs rather than sweep-evals' fixed-seq-len + # inputs (isl/osl/max-model-len, which agentic rows don't have). + sweep-agentic-evals: + needs: [setup, canary-select, canary-sweep] + if: >- + ${{ + !cancelled() && + needs.setup.result == 'success' && + needs.setup.outputs.reuse-enabled != 'true' && + (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && + toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != '[]' && + toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != 'null' + }} + uses: ./.github/workflows/benchmark-tmpl.yml + name: agentic eval / + strategy: + fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast-no-canary') }} + matrix: + config: ${{ fromJson(needs.setup.outputs.search-space-config).agentic_evals }} + secrets: inherit + with: + exp-name: ${{ matrix.config.exp-name }} + runner: ${{ matrix.config.runner }} + image: ${{ matrix.config.image }} + model: ${{ matrix.config.model }} + model-prefix: ${{ matrix.config.model-prefix }} + framework: ${{ matrix.config.framework }} + precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} + tp: ${{ matrix.config.tp }} + pp: ${{ matrix.config.pp }} + dcp-size: ${{ matrix.config.dcp-size }} + pcp-size: ${{ matrix.config.pcp-size }} + ep: ${{ matrix.config.ep }} + dp-attn: ${{ matrix.config.dp-attn }} + conc: ${{ matrix.config.conc }} + kv-offloading: ${{ matrix.config.kv-offloading }} + kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} + kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} + total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} + duration: ${{ matrix.config.duration }} + isl: '0' + osl: '0' + max-model-len: '0' + spec-decoding: ${{ matrix.config.spec-decoding }} + disagg: ${{ 'false' }} + run-eval: true + eval-only: true + scenario-type: agentic-coding + sweep-multi-node-evals: needs: [setup, canary-select, canary-sweep] if: >- @@ -714,8 +766,8 @@ jobs: result-prefix: "bmk" collect-evals: - needs: [sweep-evals, sweep-multi-node-evals, setup] - if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }} + needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, setup] + if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }} uses: ./.github/workflows/collect-evals.yml secrets: inherit diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 9925adb626..c146148de1 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -18,6 +18,7 @@ SingleNodeMasterConfigEntry, MultiNodeMasterConfigEntry, ChangelogEntry, + ChangelogMatrixEntry, validate_matrix_entry, validate_master_config, validate_runner_config, @@ -1400,6 +1401,113 @@ def test_scenario_type_must_be_nonempty_and_supported(self, scenario_type): }) +# ============================================================================= +# Test ChangelogMatrixEntry +# ============================================================================= + +@pytest.fixture +def valid_agentic_eval_row(): + """Agentic SWE-bench eval row as emitted by generate_sweep_configs --evals-only.""" + return { + "image": "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", + "precision": "fp4", + "framework": "vllm", + "runner": "cluster:b300-nv", + "tp": 8, + "pp": 1, + "dcp-size": 1, + "pcp-size": 1, + "ep": 8, + "dp-attn": True, + "spec-decoding": "mtp", + "conc": 224, + "kv-offloading": "none", + "router": {"name": "vllm-router", "version": "0.1.14"}, + "total-cpu-dram-gb": 0, + "duration": 3600, + "exp-name": "dsv4_tp8_conc224_kvnone_spec-mtp", + "scenario-type": "agentic-coding", + "run-eval": True, + "eval-only": True, + } + + +@pytest.fixture +def valid_changelog_metadata(): + return { + "base_ref": "base", + "head_ref": "head", + "entries": [{ + "config-keys": ["test-config"], + "description": ["Test entry"], + "pr-link": "https://github.com/SemiAnalysisAI/InferenceX/pull/1", + }], + } + + +class TestChangelogMatrixEntry: + """Tests for the final search-space contract consumed by run-sweep.yml.""" + + def test_agentic_eval_rows_validate_in_agentic_evals( + self, valid_agentic_eval_row, valid_changelog_metadata, + ): + entry = ChangelogMatrixEntry.model_validate({ + "agentic_evals": [valid_agentic_eval_row], + "changelog_metadata": valid_changelog_metadata, + }) + + assert entry.agentic_evals[0].scenario_type == "agentic-coding" + assert entry.agentic_evals[0].run_eval is True + assert entry.agentic_evals[0].eval_only is True + + def test_evals_bucket_rejects_agentic_rows( + self, valid_agentic_eval_row, valid_changelog_metadata, + ): + """The `evals` bucket is dispatched with fixed-seq-len inputs + (isl/osl/max-model-len), so agentic rows must never validate there.""" + with pytest.raises(ValueError): + ChangelogMatrixEntry.model_validate({ + "evals": [valid_agentic_eval_row], + "changelog_metadata": valid_changelog_metadata, + }) + + def test_evals_bucket_accepts_fixed_seq_len_rows( + self, valid_single_node_matrix_entry, valid_changelog_metadata, + ): + eval_row = { + **valid_single_node_matrix_entry, + "run-eval": True, + "eval-only": True, + } + entry = ChangelogMatrixEntry.model_validate({ + "evals": [eval_row], + "changelog_metadata": valid_changelog_metadata, + }) + + assert entry.evals[0].run_eval is True + assert entry.evals[0].eval_only is True + + def test_agentic_benchmark_rows_dump_without_eval_flags( + self, valid_agentic_eval_row, valid_changelog_metadata, + ): + """Benchmark rows omit run-eval/eval-only, and exclude_none must keep + them out of the dump so the dispatched row shape is unchanged.""" + benchmark_row = { + k: v for k, v in valid_agentic_eval_row.items() + if k not in ("run-eval", "eval-only") + } + entry = ChangelogMatrixEntry.model_validate({ + "single_node": {"agentic": [benchmark_row]}, + "changelog_metadata": valid_changelog_metadata, + }) + dumped = entry.model_dump(by_alias=True, exclude_none=True) + + assert "run-eval" not in dumped["single_node"]["agentic"][0] + assert "eval-only" not in dumped["single_node"]["agentic"][0] + + # ============================================================================= # Test load_config_files # ============================================================================= diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 6b7c671289..85a17cc632 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -291,6 +291,10 @@ class SingleNodeAgenticMatrixEntry(BaseModel): duration: int = Field(alias=Fields.DURATION.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + # Agentic eval rows (SWE-bench) carry run-eval/eval-only; benchmark rows + # omit them, and exclude_none keeps them out of dumped benchmark output. + run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value) + eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value) @model_validator(mode='after') def validate_kv_offload_fields(self): @@ -890,6 +894,13 @@ class ChangelogMatrixEntry(BaseModel): multi_node: dict[str, list[Union[MultiNodeMatrixEntry, MultiNodeAgenticMatrixEntry]] ] = Field(default_factory=dict) evals: list[SingleNodeMatrixEntry] = Field(default_factory=list) + # Agentic (SWE-bench) eval rows live in their own bucket rather than a + # union inside `evals`: each bucket maps 1:1 to a run-sweep.yml job with a + # static input block, so an agentic row can never reach the fixed-seq-len + # eval dispatch (which reads isl/osl/max-model-len and would launch the + # wrong benchmark script). + agentic_evals: list[SingleNodeAgenticMatrixEntry] = Field( + default_factory=list) multinode_evals: list[MultiNodeMatrixEntry] = Field(default_factory=list) changelog_metadata: ChangelogMetadata diff --git a/utils/process_changelog.py b/utils/process_changelog.py index 11a810e734..c4b226ada0 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -137,6 +137,7 @@ def main(): "single_node": defaultdict(list), "multi_node": defaultdict(list), "evals": [], + "agentic_evals": [], "multinode_evals": [], "changelog_metadata": { "base_ref": args.base_ref, @@ -271,7 +272,18 @@ def main(): seq_len_str = seq_len_to_str(result["isl"], result["osl"]) final_results["single_node"][seq_len_str].append(result) - final_results["evals"] = [e for e in all_eval_results if e.get("prefill") is None] + # Agentic eval rows go to their own bucket so run-sweep.yml can dispatch + # them with agentic inputs (scenario-type, kv-offloading, ...) instead of + # the fixed-seq-len inputs (isl/osl/max-model-len) they don't have. + single_node_evals = [e for e in all_eval_results if e.get("prefill") is None] + final_results["evals"] = [ + e for e in single_node_evals + if e.get("scenario-type") != "agentic-coding" + ] + final_results["agentic_evals"] = [ + e for e in single_node_evals + if e.get("scenario-type") == "agentic-coding" + ] final_results["multinode_evals"] = [e for e in all_eval_results if e.get("prefill") is not None] # Validate final results structure diff --git a/utils/recover_failed_ingest.py b/utils/recover_failed_ingest.py index f8da9cdffb..f6bb686098 100644 --- a/utils/recover_failed_ingest.py +++ b/utils/recover_failed_ingest.py @@ -492,6 +492,7 @@ def build_config( ) + len(config.get("multi_node", {}).get("agentic", []) or []), "eval_jobs": len(config.get("evals", []) or []) + + len(config.get("agentic_evals", []) or []) + len(config.get("multinode_evals", []) or []), } diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index 48f58ef305..b5cde2117c 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -583,3 +583,107 @@ def fake_run(command, **kwargs): assert "--all-evals" not in commands[2] assert _scenario_values(commands[2]) == ["fixed-seq-len"] json.loads(capsys.readouterr().out) + + +def test_eval_rows_split_into_fixed_and_agentic_buckets( + monkeypatch, + capsys, +): + """Realistic eval rows must pass final validation and land in the bucket + matching their dispatch job: fixed-seq-len rows in `evals`, agentic + (SWE-bench) rows in `agentic_evals`.""" + added_yaml = """ +- config-keys: + - test-config + description: + - Mixed fixed-seq-len and agentic eval selection + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1 +""" + + fixed_eval_row = { + "image": "vllm/vllm-openai:v0.11.0", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", + "precision": "fp4", + "framework": "vllm", + "spec-decoding": "mtp", + "runner": "cluster:b300-nv", + "isl": 8192, + "osl": 1024, + "tp": 8, + "pp": 1, + "dcp-size": 1, + "pcp-size": 1, + "ep": 8, + "dp-attn": True, + "conc": 224, + "max-model-len": 10240, + "exp-name": "dsv4_8k1k_fixed_eval", + "disagg": False, + "run-eval": True, + "eval-only": True, + } + agentic_eval_row = { + "image": "vllm/vllm-openai:v0.11.0", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", + "precision": "fp4", + "framework": "vllm", + "runner": "cluster:b300-nv", + "tp": 8, + "pp": 1, + "dcp-size": 1, + "pcp-size": 1, + "ep": 8, + "dp-attn": True, + "spec-decoding": "mtp", + "conc": 224, + "kv-offloading": "none", + "router": {"name": "vllm-router", "version": "0.1.14"}, + "total-cpu-dram-gb": 0, + "duration": 3600, + "exp-name": "dsv4_agentic_eval", + "scenario-type": "agentic-coding", + "run-eval": True, + "eval-only": True, + } + + monkeypatch.setattr( + process_changelog, + "get_added_lines", + lambda *_: added_yaml, + ) + monkeypatch.setattr( + process_changelog, + "load_config_files", + lambda _: {"test-config": {}}, + ) + + def fake_run(command, **kwargs): + if "--evals-only" in command: + stdout = json.dumps([fixed_eval_row, agentic_eval_row]) + else: + stdout = "[]" + return SimpleNamespace(stdout=stdout) + + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr(sys, "argv", [ + "process_changelog.py", + "--base-ref", "base", + "--head-ref", "head", + "--changelog-file", "perf-changelog.yaml", + ]) + + process_changelog.main() + + output = json.loads(capsys.readouterr().out) + assert [row["exp-name"] for row in output["evals"]] == [ + "dsv4_8k1k_fixed_eval" + ] + assert [row["exp-name"] for row in output["agentic_evals"]] == [ + "dsv4_agentic_eval" + ] + assert output["multinode_evals"] == [] + assert output["agentic_evals"][0]["scenario-type"] == "agentic-coding" + assert output["agentic_evals"][0]["run-eval"] is True + assert output["agentic_evals"][0]["eval-only"] is True From 4104a552702ee65cf64cc5d4885f34ec4426b834 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:55:03 +0800 Subject: [PATCH 2/4] test: trim agentic eval bucket tests to essential coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse fixtures into module constants, merge the accept/reject checks into one test, and drop redundant cases; added test lines go from 212 to 100 with the two core guards kept (agentic rows validate only in agentic_evals; realistic rows split end-to-end through the real schema). 中文：精简 agentic 评估分桶测试——fixture 合并为模块常量，接受/拒绝检查合并 为单个测试并移除冗余用例，新增测试行数从 212 降至 100，保留两项核心守卫 （agentic 行仅能通过 agentic_evals 校验；真实行形状端到端经过真实 schema 拆分）。 --- utils/matrix_logic/test_validation.py | 112 ++++++-------------------- utils/test_process_changelog.py | 90 +++++---------------- 2 files changed, 45 insertions(+), 157 deletions(-) diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index c146148de1..3d25d03eef 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -1405,108 +1405,44 @@ def test_scenario_type_must_be_nonempty_and_supported(self, scenario_type): # Test ChangelogMatrixEntry # ============================================================================= -@pytest.fixture -def valid_agentic_eval_row(): - """Agentic SWE-bench eval row as emitted by generate_sweep_configs --evals-only.""" - return { - "image": "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec", - "model": "deepseek-ai/DeepSeek-V4-Pro", - "model-prefix": "dsv4", - "precision": "fp4", - "framework": "vllm", - "runner": "cluster:b300-nv", - "tp": 8, - "pp": 1, - "dcp-size": 1, - "pcp-size": 1, - "ep": 8, - "dp-attn": True, - "spec-decoding": "mtp", - "conc": 224, - "kv-offloading": "none", - "router": {"name": "vllm-router", "version": "0.1.14"}, - "total-cpu-dram-gb": 0, - "duration": 3600, - "exp-name": "dsv4_tp8_conc224_kvnone_spec-mtp", - "scenario-type": "agentic-coding", - "run-eval": True, - "eval-only": True, - } - - -@pytest.fixture -def valid_changelog_metadata(): - return { - "base_ref": "base", - "head_ref": "head", - "entries": [{ - "config-keys": ["test-config"], - "description": ["Test entry"], - "pr-link": "https://github.com/SemiAnalysisAI/InferenceX/pull/1", - }], - } +AGENTIC_EVAL_ROW = { + "image": "vllm/vllm-openai:nightly", "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", "precision": "fp4", "framework": "vllm", + "runner": "cluster:b300-nv", "tp": 8, "pp": 1, "dcp-size": 1, + "pcp-size": 1, "ep": 8, "dp-attn": True, "spec-decoding": "mtp", + "conc": 224, "kv-offloading": "none", "total-cpu-dram-gb": 0, + "duration": 3600, "exp-name": "dsv4_tp8_conc224_kvnone_spec-mtp", + "scenario-type": "agentic-coding", "run-eval": True, "eval-only": True, +} + +CHANGELOG_METADATA = { + "base_ref": "base", "head_ref": "head", + "entries": [{ + "config-keys": ["test-config"], "description": ["Test entry"], + "pr-link": "https://github.com/SemiAnalysisAI/InferenceX/pull/1", + }], +} class TestChangelogMatrixEntry: """Tests for the final search-space contract consumed by run-sweep.yml.""" - def test_agentic_eval_rows_validate_in_agentic_evals( - self, valid_agentic_eval_row, valid_changelog_metadata, - ): + def test_agentic_eval_rows_live_in_agentic_evals_only(self): + """`evals` is dispatched with fixed-seq-len inputs (isl/osl/ + max-model-len), so agentic rows must only validate in agentic_evals.""" entry = ChangelogMatrixEntry.model_validate({ - "agentic_evals": [valid_agentic_eval_row], - "changelog_metadata": valid_changelog_metadata, + "agentic_evals": [AGENTIC_EVAL_ROW], + "changelog_metadata": CHANGELOG_METADATA, }) - - assert entry.agentic_evals[0].scenario_type == "agentic-coding" assert entry.agentic_evals[0].run_eval is True assert entry.agentic_evals[0].eval_only is True - def test_evals_bucket_rejects_agentic_rows( - self, valid_agentic_eval_row, valid_changelog_metadata, - ): - """The `evals` bucket is dispatched with fixed-seq-len inputs - (isl/osl/max-model-len), so agentic rows must never validate there.""" with pytest.raises(ValueError): ChangelogMatrixEntry.model_validate({ - "evals": [valid_agentic_eval_row], - "changelog_metadata": valid_changelog_metadata, + "evals": [AGENTIC_EVAL_ROW], + "changelog_metadata": CHANGELOG_METADATA, }) - def test_evals_bucket_accepts_fixed_seq_len_rows( - self, valid_single_node_matrix_entry, valid_changelog_metadata, - ): - eval_row = { - **valid_single_node_matrix_entry, - "run-eval": True, - "eval-only": True, - } - entry = ChangelogMatrixEntry.model_validate({ - "evals": [eval_row], - "changelog_metadata": valid_changelog_metadata, - }) - - assert entry.evals[0].run_eval is True - assert entry.evals[0].eval_only is True - - def test_agentic_benchmark_rows_dump_without_eval_flags( - self, valid_agentic_eval_row, valid_changelog_metadata, - ): - """Benchmark rows omit run-eval/eval-only, and exclude_none must keep - them out of the dump so the dispatched row shape is unchanged.""" - benchmark_row = { - k: v for k, v in valid_agentic_eval_row.items() - if k not in ("run-eval", "eval-only") - } - entry = ChangelogMatrixEntry.model_validate({ - "single_node": {"agentic": [benchmark_row]}, - "changelog_metadata": valid_changelog_metadata, - }) - dumped = entry.model_dump(by_alias=True, exclude_none=True) - - assert "run-eval" not in dumped["single_node"]["agentic"][0] - assert "eval-only" not in dumped["single_node"]["agentic"][0] - # ============================================================================= # Test load_config_files diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index b5cde2117c..476bec0742 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -599,91 +599,43 @@ def test_eval_rows_split_into_fixed_and_agentic_buckets( - Mixed fixed-seq-len and agentic eval selection pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1 """ - - fixed_eval_row = { + common = { "image": "vllm/vllm-openai:v0.11.0", - "model": "deepseek-ai/DeepSeek-V4-Pro", - "model-prefix": "dsv4", - "precision": "fp4", - "framework": "vllm", - "spec-decoding": "mtp", - "runner": "cluster:b300-nv", - "isl": 8192, - "osl": 1024, - "tp": 8, - "pp": 1, - "dcp-size": 1, - "pcp-size": 1, - "ep": 8, - "dp-attn": True, - "conc": 224, - "max-model-len": 10240, - "exp-name": "dsv4_8k1k_fixed_eval", - "disagg": False, - "run-eval": True, - "eval-only": True, + "model": "deepseek-ai/DeepSeek-V4-Pro", "model-prefix": "dsv4", + "precision": "fp4", "framework": "vllm", "spec-decoding": "mtp", + "runner": "cluster:b300-nv", "tp": 8, "pp": 1, "dcp-size": 1, + "pcp-size": 1, "ep": 8, "dp-attn": True, "conc": 224, + "run-eval": True, "eval-only": True, + } + fixed_eval_row = { + **common, "isl": 8192, "osl": 1024, "max-model-len": 10240, + "disagg": False, "exp-name": "fixed_eval", } agentic_eval_row = { - "image": "vllm/vllm-openai:v0.11.0", - "model": "deepseek-ai/DeepSeek-V4-Pro", - "model-prefix": "dsv4", - "precision": "fp4", - "framework": "vllm", - "runner": "cluster:b300-nv", - "tp": 8, - "pp": 1, - "dcp-size": 1, - "pcp-size": 1, - "ep": 8, - "dp-attn": True, - "spec-decoding": "mtp", - "conc": 224, - "kv-offloading": "none", - "router": {"name": "vllm-router", "version": "0.1.14"}, - "total-cpu-dram-gb": 0, - "duration": 3600, - "exp-name": "dsv4_agentic_eval", - "scenario-type": "agentic-coding", - "run-eval": True, - "eval-only": True, + **common, "kv-offloading": "none", "total-cpu-dram-gb": 0, + "duration": 3600, "scenario-type": "agentic-coding", + "exp-name": "agentic_eval", } monkeypatch.setattr( - process_changelog, - "get_added_lines", - lambda *_: added_yaml, - ) + process_changelog, "get_added_lines", lambda *_: added_yaml) monkeypatch.setattr( - process_changelog, - "load_config_files", - lambda _: {"test-config": {}}, - ) + process_changelog, "load_config_files", lambda _: {"test-config": {}}) def fake_run(command, **kwargs): - if "--evals-only" in command: - stdout = json.dumps([fixed_eval_row, agentic_eval_row]) - else: - stdout = "[]" - return SimpleNamespace(stdout=stdout) + is_evals = "--evals-only" in command + rows = [fixed_eval_row, agentic_eval_row] if is_evals else [] + return SimpleNamespace(stdout=json.dumps(rows)) monkeypatch.setattr(subprocess, "run", fake_run) monkeypatch.setattr(sys, "argv", [ - "process_changelog.py", - "--base-ref", "base", - "--head-ref", "head", + "process_changelog.py", "--base-ref", "base", "--head-ref", "head", "--changelog-file", "perf-changelog.yaml", ]) process_changelog.main() output = json.loads(capsys.readouterr().out) - assert [row["exp-name"] for row in output["evals"]] == [ - "dsv4_8k1k_fixed_eval" - ] - assert [row["exp-name"] for row in output["agentic_evals"]] == [ - "dsv4_agentic_eval" - ] + assert [r["exp-name"] for r in output["evals"]] == ["fixed_eval"] + assert [r["exp-name"] for r in output["agentic_evals"]] == ["agentic_eval"] assert output["multinode_evals"] == [] - assert output["agentic_evals"][0]["scenario-type"] == "agentic-coding" - assert output["agentic_evals"][0]["run-eval"] is True - assert output["agentic_evals"][0]["eval-only"] is True From 622ede7b549807aa137a3ea55c7485492839c701 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:55:03 +0800 Subject: [PATCH 3/4] chore: add changelog entry to validate agentic eval dispatch end-to-end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eval-only, agentic-coding-scoped entry for dsv4-fp4-b300-vllm-agentic so the PR sweep (and the on-push main sweep after merge) runs exactly one job: the DEP8 top-concurrency SWE-bench eval through the new sweep-agentic-evals dispatch, confirming the fix for the eval-only failures first seen on #2258/#2259. 中文：新增 changelog 条目以端到端验证 agentic 评估调度——针对 dsv4-fp4-b300-vllm-agentic 的 evals-only、仅 agentic-coding 场景条目，使 PR 扫描（及合并后 main 分支的 push 触发扫描）只运行一个任务：DEP8 最高并发 的 SWE-bench 评估经由新的 sweep-agentic-evals 调度执行，确认对 #2258/#2259 上首次出现的 eval-only 失败的修复。 --- perf-changelog.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b358f10bf1..60f774ccde 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4933,3 +4933,12 @@ description: - "Add MiniMax M3 NVFP4 B300 Dynamo-vLLM disaggregated EAGLE3 recipes" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2182 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic + description: + - "Validate the agentic SWE-bench eval dispatch path (agentic_evals bucket + sweep-agentic-evals job, fixing the eval-only failures first seen on #2258/#2259): re-run only the DEP8 top-concurrency SWE-bench eval; no throughput re-run." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2267 + evals-only: true + scenario-type: + - agentic-coding From 0d8505cb1968abe360ba9cae984650cf4fb3055f Mon Sep 17 00:00:00 2001 From: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com> Date: Sat, 18 Jul 2026 01:18:52 -0700 Subject: [PATCH 4/4] ws --- perf-changelog.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 60f774ccde..b358f10bf1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4933,12 +4933,3 @@ description: - "Add MiniMax M3 NVFP4 B300 Dynamo-vLLM disaggregated EAGLE3 recipes" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2182 - -- config-keys: - - dsv4-fp4-b300-vllm-agentic - description: - - "Validate the agentic SWE-bench eval dispatch path (agentic_evals bucket + sweep-agentic-evals job, fixing the eval-only failures first seen on #2258/#2259): re-run only the DEP8 top-concurrency SWE-bench eval; no throughput re-run." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2267 - evals-only: true - scenario-type: - - agentic-coding