diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 011353c68a..041bc6c165 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -628,6 +628,58 @@ jobs: run-eval: true eval-only: true + # Agentic (SWE-bench) eval rows carry the agentic input shape, so they are + # dispatched with sweep-agentic's inputs rather than sweep-evals' fixed-seq-len + # inputs (isl/osl/max-model-len, which agentic rows don't have). + sweep-agentic-evals: + needs: [setup, canary-select, canary-sweep] + if: >- + ${{ + !cancelled() && + needs.setup.result == 'success' && + needs.setup.outputs.reuse-enabled != 'true' && + (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && + toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != '[]' && + toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != 'null' + }} + uses: ./.github/workflows/benchmark-tmpl.yml + name: agentic eval / + strategy: + fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast-no-canary') }} + matrix: + config: ${{ fromJson(needs.setup.outputs.search-space-config).agentic_evals }} + secrets: inherit + with: + exp-name: ${{ matrix.config.exp-name }} + runner: ${{ matrix.config.runner }} + image: ${{ matrix.config.image }} + model: ${{ matrix.config.model }} + model-prefix: ${{ matrix.config.model-prefix }} + framework: ${{ matrix.config.framework }} + precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} + tp: ${{ matrix.config.tp }} + pp: ${{ matrix.config.pp }} + dcp-size: ${{ matrix.config.dcp-size }} + pcp-size: ${{ matrix.config.pcp-size }} + ep: ${{ matrix.config.ep }} + dp-attn: ${{ matrix.config.dp-attn }} + conc: ${{ matrix.config.conc }} + kv-offloading: ${{ matrix.config.kv-offloading }} + kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} + kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} + total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} + duration: ${{ matrix.config.duration }} + isl: '0' + osl: '0' + max-model-len: '0' + spec-decoding: ${{ matrix.config.spec-decoding }} + disagg: ${{ 'false' }} + run-eval: true + eval-only: true + scenario-type: agentic-coding + sweep-multi-node-evals: needs: [setup, canary-select, canary-sweep] if: >- @@ -714,8 +766,8 @@ jobs: result-prefix: "bmk" collect-evals: - needs: [sweep-evals, sweep-multi-node-evals, setup] - if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }} + needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, setup] + if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }} uses: ./.github/workflows/collect-evals.yml secrets: inherit diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 9925adb626..3d25d03eef 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -18,6 +18,7 @@ SingleNodeMasterConfigEntry, MultiNodeMasterConfigEntry, ChangelogEntry, + ChangelogMatrixEntry, validate_matrix_entry, validate_master_config, validate_runner_config, @@ -1400,6 +1401,49 @@ def test_scenario_type_must_be_nonempty_and_supported(self, scenario_type): }) +# ============================================================================= +# Test ChangelogMatrixEntry +# ============================================================================= + +AGENTIC_EVAL_ROW = { + "image": "vllm/vllm-openai:nightly", "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", "precision": "fp4", "framework": "vllm", + "runner": "cluster:b300-nv", "tp": 8, "pp": 1, "dcp-size": 1, + "pcp-size": 1, "ep": 8, "dp-attn": True, "spec-decoding": "mtp", + "conc": 224, "kv-offloading": "none", "total-cpu-dram-gb": 0, + "duration": 3600, "exp-name": "dsv4_tp8_conc224_kvnone_spec-mtp", + "scenario-type": "agentic-coding", "run-eval": True, "eval-only": True, +} + +CHANGELOG_METADATA = { + "base_ref": "base", "head_ref": "head", + "entries": [{ + "config-keys": ["test-config"], "description": ["Test entry"], + "pr-link": "https://github.com/SemiAnalysisAI/InferenceX/pull/1", + }], +} + + +class TestChangelogMatrixEntry: + """Tests for the final search-space contract consumed by run-sweep.yml.""" + + def test_agentic_eval_rows_live_in_agentic_evals_only(self): + """`evals` is dispatched with fixed-seq-len inputs (isl/osl/ + max-model-len), so agentic rows must only validate in agentic_evals.""" + entry = ChangelogMatrixEntry.model_validate({ + "agentic_evals": [AGENTIC_EVAL_ROW], + "changelog_metadata": CHANGELOG_METADATA, + }) + assert entry.agentic_evals[0].run_eval is True + assert entry.agentic_evals[0].eval_only is True + + with pytest.raises(ValueError): + ChangelogMatrixEntry.model_validate({ + "evals": [AGENTIC_EVAL_ROW], + "changelog_metadata": CHANGELOG_METADATA, + }) + + # ============================================================================= # Test load_config_files # ============================================================================= diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 6b7c671289..85a17cc632 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -291,6 +291,10 @@ class SingleNodeAgenticMatrixEntry(BaseModel): duration: int = Field(alias=Fields.DURATION.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + # Agentic eval rows (SWE-bench) carry run-eval/eval-only; benchmark rows + # omit them, and exclude_none keeps them out of dumped benchmark output. + run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value) + eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value) @model_validator(mode='after') def validate_kv_offload_fields(self): @@ -890,6 +894,13 @@ class ChangelogMatrixEntry(BaseModel): multi_node: dict[str, list[Union[MultiNodeMatrixEntry, MultiNodeAgenticMatrixEntry]] ] = Field(default_factory=dict) evals: list[SingleNodeMatrixEntry] = Field(default_factory=list) + # Agentic (SWE-bench) eval rows live in their own bucket rather than a + # union inside `evals`: each bucket maps 1:1 to a run-sweep.yml job with a + # static input block, so an agentic row can never reach the fixed-seq-len + # eval dispatch (which reads isl/osl/max-model-len and would launch the + # wrong benchmark script). + agentic_evals: list[SingleNodeAgenticMatrixEntry] = Field( + default_factory=list) multinode_evals: list[MultiNodeMatrixEntry] = Field(default_factory=list) changelog_metadata: ChangelogMetadata diff --git a/utils/process_changelog.py b/utils/process_changelog.py index 11a810e734..c4b226ada0 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -137,6 +137,7 @@ def main(): "single_node": defaultdict(list), "multi_node": defaultdict(list), "evals": [], + "agentic_evals": [], "multinode_evals": [], "changelog_metadata": { "base_ref": args.base_ref, @@ -271,7 +272,18 @@ def main(): seq_len_str = seq_len_to_str(result["isl"], result["osl"]) final_results["single_node"][seq_len_str].append(result) - final_results["evals"] = [e for e in all_eval_results if e.get("prefill") is None] + # Agentic eval rows go to their own bucket so run-sweep.yml can dispatch + # them with agentic inputs (scenario-type, kv-offloading, ...) instead of + # the fixed-seq-len inputs (isl/osl/max-model-len) they don't have. + single_node_evals = [e for e in all_eval_results if e.get("prefill") is None] + final_results["evals"] = [ + e for e in single_node_evals + if e.get("scenario-type") != "agentic-coding" + ] + final_results["agentic_evals"] = [ + e for e in single_node_evals + if e.get("scenario-type") == "agentic-coding" + ] final_results["multinode_evals"] = [e for e in all_eval_results if e.get("prefill") is not None] # Validate final results structure diff --git a/utils/recover_failed_ingest.py b/utils/recover_failed_ingest.py index f8da9cdffb..f6bb686098 100644 --- a/utils/recover_failed_ingest.py +++ b/utils/recover_failed_ingest.py @@ -492,6 +492,7 @@ def build_config( ) + len(config.get("multi_node", {}).get("agentic", []) or []), "eval_jobs": len(config.get("evals", []) or []) + + len(config.get("agentic_evals", []) or []) + len(config.get("multinode_evals", []) or []), } diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index 48f58ef305..476bec0742 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -583,3 +583,59 @@ def fake_run(command, **kwargs): assert "--all-evals" not in commands[2] assert _scenario_values(commands[2]) == ["fixed-seq-len"] json.loads(capsys.readouterr().out) + + +def test_eval_rows_split_into_fixed_and_agentic_buckets( + monkeypatch, + capsys, +): + """Realistic eval rows must pass final validation and land in the bucket + matching their dispatch job: fixed-seq-len rows in `evals`, agentic + (SWE-bench) rows in `agentic_evals`.""" + added_yaml = """ +- config-keys: + - test-config + description: + - Mixed fixed-seq-len and agentic eval selection + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1 +""" + common = { + "image": "vllm/vllm-openai:v0.11.0", + "model": "deepseek-ai/DeepSeek-V4-Pro", "model-prefix": "dsv4", + "precision": "fp4", "framework": "vllm", "spec-decoding": "mtp", + "runner": "cluster:b300-nv", "tp": 8, "pp": 1, "dcp-size": 1, + "pcp-size": 1, "ep": 8, "dp-attn": True, "conc": 224, + "run-eval": True, "eval-only": True, + } + fixed_eval_row = { + **common, "isl": 8192, "osl": 1024, "max-model-len": 10240, + "disagg": False, "exp-name": "fixed_eval", + } + agentic_eval_row = { + **common, "kv-offloading": "none", "total-cpu-dram-gb": 0, + "duration": 3600, "scenario-type": "agentic-coding", + "exp-name": "agentic_eval", + } + + monkeypatch.setattr( + process_changelog, "get_added_lines", lambda *_: added_yaml) + monkeypatch.setattr( + process_changelog, "load_config_files", lambda _: {"test-config": {}}) + + def fake_run(command, **kwargs): + is_evals = "--evals-only" in command + rows = [fixed_eval_row, agentic_eval_row] if is_evals else [] + return SimpleNamespace(stdout=json.dumps(rows)) + + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr(sys, "argv", [ + "process_changelog.py", "--base-ref", "base", "--head-ref", "head", + "--changelog-file", "perf-changelog.yaml", + ]) + + process_changelog.main() + + output = json.loads(capsys.readouterr().out) + assert [r["exp-name"] for r in output["evals"]] == ["fixed_eval"] + assert [r["exp-name"] for r in output["agentic_evals"]] == ["agentic_eval"] + assert output["multinode_evals"] == []