Skip to content

Commit e223e5b

Browse files
adibarraadibarra
committed
Merge remote-tracking branch 'origin/main' into feat/gpqa-aime26-evals
2 parents 212a8e7 + 673a601 commit e223e5b

62 files changed

Lines changed: 17329 additions & 12708 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/collectivex-sweep.yml‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ on:
2323
description: Keep only shards whose expert-parallel degree is in this comma-list (8 keeps EP8 only, dropping EP16 so GB SKUs co-schedule with 8-GPU SKUs; blank = all)
2424
type: string
2525
default: ''
26+
modes:
27+
description: "Keep only shards in this comma-list of modes (normal, low-latency); low-latency is a decode-only EP8 addition; blank = all"
28+
type: string
29+
default: ''
2630
concurrency:
2731
group: cx-${{ github.ref }}-${{ inputs.backend }}-${{ inputs.only_sku }}
2832
cancel-in-progress: false
@@ -44,12 +48,14 @@ jobs:
4448
INPUT_ONLY_SKU: ${{ inputs.only_sku }}
4549
INPUT_EXCLUDE_SKUS: ${{ inputs.exclude_skus }}
4650
INPUT_EP_SIZES: ${{ inputs.ep_sizes }}
51+
INPUT_MODES: ${{ inputs.modes }}
4752
run: |
4853
set -euo pipefail
4954
args=(--backend "$INPUT_BACKEND")
5055
[ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
5156
[ -n "$INPUT_EXCLUDE_SKUS" ] && args+=(--exclude-skus "$INPUT_EXCLUDE_SKUS")
5257
[ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES")
58+
[ -n "$INPUT_MODES" ] && args+=(--modes "$INPUT_MODES")
5359
python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
5460
python3 - "$GITHUB_OUTPUT" <<'PY'
5561
import json
@@ -94,10 +100,10 @@ jobs:
94100
COLLX_SHARD_SKU: ${{ matrix.sku }}
95101
COLLECTIVEX_CANONICAL_GHA: '1'
96102
COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
97-
# Consolidated shards run one bounded build-group in one Slurm allocation.
98-
# MI300X's compute partition has a 180-minute ceiling; the other production
99-
# pools accept 300 minutes. Allocations release as soon as the shard finishes.
100-
COLLX_TIME: ${{ matrix.sku == 'mi300x' && '180' || '300' }}
103+
# Consolidated shards run one bounded build-group in one Slurm allocation;
104+
# every production pool accepts 300 minutes. Allocations release as soon as
105+
# the shard finishes.
106+
COLLX_TIME: '300'
101107
COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_${{ matrix.id }}
102108
COLLX_JOB_PARENT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) || '/tmp' }}
103109
COLLX_JOB_ROOT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) }}

‎.github/workflows/run-sweep.yml‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,58 @@ jobs:
628628
run-eval: true
629629
eval-only: true
630630

631+
# Agentic (SWE-bench) eval rows carry the agentic input shape, so they are
632+
# dispatched with sweep-agentic's inputs rather than sweep-evals' fixed-seq-len
633+
# inputs (isl/osl/max-model-len, which agentic rows don't have).
634+
sweep-agentic-evals:
635+
needs: [setup, canary-select, canary-sweep]
636+
if: >-
637+
${{
638+
!cancelled() &&
639+
needs.setup.result == 'success' &&
640+
needs.setup.outputs.reuse-enabled != 'true' &&
641+
(needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') &&
642+
toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != '[]' &&
643+
toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != 'null'
644+
}}
645+
uses: ./.github/workflows/benchmark-tmpl.yml
646+
name: agentic eval /
647+
strategy:
648+
fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast-no-canary') }}
649+
matrix:
650+
config: ${{ fromJson(needs.setup.outputs.search-space-config).agentic_evals }}
651+
secrets: inherit
652+
with:
653+
exp-name: ${{ matrix.config.exp-name }}
654+
runner: ${{ matrix.config.runner }}
655+
image: ${{ matrix.config.image }}
656+
model: ${{ matrix.config.model }}
657+
model-prefix: ${{ matrix.config.model-prefix }}
658+
framework: ${{ matrix.config.framework }}
659+
precision: ${{ matrix.config.precision }}
660+
router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }}
661+
kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }}
662+
tp: ${{ matrix.config.tp }}
663+
pp: ${{ matrix.config.pp }}
664+
dcp-size: ${{ matrix.config.dcp-size }}
665+
pcp-size: ${{ matrix.config.pcp-size }}
666+
ep: ${{ matrix.config.ep }}
667+
dp-attn: ${{ matrix.config.dp-attn }}
668+
conc: ${{ matrix.config.conc }}
669+
kv-offloading: ${{ matrix.config.kv-offloading }}
670+
kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }}
671+
kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }}
672+
total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }}
673+
duration: ${{ matrix.config.duration }}
674+
isl: '0'
675+
osl: '0'
676+
max-model-len: '0'
677+
spec-decoding: ${{ matrix.config.spec-decoding }}
678+
disagg: ${{ 'false' }}
679+
run-eval: true
680+
eval-only: true
681+
scenario-type: agentic-coding
682+
631683
sweep-multi-node-evals:
632684
needs: [setup, canary-select, canary-sweep]
633685
if: >-
@@ -714,8 +766,8 @@ jobs:
714766
result-prefix: "bmk"
715767

716768
collect-evals:
717-
needs: [sweep-evals, sweep-multi-node-evals, setup]
718-
if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }}
769+
needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, setup]
770+
if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }}
719771
uses: ./.github/workflows/collect-evals.yml
720772
secrets: inherit
721773

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ This pace of software advancement creates a challenge: benchmarks conducted at a
5959
| MI300X | ✅ |
6060
| H200 | ✅ |
6161
| H100 | ✅ |
62+
| TPUv7x Ironwood Ghostfish | Coming Soon 🔜 |
6263
| MI455 UALoE72 | Coming Soon 🔜 |
6364
| Vera Rubin NVL72 | Coming Soon 🔜 |
6465
| Rubin NVL8 | Coming Soon 🔜 |

‎README_zh.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ SGLang、vLLM、TensorRT-LLM、CUDA、ROCm 等 AI 软件通过核函式優化、
5959
| MI300X | ✅ |
6060
| H200 | ✅ |
6161
| H100 | ✅ |
62+
| TPUv7x Ironwood Ghostfish | Coming Soon 🔜 |
6263
| MI455 UALoE72 | Coming Soon 🔜 |
6364
| Vera Rubin NVL72 | Coming Soon 🔜 |
6465
| Rubin NVL8 | Coming Soon 🔜 |

‎benchmarks/benchmark_lib.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1668,7 +1668,7 @@ resolve_trace_source() {
16681668
# WEKA_LOADER_OVERRIDE.
16691669
local default_loader
16701670
case "${MODEL_PREFIX:-}" in
1671-
dsv4*|minimaxm3*)
1671+
dsv4*|glm5.2*|minimaxm3*)
16721672
default_loader="semianalysis_cc_traces_weka_062126"
16731673
;;
16741674
*)
@@ -1788,8 +1788,11 @@ build_replay_cmd() {
17881788
# only after those requests drain and resumes from the resulting live state.
17891789
REPLAY_CMD+=" --agentic-cache-warmup-duration 600"
17901790
# Give long-context warmup requests up to 30 minutes to drain before
1791+
# declaring warmup failed. Recipes whose saturation arms carry a larger
1792+
# in-flight working set may override via AGENTIC_WARMUP_GRACE_PERIOD
1793+
# (grace is a maximum wait, not a fixed sleep — drain exits when done).
17911794
# cancelling any remaining requests and starting profiling.
1792-
REPLAY_CMD+=" --warmup-grace-period 1800"
1795+
REPLAY_CMD+=" --warmup-grace-period ${AGENTIC_WARMUP_GRACE_PERIOD:-1800}"
17931796
# Use server-reported usage fields (prompt_tokens / completion_tokens) for
17941797
# ISL/OSL instead of client-side tokenizer.encode(). Auto-enables
17951798
# stream_options.include_usage on the OpenAI chat endpoint. Skips the

benchmarks/multi_node/glm5_fp8_mi355x_sglang-disagg.sh renamed to benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env bash
22

3-
source "$(dirname "$0")/../benchmark_lib.sh"
3+
source "$(dirname "$0")/../../benchmark_lib.sh"
44

55
check_env_vars \
66
CONC_LIST \

‎benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p1d-dep2-tp4-eagle3-8k1k.yaml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: "minimax-m3-vllm-disagg-b300-1p1d-dep2-tp4-fp4-8k1k-eagle3"
2+
3+
model:
4+
path: "nvidia/MiniMax-M3-NVFP4"
5+
container: "vllm/vllm-openai:nightly-8e981630c9336233ca9de91452f68918bddbc4e2"
6+
precision: fp4
7+
8+
resources:
9+
gpu_type: b300
10+
gpus_per_node: 8
11+
prefill_nodes: 1
12+
decode_nodes: 0
13+
prefill_workers: 1
14+
decode_workers: 1
15+
gpus_per_prefill: 2
16+
gpus_per_decode: 4
17+
18+
dynamo:
19+
install: true
20+
version: 1.3.0.dev20260710
21+
22+
frontend:
23+
type: dynamo
24+
enable_multiple_frontends: false
25+
26+
backend:
27+
type: vllm
28+
connector: null
29+
allow_prefill_decode_colocation: true
30+
allow_prefill_decode_colocation_across_nodes: true
31+
32+
prefill_environment:
33+
VLLM_FLOAT32_MATMUL_PRECISION: high
34+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm
35+
UCX_TLS: cuda_copy,cuda_ipc,rc
36+
37+
decode_environment:
38+
VLLM_FLOAT32_MATMUL_PRECISION: high
39+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm
40+
UCX_TLS: cuda_copy,cuda_ipc,rc
41+
42+
vllm_config:
43+
prefill:
44+
no-enable-flashinfer-autotune: true
45+
tensor-parallel-size: 1
46+
pipeline-parallel-size: 1
47+
data-parallel-size: 2
48+
data-parallel-rpc-port: 13345
49+
enable-expert-parallel: true
50+
trust-remote-code: true
51+
no-enable-prefix-caching: true
52+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
53+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
54+
block-size: 128
55+
gpu-memory-utilization: 0.95
56+
max-model-len: 9472
57+
language-model-only: true
58+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}'
59+
stream-interval: 32
60+
max-cudagraph-capture-size: 2048
61+
max-num-batched-tokens: 16384
62+
63+
decode:
64+
no-enable-flashinfer-autotune: true
65+
tensor-parallel-size: 4
66+
pipeline-parallel-size: 1
67+
enable-expert-parallel: false
68+
trust-remote-code: true
69+
no-enable-prefix-caching: true
70+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
71+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
72+
block-size: 128
73+
gpu-memory-utilization: 0.95
74+
max-model-len: 9472
75+
language-model-only: true
76+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}'
77+
stream-interval: 32
78+
max-num-seqs: 1024
79+
max-num-batched-tokens: 16384
80+
max-cudagraph-capture-size: 2048
81+
82+
health_check:
83+
max_attempts: 360
84+
interval_seconds: 10
85+
86+
benchmark:
87+
type: sa-bench
88+
isl: 8192
89+
osl: 1024
90+
req_rate: inf
91+
num_warmup_mult: 0
92+
random_range_ratio: 0.8
93+
use_chat_template: true
94+
concurrencies: "64"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p2d-dep2-tp4-eagle3-8k1k.yaml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: "minimax-m3-vllm-disagg-b300-1p2d-dep2-tp4-fp4-8k1k-eagle3"
2+
3+
model:
4+
path: "nvidia/MiniMax-M3-NVFP4"
5+
container: "vllm/vllm-openai:nightly-8e981630c9336233ca9de91452f68918bddbc4e2"
6+
precision: fp4
7+
8+
resources:
9+
gpu_type: b300
10+
gpus_per_node: 8
11+
prefill_nodes: 1
12+
decode_nodes: 1
13+
prefill_workers: 1
14+
decode_workers: 2
15+
gpus_per_prefill: 2
16+
gpus_per_decode: 4
17+
18+
dynamo:
19+
install: true
20+
version: 1.3.0.dev20260710
21+
22+
frontend:
23+
type: dynamo
24+
enable_multiple_frontends: false
25+
26+
backend:
27+
type: vllm
28+
connector: null
29+
allow_prefill_decode_colocation: true
30+
allow_prefill_decode_colocation_across_nodes: true
31+
32+
prefill_environment:
33+
VLLM_FLOAT32_MATMUL_PRECISION: high
34+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm
35+
UCX_TLS: cuda_copy,cuda_ipc,rc
36+
37+
decode_environment:
38+
VLLM_FLOAT32_MATMUL_PRECISION: high
39+
VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm
40+
UCX_TLS: cuda_copy,cuda_ipc,rc
41+
42+
vllm_config:
43+
prefill:
44+
no-enable-flashinfer-autotune: true
45+
tensor-parallel-size: 1
46+
pipeline-parallel-size: 1
47+
data-parallel-size: 2
48+
data-parallel-rpc-port: 13345
49+
enable-expert-parallel: true
50+
trust-remote-code: true
51+
no-enable-prefix-caching: true
52+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
53+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
54+
block-size: 128
55+
gpu-memory-utilization: 0.95
56+
max-model-len: 9472
57+
language-model-only: true
58+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}'
59+
stream-interval: 32
60+
max-cudagraph-capture-size: 2048
61+
max-num-batched-tokens: 16384
62+
63+
decode:
64+
no-enable-flashinfer-autotune: true
65+
tensor-parallel-size: 4
66+
pipeline-parallel-size: 1
67+
enable-expert-parallel: false
68+
trust-remote-code: true
69+
no-enable-prefix-caching: true
70+
kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
71+
attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}'
72+
block-size: 128
73+
gpu-memory-utilization: 0.95
74+
max-model-len: 9472
75+
language-model-only: true
76+
speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}'
77+
stream-interval: 32
78+
max-num-seqs: 1024
79+
max-num-batched-tokens: 16384
80+
max-cudagraph-capture-size: 2048
81+
82+
health_check:
83+
max_attempts: 360
84+
interval_seconds: 10
85+
86+
benchmark:
87+
type: sa-bench
88+
isl: 8192
89+
osl: 1024
90+
req_rate: inf
91+
num_warmup_mult: 0
92+
random_range_ratio: 0.8
93+
use_chat_template: true
94+
concurrencies: "4x32x64"

0 commit comments

Comments
 (0)