diff --git a/benchmarks/multi_node/glm5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh similarity index 97% rename from benchmarks/multi_node/glm5_fp8_mi355x_sglang-disagg.sh rename to benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh index 7cbc6afe71..d92dad7503 100755 --- a/benchmarks/multi_node/glm5_fp8_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../benchmark_lib.sh" +source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars \ CONC_LIST \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x.sh index 4e0d507c60..76e03b25ed 100644 --- a/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -x -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x_atom.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x_atom.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x_atom.sh index b1d1b61c82..eaf61277c2 100644 --- a/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5.1_fp4_mi355x_atom.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp4_b200.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200.sh index a1ae270219..f96acd276d 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200_mtp.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp4_b200_mtp.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200_mtp.sh index b42dfbefad..4a71ac127c 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b200_mtp.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300.sh similarity index 98% rename from benchmarks/single_node/fixed_seq_len/glm5_fp4_b300.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300.sh index b85db6b8b1..80fd4e15fe 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300.sh @@ -4,7 +4,7 @@ # does not have a B300-specific recipe, so this script reuses the existing # GLM-5 FP4 B200 SGLang recipe as-is until B300-specific tuning is available. -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300_mtp.sh similarity index 98% rename from benchmarks/single_node/fixed_seq_len/glm5_fp4_b300_mtp.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300_mtp.sh index 7a859fc546..841381e826 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp4_b300_mtp.sh @@ -4,7 +4,7 @@ # does not have a B300-specific recipe, so this script reuses the existing # GLM5 FP8 B200 SGLang recipe as-is until B300-specific tuning is available. -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_b200.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200.sh index ebcecc5e57..7f5d3f5fc2 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200_mtp.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_b200_mtp.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200_mtp.sh index a7c627f46a..3a8384d0e1 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b200_mtp.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300.sh similarity index 98% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_b300.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300.sh index e5a0b2c29f..709ed2d417 100644 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300.sh @@ -4,7 +4,7 @@ # does not have a B300-specific recipe, so this script reuses the existing # GLM5 FP8 B200 SGLang recipe as-is until B300-specific tuning is available. -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300_mtp.sh similarity index 98% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_b300_mtp.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300_mtp.sh index 7a859fc546..841381e826 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_b300_mtp.sh @@ -4,7 +4,7 @@ # does not have a B300-specific recipe, so this script reuses the existing # GLM5 FP8 B200 SGLang recipe as-is until B300-specific tuning is available. -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_h200.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200.sh index 266587de96..8dced28d48 100644 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200_mtp.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_h200_mtp.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200_mtp.sh index 133d757dca..d2d9cc507a 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_h200_mtp.sh @@ -6,7 +6,7 @@ # nsa/trtllm-mha) since those backends are Blackwell-specific and not # applicable to Hopper. -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x.sh index 0564ef8d87..6b2d48d23f 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x_mtp.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x_mtp.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x_mtp.sh index fb77d84c2f..8beaedf946 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi325x_mtp.sh @@ -3,7 +3,7 @@ # GLM-5 FP8 on MI325X with EAGLE / MTP speculative decoding. # Mirrors glm5_fp8_mi325x.sh and adds the speculative-* flags. -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x.sh index 21defe90c1..f130cf9c9a 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_atom.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_atom.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_atom.sh index b1d1b61c82..eaf61277c2 100644 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_atom.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_atom.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_mtp.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_mtp.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_mtp.sh index 90fa04f5df..80a8488e9f 100755 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/glm5_fp8_mi355x_mtp.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -x -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 7d8df0e03c..d829459643 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -462,119 +462,6 @@ qwen3.5-fp8-mi300x-sglang: search-space: - { tp: 8, conc-start: 4, conc-end: 64 } -glm5-fp8-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi355x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256 } - -glm5-fp8-mi355x-sglang-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi355x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } - -glm5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi355x-disagg - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 CI smoke sweep; dp-attn false (NSA / MoRI path) - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -glm5-fp8-mi355x-atom: - image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi355x - precision: fp8 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256 } - - { tp: 8, conc-start: 4, conc-end: 256 } - -glm5.1-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/GLM-5.1-MXFP4 - model-prefix: glm5.1 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, conc-start: 4, conc-end: 16 } - -glm5.1-fp4-mi355x-atom: - image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post - model: amd/GLM-5.1-MXFP4 - model-prefix: glm5.1 - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256 } - kimik2.5-int4-mi355x-vllm: image: vllm/vllm-openai-rocm:v0.24.0 model: moonshotai/Kimi-K2.5 @@ -1476,43 +1363,6 @@ qwen3.5-fp8-mi325x-sglang-mtp: search-space: - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } -glm5-fp8-mi325x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi325x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -glm5-fp8-mi325x-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi325x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -# ============================================================================ -# Net-new agentic recipes from chore/agentx-v0.3 (no overlap with main entries). -# Recipes that ALREADY existed on main were intentionally left at main's version -# to preserve main behavior; PR-branch modifications to those recipes are NOT -# brought in here. -# ============================================================================ - qwen3.5-fp8-mi355x-sglang-agentic-hicache: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260521 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/configs/deprecated/amd-glm5-glm5.1-master.yaml b/configs/deprecated/amd-glm5-glm5.1-master.yaml new file mode 100644 index 0000000000..466975ddcf --- /dev/null +++ b/configs/deprecated/amd-glm5-glm5.1-master.yaml @@ -0,0 +1,160 @@ +# Deprecated GLM-5 / GLM-5.1 entries archived from amd-master.yaml. +# Removed from the active master config so sweep generation no longer selects them. + +glm5-fp8-mi355x-sglang: + image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: mi355x + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-start: 4, conc-end: 256 } + + +glm5-fp8-mi355x-sglang-mtp: + image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: mi355x + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } + - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } + + +glm5-fp8-mi355x-sglang-disagg: + image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: mi355x-disagg + precision: fp8 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1P+1D TP8/EP1 CI smoke sweep; dp-attn false (NSA / MoRI path) + - spec-decoding: "none" + conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + + +glm5-fp8-mi355x-atom: + image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: mi355x + precision: fp8 + framework: atom + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-start: 4, conc-end: 256 } + - { tp: 8, conc-start: 4, conc-end: 256 } + + +glm5.1-fp4-mi355x-sglang: + image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 + model: amd/GLM-5.1-MXFP4 + model-prefix: glm5.1 + runner: mi355x + precision: fp4 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 2, conc-start: 4, conc-end: 256 } + - { tp: 4, conc-start: 4, conc-end: 16 } + + +glm5.1-fp4-mi355x-atom: + image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post + model: amd/GLM-5.1-MXFP4 + model-prefix: glm5.1 + runner: mi355x + precision: fp4 + framework: atom + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-start: 4, conc-end: 256 } + + +glm5-fp8-mi325x-sglang: + image: lmsysorg/sglang:v0.5.12-rocm720-mi30x + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: mi325x + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64 } + + +glm5-fp8-mi325x-sglang-mtp: + image: lmsysorg/sglang:v0.5.12-rocm720-mi30x + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: mi325x + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } + +# ============================================================================ +# Net-new agentic recipes from chore/agentx-v0.3 (no overlap with main entries). +# Recipes that ALREADY existed on main were intentionally left at main's version +# to preserve main behavior; PR-branch modifications to those recipes are NOT +# brought in here. +# ============================================================================ + diff --git a/configs/deprecated/nvidia-glm5-glm5.1-master.yaml b/configs/deprecated/nvidia-glm5-glm5.1-master.yaml new file mode 100644 index 0000000000..9c275f3ed1 --- /dev/null +++ b/configs/deprecated/nvidia-glm5-glm5.1-master.yaml @@ -0,0 +1,2066 @@ +# Deprecated GLM-5 / GLM-5.1 entries archived from nvidia-master.yaml. +# Removed from the active master config so sweep generation no longer selects them. + +glm5-fp8-b200-sglang: + image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: b200 + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } + + +glm5-fp8-b200-sglang-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: b200 + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } + + # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 + # does not have a B300-specific recipe, so this config reuses the existing GLM5 FP8 + # B200 SGLang recipe as-is until B300-specific tuning is available. + + +glm5-fp8-b300-sglang: + image: lmsysorg/sglang:v0.5.12-cu130 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: b300 + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } + + +glm5-fp8-b300-sglang-mtp: + image: lmsysorg/sglang:v0.5.12-cu130 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: b300 + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } + + +glm5-fp4-b200-sglang: + image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: b200 + precision: fp4 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } + + +glm5-fp4-b200-sglang-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: b200 + precision: fp4 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } + + + # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5 + # does not have a B300-specific recipe, so this config reuses the existing + # GLM-5 FP4 B200 SGLang recipe as-is until B300-specific tuning is available. +glm5-fp4-b300-sglang: + image: lmsysorg/sglang:v0.5.11-cu130 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: b300 + precision: fp4 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } + + +glm5-fp4-b300-sglang-mtp: + image: lmsysorg/sglang:v0.5.11-cu130 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: b300 + precision: fp4 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } + + +glm5-fp4-gb300-dynamo-trt: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # STP configurations + - conc-list: [ 10 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch2_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch2_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 25 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch4_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 50 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch8_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 100 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch16_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 308 ] + prefill: + num-worker: 3 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx3dep2_gen1dep32_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx3dep2_gen1dep32_batch8_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 615 ] + prefill: + num-worker: 6 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx6dep2_gen1dep32_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx6dep2_gen1dep32_batch16_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 1076 ] + prefill: + num-worker: 9 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx9dep2_gen1dep16_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx9dep2_gen1dep16_batch64_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [ 1229 ] + prefill: + num-worker: 11 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx11dep2_gen1dep32_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx11dep2_gen1dep32_batch32_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 2151 ] + prefill: + num-worker: 15 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx15dep2_gen1dep16_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx15dep2_gen1dep16_batch128_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + + +glm5-fp4-gb200-dynamo-trt: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: gb200 + precision: fp4 + framework: dynamo-trt + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # STP configurations + - conc-list: [ 5 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 10 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 25 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 50 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 105 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 308 ] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 615 ] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 1127 ] + prefill: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [ 1229 ] + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 2151 ] + prefill: + num-worker: 9 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + + +glm5-fp4-gb200-dynamo-trt-mtp: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: gb200 + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # MTP configurations + - spec-decoding: "mtp" + conc-list: [ 5 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 15 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 30 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 90 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 180 ] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 333 ] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 615 ] + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 1127 ] + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 666 ] + prefill: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 2253 ] + prefill: + num-worker: 12 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + + +glm5-fp8-h200-sglang: + image: lmsysorg/sglang:v0.5.12-cu130 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: h200-dgxc + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64 } + + +glm5-fp8-h200-sglang-mtp: + image: lmsysorg/sglang:v0.5.12-cu130 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: h200 + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp } + + +glm5-fp4-gb200-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.13.post1-cu130 + model: nvidia/GLM-5.1-NVFP4 + model-prefix: glm5.1 + runner: gb200 + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1p1d dep16. 5 nodes (1P + 4D). + - spec-decoding: "mtp" + conc-list: [109] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 2p1d dep16. 6 nodes (2P + 4D). + - spec-decoding: "mtp" + conc-list: [398] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 4p1d dep16. 8 nodes (4P + 4D). + - spec-decoding: "mtp" + conc-list: [853] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 8p1d dep16. 12 nodes (8P + 4D). + - spec-decoding: "mtp" + conc-list: [2132] + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[3]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p4d tp4. 5 nodes. + - spec-decoding: "mtp" + conc-list: [104] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p4d tp4, conc16. 5 nodes. + - spec-decoding: "mtp" + conc-list: [82] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4. 9 nodes. + - spec-decoding: "mtp" + conc-list: [87] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4, conc4. 9 nodes. + - spec-decoding: "mtp" + conc-list: [47] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4. 17 nodes. + - spec-decoding: "mtp" + conc-list: [74] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4, conc1. 17 nodes. + - spec-decoding: "mtp" + conc-list: [26] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[5]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + + +glm5-fp8-b200-dynamo-sglang: + image: lmsysorg/sglang:v0.5.11-cu130 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: b200-dgxc + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [560] + prefill: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + - conc-list: [240] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + - conc-list: [224] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_hightpt[2]" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + - conc-list: [256] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[0]" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [256] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[1]" + decode: + num-worker: 3 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [200] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[2]" + decode: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [128] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[3]" + decode: + num-worker: 5 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [64] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[4]" + decode: + num-worker: 7 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [12] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[5]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + + +glm5-fp4-gb200-dynamo-sglang: + image: lmsysorg/sglang:v0.5.11-cu130 + model: nvidia/GLM-5.1-NVFP4 + model-prefix: glm5.1 + runner: gb200 + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 4p1d wide-EP. 12 nodes. + - conc-list: [989] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # 4p1d wide-EP, mrr512. 12 nodes. + - conc-list: [686] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # 6p1d wide-EP. 14 nodes. + - conc-list: [1497] + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # 10p1d wide-EP. 18 nodes. + - conc-list: [2674] + prefill: + num-worker: 10 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p4d. 5 nodes. + - conc-list: [133] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p6d. 7 nodes. + - conc-list: [146] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" + decode: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: false + # 1p6d, mrr16. 7 nodes. + - conc-list: [103] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + decode: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d. 9 nodes. + - conc-list: [130] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d. 17 nodes. + - conc-list: [113] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d, mrr1 (single-stream). 17 nodes. + - conc-list: [23] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_5.yaml" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + + +glm5-fp4-gb300-dynamo-sglang: + image: lmsysorg/sglang:v0.5.11-cu130 + model: nvidia/GLM-5.1-NVFP4 + model-prefix: glm5 + runner: gb300-nv + precision: fp4 + framework: dynamo-sglang + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 5p1d wide-EP. 13 nodes (5P @ TP=4 + 1D @ TP=32 on 8 nodes). + - conc-list: [2048] + prefill: + num-worker: 5 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[0]" + router: { name: dynamo-router, version: "0.8.0" } + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # 7p1d wide-EP. 15 nodes. + - conc-list: [3072] + prefill: + num-worker: 7 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[1]" + router: { name: dynamo-router, version: "0.8.0" } + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # 10p1d wide-EP. 18 nodes. + - conc-list: [4096] + prefill: + num-worker: 10 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[2]" + router: { name: dynamo-router, version: "0.8.0" } + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p3d. 4 nodes (1P + 3 D workers @ 1 node each). + - conc-list: [128] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + router: { name: dynamo-router, version: "1.1.0" } + decode: + num-worker: 3 + tp: 4 + ep: 1 + dp-attn: false + # 1p5d. 6 nodes. + - conc-list: [64] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" + router: { name: dynamo-router, version: "1.1.0" } + decode: + num-worker: 5 + tp: 4 + ep: 1 + dp-attn: false + # 1p9d. 10 nodes. + - conc-list: [32] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + router: { name: dynamo-router, version: "1.1.0" } + decode: + num-worker: 9 + tp: 4 + ep: 1 + dp-attn: false + # 1p15d. 16 nodes. + - conc-list: [16] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml" + router: { name: dynamo-router, version: "1.1.0" } + decode: + num-worker: 15 + tp: 4 + ep: 1 + dp-attn: false + # 1p17d. 18 nodes. + - conc-list: [12] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml" + router: { name: dynamo-router, version: "1.1.0" } + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + + + # ---------- 1k1k high-throughput (wide-EP TP=32 decode) ---------- +glm5-fp8-gb200-dynamo-sglang: + image: lmsysorg/sglang:v0.5.12 + model: zai-org/GLM-5.1-FP8 + model-prefix: glm5.1 + runner: gb200 + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 3p1d wide-EP (dep32, mrr256). 14 nodes. + - conc-list: [519] + prefill: + num-worker: 3 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # 4p1d wide-EP (dep16). 12 nodes. + - conc-list: [1484] + prefill: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 5p1d wide-EP (dep32). 18 nodes. + - conc-list: [1688] + prefill: + num-worker: 5 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # 7p1d wide-EP (dep16). 18 nodes. + - conc-list: [2699] + prefill: + num-worker: 7 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=8 decode workers) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p8d, mrr1 (single-stream). 18 nodes. + - conc-list: [8] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d, mrr16. 18 nodes. + - conc-list: [90] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d, mrr4. 18 nodes. + - conc-list: [9] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + + + # ---------- 1k1k high-throughput (wide-EP decode) ---------- +glm5-fp8-gb200-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.13.post1-cu130 + model: zai-org/GLM-5.1-FP8 + model-prefix: glm5.1 + runner: gb200 + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + # ---------- 8k1k high-throughput (wide-EP TP=16 decode, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p1d dep16. 6 nodes (2P + 4D). + - spec-decoding: "mtp" + conc-list: [137] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 2p1d dep16. 8 nodes (4P + 4D). + - spec-decoding: "mtp" + conc-list: [620] + prefill: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 4p1d dep16. 12 nodes (8P + 4D). + - spec-decoding: "mtp" + conc-list: [1305] + prefill: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # ---------- 8k1k low-latency (TP=8 decode workers, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p4d tp8. 10 nodes. + - spec-decoding: "mtp" + conc-list: [136] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + decode: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: false + # 1p4d tp8, conc56. 10 nodes. + - spec-decoding: "mtp" + conc-list: [56] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + decode: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d tp8. 18 nodes. + - spec-decoding: "mtp" + conc-list: [120] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d tp8, conc12. 18 nodes. + - spec-decoding: "mtp" + conc-list: [12] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + # 1p8d tp8, conc31. 18 nodes. + - spec-decoding: "mtp" + conc-list: [31] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + decode: + num-worker: 8 + tp: 8 + ep: 1 + dp-attn: false + + +glm5-fp4-gb300-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.13.post1-cu130 + model: nvidia/GLM-5.1-NVFP4 + model-prefix: glm5.1 + runner: gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 4p1d dep16. 6 nodes (2P + 4D). + - spec-decoding: "mtp" + conc-list: [364] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 8p1d dep16. 8 nodes (4P + 4D). + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 8 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p2d tp4. 3 nodes. + - spec-decoding: "mtp" + conc-list: [66] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + # 1p2d tp4, conc16. 3 nodes. + - spec-decoding: "mtp" + conc-list: [41] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + # 1p4d tp4. 5 nodes. + - spec-decoding: "mtp" + conc-list: [55] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p4d tp4, conc4. 5 nodes. + - spec-decoding: "mtp" + conc-list: [23] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4. 9 nodes. + - spec-decoding: "mtp" + conc-list: [44] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4, conc1. 9 nodes. + - spec-decoding: "mtp" + conc-list: [12] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[5]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + + + # ---------- 1k1k high-throughput (wide-EP decode, MTP) ---------- +glm5-fp8-gb300-dynamo-sglang: + image: lmsysorg/sglang:v0.5.11-cu130 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: gb300-nv + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [2800] + prefill: + num-worker: 14 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [1700] + prefill: + num-worker: 12 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 24 + ep: 24 + dp-attn: true + - conc-list: [1300] + prefill: + num-worker: 10 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [900] + prefill: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[3]" + decode: + num-worker: 1 + tp: 40 + ep: 40 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers) ---------- + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [150] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[0]" + decode: + num-worker: 9 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [128, 64, 32] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[1]" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [24] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[2]" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + + + # ---------- 1k1k high-throughput (wide-EP decode) ---------- +glm5-fp8-gb300-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.11-cu130 + model: zai-org/GLM-5.1-FP8 + model-prefix: glm5.1 + runner: gb300-nv + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [2800] + prefill: + num-worker: 14 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1700] + prefill: + num-worker: 12 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml" + decode: + num-worker: 1 + tp: 24 + ep: 24 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1300] + prefill: + num-worker: 10 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [900] + prefill: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml" + decode: + num-worker: 1 + tp: 40 + ep: 40 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers, EAGLE MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [150] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" + decode: + num-worker: 9 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [128, 64, 32] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [24] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + + +glm5-fp4-gb300-dynamo-trt-mtp: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # MTP configurations + - spec-decoding: "mtp" + conc-list: [ 5 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 15 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 30 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 84 ] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 180 ] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 333 ] + prefill: + num-worker: 6 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 615 ] + prefill: + num-worker: 10 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 1229 ] + prefill: + num-worker: 14 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 615 ] + prefill: + num-worker: 11 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 2253 ] + prefill: + num-worker: 21 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d63d717dba..e29ac8fce8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1258,10 +1258,10 @@ qwen3.5-fp4-b200-sglang-mtp: - { tp: 4, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - { tp: 2, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } -glm5-fp8-b200-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 +qwen3.5-fp8-b200-sglang-mtp: + image: lmsysorg/sglang:v0.5.14-cu130 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 runner: b200 precision: fp8 framework: sglang @@ -1271,13 +1271,14 @@ glm5-fp8-b200-sglang: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } + - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } -glm5-fp8-b200-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: b200 +qwen3.5-fp8-b300-sglang-mtp: + image: lmsysorg/sglang:v0.5.12-cu130 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: b300 precision: fp8 framework: sglang multinode: false @@ -1286,16 +1287,12 @@ glm5-fp8-b200-sglang-mtp: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 - # does not have a B300-specific recipe, so this config reuses the existing GLM5 FP8 - # B200 SGLang recipe as-is until B300-specific tuning is available. + - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } -glm5-fp8-b300-sglang: +qwen3.5-fp8-b300-sglang: image: lmsysorg/sglang:v0.5.12-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 runner: b300 precision: fp8 framework: sglang @@ -1305,14 +1302,14 @@ glm5-fp8-b300-sglang: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } -glm5-fp8-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 +qwen3.5-fp4-b300-sglang: + image: lmsysorg/sglang:v0.5.14-cu130 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 runner: b300 - precision: fp8 + precision: fp4 framework: sglang multinode: false scenarios: @@ -1320,13 +1317,14 @@ glm5-fp8-b300-sglang-mtp: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 128 } + - { tp: 2, ep: 2, conc-start: 4, conc-end: 128 } -glm5-fp4-b200-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: b200 +qwen3.5-fp4-b300-sglang-mtp: + image: lmsysorg/sglang:v0.5.14-cu130 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: b300 precision: fp4 framework: sglang multinode: false @@ -1335,15 +1333,15 @@ glm5-fp4-b200-sglang: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 128, spec-decoding: mtp } + - { tp: 2, ep: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } -glm5-fp4-b200-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: b200 - precision: fp4 +qwen3.5-bf16-b300-sglang: + image: lmsysorg/sglang:v0.5.12-cu130 + model: Qwen/Qwen3.5-397B-A17B + model-prefix: qwen3.5 + runner: b300 + precision: bf16 framework: sglang multinode: false scenarios: @@ -1351,18 +1349,15 @@ glm5-fp4-b200-sglang-mtp: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 8, ep: 1, conc-start: 4, conc-end: 64 } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 64 } - # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5 - # does not have a B300-specific recipe, so this config reuses the existing - # GLM-5 FP4 B200 SGLang recipe as-is until B300-specific tuning is available. -glm5-fp4-b300-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 +qwen3.5-bf16-b300-sglang-mtp: + image: lmsysorg/sglang:v0.5.12-cu130 + model: Qwen/Qwen3.5-397B-A17B + model-prefix: qwen3.5 runner: b300 - precision: fp4 + precision: bf16 framework: sglang multinode: false scenarios: @@ -1370,712 +1365,117 @@ glm5-fp4-b300-sglang: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } + - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } + +kimik2.5-int4-b200-vllm: + image: vllm/vllm-openai:v0.25.0 + model: moonshotai/Kimi-K2.5 + model-prefix: kimik2.5 + runner: b200 + precision: int4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 64 } + +kimik2.5-int4-b200-vllm-agentic: + image: vllm/vllm-openai:v0.22.0 + model: moonshotai/Kimi-K2.5 + model-prefix: kimik2.5 + runner: cluster:b200-dgxc + precision: int4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32, 64, 96, 128] } -glm5-fp4-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.11-cu130 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 +kimik2.5-int4-b300-vllm: + image: vllm/vllm-openai:v0.25.0 + model: moonshotai/Kimi-K2.5 + model-prefix: kimik2.5 runner: b300 - precision: fp4 - framework: sglang + precision: int4 + framework: vllm multinode: false scenarios: fixed-seq-len: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 8, conc-start: 4, conc-end: 64 } + - { tp: 4, ep: 1, conc-start: 4, conc-end: 64 } -glm5-fp4-gb300-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: gb300-nv - precision: fp4 - framework: dynamo-trt - router: { name: dynamo-router, version: "v1.3.0-dev.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true +kimik2.5-int4-h200-vllm: + image: vllm/vllm-openai:v0.22.0 + model: moonshotai/Kimi-K2.5 + model-prefix: kimik2.5 + runner: h200 + precision: int4 + framework: vllm + multinode: false scenarios: fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # STP configurations - - conc-list: [ 10 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 25 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 50 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 100 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 308 ] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx3dep2_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx3dep2_gen1dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 615 ] - prefill: - num-worker: 6 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx6dep2_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx6dep2_gen1dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 1076 ] - prefill: - num-worker: 9 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx9dep2_gen1dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx9dep2_gen1dep16_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 1229 ] - prefill: - num-worker: 11 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx11dep2_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx11dep2_gen1dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 2151 ] - prefill: - num-worker: 15 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx15dep2_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx15dep2_gen1dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true + - { tp: 8, conc-start: 4, conc-end: 64 } -glm5-fp4-gb200-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: gb200 +kimik2.5-int4-h200-vllm-agentic: + image: vllm/vllm-openai:v0.22.0 + model: moonshotai/Kimi-K2.5 + model-prefix: kimik2.5 + runner: cluster:h200-dgxc + precision: int4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] } + +# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html +# does not have a B300-specific recipe, so this config reuses the existing +# Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. + +kimik2.5-fp4-b200-vllm: + image: vllm/vllm-openai:v0.22.0 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: b200 precision: fp4 - framework: dynamo-trt - kv-p2p-transfer: nixl - multinode: true - disagg: true + framework: vllm + multinode: false scenarios: fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # STP configurations - - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 10 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 25 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 50 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 105 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 308 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 615 ] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 1127 ] - prefill: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 1229 ] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 2151 ] - prefill: - num-worker: 9 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true + - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } + - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 } -glm5-fp4-gb200-dynamo-trt-mtp: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: gb200 +# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html +# does not have a B300-specific recipe, so this config reuses the existing +# Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. + +kimik2.5-fp4-b300-vllm: + image: vllm/vllm-openai:v0.22.0 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: b300 precision: fp4 - framework: dynamo-trt - router: { name: dynamo-router, version: "v1.3.0-dev.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - - spec-decoding: "mtp" - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 15 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 30 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 90 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 180 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 333 ] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 615 ] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 1127 ] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 666 ] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 2253 ] - prefill: - num-worker: 12 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -qwen3.5-fp8-b200-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: b200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -qwen3.5-fp8-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: b300 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -qwen3.5-fp8-b300-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: b300 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - -qwen3.5-fp4-b300-sglang: - image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: b300 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 128 } - - { tp: 2, ep: 2, conc-start: 4, conc-end: 128 } - -qwen3.5-fp4-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: b300 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 2, ep: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - -qwen3.5-bf16-b300-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: b300 - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 64 } - -qwen3.5-bf16-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: b300 - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -kimik2.5-int4-b200-vllm: - image: vllm/vllm-openai:v0.25.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: b200 - precision: int4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -kimik2.5-int4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: cluster:b200-dgxc - precision: int4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32, 64, 96, 128] } - -kimik2.5-int4-b300-vllm: - image: vllm/vllm-openai:v0.25.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: b300 - precision: int4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 64 } - -kimik2.5-int4-h200-vllm: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: h200 - precision: int4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -kimik2.5-int4-h200-vllm-agentic: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: cluster:h200-dgxc - precision: int4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] } - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html -# does not have a B300-specific recipe, so this config reuses the existing -# Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. - -kimik2.5-fp4-b200-vllm: - image: vllm/vllm-openai:v0.22.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: b200 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 } - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html -# does not have a B300-specific recipe, so this config reuses the existing -# Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. - -kimik2.5-fp4-b300-vllm: - image: vllm/vllm-openai:v0.22.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: b300 - precision: fp4 - framework: vllm - multinode: false + framework: vllm + multinode: false scenarios: fixed-seq-len: - isl: 8192 @@ -2392,1287 +1792,94 @@ dsv4-fp4-b300-vllm-mtp: image: vllm/vllm-openai:v0.25.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: cluster:b300-nv - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, conc-start: 1, conc-end: 8, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } - -qwen3.5-fp8-h200-sglang: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: h200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 64 } - -qwen3.5-fp8-h200-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: h200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp } - -glm5-fp8-h200-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: h200-dgxc - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -glm5-fp8-h200-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: h200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -dsr1-fp8-h200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h200 - precision: fp8 - framework: trt - multinode: false - # For all sequence lengths, EP=TP - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - # If CONC > 32, then DP_ATTN=true - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 32 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 64 } - -dsr1-fp8-h200-trt-mtp: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h200 - precision: fp8 - framework: trt - multinode: false - # For all sequence lengths, EP=TP, MOE_BACKEND=CUTLASS, MTP=3 (or MTP=1 when DP_ATTN=true) - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # If CONC >= 64, then DP_ATTN=true, MTP=1 - - { tp: 8, ep: 8, conc-start: 4, conc-end: 32, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 256, spec-decoding: mtp } - -dsr1-fp8-h200-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h200-multinode - precision: fp8 - framework: dynamo-trt - router: { name: dynamo-router, version: "0.8.1.post1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - - spec-decoding: "mtp" - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [4] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [16] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [32] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [64] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [128] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [256] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [512] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # Non-MTP configurations (STP) - - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [16] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [32] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [64] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [256] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [512] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsr1-fp8-h100-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h100-multinode - precision: fp8 - framework: dynamo-trt - router: { name: dynamo-router, version: "0.8.1.post3" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations (6 points) - - spec-decoding: "mtp" - conc-list: [6] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [9] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [30] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [77] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # commenting out cuz it persistently causes problems - # https://github.com/InferenceMAX/InferenceMAX/actions/runs/21769314582/job/62813105509 - # - spec-decoding: "mtp" - # conc-list: [78] - # prefill: - # num-worker: 1 - # tp: 16 - # ep: 16 - # dp-attn: true - # additional-settings: - # # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml - # - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml" - # decode: - # num-worker: 2 - # tp: 16 - # ep: 16 - # dp-attn: false - - spec-decoding: "mtp" - conc-list: [154] - prefill: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # STP configurations (5 points) - - conc-list: [6] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [9] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [30] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [154] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [308] - prefill: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -minimaxm3-fp8-h100-vllm: - image: vllm/vllm-openai:minimax-m3 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: h100 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } - -dsr1-fp8-h100-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h100-multinode - precision: fp8 - framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.0" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # # STP: Max throughput TEP (1 prefill, 1 decode) - # - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] - # prefill: - # num-worker: 1 - # tp: 16 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-tp.yaml" - # decode: - # num-worker: 1 - # tp: 16 - # ep: 1 - # dp-attn: false - # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - # - conc-list: [1, 2, 4, 8, 16, 32, 64] - # prefill: - # num-worker: 1 - # tp: 16 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-dep.yaml" - # decode: - # num-worker: 1 - # tp: 16 - # ep: 16 - # dp-attn: true - # MTP: Max throughput TEP (1 prefill, 1 decode) - - spec-decoding: "mtp" - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-tp-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - - spec-decoding: "mtp" - conc-list: [1, 2, 4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -minimaxm3-fp8-h200-vllm: - image: vllm/vllm-openai:minimax-m3 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: h200 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 32 } - - { tp: 4, ep: 4, conc-start: 1, conc-end: 256 } - - { tp: 8, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512 } - -dsr1-fp4-gb200-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 - model: nvidia/DeepSeek-R1-0528-NVFP4-v2 - model-prefix: dsr1 - runner: gb200 - precision: fp4 - framework: dynamo-trt - router: { name: dynamo-router, version: "0.8.1.post2" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations (spec_decoding="mtp") - - spec-decoding: "mtp" - conc-list: [ 4, 8, 12, 24, 48 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 180 ] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 1229 ] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 666 ] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 4301 ] - prefill: - num-worker: 11 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [ 12, 44, 76 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 333 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 1229 ] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 2253 ] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 4096 ] - prefill: - num-worker: 10 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -dsr1-fp8-gb200-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: gb200 - precision: fp8 - framework: dynamo-trt - router: { name: dynamo-router, version: "0.8.1.post2" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [666] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [666] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [333] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [333] - prefill: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [90] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [15] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [6] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - # 8k1k STP configs - - conc-list: [1229] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [666] - prefill: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [615] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [333] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [63] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [18] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [6] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false + runner: cluster:b300-nv + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } + - { tp: 8, conc-start: 1, conc-end: 8, spec-decoding: mtp } + - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } + - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } -dsr1-fp8-gb200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: gb200 +qwen3.5-fp8-h200-sglang: + image: lmsysorg/sglang:v0.5.14-cu130 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: h200 precision: fp8 - framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true + framework: sglang + multinode: false scenarios: fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # "Low latency" (1 prefill worker at TP8 and 1 decode worker at TP8) - - conc-list: [4, 8, 16] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/low-latency.yaml - - "CONFIG_FILE=recipes/gb200-fp8/8k1k/low-latency.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - # "Mid curve" (5 prefill workers at DEP8 and 1 decode worker at DEP32) - - conc-list: [512, 1024, 2048, 6144] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/mid-curve.yaml - - "CONFIG_FILE=recipes/gb200-fp8/8k1k/mid-curve.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true + - { tp: 8, ep: 8, conc-start: 4, conc-end: 64 } - # "Max throughput" (6 prefill workers at DEP8 and 1 decode worker at DEP24) - - conc-list: [2048, 4096, 6144] - prefill: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/max_tpt.yaml - - "CONFIG_FILE=recipes/gb200-fp8/8k1k/max_tpt.yaml" - decode: - num-worker: 1 - tp: 24 - ep: 24 - dp-attn: true +qwen3.5-fp8-h200-sglang-mtp: + image: lmsysorg/sglang:v0.5.14-cu130 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: h200 + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp } -dsr1-fp8-gb300-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130 +dsr1-fp8-h200-trt: + image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 - runner: gb300 + runner: h200 precision: fp8 - framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.0" } - kv-p2p-transfer: nixl - multinode: true - disagg: true + framework: trt + multinode: false + # For all sequence lengths, EP=TP scenarios: fixed-seq-len: - isl: 8192 osl: 1024 + # If CONC > 32, then DP_ATTN=true search-space: - # "Low latency" (1 prefill worker at TP4 and 1 decode worker at TP4) - - conc-list: [4, 8] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/low-latency.yaml - - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/low-latency.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - # "Mid curve" (5 prefill workers at DEP8 and 1 decode worker at DEP32) - - conc-list: [128, 256, 512, 1024] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/mid.yaml - - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/mid.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - # "Max throughput" (6 prefill workers at DEP8 and 1 decode worker at DEP24) - - conc-list: [2048, 4096] - prefill: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/max.yaml - - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/max.yaml" - decode: - num-worker: 1 - tp: 24 - ep: 24 - dp-attn: true + - { tp: 8, ep: 8, conc-start: 4, conc-end: 32 } + - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 64 } -dsr1-fp4-gb200-dynamo-sglang: - image: "lmsysorg/sglang:v0.5.8-cu130" - model: nvidia/DeepSeek-R1-0528-NVFP4-v2 +dsr1-fp8-h200-trt-mtp: + image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 + model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 - runner: gb200 - precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true + runner: h200 + precision: fp8 + framework: trt + multinode: false + # For all sequence lengths, EP=TP, MOE_BACKEND=CUTLASS, MTP=3 (or MTP=1 when DP_ATTN=true) scenarios: fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # Low latency (1 prefill node, 4 decode nodes) - - spec-decoding: "none" - conc-list: [ 4, 8 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/8k1k/low-latency.yaml" - decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - - # Mid curve (6 prefill nodes, 12 decode nodes) - - spec-decoding: "none" - conc-list: [ 512, 2048, 4096 ] - prefill: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/8k1k/mid-curve.yaml" - decode: - num-worker: 1 - tp: 48 - ep: 48 - dp-attn: true - - # Max throughput (10 prefill nodes, 8 decode nodes) - - spec-decoding: "none" - conc-list: [ 2048 ] - prefill: - num-worker: 10 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/8k1k/max-tpt.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true + # If CONC >= 64, then DP_ATTN=true, MTP=1 + - { tp: 8, ep: 8, conc-start: 4, conc-end: 32, spec-decoding: mtp } + - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 256, spec-decoding: mtp } -dsr1-fp4-gb300-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 - model: nvidia/DeepSeek-R1-0528-NVFP4-v2 +dsr1-fp8-h200-dynamo-trt: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1 + model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 - runner: gb300 - precision: fp4 + runner: h200-multinode + precision: fp8 framework: dynamo-trt - router: { name: dynamo-router, version: "0.8.1.post2" } + router: { name: dynamo-router, version: "0.8.1.post1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -3681,312 +1888,278 @@ dsr1-fp4-gb300-dynamo-trt: - isl: 8192 osl: 1024 search-space: - # MTP configurations (spec_decoding="mtp") + # MTP configurations - spec-decoding: "mtp" - conc-list: [33] + conc-list: [1] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true + tp: 8 + ep: 8 + dp-attn: false + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml" + decode: + num-worker: 7 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [4] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml" decode: - num-worker: 3 + num-worker: 7 tp: 8 ep: 8 dp-attn: false - spec-decoding: "mtp" - conc-list: [5] + conc-list: [8] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml" decode: - num-worker: 4 + num-worker: 6 tp: 8 ep: 8 dp-attn: false - spec-decoding: "mtp" - conc-list: [12, 24] + conc-list: [16] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml" decode: - num-worker: 4 + num-worker: 3 tp: 8 ep: 8 dp-attn: false - spec-decoding: "mtp" - conc-list: [180] + conc-list: [32] prefill: - num-worker: 4 - tp: 2 - ep: 2 - dp-attn: true + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml" decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: false - spec-decoding: "mtp" - conc-list: [308] + conc-list: [64] prefill: - num-worker: 8 - tp: 2 - ep: 2 - dp-attn: true + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 8 + ep: 8 dp-attn: true - spec-decoding: "mtp" - conc-list: [2253] + conc-list: [128] prefill: - num-worker: 10 - tp: 2 - ep: 2 - dp-attn: true + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - spec-decoding: "mtp" - conc-list: [666] + conc-list: [256] prefill: - num-worker: 10 - tp: 2 - ep: 2 - dp-attn: true + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - spec-decoding: "mtp" - conc-list: [1127] + conc-list: [512] prefill: - num-worker: 13 - tp: 2 - ep: 2 - dp-attn: true + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [72] + # Non-MTP configurations (STP) + - conc-list: [1] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml" decode: - num-worker: 3 + num-worker: 7 tp: 8 ep: 8 dp-attn: false - - conc-list: [5] + - conc-list: [4] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml" decode: - num-worker: 4 + num-worker: 7 tp: 8 ep: 8 dp-attn: false - - conc-list: [12] + - conc-list: [8] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml" decode: - num-worker: 4 + num-worker: 6 tp: 8 ep: 8 dp-attn: false - - conc-list: [5, 15, 30] + - conc-list: [16] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: false - - conc-list: [666] - prefill: - num-worker: 7 - tp: 2 - ep: 2 - dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml" decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [1229] + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [32] prefill: - num-worker: 9 - tp: 2 - ep: 2 - dp-attn: true + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml" decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [3228] + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [64] prefill: - num-worker: 11 - tp: 2 - ep: 2 - dp-attn: true + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml" decode: num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - - conc-list: [2253] - prefill: - num-worker: 14 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true -dsr1-fp4-gb300-dynamo-sglang: - image: "lmsysorg/sglang:v0.5.8.post1-cu130-runtime" - model: nvidia/DeepSeek-R1-0528-NVFP4-v2 - model-prefix: dsr1 - runner: gb300 - precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Low latency (1 prefill node, 4 decode nodes) - - spec-decoding: "none" - conc-list: [ 4, 8, 32, 64 ] + - conc-list: [128] prefill: num-worker: 1 - tp: 4 - ep: 1 + tp: 8 + ep: 8 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/8k1k/low_latency.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml" decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - - # Mid curve (6 prefill nodes, 12 decode nodes) - - spec-decoding: "none" - conc-list: [ 512, 2048, 4096 ] + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [256] prefill: - num-worker: 6 - tp: 4 - ep: 1 + num-worker: 5 + tp: 8 + ep: 8 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/8k1k/mid_curve.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml" decode: - num-worker: 1 - tp: 48 - ep: 48 + num-worker: 3 + tp: 8 + ep: 8 dp-attn: true - - # Max throughput (10 prefill nodes, 8 decode nodes) - - spec-decoding: "none" - conc-list: [ 2048 ] + - conc-list: [512] prefill: - num-worker: 10 - tp: 4 - ep: 1 + num-worker: 3 + tp: 8 + ep: 8 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/8k1k/max_tpt.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 8 + ep: 8 dp-attn: true -dsr1-fp8-gb300-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 +dsr1-fp8-h100-dynamo-trt: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3 model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 - runner: gb300-nv + runner: h100-multinode precision: fp8 framework: dynamo-trt - router: { name: dynamo-router, version: "v1.3.0-dev.1" } + router: { name: dynamo-router, version: "0.8.1.post3" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -3995,201 +2168,192 @@ dsr1-fp8-gb300-dynamo-trt: - isl: 8192 osl: 1024 search-space: - # MTP configurations (spec_decoding="mtp") + # MTP configurations (6 points) - spec-decoding: "mtp" - conc-list: [8] + conc-list: [6] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 16 + ep: 16 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml" decode: - num-worker: 4 - tp: 8 - ep: 8 + num-worker: 3 + tp: 16 + ep: 16 dp-attn: false - spec-decoding: "mtp" - conc-list: [24] + conc-list: [9] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 16 + ep: 16 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml" decode: - num-worker: 4 - tp: 8 - ep: 8 + num-worker: 3 + tp: 16 + ep: 16 dp-attn: false - spec-decoding: "mtp" - conc-list: [333] + conc-list: [30] prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml" - decode: num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [666] - prefill: - num-worker: 8 - tp: 4 - ep: 4 + tp: 16 + ep: 16 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml" decode: - num-worker: 1 + num-worker: 3 tp: 16 ep: 16 - dp-attn: true + dp-attn: false - spec-decoding: "mtp" - conc-list: [1229] + conc-list: [77] prefill: - num-worker: 10 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml" - decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - # STP configurations (no spec_decoding) - - conc-list: [4] + # commenting out cuz it persistently causes problems + # https://github.com/InferenceMAX/InferenceMAX/actions/runs/21769314582/job/62813105509 + # - spec-decoding: "mtp" + # conc-list: [78] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 16 + # dp-attn: true + # additional-settings: + # # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml + # - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml" + # decode: + # num-worker: 2 + # tp: 16 + # ep: 16 + # dp-attn: false + - spec-decoding: "mtp" + conc-list: [154] prefill: - num-worker: 1 - tp: 4 - ep: 4 + num-worker: 2 + tp: 16 + ep: 16 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml" decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [24] - prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 16 + ep: 16 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [36] + # STP configurations (5 points) + - conc-list: [6] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 16 + ep: 16 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml" decode: - num-worker: 4 - tp: 8 - ep: 8 + num-worker: 3 + tp: 16 + ep: 16 dp-attn: false - - conc-list: [512] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [666] + - conc-list: [9] prefill: - num-worker: 4 - tp: 4 - ep: 4 + num-worker: 1 + tp: 16 + ep: 16 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml" decode: - num-worker: 1 + num-worker: 3 tp: 16 ep: 16 - dp-attn: true - - conc-list: [1229] + dp-attn: false + - conc-list: [30] prefill: - num-worker: 7 - tp: 4 - ep: 4 + num-worker: 1 + tp: 16 + ep: 16 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml" decode: + num-worker: 3 + tp: 16 + ep: 16 + dp-attn: false + - conc-list: [154] + prefill: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - conc-list: [2151] + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml" + decode: + num-worker: 2 + tp: 16 + ep: 16 + dp-attn: false + - conc-list: [308] prefill: - num-worker: 7 - tp: 4 - ep: 4 + num-worker: 2 + tp: 16 + ep: 16 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true -dsr1-fp8-h200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130 + +minimaxm3-fp8-h100-vllm: + image: vllm/vllm-openai:minimax-m3 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: h100 + precision: fp8 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 1, conc-end: 64 } + - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } + +dsr1-fp8-h100-dynamo-sglang: + image: lmsysorg/sglang:v0.5.8-cu130 model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 - runner: h200-multinode + runner: h100-multinode precision: fp8 framework: dynamo-sglang router: { name: dynamo-router, version: "0.8.0" } @@ -4201,245 +2365,271 @@ dsr1-fp8-h200-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - # STP: Low latency TEP (1 prefill, 7 decode) - - spec-decoding: "none" - conc-list: [1, 4, 8] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs4-1p7d.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 1 - dp-attn: false - # STP: TEP (1 prefill, 6 decode) - - spec-decoding: "none" - conc-list: [4, 8, 16] + # # STP: Max throughput TEP (1 prefill, 1 decode) + # - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-tp.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + # - conc-list: [1, 2, 4, 8, 16, 32, 64] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-dep.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 16 + # dp-attn: true + # MTP: Max throughput TEP (1 prefill, 1 decode) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128] prefill: num-worker: 1 - tp: 8 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs8-1p6d.yaml" + - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-tp-mtp.yaml" decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # STP: TEP (1 prefill, 3 decode) - - spec-decoding: "none" - conc-list: [8, 16, 32] - prefill: num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs16-1p3d.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # STP: TEP (2 prefill, 3 decode) - - spec-decoding: "none" - conc-list: [32, 64, 128] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs64-2p3d.yaml" - decode: - num-worker: 3 - tp: 8 + tp: 16 ep: 1 dp-attn: false - # STP: High throughput DEP (1 prefill, 1 decode, dp-attention) - - spec-decoding: "none" - conc-list: [64, 128, 256] + # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64] prefill: num-worker: 1 - tp: 8 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs128-1p1d-dep.yaml" + - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - # MTP: Low latency TEP (1 prefill, 7 decode) + +minimaxm3-fp8-h200-vllm: + image: vllm/vllm-openai:minimax-m3 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: h200 + precision: fp8 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-start: 1, conc-end: 32 } + - { tp: 4, ep: 4, conc-start: 1, conc-end: 256 } + - { tp: 8, conc-start: 1, conc-end: 128 } + - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } + - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512 } + +dsr1-fp4-gb200-dynamo-trt: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 + model: nvidia/DeepSeek-R1-0528-NVFP4-v2 + model-prefix: dsr1 + runner: gb200 + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post2" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # MTP configurations (spec_decoding="mtp") - spec-decoding: "mtp" - conc-list: [1, 4, 8] + conc-list: [ 4, 8, 12, 24, 48 ] prefill: num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false + tp: 4 + ep: 4 + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs4-1p7d-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml" decode: - num-worker: 7 + num-worker: 4 tp: 8 - ep: 1 + ep: 8 dp-attn: false - # MTP: TEP (1 prefill, 6 decode) - spec-decoding: "mtp" - conc-list: [2, 4, 8, 16, 32] + conc-list: [ 180 ] prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs8-1p6d-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml" decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # MTP: TEP (1 prefill, 3 decode) - - spec-decoding: "mtp" - conc-list: [4, 8, 16, 32, 64] - prefill: num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 1229 ] + prefill: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs16-1p3d-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml" decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # MTP: TEP (2 prefill, 3 decode) + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true - spec-decoding: "mtp" - conc-list: [32, 64, 128] + conc-list: [ 666 ] prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs64-2p3d-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml" decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # MTP: High throughput DEP (1 prefill, 1 decode, dp-attention) + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true - spec-decoding: "mtp" - conc-list: [32, 64, 128, 256, 512] + conc-list: [ 4301 ] prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false + num-worker: 11 + tp: 4 + ep: 4 + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs128-1p1d-dep-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true -dsr1-fp4-b200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130-runtime - model: deepseek-r1-fp4 - model-prefix: dsr1 - runner: b200-multinode - precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.0" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Non-MTP configurations - - conc-list: [64, 128] + + # Non-MTP configurations (default spec_decoding="none") + - conc-list: [ 12, 44, 76 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[0]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - conc-list: [8] + - conc-list: [ 5 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[1]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" decode: - num-worker: 5 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - conc-list: [4, 128] + - conc-list: [ 333 ] prefill: num-worker: 2 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[2]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml" decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4, 8, 16, 64] + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 1229 ] prefill: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml" + decode: num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 2253 ] + prefill: + num-worker: 8 tp: 4 - ep: 1 - dp-attn: false + ep: 4 + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:override_stp_tp4" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [1024, 2048] + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [ 4096 ] prefill: - num-worker: 7 + num-worker: 10 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:override_stp_maxtpt_7p2d" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 16 + ep: 16 dp-attn: true -dsr1-fp8-b200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64 + +dsr1-fp8-gb200-dynamo-trt: + image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 - runner: b200-multinode + runner: gb200 precision: fp8 - framework: dynamo-sglang + framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post2" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -4448,252 +2638,219 @@ dsr1-fp8-b200-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - # STP low-latency: resolved from 8k1k.yaml zip_override_stp_lowlat - - conc-list: [128] + - spec-decoding: "mtp" + conc-list: [666] prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_0.yaml" - router: { name: dynamo-router, version: "0.9.1" } - decode: num-worker: 3 tp: 8 - ep: 1 - dp-attn: false - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_1.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml" decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [8, 16, 32, 64, 128] - prefill: num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_2.yaml" - router: { name: dynamo-router, version: "0.9.1" } - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # STP max-throughput: resolved from 8k1k.yaml zip_override_stp_maxtpt - - conc-list: [288] + - spec-decoding: "mtp" + conc-list: [666] prefill: - num-worker: 1 + num-worker: 5 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 16 + ep: 16 dp-attn: true - - conc-list: [160, 288] + - spec-decoding: "mtp" + conc-list: [333] prefill: - num-worker: 1 + num-worker: 3 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - - conc-list: [512] + - spec-decoding: "mtp" + conc-list: [333] prefill: - num-worker: 2 + num-worker: 4 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [1024] + - spec-decoding: "mtp" + conc-list: [90] prefill: - num-worker: 3 + num-worker: 2 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - -dsr1-fp8-b200-dynamo-sglang-mtp: - image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b200-multinode - precision: fp8 - framework: dynamo-sglang - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP low-latency: resolved from 8k1k.yaml zip_override_mtp_lowlat - spec-decoding: "mtp" - conc-list: [128] + conc-list: [15] prefill: num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml" decode: num-worker: 3 tp: 8 - ep: 1 + ep: 8 dp-attn: false - spec-decoding: "mtp" - conc-list: [128] + conc-list: [6] prefill: num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml" decode: - num-worker: 4 + num-worker: 3 tp: 8 - ep: 1 + ep: 8 dp-attn: false - - spec-decoding: "mtp" - conc-list: [8, 16, 32, 64, 128] + # 8k1k STP configs + - conc-list: [1229] prefill: - num-worker: 1 + num-worker: 5 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml" decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # MTP max-throughput: resolved from 8k1k.yaml zip_override_mtp_maxtpt - - spec-decoding: "mtp" - conc-list: [288] - prefill: num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [666] + prefill: + num-worker: 4 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 32 + ep: 32 dp-attn: true - - spec-decoding: "mtp" - conc-list: [160, 288] + - conc-list: [615] prefill: - num-worker: 1 + num-worker: 3 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - - spec-decoding: "mtp" - conc-list: [512] + - conc-list: [333] prefill: num-worker: 2 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml" decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [63] + prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - - spec-decoding: "mtp" - conc-list: [1024] - prefill: + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml" + decode: num-worker: 3 tp: 8 - ep: 1 + ep: 8 + dp-attn: false + - conc-list: [18] + prefill: + num-worker: 1 + tp: 8 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml" - router: { name: dynamo-router, version: "0.9.1" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml" decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [6] + prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml + - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false -dsr1-fp4-b200-dynamo-sglang-mtp: - image: "lmsysorg/sglang:v0.5.12.post1" - model: deepseek-r1-fp4 +dsr1-fp8-gb200-dynamo-sglang: + image: lmsysorg/sglang:v0.5.8.post1-cu130 + model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 - runner: b200-multinode - precision: fp4 + runner: gb200 + precision: fp8 framework: dynamo-sglang - router: { name: dynamo-router, version: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" } + router: { name: dynamo-router, version: "0.8.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -4702,151 +2859,190 @@ dsr1-fp4-b200-dynamo-sglang-mtp: - isl: 8192 osl: 1024 search-space: - # 1p5d low-latency (decode-heavy). - - spec-decoding: "mtp" - conc-list: [4, 8, 16, 32] + # "Low latency" (1 prefill worker at TP8 and 1 decode worker at TP8) + - conc-list: [4, 8, 16] prefill: num-worker: 1 - tp: 4 + tp: 8 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/low-latency.yaml + - "CONFIG_FILE=recipes/gb200-fp8/8k1k/low-latency.yaml" decode: - num-worker: 5 + num-worker: 1 tp: 8 ep: 1 dp-attn: false - # 1p3d low-latency. - - spec-decoding: "mtp" - conc-list: [32, 64] + + # "Mid curve" (5 prefill workers at DEP8 and 1 decode worker at DEP32) + - conc-list: [512, 1024, 2048, 6144] prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/mid-curve.yaml + - "CONFIG_FILE=recipes/gb200-fp8/8k1k/mid-curve.yaml" decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # 1p1d low-latency. - - spec-decoding: "mtp" - conc-list: [32] - prefill: num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false + tp: 32 + ep: 32 + dp-attn: true + + # "Max throughput" (6 prefill workers at DEP8 and 1 decode worker at DEP24) + - conc-list: [2048, 4096, 6144] + prefill: + num-worker: 6 + tp: 8 + ep: 8 + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/max_tpt.yaml + - "CONFIG_FILE=recipes/gb200-fp8/8k1k/max_tpt.yaml" decode: num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - # MTP2 high-throughput (DEP4 prefill / DEP8 decode), one Pareto point each. - # 1p1d throughput. - - spec-decoding: "mtp" - conc-list: [512] + tp: 24 + ep: 24 + dp-attn: true + +dsr1-fp8-gb300-dynamo-sglang: + image: lmsysorg/sglang:v0.5.8.post1-cu130 + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: gb300 + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # "Low latency" (1 prefill worker at TP4 and 1 decode worker at TP4) + - conc-list: [4, 8] prefill: num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/low-latency.yaml + - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/low-latency.yaml" decode: num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + + # "Mid curve" (5 prefill workers at DEP8 and 1 decode worker at DEP32) + - conc-list: [128, 256, 512, 1024] + prefill: + num-worker: 5 tp: 8 ep: 8 dp-attn: true - # 2p1d throughput. - - spec-decoding: "mtp" - conc-list: [768] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/mid.yaml + - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/mid.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - # 3p1d throughput. - - spec-decoding: "mtp" - conc-list: [1024] + + # "Max throughput" (6 prefill workers at DEP8 and 1 decode worker at DEP24) + - conc-list: [2048, 4096] prefill: - num-worker: 3 - tp: 4 - ep: 4 + num-worker: 6 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/max.yaml + - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/max.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 24 + ep: 24 dp-attn: true - # 4p1d throughput. - - spec-decoding: "mtp" - conc-list: [512] + +dsr1-fp4-gb200-dynamo-sglang: + image: "lmsysorg/sglang:v0.5.8-cu130" + model: nvidia/DeepSeek-R1-0528-NVFP4-v2 + model-prefix: dsr1 + runner: gb200 + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # Low latency (1 prefill node, 4 decode nodes) + - spec-decoding: "none" + conc-list: [ 4, 8 ] prefill: - num-worker: 4 + num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml" + - "CONFIG_FILE=recipes/gb200-fp4/8k1k/low-latency.yaml" decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # 5p1d throughput. - - spec-decoding: "mtp" - conc-list: [2048] + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + + # Mid curve (6 prefill nodes, 12 decode nodes) + - spec-decoding: "none" + conc-list: [ 512, 2048, 4096 ] prefill: - num-worker: 5 + num-worker: 6 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml" + - "CONFIG_FILE=recipes/gb200-fp4/8k1k/mid-curve.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 48 + ep: 48 dp-attn: true - # 4p1d high-throughput (dedicated c2048 tune). - - spec-decoding: "mtp" - conc-list: [2048] + + # Max throughput (10 prefill nodes, 8 decode nodes) + - spec-decoding: "none" + conc-list: [ 2048 ] prefill: - num-worker: 4 + num-worker: 10 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml" + - "CONFIG_FILE=recipes/gb200-fp4/8k1k/max-tpt.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true -kimik2.5-fp4-gb200-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: gb200 +dsr1-fp4-gb300-dynamo-trt: + image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 + model: nvidia/DeepSeek-R1-0528-NVFP4-v2 + model-prefix: dsr1 + runner: gb300 precision: fp4 framework: dynamo-trt - router: { name: dynamo-router, version: "v1.3.0-dev.1" } + router: { name: dynamo-router, version: "0.8.1.post2" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -4855,356 +3051,312 @@ kimik2.5-fp4-gb200-dynamo-trt: - isl: 8192 osl: 1024 search-space: - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [ 8 ] + # MTP configurations (spec_decoding="mtp") + - spec-decoding: "mtp" + conc-list: [33] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml" decode: - num-worker: 4 + num-worker: 3 tp: 8 ep: 8 dp-attn: false - - conc-list: [ 24 ] + - spec-decoding: "mtp" + conc-list: [5] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 30 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 60 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 105 ] + - spec-decoding: "mtp" + conc-list: [12, 24] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml" decode: - num-worker: 5 - tp: 4 - ep: 4 + num-worker: 4 + tp: 8 + ep: 8 dp-attn: false - - conc-list: [ 333 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 666 ] + - spec-decoding: "mtp" + conc-list: [180] prefill: num-worker: 4 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - - conc-list: [ 1229 ] + - spec-decoding: "mtp" + conc-list: [308] prefill: - num-worker: 4 - tp: 4 - ep: 4 + num-worker: 8 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep16_batch64_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [ 1229 ] + - spec-decoding: "mtp" + conc-list: [2253] prefill: - num-worker: 6 - tp: 4 - ep: 4 + num-worker: 10 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 8 + ep: 8 dp-attn: true - - conc-list: [ 2253 ] + - spec-decoding: "mtp" + conc-list: [666] prefill: - num-worker: 7 - tp: 4 - ep: 4 + num-worker: 10 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx7dep4_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx7dep4_gen1dep16_batch128_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - conc-list: [ 4301 ] + - spec-decoding: "mtp" + conc-list: [1127] prefill: - num-worker: 9 - tp: 4 - ep: 4 + num-worker: 13 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch256_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - -kimik2.5-fp4-gb300-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: gb300 - precision: fp4 - framework: dynamo-trt - router: { name: dynamo-router, version: "v1.3.0-dev.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: # Non-MTP configurations (default spec_decoding="none") - - conc-list: [ 4 ] + - conc-list: [72] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml" decode: - num-worker: 4 + num-worker: 3 tp: 8 ep: 8 dp-attn: false - - conc-list: [ 12 ] + - conc-list: [5] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - - conc-list: [ 24 ] + - conc-list: [12] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - - conc-list: [ 30 ] + - conc-list: [5, 15, 30] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml" decode: num-worker: 5 tp: 4 ep: 4 dp-attn: false - - conc-list: [ 60 ] + - conc-list: [666] prefill: - num-worker: 1 - tp: 4 - ep: 4 + num-worker: 7 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml" decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 115 ] - prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 32 + ep: 32 dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 333 ] + - conc-list: [1229] prefill: - num-worker: 2 - tp: 4 - ep: 4 + num-worker: 9 + tp: 2 + ep: 2 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 16 + ep: 16 dp-attn: true - - conc-list: [ 615 ] + - conc-list: [3228] prefill: + num-worker: 11 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml" + decode: num-worker: 3 tp: 4 ep: 4 dp-attn: true + - conc-list: [2253] + prefill: + num-worker: 14 + tp: 2 + ep: 2 + dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx3dep4_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx3dep4_gen1dep32_batch16_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 16 + ep: 16 dp-attn: true - - conc-list: [ 1229 ] +dsr1-fp4-gb300-dynamo-sglang: + image: "lmsysorg/sglang:v0.5.8.post1-cu130-runtime" + model: nvidia/DeepSeek-R1-0528-NVFP4-v2 + model-prefix: dsr1 + runner: gb300 + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # Low latency (1 prefill node, 4 decode nodes) + - spec-decoding: "none" + conc-list: [ 4, 8, 32, 64 ] prefill: - num-worker: 5 + num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep32_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/gb300-fp4/8k1k/low_latency.yaml" decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 2253 ] + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + + # Mid curve (6 prefill nodes, 12 decode nodes) + - spec-decoding: "none" + conc-list: [ 512, 2048, 4096 ] prefill: num-worker: 6 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep16_batch128_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/gb300-fp4/8k1k/mid_curve.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 48 + ep: 48 dp-attn: true - - conc-list: [ 2151 ] + + # Max throughput (10 prefill nodes, 8 decode nodes) + - spec-decoding: "none" + conc-list: [ 2048 ] prefill: - num-worker: 8 + num-worker: 10 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx8dep4_gen1dep32_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx8dep4_gen1dep32_batch64_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/gb300-fp4/8k1k/max_tpt.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - -kimik2.5-fp4-gb200-dynamo-vllm: - image: vllm/vllm-openai:v0.21.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: gb200 - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.1" } + tp: 32 + ep: 32 + dp-attn: true + +dsr1-fp8-gb300-dynamo-trt: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: gb300-nv + precision: fp8 + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -5213,266 +3365,204 @@ kimik2.5-fp4-gb200-dynamo-vllm: - isl: 8192 osl: 1024 search-space: - - conc-list: [128] + # MTP configurations (spec_decoding="mtp") + - spec-decoding: "mtp" + conc-list: [8] prefill: num-worker: 1 - tp: 1 + tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tep4.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml" decode: num-worker: 4 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: false - - conc-list: [4, 8, 16, 32, 256] + - spec-decoding: "mtp" + conc-list: [24] prefill: num-worker: 1 - tp: 1 + tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tp8.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml" decode: num-worker: 4 tp: 8 - ep: 1 + ep: 8 dp-attn: false - - conc-list: [1024] + - spec-decoding: "mtp" + conc-list: [333] prefill: - num-worker: 3 - tp: 1 + num-worker: 6 + tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-3p1d-dep4-dep16.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml" decode: num-worker: 1 - tp: 1 - ep: 16 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [3072] + - spec-decoding: "mtp" + conc-list: [666] prefill: - num-worker: 6 - tp: 1 + num-worker: 8 + tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-6p1d-dep4-dep16.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml" decode: num-worker: 1 - tp: 1 + tp: 16 ep: 16 dp-attn: true - - conc-list: [6144] + - spec-decoding: "mtp" + conc-list: [1229] prefill: - num-worker: 8 - tp: 1 + num-worker: 10 + tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-8p1d-dep4-dep16.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml" decode: num-worker: 1 - tp: 1 + tp: 16 ep: 16 dp-attn: true - -dsv4-fp4-b200-dynamo-vllm: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b200-multinode - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.0.dev20260426" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # B200 adaptation of the DSV4 GB200 vLLM disagg recipes. Each worker - # maps to one full 8-GPU B200 node. - - conc-list: [1] + - spec-decoding: "mtp" + conc-list: [1229] prefill: - num-worker: 1 - tp: 8 - ep: 8 + num-worker: 7 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml" decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [32, 128] - prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [64] + # STP configurations (no spec_decoding) + - conc-list: [4] prefill: num-worker: 1 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 - ep: 1 + ep: 8 dp-attn: false - - conc-list: [256] + - conc-list: [24] prefill: num-worker: 1 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 ep: 8 - dp-attn: true - - conc-list: [512] + dp-attn: false + - conc-list: [36] prefill: num-worker: 1 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 ep: 8 - dp-attn: true - -dsv4-fp4-gb200-dynamo-vllm: - image: vllm/vllm-openai:v0.20.0-ubuntu2404 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.0.dev20260426" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Validated 8k/1k points mirrored from NVIDIA/srt-slurm - # aflowers/vllm-gb200-v0.20.0 history. conc-list values match each - # recipe's benchmark.concurrencies. - - # Low latency: 1 prefill (DEP=8) + 1 decode (TP=8). 5 nodes total with - # a dedicated NATS/etcd infra node. - - conc-list: [1] + dp-attn: false + - conc-list: [512] prefill: - num-worker: 1 - tp: 8 - ep: 8 + num-worker: 6 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml" decode: num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - # Low-middle curve: 1 prefill (DEP=8) + 4 decode (TP=8). 11 nodes total - # with a dedicated NATS/etcd infra node. - - conc-list: [256, 512] - prefill: - num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - # MegaMOE mid curve: 1 prefill (DEP=8) + 1 decode (DEP=8). 5 nodes - # total with a dedicated NATS/etcd infra node. - - conc-list: [256, 512, 1024] + - conc-list: [666] prefill: - num-worker: 1 - tp: 8 - ep: 8 + num-worker: 4 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - - # MegaMOE high throughput: 2 prefill (DEP=8 each) + 1 decode (DEP=8). - # 7 nodes total with a dedicated NATS/etcd infra node. - - conc-list: [4096] + - conc-list: [1229] prefill: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 7 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - - # MegaMOE max throughput: 3 prefill (DEP=8 each) + 1 decode (DEP=8). - # 9 nodes total with a dedicated NATS/etcd infra node. - - conc-list: [4096] - prefill: - num-worker: 3 - tp: 8 - ep: 8 + - conc-list: [2151] + prefill: + num-worker: 7 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml + - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - -# TODO: change image to official llmd image. -# Build source: benchmarks/llm-d/Dockerfile. -dsv4-fp4-gb200-llmd-vllm: - image: ghcr.io/ezrasilvera/llm-d-nokube-vllm:vllm0.23 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 - precision: fp4 - framework: llmd-vllm - router: { name: llm-d-router, version: "0.9.0" } +dsr1-fp8-h200-dynamo-sglang: + image: lmsysorg/sglang:v0.5.8.post1-cu130 + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: h200-multinode + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -5481,124 +3571,164 @@ dsv4-fp4-gb200-llmd-vllm: - isl: 8192 osl: 1024 search-space: - # Low latency: 1 prefill DEP8 + 1 decode TP8. + # STP: Low latency TEP (1 prefill, 7 decode) - spec-decoding: "none" - conc-list: [1] + conc-list: [1, 4, 8] prefill: num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "PREFILL_NODES=2" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-low-latency.yaml" + - "CONFIG_FILE=recipes/h200/8k1k/bs4-1p7d.yaml" decode: + num-worker: 7 + tp: 8 + ep: 1 + dp-attn: false + # STP: TEP (1 prefill, 6 decode) + - spec-decoding: "none" + conc-list: [4, 8, 16] + prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" - - # Low-middle: 1 prefill DEP8 + 4 decode TP8. + - "CONFIG_FILE=recipes/h200/8k1k/bs8-1p6d.yaml" + decode: + num-worker: 6 + tp: 8 + ep: 1 + dp-attn: false + # STP: TEP (1 prefill, 3 decode) - spec-decoding: "none" - conc-list: [256, 512] + conc-list: [8, 16, 32] prefill: num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "PREFILL_NODES=2" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-low-latency.yaml" + - "CONFIG_FILE=recipes/h200/8k1k/bs16-1p3d.yaml" decode: - num-worker: 4 + num-worker: 3 + tp: 8 + ep: 1 + dp-attn: false + # STP: TEP (2 prefill, 3 decode) + - spec-decoding: "none" + conc-list: [32, 64, 128] + prefill: + num-worker: 2 tp: 8 ep: 1 dp-attn: false additional-settings: - - "DECODE_NODES=8" - - "DECODE_WORKERS=4" - - "GPUS_PER_NODE=4" - - # Mid curve: 1 prefill DEP8 + 1 decode DEP8. + - "CONFIG_FILE=recipes/h200/8k1k/bs64-2p3d.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 1 + dp-attn: false + # STP: High throughput DEP (1 prefill, 1 decode, dp-attention) - spec-decoding: "none" - conc-list: [256, 512, 1024] + conc-list: [64, 128, 256] prefill: num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "PREFILL_NODES=2" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" + - "CONFIG_FILE=recipes/h200/8k1k/bs128-1p1d-dep.yaml" decode: num-worker: 1 - tp: 1 + tp: 8 ep: 8 dp-attn: true + # MTP: Low latency TEP (1 prefill, 7 decode) + - spec-decoding: "mtp" + conc-list: [1, 4, 8] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" - - # High throughput: 2 prefill DEP8 + 1 decode DEP8. - - spec-decoding: "none" - conc-list: [4096] + - "CONFIG_FILE=recipes/h200/8k1k/bs4-1p7d-mtp.yaml" + decode: + num-worker: 7 + tp: 8 + ep: 1 + dp-attn: false + # MTP: TEP (1 prefill, 6 decode) + - spec-decoding: "mtp" + conc-list: [2, 4, 8, 16, 32] prefill: - num-worker: 2 - tp: 1 - ep: 8 - dp-attn: true + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "PREFILL_NODES=4" - - "PREFILL_WORKERS=2" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" + - "CONFIG_FILE=recipes/h200/8k1k/bs8-1p6d-mtp.yaml" decode: + num-worker: 6 + tp: 8 + ep: 1 + dp-attn: false + # MTP: TEP (1 prefill, 3 decode) + - spec-decoding: "mtp" + conc-list: [4, 8, 16, 32, 64] + prefill: num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" - - # Max throughput: 3 prefill DEP8 + 1 decode DEP8. - - spec-decoding: "none" - conc-list: [4096] + - "CONFIG_FILE=recipes/h200/8k1k/bs16-1p3d-mtp.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 1 + dp-attn: false + # MTP: TEP (2 prefill, 3 decode) + - spec-decoding: "mtp" + conc-list: [32, 64, 128] prefill: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/h200/8k1k/bs64-2p3d-mtp.yaml" + decode: num-worker: 3 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false + # MTP: High throughput DEP (1 prefill, 1 decode, dp-attention) + - spec-decoding: "mtp" + conc-list: [32, 64, 128, 256, 512] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "PREFILL_NODES=6" - - "PREFILL_WORKERS=3" - - "GPUS_PER_NODE=4" - - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" + - "CONFIG_FILE=recipes/h200/8k1k/bs128-1p1d-dep-mtp.yaml" decode: num-worker: 1 - tp: 1 + tp: 8 ep: 8 dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "GPUS_PER_NODE=4" - -# MTP2 variant of dsv4-fp4-gb200-dynamo-vllm. Uses the vLLM 0.20.1 image -# and hand-picked 8k/1k Pareto points mirrored from NVIDIA/srt-slurm. -dsv4-fp4-gb200-dynamo-vllm-mtp2: - image: vllm/vllm-openai:v0.20.1-ubuntu2404 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 +dsr1-fp4-b200-dynamo-sglang: + image: lmsysorg/sglang:v0.5.8.post1-cu130-runtime + model: deepseek-r1-fp4 + model-prefix: dsr1 + runner: b200-multinode precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.0.dev20260426" } + framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -5607,81 +3737,80 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2: - isl: 8192 osl: 1024 search-space: - # Aggregate low latency: TP=8, max-num-seqs=4. - - conc-list: [1] - spec-decoding: mtp + # Non-MTP configurations + - conc-list: [64, 128] prefill: num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false + tp: 4 + ep: 4 + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml" + - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[0]" decode: - num-worker: 0 + num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: false - - # Low-latency bridge: 1 prefill (DEP=8) + 4 decode (TP=8), no offload. - - conc-list: [16, 32, 64] - spec-decoding: mtp + - conc-list: [8] prefill: num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[1]" + decode: + num-worker: 5 tp: 8 ep: 8 + dp-attn: false + - conc-list: [4, 128] + prefill: + num-worker: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml" + - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[2]" decode: - num-worker: 4 + num-worker: 5 tp: 8 - ep: 1 + ep: 8 dp-attn: false - - # MegaMOE mid curve: 1 prefill (DEP=8) + 1 decode (DEP=8). - # 5 nodes total with a dedicated NATS/etcd infra node. - - conc-list: [128, 256, 512, 1024] - spec-decoding: mtp + - conc-list: [4, 8, 16, 64] prefill: num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true + tp: 4 + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml" + - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:override_stp_tp4" decode: num-worker: 1 tp: 8 - ep: 8 - dp-attn: true - - # MegaMOE high throughput: 2 prefill (DEP=8 each) + 1 decode (DEP=8). - # 7 nodes total with a dedicated NATS/etcd infra node. - - conc-list: [1024] - spec-decoding: mtp + ep: 1 + dp-attn: false + - conc-list: [1024, 2048] prefill: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 7 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml" + - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:override_stp_maxtpt_7p2d" decode: - num-worker: 1 + num-worker: 2 tp: 8 ep: 8 dp-attn: true - -dsv4-fp4-gb200-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 - precision: fp4 +dsr1-fp8-b200-dynamo-sglang: + image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64 + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: b200-multinode + precision: fp8 framework: dynamo-sglang - router: { name: dynamo-router, version: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" } - kv-p2p-transfer: mooncake + kv-p2p-transfer: nixl multinode: true disagg: true scenarios: @@ -5689,128 +3818,122 @@ dsv4-fp4-gb200-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - # Low latency: 1p1d-tp8-tp8. 4 nodes. - - conc-list: [1] + # STP low-latency: resolved from 8k1k.yaml zip_override_stp_lowlat + - conc-list: [128] prefill: num-worker: 1 tp: 8 ep: 1 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_0.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_0.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: - num-worker: 1 + num-worker: 3 tp: 8 ep: 1 dp-attn: false - # 1p4d-dep8-tp8. 10 nodes. - - conc-list: [64] + - conc-list: [128] prefill: num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_1.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_1.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 4 tp: 8 ep: 1 dp-attn: false - # WideEP TP=16 decode: 1p2d-dep8-dep16. 10 nodes. - - conc-list: [256] + - conc-list: [8, 16, 32, 64, 128] prefill: num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_2.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_2.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: true - # WideEP TP=16 decode: 1p1d-dep8-dep16. 6 nodes. - - conc-list: [512] + num-worker: 6 + tp: 8 + ep: 1 + dp-attn: false + # STP max-throughput: resolved from 8k1k.yaml zip_override_stp_maxtpt + - conc-list: [288] prefill: num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # WideEP TP=16 decode: 2p1d-dep8-dep16. 8 nodes. - - conc-list: [1536] - prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # WideEP TP=16 decode: 4p1d-dep8-dep16. 12 nodes. - - conc-list: [4096] + - conc-list: [160, 288] prefill: - num-worker: 4 + num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # WideEP TP=16 decode: 5p1d-dep8-dep16. 14 nodes. - - conc-list: [8192] + - conc-list: [512] prefill: - num-worker: 5 + num-worker: 2 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # WideEP TP=12 decode: 6p1d-dep8-dep12. 15 nodes. - - conc-list: [8192] + - conc-list: [1024] prefill: - num-worker: 6 + num-worker: 3 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 - tp: 12 - ep: 12 + tp: 8 + ep: 8 dp-attn: true -qwen3.5-fp8-gb200-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: gb200 +dsr1-fp8-b200-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64 + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: b200-multinode precision: fp8 framework: dynamo-sglang - router: { name: dynamo-router, version: "46520ca59afe992fb5ef61b3197b2316f8df9b2b" } - kv-p2p-transfer: mooncake + kv-p2p-transfer: nixl multinode: true disagg: true scenarios: @@ -5818,199 +3941,129 @@ qwen3.5-fp8-gb200-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - # 1P1D STP: TP4 prefill + TP4 decode (pure tensor parallel). 2 nodes (1+1). - - spec-decoding: "none" - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + # MTP low-latency: resolved from 8k1k.yaml zip_override_mtp_lowlat + - spec-decoding: "mtp" + conc-list: [128] prefill: num-worker: 1 - tp: 4 + tp: 8 ep: 1 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: - num-worker: 1 - tp: 4 + num-worker: 3 + tp: 8 ep: 1 dp-attn: false - # 4P1D wide-EP: 4 prefill DEP4 + decode DEP16. 8 nodes (4+4). - - spec-decoding: "none" - conc-list: [1024] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # 8P1D wide-EP: 8 prefill DEP4 + decode DEP16. 12 nodes (8+4). - - spec-decoding: "none" - conc-list: [2048, 4096] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -# MTP variant of dsv4-fp4-gb200-dynamo-sglang. -dsv4-fp4-gb200-dynamo-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 - precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Low-latency baseline: 1p1d-tp8-tp8. 4 nodes. - spec-decoding: "mtp" - conc-list: [1] + conc-list: [128] prefill: num-worker: 1 tp: 8 ep: 1 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: - num-worker: 1 + num-worker: 4 tp: 8 ep: 1 dp-attn: false - # Low-latency 1p6d-dep8-tp8: 1P (DEP=8) + 6 TP=8 decode workers. 14 nodes. - # Recipe runs concurrencies=32x64x128; matrix tracks the max. - spec-decoding: "mtp" - conc-list: [128] + conc-list: [8, 16, 32, 64, 128] prefill: num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 6 tp: 8 ep: 1 dp-attn: false - # Mid curve 1p1d-dep8-dep16. 6 nodes. + # MTP max-throughput: resolved from 8k1k.yaml zip_override_mtp_maxtpt - spec-decoding: "mtp" - conc-list: [1024] + conc-list: [288] prefill: num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 2p1d-dep8-dep16. 8 nodes. - - spec-decoding: "mtp" - conc-list: [2048] - prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 3p1d-dep8-dep16. 10 nodes. - spec-decoding: "mtp" - conc-list: [3072] + conc-list: [160, 288] prefill: - num-worker: 3 + num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 4p1d-dep8-dep16. 12 nodes. - - spec-decoding: "mtp" - conc-list: [6144] - prefill: - num-worker: 4 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 5p1d-dep8-dep16. 14 nodes. - spec-decoding: "mtp" - conc-list: [8192] + conc-list: [512] prefill: - num-worker: 5 + num-worker: 2 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # Mid curve 6p1d-dep8-dep16. 16 nodes. - spec-decoding: "mtp" - conc-list: [16384] + conc-list: [1024] prefill: - num-worker: 6 + num-worker: 3 tp: 8 - ep: 8 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml + - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true -glm5-fp4-gb200-dynamo-sglang-mtp: - image: lmsysorg/sglang:v0.5.13.post1-cu130 - model: nvidia/GLM-5.1-NVFP4 - model-prefix: glm5.1 - runner: gb200 +dsr1-fp4-b200-dynamo-sglang-mtp: + image: "lmsysorg/sglang:v0.5.12.post1" + model: deepseek-r1-fp4 + model-prefix: dsr1 + runner: b200-multinode precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.2.1" } + router: { name: dynamo-router, version: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -6019,169 +4072,151 @@ glm5-fp4-gb200-dynamo-sglang-mtp: - isl: 8192 osl: 1024 search-space: - # 1p1d dep16. 5 nodes (1P + 4D). + # 1p5d low-latency (decode-heavy). - spec-decoding: "mtp" - conc-list: [109] + conc-list: [4, 8, 16, 32] prefill: num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # 2p1d dep16. 6 nodes (2P + 4D). + num-worker: 5 + tp: 8 + ep: 1 + dp-attn: false + # 1p3d low-latency. - spec-decoding: "mtp" - conc-list: [398] + conc-list: [32, 64] prefill: - num-worker: 2 + num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # 4p1d dep16. 8 nodes (4P + 4D). + num-worker: 3 + tp: 8 + ep: 1 + dp-attn: false + # 1p1d low-latency. - spec-decoding: "mtp" - conc-list: [853] + conc-list: [32] prefill: - num-worker: 4 + num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[2]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # 8p1d dep16. 12 nodes (8P + 4D). + tp: 8 + ep: 1 + dp-attn: false + # MTP2 high-throughput (DEP4 prefill / DEP8 decode), one Pareto point each. + # 1p1d throughput. - spec-decoding: "mtp" - conc-list: [2132] + conc-list: [512] prefill: - num-worker: 8 + num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[3]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # ---------- 8k1k low-latency (per-node TP=4 decode workers, MTP) ---------- - - isl: 8192 - osl: 1024 - search-space: - # 1p4d tp4. 5 nodes. + # 2p1d throughput. - spec-decoding: "mtp" - conc-list: [104] + conc-list: [768] prefill: - num-worker: 1 + num-worker: 2 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml" decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - # 1p4d tp4, conc16. 5 nodes. - - spec-decoding: "mtp" - conc-list: [82] - prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" - decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - # 1p8d tp4. 9 nodes. + # 3p1d throughput. - spec-decoding: "mtp" - conc-list: [87] + conc-list: [1024] prefill: - num-worker: 1 + num-worker: 3 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml" decode: - num-worker: 8 - tp: 4 - ep: 1 - dp-attn: false - # 1p8d tp4, conc4. 9 nodes. + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 4p1d throughput. - spec-decoding: "mtp" - conc-list: [47] + conc-list: [512] prefill: - num-worker: 1 + num-worker: 4 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml" decode: - num-worker: 8 - tp: 4 - ep: 1 - dp-attn: false - # 1p16d tp4. 17 nodes. + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 5p1d throughput. - spec-decoding: "mtp" - conc-list: [74] + conc-list: [2048] prefill: - num-worker: 1 + num-worker: 5 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml" decode: - num-worker: 16 - tp: 4 - ep: 1 - dp-attn: false - # 1p16d tp4, conc1. 17 nodes. + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 4p1d high-throughput (dedicated c2048 tune). - spec-decoding: "mtp" - conc-list: [26] + conc-list: [2048] prefill: - num-worker: 1 + num-worker: 4 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[5]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml" decode: - num-worker: 16 - tp: 4 - ep: 1 - dp-attn: false - # ---------- 1k1k high-throughput (wide-EP decode, MTP) ---------- -dsv4-fp4-b300-dynamo-vllm: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b300 + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + +kimik2.5-fp4-gb200-dynamo-trt: + image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: gb200 precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.0.dev20260426" } + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -6190,151 +4225,184 @@ dsv4-fp4-b300-dynamo-vllm: - isl: 8192 osl: 1024 search-space: - # B300 adaptation of the DSV4 B200/GB200 vLLM disagg recipes. Each - # prefill/decode worker maps to one full 8-GPU B300 node. - - conc-list: [1, 32, 64, 128] + # Non-MTP configurations (default spec_decoding="none") + - conc-list: [ 8 ] prefill: num-worker: 1 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 - ep: 1 + ep: 8 dp-attn: false - - conc-list: [256, 1024] + - conc-list: [ 24 ] prefill: num-worker: 1 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 ep: 8 - dp-attn: true - - conc-list: [4096] + dp-attn: false + - conc-list: [ 5 ] prefill: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 30 ] + prefill: num-worker: 1 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: true - -dsv4-fp4-gb300-dynamo-vllm: - image: vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.0.dev20260426" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [192] + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 60 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" decode: - num-worker: 6 + num-worker: 5 tp: 4 - ep: 1 + ep: 4 dp-attn: false - - conc-list: [18] + - conc-list: [ 105 ] prefill: num-worker: 1 tp: 4 ep: 4 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p9d-tep4-tp4.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" decode: - num-worker: 9 + num-worker: 5 tp: 4 - ep: 1 + ep: 4 dp-attn: false - - conc-list: [4096] + - conc-list: [ 333 ] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 666 ] prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [4096] + - conc-list: [ 1229 ] prefill: - num-worker: 5 + num-worker: 4 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep16_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep16_batch64_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - - conc-list: [4096] + - conc-list: [ 1229 ] prefill: num-worker: 6 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [3072] + - conc-list: [ 2253 ] prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx7dep4_gen1dep16_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx7dep4_gen1dep16_batch128_eplb0_mtp0.yaml" decode: - num-worker: 2 + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [ 4301 ] + prefill: + num-worker: 9 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch256_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch256_eplb0_mtp0.yaml" + decode: + num-worker: 1 tp: 16 ep: 16 dp-attn: true -dsv4-fp4-gb300-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300-nv +kimik2.5-fp4-gb300-dynamo-trt: + image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: gb300 precision: fp4 framework: dynamo-trt - router: { name: dynamo-router, version: "v1.3.0-deepseek-v4-dev.1" } + router: { name: dynamo-router, version: "v1.3.0-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -6343,197 +4411,170 @@ dsv4-fp4-gb300-dynamo-trt: - isl: 8192 osl: 1024 search-space: - - conc-list: [4] + # Non-MTP configurations (default spec_decoding="none") + - conc-list: [ 4 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" decode: num-worker: 4 tp: 8 ep: 8 dp-attn: false - - conc-list: [5] + - conc-list: [ 12 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml" decode: - num-worker: 5 - tp: 4 - ep: 4 + num-worker: 4 + tp: 8 + ep: 8 dp-attn: false - - conc-list: [15] + - conc-list: [ 24 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" decode: - num-worker: 5 - tp: 4 - ep: 4 + num-worker: 4 + tp: 8 + ep: 8 dp-attn: false - - conc-list: [25] + - conc-list: [ 30 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" decode: num-worker: 5 tp: 4 ep: 4 dp-attn: false - - conc-list: [55] + - conc-list: [ 60 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" decode: num-worker: 5 tp: 4 ep: 4 dp-attn: false - - conc-list: [154] + - conc-list: [ 115 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [308] - prefill: - num-worker: 2 + num-worker: 5 tp: 4 ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [615] + dp-attn: false + - conc-list: [ 333 ] prefill: - num-worker: 4 + num-worker: 2 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb384_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - - conc-list: [1127] + - conc-list: [ 615 ] prefill: - num-worker: 6 + num-worker: 3 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb384_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx3dep4_gen1dep32_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx3dep4_gen1dep32_batch16_eplb0_mtp0.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - - conc-list: [1229] - prefill: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [2253] + - conc-list: [ 1229 ] prefill: - num-worker: 6 + num-worker: 5 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep8_batch256_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep8_batch256_eplb384_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep32_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep32_batch32_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [2253] + - conc-list: [ 2253 ] prefill: - num-worker: 9 + num-worker: 6 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb384_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep16_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep16_batch128_eplb0_mtp0.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - conc-list: [4301] + - conc-list: [ 2151 ] prefill: - num-worker: 10 + num-worker: 8 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx10dep4_gen1dep8_batch512_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx10dep4_gen1dep8_batch512_eplb384_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx8dep4_gen1dep32_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx8dep4_gen1dep32_batch64_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true -dsv4-fp4-gb300-dynamo-trt-mtp: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300-nv +kimik2.5-fp4-gb200-dynamo-vllm: + image: vllm/vllm-openai:v0.21.0 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: gb200 precision: fp4 - framework: dynamo-trt - router: { name: dynamo-router, version: "v1.3.0-deepseek-v4-dev.1" } + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -6542,211 +4583,267 @@ dsv4-fp4-gb300-dynamo-trt-mtp: - isl: 8192 osl: 1024 search-space: - - spec-decoding: "mtp" - conc-list: [8] + - conc-list: [128] prefill: num-worker: 1 - tp: 4 + tp: 1 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tep4.yaml" decode: num-worker: 4 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: false - - spec-decoding: "mtp" - conc-list: [10] + - conc-list: [4, 8, 16, 32, 256] prefill: num-worker: 1 - tp: 4 + tp: 1 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tp8.yaml" decode: - num-worker: 5 - tp: 4 - ep: 4 + num-worker: 4 + tp: 8 + ep: 1 dp-attn: false - - spec-decoding: "mtp" - conc-list: [15] + - conc-list: [1024] prefill: - num-worker: 1 - tp: 4 + num-worker: 3 + tp: 1 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-3p1d-dep4-dep16.yaml" decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [30] - prefill: num-worker: 1 - tp: 4 + tp: 1 + ep: 16 + dp-attn: true + - conc-list: [3072] + prefill: + num-worker: 6 + tp: 1 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-6p1d-dep4-dep16.yaml" decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [84] + num-worker: 1 + tp: 1 + ep: 16 + dp-attn: true + - conc-list: [6144] prefill: - num-worker: 2 - tp: 4 + num-worker: 8 + tp: 1 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch2_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch2_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-8p1d-dep4-dep16.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 1 + ep: 16 dp-attn: true - - spec-decoding: "mtp" - conc-list: [180] + +dsv4-fp4-b200-dynamo-vllm: + image: vllm/vllm-openai:v0.23.0 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: b200-multinode + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # B200 adaptation of the DSV4 GB200 vLLM disagg recipes. Each worker + # maps to one full 8-GPU B200 node. + - conc-list: [1] prefill: - num-worker: 3 - tp: 4 - ep: 4 + num-worker: 1 + tp: 8 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx3dep4_gen1dep32_batch4_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx3dep4_gen1dep32_batch4_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [333] + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [32, 128] prefill: - num-worker: 4 - tp: 4 - ep: 4 + num-worker: 1 + tp: 8 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [64] + prefill: + num-worker: 1 + tp: 8 + ep: 8 dp-attn: true - - spec-decoding: "mtp" - conc-list: [615] + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - conc-list: [256] prefill: - num-worker: 8 - tp: 4 - ep: 4 + num-worker: 1 + tp: 8 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep32_batch16_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep32_batch16_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 8 + ep: 8 dp-attn: true - - spec-decoding: "mtp" - conc-list: [666] + - conc-list: [512] prefill: - num-worker: 6 - tp: 4 - ep: 4 + num-worker: 1 + tp: 8 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] + +dsv4-fp4-gb200-dynamo-vllm: + image: vllm/vllm-openai:v0.20.0-ubuntu2404 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb200 + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # Validated 8k/1k points mirrored from NVIDIA/srt-slurm + # aflowers/vllm-gb200-v0.20.0 history. conc-list values match each + # recipe's benchmark.concurrencies. + + # Low latency: 1 prefill (DEP=8) + 1 decode (TP=8). 5 nodes total with + # a dedicated NATS/etcd infra node. + - conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + + # Low-middle curve: 1 prefill (DEP=8) + 4 decode (TP=8). 11 nodes total + # with a dedicated NATS/etcd infra node. + - conc-list: [256, 512] prefill: - num-worker: 7 - tp: 4 - ep: 4 + num-worker: 1 + tp: 8 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep8_batch128_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep8_batch128_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml" decode: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: false + + # MegaMOE mid curve: 1 prefill (DEP=8) + 1 decode (DEP=8). 5 nodes + # total with a dedicated NATS/etcd infra node. + - conc-list: [256, 512, 1024] + prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 10 - tp: 4 - ep: 4 - dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx10dep4_gen1dep16_batch64_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx10dep4_gen1dep16_batch64_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - - spec-decoding: "mtp" - conc-list: [2253] + + # MegaMOE high throughput: 2 prefill (DEP=8 each) + 1 decode (DEP=8). + # 7 nodes total with a dedicated NATS/etcd infra node. + - conc-list: [4096] prefill: - num-worker: 9 - tp: 4 - ep: 4 + num-worker: 2 + tp: 8 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx9dep4_gen1dep8_batch256_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx9dep4_gen1dep8_batch256_eplb384_mtp1.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - - spec-decoding: "mtp" - conc-list: [4301] + + # MegaMOE max throughput: 3 prefill (DEP=8 each) + 1 decode (DEP=8). + # 9 nodes total with a dedicated NATS/etcd infra node. + - conc-list: [4096] prefill: - num-worker: 12 - tp: 4 - ep: 4 + num-worker: 3 + tp: 8 + ep: 8 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep8_batch512_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep8_batch512_eplb384_mtp1.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true -dsv4-fp4-gb300-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd +# TODO: change image to official llmd image. +# Build source: benchmarks/llm-d/Dockerfile. +dsv4-fp4-gb200-llmd-vllm: + image: ghcr.io/ezrasilvera/llm-d-nokube-vllm:vllm0.23 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: gb300 + runner: gb200 precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "81d0555ee23519cea80a42b4fe824e30368b7300" } - kv-p2p-transfer: mooncake + framework: llmd-vllm + router: { name: llm-d-router, version: "0.9.0" } + kv-p2p-transfer: nixl multinode: true disagg: true scenarios: @@ -6754,115 +4851,207 @@ dsv4-fp4-gb300-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - # WideEP TP=16 decode: 1p1d-dep4-dep16. 5 nodes (4P + 16D = 20 GPUs). - - conc-list: [1024] + # Low latency: 1 prefill DEP8 + 1 decode TP8. + - spec-decoding: "none" + conc-list: [1] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml" + - "PREFILL_NODES=2" + - "GPUS_PER_NODE=4" + - "CONFIG_FILE=dsv4-fp4-gb200-low-latency.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Low concurrency: 1p1d-tp4-tp4. 2 nodes. - - conc-list: [1] - prefill: - num-worker: 1 - tp: 4 + tp: 8 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml" - decode: + - "DECODE_NODES=2" + - "GPUS_PER_NODE=4" + + # Low-middle: 1 prefill DEP8 + 4 decode TP8. + - spec-decoding: "none" + conc-list: [256, 512] + prefill: num-worker: 1 - tp: 4 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + - "GPUS_PER_NODE=4" + - "CONFIG_FILE=dsv4-fp4-gb200-low-latency.yaml" + decode: + num-worker: 4 + tp: 8 ep: 1 dp-attn: false - # --- Weiliang wide-EP sweep (srt-slurm PR#173), 18 nodes total --- - # EP=12: 15P+3D, conc=12000. - - conc-list: [12000] + additional-settings: + - "DECODE_NODES=8" + - "DECODE_WORKERS=4" + - "GPUS_PER_NODE=4" + + # Mid curve: 1 prefill DEP8 + 1 decode DEP8. + - spec-decoding: "none" + conc-list: [256, 512, 1024] prefill: - num-worker: 15 - tp: 4 - ep: 4 + num-worker: 1 + tp: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml" + - "PREFILL_NODES=2" + - "GPUS_PER_NODE=4" + - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" decode: num-worker: 1 - tp: 12 - ep: 12 + tp: 1 + ep: 8 dp-attn: true - # EP=16: 14P+4D, conc=8192. - - conc-list: [8192] + additional-settings: + - "DECODE_NODES=2" + - "GPUS_PER_NODE=4" + + # High throughput: 2 prefill DEP8 + 1 decode DEP8. + - spec-decoding: "none" + conc-list: [4096] prefill: - num-worker: 14 - tp: 4 - ep: 4 + num-worker: 2 + tp: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml" + - "PREFILL_NODES=4" + - "PREFILL_WORKERS=2" + - "GPUS_PER_NODE=4" + - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 1 + ep: 8 dp-attn: true - # EP=24: 12P+6D, conc=3000. - - conc-list: [3000] + additional-settings: + - "DECODE_NODES=2" + - "GPUS_PER_NODE=4" + + # Max throughput: 3 prefill DEP8 + 1 decode DEP8. + - spec-decoding: "none" + conc-list: [4096] prefill: - num-worker: 12 - tp: 4 - ep: 4 + num-worker: 3 + tp: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml" + - "PREFILL_NODES=6" + - "PREFILL_WORKERS=3" + - "GPUS_PER_NODE=4" + - "CONFIG_FILE=dsv4-fp4-gb200-mid-curve-megamoe.yaml" decode: num-worker: 1 - tp: 24 - ep: 24 + tp: 1 + ep: 8 dp-attn: true - # EP=32: 10P+8D, conc=2500. - - conc-list: [2500] + additional-settings: + - "DECODE_NODES=2" + - "GPUS_PER_NODE=4" + +# MTP2 variant of dsv4-fp4-gb200-dynamo-vllm. Uses the vLLM 0.20.1 image +# and hand-picked 8k/1k Pareto points mirrored from NVIDIA/srt-slurm. +dsv4-fp4-gb200-dynamo-vllm-mtp2: + image: vllm/vllm-openai:v0.20.1-ubuntu2404 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb200 + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # Aggregate low latency: TP=8, max-num-seqs=4. + - conc-list: [1] + spec-decoding: mtp prefill: - num-worker: 10 - tp: 4 - ep: 4 + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + + # Low-latency bridge: 1 prefill (DEP=8) + 4 decode (TP=8), no offload. + - conc-list: [16, 32, 64] + spec-decoding: mtp + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 1 + dp-attn: false + + # MegaMOE mid curve: 1 prefill (DEP=8) + 1 decode (DEP=8). + # 5 nodes total with a dedicated NATS/etcd infra node. + - conc-list: [128, 256, 512, 1024] + spec-decoding: mtp + prefill: + num-worker: 1 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 8 + ep: 8 dp-attn: true - # EP=40: 8P+10D, conc=2048. - - conc-list: [2048] + + # MegaMOE high throughput: 2 prefill (DEP=8 each) + 1 decode (DEP=8). + # 7 nodes total with a dedicated NATS/etcd infra node. + - conc-list: [1024] + spec-decoding: mtp prefill: - num-worker: 8 - tp: 4 - ep: 4 + num-worker: 2 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml" decode: num-worker: 1 - tp: 40 - ep: 40 + tp: 8 + ep: 8 dp-attn: true -glm5-fp8-b200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: b200-dgxc - precision: fp8 +dsv4-fp4-gb200-dynamo-sglang: + image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb200 + precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.0" } - kv-p2p-transfer: nixl + router: { name: dynamo-router, version: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" } + kv-p2p-transfer: mooncake multinode: true disagg: true scenarios: @@ -6870,133 +5059,127 @@ glm5-fp8-b200-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - - conc-list: [560] + # Low latency: 1p1d-tp8-tp8. 4 nodes. + - conc-list: [1] prefill: - num-worker: 2 + num-worker: 1 tp: 8 ep: 1 - dp-attn: true + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_hightpt[0]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml" decode: num-worker: 1 tp: 8 ep: 1 - dp-attn: true - - conc-list: [240] + dp-attn: false + # 1p4d-dep8-tp8. 10 nodes. + - conc-list: [64] prefill: num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_hightpt[1]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 ep: 1 - dp-attn: true - - conc-list: [224] + dp-attn: false + # WideEP TP=16 decode: 1p2d-dep8-dep16. 10 nodes. + - conc-list: [256] prefill: num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_hightpt[2]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml" decode: num-worker: 2 - tp: 8 - ep: 1 + tp: 16 + ep: 16 dp-attn: true - - conc-list: [256] + # WideEP TP=16 decode: 1p1d-dep8-dep16. 6 nodes. + - conc-list: [512] prefill: num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[0]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml" decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [256] - prefill: num-worker: 1 - tp: 8 - ep: 1 + tp: 16 + ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[1]" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [200] + # WideEP TP=16 decode: 2p1d-dep8-dep16. 8 nodes. + - conc-list: [1536] prefill: - num-worker: 1 + num-worker: 2 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[2]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml" decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [128] - prefill: num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # WideEP TP=16 decode: 4p1d-dep8-dep16. 12 nodes. + - conc-list: [4096] + prefill: + num-worker: 4 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[3]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml" decode: - num-worker: 5 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [64] - prefill: num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # WideEP TP=16 decode: 5p1d-dep8-dep16. 14 nodes. + - conc-list: [8192] + prefill: + num-worker: 5 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[4]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml" decode: - num-worker: 7 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [12] - prefill: num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # WideEP TP=12 decode: 6p1d-dep8-dep12. 15 nodes. + - conc-list: [8192] + prefill: + num-worker: 6 tp: 8 - ep: 1 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[5]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml" decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false + num-worker: 1 + tp: 12 + ep: 12 + dp-attn: true -# MTP variant of dsv4-fp4-gb300-dynamo-sglang. -dsv4-fp4-gb300-dynamo-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-20260527-14f81a67 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300 - precision: fp4 +qwen3.5-fp8-gb200-dynamo-sglang: + image: lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: gb200 + precision: fp8 framework: dynamo-sglang - router: { name: dynamo-router, version: "81d0555ee23519cea80a42b4fe824e30368b7300" } + router: { name: dynamo-router, version: "46520ca59afe992fb5ef61b3197b2316f8df9b2b" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -7005,345 +5188,259 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: - isl: 8192 osl: 1024 search-space: - # Low-latency baseline: 1p1d-tp4-tp4. 2 nodes. - - spec-decoding: "mtp" - conc-list: [1] + # 1P1D STP: TP4 prefill + TP4 decode (pure tensor parallel). 2 nodes (1+1). + - spec-decoding: "none" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - # Low-latency 1p6d-dep4-tp4: 1P (DEP=4) + 6 TP=4 decode workers. 7 nodes. - # Recipe runs concurrencies=8x32x64; matrix tracks the max. - - spec-decoding: "mtp" - conc-list: [64] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml" - decode: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - # Mid curve 1p1d-dep4-dep8. 3 nodes. - - spec-decoding: "mtp" - conc-list: [256] + # 4P1D wide-EP: 4 prefill DEP4 + decode DEP16. 8 nodes (4+4). + - spec-decoding: "none" + conc-list: [1024] prefill: - num-worker: 1 + num-worker: 4 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - # Mid curve 1p1d-dep4-dep16. 5 nodes. - - spec-decoding: "mtp" - conc-list: [256] + # 8P1D wide-EP: 8 prefill DEP4 + decode DEP16. 12 nodes (8+4). + - spec-decoding: "none" + conc-list: [2048, 4096] prefill: - num-worker: 1 + num-worker: 8 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - # Mid curve 2p1d-dep4-dep8. 4 nodes. + +# MTP variant of dsv4-fp4-gb200-dynamo-sglang. +dsv4-fp4-gb200-dynamo-sglang-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb200 + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # Low-latency baseline: 1p1d-tp8-tp8. 4 nodes. - spec-decoding: "mtp" - conc-list: [512] + conc-list: [1] prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml" decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + # Low-latency 1p6d-dep8-tp8: 1P (DEP=8) + 6 TP=8 decode workers. 14 nodes. + # Recipe runs concurrencies=32x64x128; matrix tracks the max. + - spec-decoding: "mtp" + conc-list: [128] + prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - # Mid curve 4p1d-dep4-dep8. 6 nodes. + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml" + decode: + num-worker: 6 + tp: 8 + ep: 1 + dp-attn: false + # Mid curve 1p1d-dep8-dep16. 6 nodes. - spec-decoding: "mtp" conc-list: [1024] prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml" - decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - # High concurrency 6p1d-dep4-dep8. 8 nodes. - - spec-decoding: "mtp" - conc-list: [4096] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - # High concurrency 8p1d-dep4-dep8. 10 nodes. + # Mid curve 2p1d-dep8-dep16. 8 nodes. - spec-decoding: "mtp" - conc-list: [8192] + conc-list: [2048] prefill: - num-worker: 8 - tp: 4 - ep: 4 + num-worker: 2 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 16 + ep: 16 dp-attn: true - -kimik2.5-int4-h100-vllm: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: cluster:h100-dgxc - precision: int4 - framework: vllm - multinode: false - scenarios: - # H100 has 80 GB HBM per GPU (smallest in this set); the KV cliff arrives - # early. Sweep saturates conc=20 to keep total HBM headroom. - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 12, 16, 20] } - -qwen3.5-fp8-h100-sglang: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: h100 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 8 } - - { tp: 8, ep: 8, conc-start: 16, conc-end: 256 } - -qwen3.5-fp8-h100-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: h100 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 32, spec-decoding: mtp } - -glm5-fp4-gb200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 - model: nvidia/GLM-5.1-NVFP4 - model-prefix: glm5.1 - runner: gb200 - precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "1.2.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 4p1d wide-EP. 12 nodes. - - conc-list: [989] + # Mid curve 3p1d-dep8-dep16. 10 nodes. + - spec-decoding: "mtp" + conc-list: [3072] prefill: - num-worker: 4 - tp: 4 - ep: 4 + num-worker: 3 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 16 + ep: 16 dp-attn: true - # 4p1d wide-EP, mrr512. 12 nodes. - - conc-list: [686] + # Mid curve 4p1d-dep8-dep16. 12 nodes. + - spec-decoding: "mtp" + conc-list: [6144] prefill: num-worker: 4 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 16 + ep: 16 dp-attn: true - # 6p1d wide-EP. 14 nodes. - - conc-list: [1497] + # Mid curve 5p1d-dep8-dep16. 14 nodes. + - spec-decoding: "mtp" + conc-list: [8192] prefill: - num-worker: 6 - tp: 4 - ep: 4 + num-worker: 5 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 16 + ep: 16 dp-attn: true - # 10p1d wide-EP. 18 nodes. - - conc-list: [2674] + # Mid curve 6p1d-dep8-dep16. 16 nodes. + - spec-decoding: "mtp" + conc-list: [16384] prefill: - num-worker: 10 - tp: 4 - ep: 4 + num-worker: 6 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_3.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 16 + ep: 16 dp-attn: true - # ---------- 8k1k low-latency (per-node TP=4 decode workers) ---------- + + # ---------- 1k1k high-throughput (wide-EP decode, MTP) ---------- +dsv4-fp4-b300-dynamo-vllm: + image: vllm/vllm-openai:v0.23.0 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: b300 + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # 1p4d. 5 nodes. - - conc-list: [133] + # B300 adaptation of the DSV4 B200/GB200 vLLM disagg recipes. Each + # prefill/decode worker maps to one full 8-GPU B300 node. + - conc-list: [1, 32, 64, 128] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml" decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - # 1p6d. 7 nodes. - - conc-list: [146] - prefill: num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" - decode: - num-worker: 6 - tp: 4 + tp: 8 ep: 1 dp-attn: false - # 1p6d, mrr16. 7 nodes. - - conc-list: [103] + - conc-list: [256, 1024] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml" decode: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - # 1p8d. 9 nodes. - - conc-list: [130] - prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml" - decode: - num-worker: 8 - tp: 4 - ep: 1 - dp-attn: false - # 1p16d. 17 nodes. - - conc-list: [113] + - conc-list: [4096] prefill: - num-worker: 1 - tp: 4 - ep: 4 + num-worker: 2 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml" decode: - num-worker: 16 - tp: 4 - ep: 1 - dp-attn: false - # 1p16d, mrr1 (single-stream). 17 nodes. - - conc-list: [23] - prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_5.yaml" - decode: - num-worker: 16 - tp: 4 - ep: 1 - dp-attn: false - # ---------- 1k1k high-throughput (wide-EP TP=32 decode) ---------- -qwen3.5-fp4-gb300-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: gb300 + +dsv4-fp4-gb300-dynamo-vllm: + image: vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb300-nv precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "1.1.0" } + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } + kv-p2p-transfer: nixl multinode: true disagg: true scenarios: @@ -7351,83 +5448,93 @@ qwen3.5-fp4-gb300-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - # 1P1D TP4: 1 prefill worker at TP4 and 1 decode worker at TP4 - # Pure tensor parallel (STP), 8k1k baseline-low-latency sweep. - # Total: 8 GB300 GPUs. - - spec-decoding: "none" - kv-p2p-transfer: mooncake - conc-list: [1, 4, 8, 16, 32, 64, 256] + - conc-list: [192] prefill: num-worker: 1 tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml" + decode: + num-worker: 6 + tp: 4 ep: 1 dp-attn: false + - conc-list: [18] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p9d-tep4-tp4.yaml" decode: - num-worker: 1 + num-worker: 9 tp: 4 ep: 1 dp-attn: false - # 5P1D wide-EP: 5 prefill workers @ DEP4 + 1 decode worker @ DEP16. - # NIXL transfer. Total: 36 GB300 GPUs (5*4 + 4*4). - - spec-decoding: "none" - kv-p2p-transfer: nixl - conc-list: [2048] + - conc-list: [4096] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [4096] prefill: num-worker: 5 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # 6P1D wide-EP: 6 prefill workers @ DEP4 + 1 decode worker @ DEP16. - # Mooncake transfer. Total: 40 GB300 GPUs (6*4 + 4*4). - - spec-decoding: "none" - kv-p2p-transfer: mooncake - conc-list: [5120] + - conc-list: [4096] prefill: num-worker: 6 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # 7P1D wide-EP: 7 prefill workers @ DEP4 + 1 decode worker @ DEP16. - # Mooncake transfer. Total: 44 GB300 GPUs (7*4 + 4*4). - - spec-decoding: "none" - kv-p2p-transfer: mooncake - conc-list: [5120] + - conc-list: [3072] prefill: num-worker: 7 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml" decode: - num-worker: 1 + num-worker: 2 tp: 16 ep: 16 dp-attn: true -glm5-fp4-gb300-dynamo-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 - model: nvidia/GLM-5.1-NVFP4 - model-prefix: glm5 +dsv4-fp4-gb300-dynamo-trt: + image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 runner: gb300-nv precision: fp4 - framework: dynamo-sglang + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-deepseek-v4-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -7436,654 +5543,713 @@ glm5-fp4-gb300-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - # 5p1d wide-EP. 13 nodes (5P @ TP=4 + 1D @ TP=32 on 8 nodes). - - conc-list: [2048] + - conc-list: [4] prefill: - num-worker: 5 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[0]" - router: { name: dynamo-router, version: "0.8.0" } - decode: num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # 7p1d wide-EP. 15 nodes. - - conc-list: [3072] - prefill: - num-worker: 7 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[1]" - router: { name: dynamo-router, version: "0.8.0" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # 10p1d wide-EP. 18 nodes. - - conc-list: [4096] + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [5] prefill: - num-worker: 10 + num-worker: 1 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[2]" - router: { name: dynamo-router, version: "0.8.0" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # ---------- 8k1k low-latency (per-node TP=4 decode workers) ---------- - - isl: 8192 - osl: 1024 - search-space: - # 1p3d. 4 nodes (1P + 3 D workers @ 1 node each). - - conc-list: [128] + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [15] prefill: num-worker: 1 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" - router: { name: dynamo-router, version: "1.1.0" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml" decode: - num-worker: 3 + num-worker: 5 tp: 4 - ep: 1 + ep: 4 dp-attn: false - # 1p5d. 6 nodes. - - conc-list: [64] + - conc-list: [25] prefill: num-worker: 1 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" - router: { name: dynamo-router, version: "1.1.0" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" decode: num-worker: 5 tp: 4 - ep: 1 + ep: 4 dp-attn: false - # 1p9d. 10 nodes. - - conc-list: [32] + - conc-list: [55] prefill: num-worker: 1 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" - router: { name: dynamo-router, version: "1.1.0" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" decode: - num-worker: 9 + num-worker: 5 tp: 4 - ep: 1 + ep: 4 dp-attn: false - # 1p15d. 16 nodes. - - conc-list: [16] + - conc-list: [154] prefill: num-worker: 1 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml" - router: { name: dynamo-router, version: "1.1.0" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml" decode: - num-worker: 15 - tp: 4 - ep: 1 - dp-attn: false - # 1p17d. 18 nodes. - - conc-list: [12] - prefill: num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [308] + prefill: + num-worker: 2 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml" - router: { name: dynamo-router, version: "1.1.0" } + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb384_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb384_mtp0.yaml" decode: - num-worker: 17 + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [615] + prefill: + num-worker: 4 tp: 4 - ep: 1 - dp-attn: false - # ---------- 1k1k high-throughput (wide-EP TP=32 decode) ---------- -glm5-fp8-gb200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.12 - model: zai-org/GLM-5.1-FP8 - model-prefix: glm5.1 - runner: gb200 - precision: fp8 - framework: dynamo-sglang - router: { name: dynamo-router, version: "1.2.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 3p1d wide-EP (dep32, mrr256). 14 nodes. - - conc-list: [519] + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb384_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb384_mtp0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [1127] prefill: - num-worker: 3 - tp: 8 - ep: 1 + num-worker: 6 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb384_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb384_mtp0.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - # 4p1d wide-EP (dep16). 12 nodes. - - conc-list: [1484] + - conc-list: [1229] prefill: - num-worker: 4 - tp: 8 - ep: 1 + num-worker: 5 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb384_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb384_mtp0.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - # 5p1d wide-EP (dep32). 18 nodes. - - conc-list: [1688] + - conc-list: [2253] prefill: - num-worker: 5 + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep8_batch256_eplb384_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep8_batch256_eplb384_mtp0.yaml" + decode: + num-worker: 1 tp: 8 - ep: 1 + ep: 8 + dp-attn: true + - conc-list: [2253] + prefill: + num-worker: 9 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb384_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb384_mtp0.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 16 + ep: 16 dp-attn: true - # 7p1d wide-EP (dep16). 18 nodes. - - conc-list: [2699] + - conc-list: [4301] prefill: - num-worker: 7 - tp: 8 - ep: 1 + num-worker: 10 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx10dep4_gen1dep8_batch512_eplb384_mtp0.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx10dep4_gen1dep8_batch512_eplb384_mtp0.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # ---------- 8k1k low-latency (per-node TP=8 decode workers) ---------- + +dsv4-fp4-gb300-dynamo-trt-mtp: + image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-deepseek-v4-dev.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # 1p8d, mrr1 (single-stream). 18 nodes. - - conc-list: [8] + - spec-decoding: "mtp" + conc-list: [8] prefill: num-worker: 1 - tp: 8 - ep: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" decode: - num-worker: 8 + num-worker: 4 tp: 8 - ep: 1 + ep: 8 dp-attn: false - # 1p8d, mrr16. 18 nodes. - - conc-list: [90] + - spec-decoding: "mtp" + conc-list: [10] prefill: num-worker: 1 - tp: 8 - ep: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" decode: - num-worker: 8 - tp: 8 - ep: 1 + num-worker: 5 + tp: 4 + ep: 4 dp-attn: false - # 1p8d, mrr4. 18 nodes. - - conc-list: [9] + - spec-decoding: "mtp" + conc-list: [15] prefill: num-worker: 1 - tp: 8 - ep: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" decode: - num-worker: 8 - tp: 8 - ep: 1 + num-worker: 5 + tp: 4 + ep: 4 dp-attn: false - # ---------- 1k1k high-throughput (wide-EP decode) ---------- -glm5-fp8-gb200-dynamo-sglang-mtp: - image: lmsysorg/sglang:v0.5.13.post1-cu130 - model: zai-org/GLM-5.1-FP8 - model-prefix: glm5.1 - runner: gb200 - precision: fp8 - framework: dynamo-sglang - router: { name: dynamo-router, version: "1.2.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # ---------- 8k1k high-throughput (wide-EP TP=16 decode, MTP) ---------- - - isl: 8192 - osl: 1024 - search-space: - # 1p1d dep16. 6 nodes (2P + 4D). - spec-decoding: "mtp" - conc-list: [137] + conc-list: [30] prefill: num-worker: 1 - tp: 8 - ep: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # 2p1d dep16. 8 nodes (4P + 4D). + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false - spec-decoding: "mtp" - conc-list: [620] + conc-list: [84] prefill: num-worker: 2 - tp: 8 - ep: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch2_eplb384_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch2_eplb384_mtp3.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 32 + ep: 32 dp-attn: true - # 4p1d dep16. 12 nodes (8P + 4D). - spec-decoding: "mtp" - conc-list: [1305] + conc-list: [180] prefill: - num-worker: 4 - tp: 8 - ep: 1 + num-worker: 3 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[2]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx3dep4_gen1dep32_batch4_eplb384_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx3dep4_gen1dep32_batch4_eplb384_mtp3.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 32 + ep: 32 dp-attn: true - # ---------- 8k1k low-latency (TP=8 decode workers, MTP) ---------- - - isl: 8192 - osl: 1024 - search-space: - # 1p4d tp8. 10 nodes. - spec-decoding: "mtp" - conc-list: [136] + conc-list: [333] prefill: - num-worker: 1 - tp: 8 - ep: 1 + num-worker: 4 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb384_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb384_mtp3.yaml" decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - # 1p4d tp8, conc56. 10 nodes. + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true - spec-decoding: "mtp" - conc-list: [56] + conc-list: [615] prefill: - num-worker: 1 - tp: 8 - ep: 1 + num-worker: 8 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep32_batch16_eplb384_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep32_batch16_eplb384_mtp3.yaml" decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - # 1p8d tp8. 18 nodes. + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true - spec-decoding: "mtp" - conc-list: [120] + conc-list: [666] prefill: - num-worker: 1 - tp: 8 - ep: 1 + num-worker: 6 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb384_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb384_mtp3.yaml" decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - # 1p8d tp8, conc12. 18 nodes. + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true - spec-decoding: "mtp" - conc-list: [12] + conc-list: [1229] prefill: - num-worker: 1 - tp: 8 - ep: 1 + num-worker: 7 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep8_batch128_eplb384_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep8_batch128_eplb384_mtp3.yaml" decode: - num-worker: 8 + num-worker: 1 tp: 8 - ep: 1 - dp-attn: false - # 1p8d tp8, conc31. 18 nodes. + ep: 8 + dp-attn: true - spec-decoding: "mtp" - conc-list: [31] + conc-list: [1229] prefill: - num-worker: 1 - tp: 8 - ep: 1 + num-worker: 10 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx10dep4_gen1dep16_batch64_eplb384_mtp3.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx10dep4_gen1dep16_batch64_eplb384_mtp3.yaml" decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - -glm5-fp4-gb300-dynamo-sglang-mtp: - image: lmsysorg/sglang:v0.5.13.post1-cu130 - model: nvidia/GLM-5.1-NVFP4 - model-prefix: glm5.1 - runner: gb300-nv - precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.0" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 4p1d dep16. 6 nodes (2P + 4D). + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true - spec-decoding: "mtp" - conc-list: [364] + conc-list: [2253] prefill: - num-worker: 4 - tp: 2 - ep: 2 + num-worker: 9 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx9dep4_gen1dep8_batch256_eplb384_mtp1.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx9dep4_gen1dep8_batch256_eplb384_mtp1.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # 8p1d dep16. 8 nodes (4P + 4D). - spec-decoding: "mtp" - conc-list: [512] + conc-list: [4301] prefill: - num-worker: 8 - tp: 2 - ep: 2 + num-worker: 12 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep8_batch512_eplb384_mtp1.yaml + - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep8_batch512_eplb384_mtp1.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: true - # ---------- 8k1k low-latency (per-node TP=4 decode workers, MTP) ---------- + +dsv4-fp4-gb300-dynamo-sglang: + image: lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb300 + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "81d0555ee23519cea80a42b4fe824e30368b7300" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # 1p2d tp4. 3 nodes. - - spec-decoding: "mtp" - conc-list: [66] + # WideEP TP=16 decode: 1p1d-dep4-dep16. 5 nodes (4P + 16D = 20 GPUs). + - conc-list: [1024] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml" decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - # 1p2d tp4, conc16. 3 nodes. - - spec-decoding: "mtp" - conc-list: [41] - prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 16 + ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" - decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - # 1p4d tp4. 5 nodes. - - spec-decoding: "mtp" - conc-list: [55] + # Low concurrency: 1p1d-tp4-tp4. 2 nodes. + - conc-list: [1] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" - decode: - num-worker: 4 tp: 4 ep: 1 dp-attn: false - # 1p4d tp4, conc4. 5 nodes. - - spec-decoding: "mtp" - conc-list: [23] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml" decode: - num-worker: 4 + num-worker: 1 tp: 4 ep: 1 dp-attn: false - # 1p8d tp4. 9 nodes. - - spec-decoding: "mtp" - conc-list: [44] + # --- Weiliang wide-EP sweep (srt-slurm PR#173), 18 nodes total --- + # EP=12: 15P+3D, conc=12000. + - conc-list: [12000] prefill: - num-worker: 1 - tp: 2 - ep: 2 + num-worker: 15 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml" decode: - num-worker: 8 - tp: 4 - ep: 1 - dp-attn: false - # 1p8d tp4, conc1. 9 nodes. - - spec-decoding: "mtp" - conc-list: [12] - prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 12 + ep: 12 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[5]" - decode: - num-worker: 8 - tp: 4 - ep: 1 - dp-attn: false - # ---------- 1k1k high-throughput (wide-EP decode, MTP) ---------- -glm5-fp8-gb300-dynamo-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: gb300-nv - precision: fp8 - framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.0" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [2800] + # EP=16: 14P+4D, conc=8192. + - conc-list: [8192] prefill: num-worker: 14 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[0]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - - conc-list: [1700] + # EP=24: 12P+6D, conc=3000. + - conc-list: [3000] prefill: num-worker: 12 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[1]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml" decode: num-worker: 1 tp: 24 ep: 24 dp-attn: true - - conc-list: [1300] + # EP=32: 10P+8D, conc=2500. + - conc-list: [2500] prefill: num-worker: 10 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[2]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml" decode: num-worker: 1 tp: 32 ep: 32 dp-attn: true - - conc-list: [900] + # EP=40: 8P+10D, conc=2048. + - conc-list: [2048] prefill: num-worker: 8 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[3]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml" decode: num-worker: 1 tp: 40 ep: 40 dp-attn: true - # ---------- 8k1k low-latency (per-node TP=4 decode workers) ---------- + +# MTP variant of dsv4-fp4-gb300-dynamo-sglang. +dsv4-fp4-gb300-dynamo-sglang-mtp: + image: lmsysorg/sglang:nightly-dev-20260527-14f81a67 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb300 + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "81d0555ee23519cea80a42b4fe824e30368b7300" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + fixed-seq-len: - isl: 8192 osl: 1024 search-space: - - conc-list: [150] + # Low-latency baseline: 1p1d-tp4-tp4. 2 nodes. + - spec-decoding: "mtp" + conc-list: [1] prefill: num-worker: 1 tp: 4 ep: 1 - dp-attn: true + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[0]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml" decode: - num-worker: 9 + num-worker: 1 tp: 4 ep: 1 dp-attn: false - - conc-list: [128, 64, 32] + # Low-latency 1p6d-dep4-tp4: 1P (DEP=4) + 6 TP=4 decode workers. 7 nodes. + # Recipe runs concurrencies=8x32x64; matrix tracks the max. + - spec-decoding: "mtp" + conc-list: [64] prefill: num-worker: 1 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[1]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml" decode: - num-worker: 17 + num-worker: 6 tp: 4 ep: 1 dp-attn: false - - conc-list: [24] + # Mid curve 1p1d-dep4-dep8. 3 nodes. + - spec-decoding: "mtp" + conc-list: [256] prefill: num-worker: 1 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[2]" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml" decode: - num-worker: 17 + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # Mid curve 1p1d-dep4-dep16. 5 nodes. + - spec-decoding: "mtp" + conc-list: [256] + prefill: + num-worker: 1 tp: 4 - ep: 1 - dp-attn: false - # ---------- 1k1k high-throughput (wide-EP decode) ---------- -glm5-fp8-gb300-dynamo-sglang-mtp: - image: lmsysorg/sglang:v0.5.11-cu130 - model: zai-org/GLM-5.1-FP8 - model-prefix: glm5.1 - runner: gb300-nv + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # Mid curve 2p1d-dep4-dep8. 4 nodes. + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # Mid curve 4p1d-dep4-dep8. 6 nodes. + - spec-decoding: "mtp" + conc-list: [1024] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # High concurrency 6p1d-dep4-dep8. 8 nodes. + - spec-decoding: "mtp" + conc-list: [4096] + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # High concurrency 8p1d-dep4-dep8. 10 nodes. + - spec-decoding: "mtp" + conc-list: [8192] + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + +kimik2.5-int4-h100-vllm: + image: vllm/vllm-openai:v0.22.0 + model: moonshotai/Kimi-K2.5 + model-prefix: kimik2.5 + runner: cluster:h100-dgxc + precision: int4 + framework: vllm + multinode: false + scenarios: + # H100 has 80 GB HBM per GPU (smallest in this set); the KV cliff arrives + # early. Sweep saturates conc=20 to keep total HBM headroom. + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 12, 16, 20] } + +qwen3.5-fp8-h100-sglang: + image: lmsysorg/sglang:v0.5.14-cu130 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: h100 + precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 1, conc-end: 8 } + - { tp: 8, ep: 8, conc-start: 16, conc-end: 256 } + +qwen3.5-fp8-h100-sglang-mtp: + image: lmsysorg/sglang:v0.5.14-cu130 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: h100 precision: fp8 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 8, conc-start: 4, conc-end: 32, spec-decoding: mtp } + + # ---------- 1k1k high-throughput (wide-EP TP=32 decode) ---------- +qwen3.5-fp4-gb300-dynamo-sglang: + image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: gb300 + precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.2.1" } - kv-p2p-transfer: nixl + router: { name: dynamo-router, version: "1.1.0" } multinode: true disagg: true scenarios: @@ -8091,108 +6257,76 @@ glm5-fp8-gb300-dynamo-sglang-mtp: - isl: 8192 osl: 1024 search-space: - - spec-decoding: "mtp" - conc-list: [2800] + # 1P1D TP4: 1 prefill worker at TP4 and 1 decode worker at TP4 + # Pure tensor parallel (STP), 8k1k baseline-low-latency sweep. + # Total: 8 GB300 GPUs. + - spec-decoding: "none" + kv-p2p-transfer: mooncake + conc-list: [1, 4, 8, 16, 32, 64, 256] prefill: - num-worker: 14 + num-worker: 1 tp: 4 ep: 1 - dp-attn: true + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1700] - prefill: - num-worker: 12 tp: 4 ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml" - decode: - num-worker: 1 - tp: 24 - ep: 24 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1300] + dp-attn: false + # 5P1D wide-EP: 5 prefill workers @ DEP4 + 1 decode worker @ DEP16. + # NIXL transfer. Total: 36 GB300 GPUs (5*4 + 4*4). + - spec-decoding: "none" + kv-p2p-transfer: nixl + conc-list: [2048] prefill: - num-worker: 10 + num-worker: 5 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml" decode: num-worker: 1 - tp: 32 - ep: 32 + tp: 16 + ep: 16 dp-attn: true - - spec-decoding: "mtp" - conc-list: [900] + # 6P1D wide-EP: 6 prefill workers @ DEP4 + 1 decode worker @ DEP16. + # Mooncake transfer. Total: 40 GB300 GPUs (6*4 + 4*4). + - spec-decoding: "none" + kv-p2p-transfer: mooncake + conc-list: [5120] prefill: - num-worker: 8 + num-worker: 6 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml" decode: num-worker: 1 - tp: 40 - ep: 40 - dp-attn: true - # ---------- 8k1k low-latency (per-node TP=4 decode workers, EAGLE MTP) ---------- - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [150] - prefill: - num-worker: 1 - tp: 4 - ep: 1 + tp: 16 + ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" - decode: - num-worker: 9 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [128, 64, 32] + # 7P1D wide-EP: 7 prefill workers @ DEP4 + 1 decode worker @ DEP16. + # Mooncake transfer. Total: 44 GB300 GPUs (7*4 + 4*4). + - spec-decoding: "none" + kv-p2p-transfer: mooncake + conc-list: [5120] prefill: - num-worker: 1 + num-worker: 7 tp: 4 - ep: 1 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml" decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [24] - prefill: num-worker: 1 - tp: 4 - ep: 1 + tp: 16 + ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml" - decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false + # ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ---------- qwen3.5-fp8-b300-sglang-agentic-hicache: image: lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd @@ -9529,174 +7663,6 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true -glm5-fp4-gb300-dynamo-trt-mtp: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: gb300-nv - precision: fp4 - framework: dynamo-trt - router: { name: dynamo-router, version: "v1.3.0-dev.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - - spec-decoding: "mtp" - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 15 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 30 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 84 ] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 180 ] - prefill: - num-worker: 4 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 333 ] - prefill: - num-worker: 6 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 615 ] - prefill: - num-worker: 10 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 1229 ] - prefill: - num-worker: 14 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 615 ] - prefill: - num-worker: 11 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 2253 ] - prefill: - num-worker: 21 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro