From 2aeafb4d314af6e3f7f27d0c4a0f9c736ba56524 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 9 Jun 2026 14:23:00 -0700 Subject: [PATCH 01/20] add dsv4 1k1k --- .github/configs/nvidia-master.yaml | 85 +++++++++ .../1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml | 160 +++++++++++++++++ .../1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml | 161 ++++++++++++++++++ .../1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml | 160 +++++++++++++++++ .../1k1k/disagg-low-latency-dep4-mtp.yaml | 148 ++++++++++++++++ .../1k1k/disagg-low-latency-tp4-mtp.yaml | 134 +++++++++++++++ perf-changelog.yaml | 11 ++ 7 files changed, 859 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index a02749d4d1..54e8758335 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -9217,6 +9217,91 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 8 dp-attn: true +dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: + image: lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb300-cw + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [8192] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [8192] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [8192] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [64] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [2] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + kimik2.5-int4-h100-vllm: image: vllm/vllm-openai:v0.20.2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml new file mode 100644 index 0000000000..80a9d1455e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml @@ -0,0 +1,160 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-1p1d-dep16-conc8192" + +slurm: + time_limit: "03:00:00" + +sbatch_directives: + cpus-per-task: '144' + mem: '0' + +dynamo: + hash: "34d55a596fb8d3d44daefe425ec1e303131f4d2c" + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +model: + path: dsv4-pro + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + precision: fp4 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_DISABLE_REUSE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_FAST_MASK_EP: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_FIX_HASH_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: '1' + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '0' + + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + + decode_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_DISABLE_REUSE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_USE_FAST_MASK_EP: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_FIX_HASH_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: '1' + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '0' + + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. + + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: deepep + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: deepep + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 18432 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8192" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml new file mode 100644 index 0000000000..133beff1cf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml @@ -0,0 +1,161 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-1p1d-dep8-conc8192" + +slurm: + time_limit: "03:00:00" + +sbatch_directives: + cpus-per-task: '144' + mem: '0' + +dynamo: + hash: "34d55a596fb8d3d44daefe425ec1e303131f4d2c" + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +model: + path: dsv4-pro + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + precision: fp4 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_DISABLE_REUSE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_FAST_MASK_EP: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_FIX_HASH_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: '1' + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '0' + + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + + decode_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_DISABLE_REUSE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_USE_FAST_MASK_EP: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_FIX_HASH_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048' + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: '1' + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '0' + + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: deepep + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: deepep + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.94 + max-running-requests: 1536 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8192" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml new file mode 100644 index 0000000000..5176c28f23 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml @@ -0,0 +1,160 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-2p1d-dep16-conc8192" + +slurm: + time_limit: "03:00:00" + +sbatch_directives: + cpus-per-task: '144' + mem: '0' + +dynamo: + hash: "34d55a596fb8d3d44daefe425ec1e303131f4d2c" + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +model: + path: dsv4-pro + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + precision: fp4 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_DISABLE_REUSE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_FAST_MASK_EP: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_FIX_HASH_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: '1' + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '0' + + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + + decode_environment: + PYTHONUNBUFFERED: '1' + SGLANG_RADIX_DISABLE_REUSE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_USE_JIT_NORM: '1' + SGLANG_OPT_USE_JIT_INDEXER_METADATA: '1' + SGLANG_OPT_USE_TOPK_V2: '1' + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_USE_FAST_MASK_EP: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_FIX_HASH_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: '1' + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '0' + + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. + + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: deepep + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: deepep + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 18432 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8192" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml new file mode 100644 index 0000000000..08aad8c434 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml @@ -0,0 +1,148 @@ +base: + name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-dep4" + + frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + + model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + + resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 1 + decode_workers: 1 + + backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "deepep" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + + benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +zip_override_1k1k_lowlat: + resources: + decode_nodes: [1, 2, 4, 6] + decode_workers: [1, 2, 4, 6] + backend: + sglang_config: + decode: + max-running-requests: [64, 64, 64, 64] + cuda-graph-max-bs: [64, 64, 64, 64] + benchmark: + concurrencies: "1x2x4x8x16x32x64" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml new file mode 100644 index 0000000000..e7f5f0f345 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml @@ -0,0 +1,134 @@ +base: + name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-tp4" + + frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + + model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + + resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 1 + decode_workers: 1 + + backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 8 + cuda-graph-max-bs: 8 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + + benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +zip_override_1k1k_lowlat: + resources: + decode_nodes: [1, 2, 4, 6] + decode_workers: [1, 2, 4, 6] + backend: + sglang_config: + decode: + max-running-requests: [8, 8, 8, 8] + cuda-graph-max-bs: [8, 8, 8, 8] + benchmark: + concurrencies: "1x2" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5622173f16..80cbcbb218 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3531,3 +3531,14 @@ - "The Rust frontend replaces only the Python serving/API layer (HTTP, tokenization, scheduling glue, detokenization) and spawns the same Python EngineCore, so GPU kernels/attention/MoE GEMM/KV cache are untouched" - "A/B sweep (28 single-node points, 1k1k + 8k1k, TP 1/2/4) vs the Python-frontend baseline (run 26696260751): throughput Pareto-neutral (peak tok/s/GPU within <1.5%, frontiers coincident) and TPOT flat (+-0.5%); TTFT improves ~8% at 1k1k and ~22% at 8k1k (every point), the expected signature of lower frontend CPU latency before first token, scaling with input length" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1634 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k + description: + - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang MTP 1k/1k coverage using lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + - "New top-level config (sibling of dsv4-fp4-gb300-dynamo-sglang-mtp which stays on the older 8k/1k image nightly-dev-20260527-14f81a67); separate entry is required because the launcher builds one squashfs per top-level image and the 1k/1k recipes pin a newer container string" + - "Wires 5 disagg recipes adapted from NVIDIA/srt-slurm PR #177 (recipes/dsv4-pro/sglang/gb300-fp4/1k1k/disagg/mtp/), staged locally under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16) plus 2 low-latency templates (disagg-low-latency-dep4-mtp matrix-point at conc=64, disagg-low-latency-tp4-mtp matrix-point at conc=2)" + - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 5 recipes" + - "The 2 low-latency recipe yamls retain their upstream container reference (lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e); that image is not currently on Docker Hub, so the squash-import step would 404 without a launcher-side container alias or an upstream image push" + - "Matrix tracks one representative point per low-latency template; the recipe itself uses srt-slurm's base+zip_override_1k1k_lowlat syntax to internally generate 4 decode_nodes variants (1/2/4/6) per template" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX From 984a91ed5be78f1baf3f806f84d58f51e9437f43 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 9 Jun 2026 16:29:57 -0700 Subject: [PATCH 02/20] update configs --- .github/configs/nvidia-master.yaml | 44 +++++- .../disagg-low-latency-1p1d-tp4-tp4-mtp.yaml | 120 ++++++++++++++++ .../disagg-low-latency-1p2d-tp4-tp4-mtp.yaml | 120 ++++++++++++++++ .../disagg-low-latency-1p4d-tp4-tp4-mtp.yaml | 120 ++++++++++++++++ .../disagg-low-latency-1p6d-tp4-tp4-mtp.yaml | 120 ++++++++++++++++ .../1k1k/disagg-low-latency-tp4-mtp.yaml | 134 ------------------ perf-changelog.yaml | 9 +- 7 files changed, 527 insertions(+), 140 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 54e8758335..0a71d9d0ee 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -9295,12 +9295,54 @@ dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false + - spec-decoding: "mtp" + conc-list: [2] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [2] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [2] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml" + decode: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: false kimik2.5-int4-h100-vllm: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml new file mode 100644 index 0000000000..97935ebec7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -0,0 +1,120 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-1p1d-tp4-tp4" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 1 + decode_workers: 1 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 8 + cuda-graph-max-bs: 8 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml new file mode 100644 index 0000000000..454c5cd09f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml @@ -0,0 +1,120 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-1p2d-tp4-tp4" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 2 + decode_workers: 2 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 8 + cuda-graph-max-bs: 8 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml new file mode 100644 index 0000000000..6b77bf1132 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml @@ -0,0 +1,120 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-1p4d-tp4-tp4" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 4 + decode_workers: 4 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 8 + cuda-graph-max-bs: 8 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml new file mode 100644 index 0000000000..cb657a956f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml @@ -0,0 +1,120 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-1p6d-tp4-tp4" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 6 + decode_workers: 6 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 8 + cuda-graph-max-bs: 8 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml deleted file mode 100644 index e7f5f0f345..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-tp4-mtp.yaml +++ /dev/null @@ -1,134 +0,0 @@ -base: - name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-tp4" - - frontend: - type: sglang - enable_multiple_frontends: false - args: - policy: "cache_aware" - - model: - path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" - precision: "mxfp4" - - resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 1 - decode_workers: 1 - - backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_TOPK_V2: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_TOPK_V2: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.90 - max-running-requests: 8 - cuda-graph-max-bs: 8 - chunked-prefill-size: 4096 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - swa-full-tokens-ratio: 0.1 - context-length: 4096 - - benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1x2" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -zip_override_1k1k_lowlat: - resources: - decode_nodes: [1, 2, 4, 6] - decode_workers: [1, 2, 4, 6] - backend: - sglang_config: - decode: - max-running-requests: [8, 8, 8, 8] - cuda-graph-max-bs: [8, 8, 8, 8] - benchmark: - concurrencies: "1x2" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 80cbcbb218..4a79939151 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3537,8 +3537,7 @@ description: - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang MTP 1k/1k coverage using lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" - "New top-level config (sibling of dsv4-fp4-gb300-dynamo-sglang-mtp which stays on the older 8k/1k image nightly-dev-20260527-14f81a67); separate entry is required because the launcher builds one squashfs per top-level image and the 1k/1k recipes pin a newer container string" - - "Wires 5 disagg recipes adapted from NVIDIA/srt-slurm PR #177 (recipes/dsv4-pro/sglang/gb300-fp4/1k1k/disagg/mtp/), staged locally under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16) plus 2 low-latency templates (disagg-low-latency-dep4-mtp matrix-point at conc=64, disagg-low-latency-tp4-mtp matrix-point at conc=2)" - - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 5 recipes" - - "The 2 low-latency recipe yamls retain their upstream container reference (lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e); that image is not currently on Docker Hub, so the squash-import step would 404 without a launcher-side container alias or an upstream image push" - - "Matrix tracks one representative point per low-latency template; the recipe itself uses srt-slurm's base+zip_override_1k1k_lowlat syntax to internally generate 4 decode_nodes variants (1/2/4/6) per template" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX + - "Wires 8 disagg recipes adapted from NVIDIA/srt-slurm PR #177 (recipes/dsv4-pro/sglang/gb300-fp4/1k1k/disagg/mtp/), staged locally under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16), 1 low-latency dep4 template (disagg-low-latency-dep4-mtp at conc=64), and 4 flat low-latency tp4 recipes (1p1d/1p2d/1p4d/1p6d at conc=2)" + - "Flat tp4 split (1p1d/1p2d/1p4d/1p6d-tp4-tp4) replaces the upstream disagg-low-latency-tp4-mtp.yaml template (base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]); the templated form silently broke under the gb300-cw launcher because its top-level name-injection (`sed -i \"s/^name:.*/name: ...\"`) doesn't match base.name's 2-space indent, prepending a duplicate top-level name: key. Single-config files match the 8k1k convention (one recipe per decode-worker count) and side-step the issue" + - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 8 recipes; recipe container strings are kept upstream-faithful (high-tput recipes pin nightly-dev-cu13-20260603-83bc7766, low-latency recipes pin nightly-dev-cu13-20260510-2473659e)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1697 From 7c91283fcd52f5c84fae78c211382dfa1f346f89 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 9 Jun 2026 21:36:48 -0700 Subject: [PATCH 03/20] split configs --- .github/configs/nvidia-master.yaml | 44 +++++- .../disagg-low-latency-1p1d-dep4-tp4-mtp.yaml | 134 ++++++++++++++++ .../disagg-low-latency-1p2d-dep4-tp4-mtp.yaml | 134 ++++++++++++++++ .../disagg-low-latency-1p4d-dep4-tp4-mtp.yaml | 134 ++++++++++++++++ .../disagg-low-latency-1p6d-dep4-tp4-mtp.yaml | 134 ++++++++++++++++ .../1k1k/disagg-low-latency-dep4-mtp.yaml | 148 ------------------ perf-changelog.yaml | 6 +- 7 files changed, 582 insertions(+), 152 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 0a71d9d0ee..a6d42207c8 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -9281,12 +9281,54 @@ dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false + - spec-decoding: "mtp" + conc-list: [64] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [64] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [64] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml" + decode: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: false - spec-decoding: "mtp" conc-list: [2] prefill: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml new file mode 100644 index 0000000000..92a7d98a59 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml @@ -0,0 +1,134 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-1p1d-dep4-tp4" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 1 + decode_workers: 1 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "deepep" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 64 + cuda-graph-max-bs: 64 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml new file mode 100644 index 0000000000..fe154f8898 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml @@ -0,0 +1,134 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-1p2d-dep4-tp4" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 2 + decode_workers: 2 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "deepep" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 64 + cuda-graph-max-bs: 64 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml new file mode 100644 index 0000000000..148d277075 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml @@ -0,0 +1,134 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-1p4d-dep4-tp4" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 4 + decode_workers: 4 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "deepep" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 64 + cuda-graph-max-bs: 64 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml new file mode 100644 index 0000000000..9e7da04585 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -0,0 +1,134 @@ +name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-1p6d-dep4-tp4" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "dsv4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 6 + decode_workers: 6 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "deepep" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 64 + cuda-graph-max-bs: 64 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml deleted file mode 100644 index 08aad8c434..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-dep4-mtp.yaml +++ /dev/null @@ -1,148 +0,0 @@ -base: - name: "dsv4-pro-gb300-disagg-1k1k-mtp-low-latency-dep4" - - frontend: - type: sglang - enable_multiple_frontends: false - args: - policy: "cache_aware" - - model: - path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" - precision: "mxfp4" - - resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 1 - decode_workers: 1 - - backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_TOPK_V2: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_TOPK_V2: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "deepep" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 4096 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 512 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 4096 - - benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -zip_override_1k1k_lowlat: - resources: - decode_nodes: [1, 2, 4, 6] - decode_workers: [1, 2, 4, 6] - backend: - sglang_config: - decode: - max-running-requests: [64, 64, 64, 64] - cuda-graph-max-bs: [64, 64, 64, 64] - benchmark: - concurrencies: "1x2x4x8x16x32x64" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4a79939151..0b8ab1cb6a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3537,7 +3537,7 @@ description: - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang MTP 1k/1k coverage using lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" - "New top-level config (sibling of dsv4-fp4-gb300-dynamo-sglang-mtp which stays on the older 8k/1k image nightly-dev-20260527-14f81a67); separate entry is required because the launcher builds one squashfs per top-level image and the 1k/1k recipes pin a newer container string" - - "Wires 8 disagg recipes adapted from NVIDIA/srt-slurm PR #177 (recipes/dsv4-pro/sglang/gb300-fp4/1k1k/disagg/mtp/), staged locally under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16), 1 low-latency dep4 template (disagg-low-latency-dep4-mtp at conc=64), and 4 flat low-latency tp4 recipes (1p1d/1p2d/1p4d/1p6d at conc=2)" - - "Flat tp4 split (1p1d/1p2d/1p4d/1p6d-tp4-tp4) replaces the upstream disagg-low-latency-tp4-mtp.yaml template (base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]); the templated form silently broke under the gb300-cw launcher because its top-level name-injection (`sed -i \"s/^name:.*/name: ...\"`) doesn't match base.name's 2-space indent, prepending a duplicate top-level name: key. Single-config files match the 8k1k convention (one recipe per decode-worker count) and side-step the issue" - - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 8 recipes; recipe container strings are kept upstream-faithful (high-tput recipes pin nightly-dev-cu13-20260603-83bc7766, low-latency recipes pin nightly-dev-cu13-20260510-2473659e)" + - "Wires 11 disagg recipes adapted from NVIDIA/srt-slurm PR #177 (recipes/dsv4-pro/sglang/gb300-fp4/1k1k/disagg/mtp/), staged locally under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16), 4 flat low-latency dep4 recipes (1p1d/1p2d/1p4d/1p6d-dep4-tp4 at conc=64), and 4 flat low-latency tp4 recipes (1p1d/1p2d/1p4d/1p6d-tp4-tp4 at conc=2)" + - "Flat dep4 and tp4 splits replace the upstream disagg-low-latency-dep4-mtp.yaml and disagg-low-latency-tp4-mtp.yaml templates (each base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]). The templated form silently broke under the gb300-cw launcher: srtctl fans the single recipe into 4 Slurm submissions per `srtctl apply`, but launch_gb300-cw.sh:282 captures the job IDs with `grep -oP '✅ Job \\K[0-9]+'` into a scalar JOB_ID, jamming 4 newline-separated IDs into one variable. Downstream uses (LOGS_DIR path interpolation at :293-294, squeue polls at :313-314, error message at :315) all corrupt, producing the observed `ERROR: Job 7079\\n7082 failed before creating log file` and orphaning the real Slurm jobs. Single-config files match the 8k1k convention (one recipe per decode-worker count) and force srtctl to emit exactly 1 job per launcher invocation" + - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 11 recipes; recipe container strings are kept upstream-faithful (high-tput recipes pin nightly-dev-cu13-20260603-83bc7766, low-latency recipes pin nightly-dev-cu13-20260510-2473659e)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1697 From 47460ef649a9df9412b6e381ca109b18244de5ee Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 11 Jun 2026 10:10:46 -0700 Subject: [PATCH 04/20] update runner --- .github/configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index a6d42207c8..f00031cdf5 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -9221,7 +9221,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: image: lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: gb300-cw + runner: gb300-nv precision: fp4 framework: dynamo-sglang multinode: true From fd84ba54e7995d04bdedde9dd9e093dc20506198 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 11 Jun 2026 10:47:30 -0700 Subject: [PATCH 05/20] Update perf-changelog.yaml --- perf-changelog.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7f117b8664..ba26f8d578 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3589,4 +3589,3 @@ - "Flat dep4 and tp4 splits replace the upstream disagg-low-latency-dep4-mtp.yaml and disagg-low-latency-tp4-mtp.yaml templates (each base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]). The templated form silently broke under the gb300-cw launcher: srtctl fans the single recipe into 4 Slurm submissions per `srtctl apply`, but launch_gb300-cw.sh:282 captures the job IDs with `grep -oP '✅ Job \\K[0-9]+'` into a scalar JOB_ID, jamming 4 newline-separated IDs into one variable. Downstream uses (LOGS_DIR path interpolation at :293-294, squeue polls at :313-314, error message at :315) all corrupt, producing the observed `ERROR: Job 7079\\n7082 failed before creating log file` and orphaning the real Slurm jobs. Single-config files match the 8k1k convention (one recipe per decode-worker count) and force srtctl to emit exactly 1 job per launcher invocation" - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 11 recipes; recipe container strings are kept upstream-faithful (high-tput recipes pin nightly-dev-cu13-20260603-83bc7766, low-latency recipes pin nightly-dev-cu13-20260510-2473659e)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1697 - \ No newline at end of file From 37100c4c5bab8cb402e74f467819b116cb06efdf Mon Sep 17 00:00:00 2001 From: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com> Date: Thu, 11 Jun 2026 12:59:10 -0700 Subject: [PATCH 06/20] Update nvidia-master.yaml --- .github/configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index c73ab5428b..d1ad86d809 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -9247,7 +9247,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: image: lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: gb300-nv + runner: gb300 precision: fp4 framework: dynamo-sglang multinode: true From ae2385e07e036a281e99909c73d9e9bf039fb6df Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 15 Jun 2026 12:06:01 -0700 Subject: [PATCH 07/20] update container --- .../deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml index 92a7d98a59..6eb56a2477 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml index 97935ebec7..4f472efc8f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml index fe154f8898..4b64913d72 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml index 454c5cd09f..5f7767c695 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml index 148d277075..936783159e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml index 6b77bf1132..65a92db6a6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml index 9e7da04585..64c6952308 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml index cb657a956f..bb51740604 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" precision: "mxfp4" resources: From 9e81ea919ad7df0aeca2758a03a9839c6f2f04d4 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 15 Jun 2026 12:08:43 -0700 Subject: [PATCH 08/20] Update perf-changelog.yaml --- perf-changelog.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0183c3b9ba..d1b0913966 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3859,4 +3859,5 @@ - "Flat dep4 and tp4 splits replace the upstream disagg-low-latency-dep4-mtp.yaml and disagg-low-latency-tp4-mtp.yaml templates (each base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]). The templated form silently broke under the gb300-cw launcher: srtctl fans the single recipe into 4 Slurm submissions per `srtctl apply`, but launch_gb300-cw.sh:282 captures the job IDs with `grep -oP '✅ Job \\K[0-9]+'` into a scalar JOB_ID, jamming 4 newline-separated IDs into one variable. Downstream uses (LOGS_DIR path interpolation at :293-294, squeue polls at :313-314, error message at :315) all corrupt, producing the observed `ERROR: Job 7079\\n7082 failed before creating log file` and orphaning the real Slurm jobs. Single-config files match the 8k1k convention (one recipe per decode-worker count) and force srtctl to emit exactly 1 job per launcher invocation" - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 11 recipes; recipe container strings are kept upstream-faithful (high-tput recipes pin nightly-dev-cu13-20260603-83bc7766, low-latency recipes pin nightly-dev-cu13-20260510-2473659e)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1697 - \ No newline at end of file + + From a87852e8bc8575e7bd7e47a4e496c28a2ff11741 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 15 Jun 2026 22:43:52 -0700 Subject: [PATCH 09/20] update image to latest --- .../deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml | 2 +- .../1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml | 2 +- .../1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml | 2 +- .../1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml | 2 +- .../1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml | 2 +- perf-changelog.yaml | 4 ++-- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml index 80a9d1455e..fed845263e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml @@ -18,7 +18,7 @@ frontend: model: path: dsv4-pro - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: fp4 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml index 133beff1cf..65f4f4d0a3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml @@ -18,7 +18,7 @@ frontend: model: path: dsv4-pro - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: fp4 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml index 5176c28f23..bcafd24bde 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml @@ -18,7 +18,7 @@ frontend: model: path: dsv4-pro - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: fp4 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml index 6eb56a2477..1262076011 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml index 4f472efc8f..346933557c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml index 4b64913d72..e7c712e500 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml index 5f7767c695..134e08b345 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml index 936783159e..97eff2d40d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml index 65a92db6a6..11e059294f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml index 64c6952308..b974668d24 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml index bb51740604..b059593e80 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "dsv4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" resources: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d1b0913966..9f27fdf339 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3853,11 +3853,11 @@ - config-keys: - dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k description: - - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang MTP 1k/1k coverage using lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766" + - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang MTP 1k/1k coverage using lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" - "New top-level config (sibling of dsv4-fp4-gb300-dynamo-sglang-mtp which stays on the older 8k/1k image nightly-dev-20260527-14f81a67); separate entry is required because the launcher builds one squashfs per top-level image and the 1k/1k recipes pin a newer container string" - "Wires 11 disagg recipes adapted from NVIDIA/srt-slurm PR #177 (recipes/dsv4-pro/sglang/gb300-fp4/1k1k/disagg/mtp/), staged locally under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16), 4 flat low-latency dep4 recipes (1p1d/1p2d/1p4d/1p6d-dep4-tp4 at conc=64), and 4 flat low-latency tp4 recipes (1p1d/1p2d/1p4d/1p6d-tp4-tp4 at conc=2)" - "Flat dep4 and tp4 splits replace the upstream disagg-low-latency-dep4-mtp.yaml and disagg-low-latency-tp4-mtp.yaml templates (each base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]). The templated form silently broke under the gb300-cw launcher: srtctl fans the single recipe into 4 Slurm submissions per `srtctl apply`, but launch_gb300-cw.sh:282 captures the job IDs with `grep -oP '✅ Job \\K[0-9]+'` into a scalar JOB_ID, jamming 4 newline-separated IDs into one variable. Downstream uses (LOGS_DIR path interpolation at :293-294, squeue polls at :313-314, error message at :315) all corrupt, producing the observed `ERROR: Job 7079\\n7082 failed before creating log file` and orphaning the real Slurm jobs. Single-config files match the 8k1k convention (one recipe per decode-worker count) and force srtctl to emit exactly 1 job per launcher invocation" - - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 11 recipes; recipe container strings are kept upstream-faithful (high-tput recipes pin nightly-dev-cu13-20260603-83bc7766, low-latency recipes pin nightly-dev-cu13-20260510-2473659e)" + - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 11 recipes; all recipes pin lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1697 From a0e8fc25e86cb49f7716ddae4b1254e6fb10b2e3 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 15 Jun 2026 22:50:59 -0700 Subject: [PATCH 10/20] add image change --- .github/configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index fc39b56fd2..17af5c8801 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -10313,7 +10313,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: dp-attn: true dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: - image: lmsysorg/sglang:nightly-dev-cu13-20260603-83bc7766 + image: lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: gb300 From 1fa0642586d09b43e31098ad5205e39d3196e8c7 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 16 Jun 2026 09:57:15 -0700 Subject: [PATCH 11/20] update nv elif --- runners/launch_gb300-nv.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index d21c91d100..aec68d7ee2 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -143,6 +143,12 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" ]]; then git checkout sa-submission-q2-2026 mkdir -p recipes/sglang/glm5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5" recipes/sglang/glm5 +elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout main + mkdir -p recipes/sglang/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" From 30fe6f2ccf5debc8ab7320f0621258f964c18c8c Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 16 Jun 2026 10:32:11 -0700 Subject: [PATCH 12/20] update cluster to nv --- .github/configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index a29dcab9c8..0a770efa42 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -10316,7 +10316,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: image: lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: gb300 + runner: gb300-nv precision: fp4 framework: dynamo-sglang multinode: true From f83ea431b0b29858a7e30c92b607be85a8e59cb4 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 16 Jun 2026 13:08:52 -0700 Subject: [PATCH 13/20] update model name --- .../sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml | 2 +- .../sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml | 2 +- .../sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml index fed845263e..caf9c51907 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml @@ -17,7 +17,7 @@ frontend: num_additional_frontends: 8 model: - path: dsv4-pro + path: deepseek-v4-pro container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: fp4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml index 65f4f4d0a3..f12fb51157 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml @@ -17,7 +17,7 @@ frontend: num_additional_frontends: 8 model: - path: dsv4-pro + path: deepseek-v4-pro container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: fp4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml index bcafd24bde..c7b02c24f5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml @@ -17,7 +17,7 @@ frontend: num_additional_frontends: 8 model: - path: dsv4-pro + path: deepseek-v4-pro container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: fp4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml index 1262076011..96305da08f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml @@ -7,7 +7,7 @@ frontend: policy: "cache_aware" model: - path: "dsv4-pro" + path: "deepseek-v4-pro" container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml index 346933557c..78d66ba66b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -7,7 +7,7 @@ frontend: policy: "cache_aware" model: - path: "dsv4-pro" + path: "deepseek-v4-pro" container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml index e7c712e500..40ea83edc2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml @@ -7,7 +7,7 @@ frontend: policy: "cache_aware" model: - path: "dsv4-pro" + path: "deepseek-v4-pro" container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml index 134e08b345..9cdd390a25 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml @@ -7,7 +7,7 @@ frontend: policy: "cache_aware" model: - path: "dsv4-pro" + path: "deepseek-v4-pro" container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml index 97eff2d40d..2d88e92e7d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml @@ -7,7 +7,7 @@ frontend: policy: "cache_aware" model: - path: "dsv4-pro" + path: "deepseek-v4-pro" container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml index 11e059294f..be9b3c9e5c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml @@ -7,7 +7,7 @@ frontend: policy: "cache_aware" model: - path: "dsv4-pro" + path: "deepseek-v4-pro" container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml index b974668d24..166bac0557 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -7,7 +7,7 @@ frontend: policy: "cache_aware" model: - path: "dsv4-pro" + path: "deepseek-v4-pro" container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml index b059593e80..87f424b1f7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml @@ -7,7 +7,7 @@ frontend: policy: "cache_aware" model: - path: "dsv4-pro" + path: "deepseek-v4-pro" container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" precision: "mxfp4" From 395f577c3ca89160c4bdcff0d0a0f18c31771c5b Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Tue, 16 Jun 2026 17:03:04 -0700 Subject: [PATCH 14/20] image update to stable --- .../sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml | 2 +- .../sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml | 2 +- .../sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml | 2 +- .../8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml | 2 +- .../8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml | 2 +- .../8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml | 2 +- .../8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml | 2 +- .../deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml | 2 +- .../sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml | 2 +- .../deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml index caf9c51907..b919971a1d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml @@ -18,7 +18,7 @@ frontend: model: path: deepseek-v4-pro - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: fp4 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml index f12fb51157..553403480a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml @@ -18,7 +18,7 @@ frontend: model: path: deepseek-v4-pro - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: fp4 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml index c7b02c24f5..f84cc28832 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml @@ -18,7 +18,7 @@ frontend: model: path: deepseek-v4-pro - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: fp4 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml index 96305da08f..cf8b3f5c36 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml index 78d66ba66b..6a668c9ed9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml index 40ea83edc2..6491339e37 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml index 9cdd390a25..c6fc487630 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml index 2d88e92e7d..aad8d62616 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml index be9b3c9e5c..ecf0aa8aa2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml index 166bac0557..6d81706b44 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml index 87f424b1f7..c22829f12e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml index 528aa5721a..14dc51f42f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-10p1d-dep4-dep32-18-c2500" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml index 32a5124c2d..137434b1dd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-12p1d-dep4-dep24-18-c3000" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml index adc6b15507..619957210f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-14p1d-dep4-dep16-18-c8192" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml index 73ad15750d..d14198bc7a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-15p1d-dep4-dep12-18-c12000" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml index 7cdb779c71..5bf66527b6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml @@ -33,7 +33,7 @@ name: "disagg-gb300-1p1d-dep4-dep16-5-c1024" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml index 9382dd6dda..23c0501749 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml @@ -33,7 +33,7 @@ name: "disagg-gb300-1p1d-tp4-tp4-2-c1" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "fp4" # See ../1k1k/disagg-gb200-1p1d-dep8-tep8.yaml for the dynamo pin diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml index 269b92e124..3cede19112 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-8p1d-dep4-dep40-18-c2048" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: "fp4" dynamo: From bff605696bab5de921126b77a296cefcbccef717 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 2 Jul 2026 11:24:50 -0700 Subject: [PATCH 15/20] Consolidate dsv4-fp4-gb300-dynamo-sglang-mtp: merge 1k1k scenarios into existing 8k1k key Move the 11 isl=1024 fixed-seq-len entries out of the standalone dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k key and append them to the existing dsv4-fp4-gb300-dynamo-sglang-mtp key so a single top-level config covers both 8k/1k and 1k/1k scenarios on GB300. Master config image stays at lmsysorg/sglang:nightly-dev-20260527-14f81a67 (unchanged); the 11 new 1k/1k recipes remain pinned to lmsysorg/sglang:v0.5.13.post1-cu130 per PR head. --- .github/configs/nvidia-master.yaml | 17 +++++------------ perf-changelog.yaml | 12 ++++++------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 1a8b7ff283..e1639cfcb7 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -10311,21 +10311,10 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: tp: 8 ep: 8 dp-attn: true - -dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: - image: lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300-nv - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - isl: 1024 osl: 1024 search-space: + # High-throughput: 1p1d-dep8-conc8192. 3 nodes. - spec-decoding: "mtp" conc-list: [8192] prefill: @@ -10340,6 +10329,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: tp: 8 ep: 8 dp-attn: true + # High-throughput: 1p1d-dep16-conc8192. 5 nodes. - spec-decoding: "mtp" conc-list: [8192] prefill: @@ -10354,6 +10344,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: tp: 16 ep: 16 dp-attn: true + # High-throughput: 2p1d-dep16-conc8192. 6 nodes. - spec-decoding: "mtp" conc-list: [8192] prefill: @@ -10368,6 +10359,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: tp: 16 ep: 16 dp-attn: true + # Low-latency dep4 splits: 1P (DEP=4) + N TP=4 decode workers. - spec-decoding: "mtp" conc-list: [64] prefill: @@ -10424,6 +10416,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k: tp: 4 ep: 1 dp-attn: false + # Low-latency tp4 splits: 1P (TP=4) + N TP=4 decode workers. - spec-decoding: "mtp" conc-list: [2] prefill: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a54c7f42c4..74acef8995 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3911,11 +3911,11 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1809 - config-keys: - - dsv4-fp4-gb300-dynamo-sglang-mtp-1k1k + - dsv4-fp4-gb300-dynamo-sglang-mtp description: - - "Add DeepSeek-V4-Pro FP4 GB300 disaggregated SGLang MTP 1k/1k coverage using lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" - - "New top-level config (sibling of dsv4-fp4-gb300-dynamo-sglang-mtp which stays on the older 8k/1k image nightly-dev-20260527-14f81a67); separate entry is required because the launcher builds one squashfs per top-level image and the 1k/1k recipes pin a newer container string" - - "Wires 11 disagg recipes adapted from NVIDIA/srt-slurm PR #177 (recipes/dsv4-pro/sglang/gb300-fp4/1k1k/disagg/mtp/), staged locally under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16), 4 flat low-latency dep4 recipes (1p1d/1p2d/1p4d/1p6d-dep4-tp4 at conc=64), and 4 flat low-latency tp4 recipes (1p1d/1p2d/1p4d/1p6d-tp4-tp4 at conc=2)" - - "Flat dep4 and tp4 splits replace the upstream disagg-low-latency-dep4-mtp.yaml and disagg-low-latency-tp4-mtp.yaml templates (each base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]). The templated form silently broke under the gb300-cw launcher: srtctl fans the single recipe into 4 Slurm submissions per `srtctl apply`, but launch_gb300-cw.sh:282 captures the job IDs with `grep -oP '✅ Job \\K[0-9]+'` into a scalar JOB_ID, jamming 4 newline-separated IDs into one variable. Downstream uses (LOGS_DIR path interpolation at :293-294, squeue polls at :313-314, error message at :315) all corrupt, producing the observed `ERROR: Job 7079\\n7082 failed before creating log file` and orphaning the real Slurm jobs. Single-config files match the 8k1k convention (one recipe per decode-worker count) and force srtctl to emit exactly 1 job per launcher invocation" - - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 11 recipes; all recipes pin lmsysorg/sglang:nightly-dev-cu13-20260615-c127ba64" + - "Add 1k/1k coverage under the existing dsv4-fp4-gb300-dynamo-sglang-mtp key so a single top-level config covers both 8k/1k and 1k/1k on GB300. Config image stays at lmsysorg/sglang:nightly-dev-20260527-14f81a67 (unchanged); the 11 new 1k/1k recipes pin lmsysorg/sglang:v0.5.13.post1-cu130" + - "Wire 11 disagg recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16), 4 flat low-latency dep4 recipes (1p1d/1p2d/1p4d/1p6d-dep4-tp4 at conc=64), and 4 flat low-latency tp4 recipes (1p1d/1p2d/1p4d/1p6d-tp4-tp4 at conc=2)" + - "Flat dep4 and tp4 splits replace the upstream disagg-low-latency-dep4-mtp.yaml and disagg-low-latency-tp4-mtp.yaml templates (each base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]). The templated form silently broke under the launcher: srtctl fans the single recipe into 4 Slurm submissions per `srtctl apply`, but the launcher captures job IDs with `grep -oP '✅ Job \\K[0-9]+'` into a scalar JOB_ID, jamming 4 newline-separated IDs into one variable and corrupting downstream LOGS_DIR interpolation and squeue polling. Single-config files match the 8k1k convention (one recipe per decode-worker count) and force srtctl to emit exactly 1 job per launcher invocation" + - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 11 new 1k/1k recipes" + - "Extend runners/launch_gb300-nv.sh so dynamo-sglang + dsv4 clones NVIDIA/srt-slurm@main and overlays the local deepseek-v4 recipe tree at job time" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1697 From 656ffbfd7a8d3f95ef60a68e678f2b8a409f493b Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 2 Jul 2026 11:29:01 -0700 Subject: [PATCH 16/20] Revert non-MTP 8k1k recipe container bumps Restore the 7 disagg-gb300-*.yaml recipes referenced by the non-MTP dsv4-fp4-gb300-dynamo-sglang config to their main-branch container tag. These bumps were unrelated to the -mtp key consolidation in this PR. --- .../8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml | 2 +- .../8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml | 2 +- .../8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml | 2 +- .../8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml | 2 +- .../deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml | 2 +- .../sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml | 2 +- .../deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml index 14dc51f42f..528aa5721a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-10p1d-dep4-dep32-18-c2500" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml index 137434b1dd..32a5124c2d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-12p1d-dep4-dep24-18-c3000" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml index 619957210f..adc6b15507 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-14p1d-dep4-dep16-18-c8192" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml index d14198bc7a..73ad15750d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-15p1d-dep4-dep12-18-c12000" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml index 5bf66527b6..7cdb779c71 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml @@ -33,7 +33,7 @@ name: "disagg-gb300-1p1d-dep4-dep16-5-c1024" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" precision: "fp4" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml index 23c0501749..9382dd6dda 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml @@ -33,7 +33,7 @@ name: "disagg-gb300-1p1d-tp4-tp4-2-c1" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" precision: "fp4" # See ../1k1k/disagg-gb200-1p1d-dep8-tep8.yaml for the dynamo pin diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml index 3cede19112..269b92e124 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml @@ -6,7 +6,7 @@ name: "disagg-gb300-8p1d-dep4-dep40-18-c2048" model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" precision: "fp4" dynamo: From d2854ced1d18141b0a78cdc56ee5c8752bc0aaeb Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 6 Jul 2026 11:44:16 -0700 Subject: [PATCH 17/20] Bump dsv4-fp4-gb300-dynamo-sglang-mtp image to latest nightly Previous tag lmsysorg/sglang:nightly-dev-20260527-14f81a67 expired on Docker Hub (404), causing every job in run 28613151268 (PR #1697) to fail at enroot import. Bump the 8 MTP recipes and the nvidia-master entry to lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e (verified live). Update the PR's perf-changelog entry to reflect the bump. Co-Authored-By: Claude Opus 4.7 --- .github/configs/nvidia-master.yaml | 2 +- .../deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml | 2 +- perf-changelog.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 60026223e0..1a4207bc3f 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -10945,7 +10945,7 @@ glm5-fp8-b200-dynamo-sglang: # MTP variant of dsv4-fp4-gb300-dynamo-sglang. dsv4-fp4-gb300-dynamo-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-20260527-14f81a67 + image: lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: gb300 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml index 39e11b7199..19de980123 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml index a4cf477ed3..924a722230 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml index be5c4cf389..971aa7a086 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml index 5657ad84d6..e941d6476b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml index f4ae3076ce..8ba94e8ac4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml index 4f9f902769..efc6521aeb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml index 17018a57ea..9436ba0ee3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml index 15578537a4..b6cf0a3b7e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" precision: "mxfp4" sbatch_directives: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8629a72c5e..053f0aa12f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4430,7 +4430,7 @@ - config-keys: - dsv4-fp4-gb300-dynamo-sglang-mtp description: - - "Add 1k/1k coverage under the existing dsv4-fp4-gb300-dynamo-sglang-mtp key so a single top-level config covers both 8k/1k and 1k/1k on GB300. Config image stays at lmsysorg/sglang:nightly-dev-20260527-14f81a67 (unchanged); the 11 new 1k/1k recipes pin lmsysorg/sglang:v0.5.13.post1-cu130" + - "Add 1k/1k coverage under the existing dsv4-fp4-gb300-dynamo-sglang-mtp key so a single top-level config covers both 8k/1k and 1k/1k on GB300. Bump config image from lmsysorg/sglang:nightly-dev-20260527-14f81a67 (expired on Docker Hub) to lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e; the 11 new 1k/1k recipes pin lmsysorg/sglang:v0.5.13.post1-cu130" - "Wire 11 disagg recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16), 4 flat low-latency dep4 recipes (1p1d/1p2d/1p4d/1p6d-dep4-tp4 at conc=64), and 4 flat low-latency tp4 recipes (1p1d/1p2d/1p4d/1p6d-tp4-tp4 at conc=2)" - "Flat dep4 and tp4 splits replace the upstream disagg-low-latency-dep4-mtp.yaml and disagg-low-latency-tp4-mtp.yaml templates (each base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]). The templated form silently broke under the launcher: srtctl fans the single recipe into 4 Slurm submissions per `srtctl apply`, but the launcher captures job IDs with `grep -oP '✅ Job \\K[0-9]+'` into a scalar JOB_ID, jamming 4 newline-separated IDs into one variable and corrupting downstream LOGS_DIR interpolation and squeue polling. Single-config files match the 8k1k convention (one recipe per decode-worker count) and force srtctl to emit exactly 1 job per launcher invocation" - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 11 new 1k/1k recipes" From 59f4a2468e2c266306874b9a05e181276140692a Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Fri, 10 Jul 2026 14:12:22 -0700 Subject: [PATCH 18/20] Bump dsv4-fp4-gb300-dynamo-sglang-mtp image to sglang 0710 nightly Pin master image: and all 19 (11 x 1k1k + 8 x 8k1k) recipe model.container values to lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05, aligning the whole config on the newer nightly and clearing the prior split between 1k1k (v0.5.13.post1-cu130) and 8k1k (07-06 nightly). --- .../sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml | 2 +- .../sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml | 2 +- .../sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml | 2 +- .../deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml | 2 +- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml index b919971a1d..c4086de6d6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml @@ -18,7 +18,7 @@ frontend: model: path: deepseek-v4-pro - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: fp4 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml index 553403480a..b014020809 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml @@ -18,7 +18,7 @@ frontend: model: path: deepseek-v4-pro - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: fp4 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml index f84cc28832..eb6e568fee 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml @@ -18,7 +18,7 @@ frontend: model: path: deepseek-v4-pro - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: fp4 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml index cf8b3f5c36..9ed30f1ece 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml index 6a668c9ed9..d387f66af5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml index 6491339e37..eb527cc74f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml index c6fc487630..bc26a190cc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p2d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml index aad8d62616..3158edadef 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml index ecf0aa8aa2..7312fd2e23 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p4d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml index 6d81706b44..0f9c74b7c1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml index c22829f12e..eb3a157252 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-low-latency-1p6d-tp4-tp4-mtp.yaml @@ -8,7 +8,7 @@ frontend: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.13.post1-cu130" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml index 19de980123..830da5fdf9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml index 924a722230..f812dfc826 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml index 971aa7a086..6ea464be7f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml index e941d6476b..4c5bc6c4dd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml index 8ba94e8ac4..b5114dd6e7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml index efc6521aeb..b92407a88b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml index 9436ba0ee3..fe06bedf10 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml index b6cf0a3b7e..1c1bdec6f1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml @@ -11,7 +11,7 @@ dynamo: model: path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" precision: "mxfp4" sbatch_directives: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c4fd7e6b46..af07ae71c3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -10389,7 +10389,7 @@ glm5-fp8-b200-dynamo-sglang: # MTP variant of dsv4-fp4-gb300-dynamo-sglang. dsv4-fp4-gb300-dynamo-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e + image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: gb300 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 02e4bdf80f..719ca890c7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4578,7 +4578,7 @@ - config-keys: - dsv4-fp4-gb300-dynamo-sglang-mtp description: - - "Add 1k/1k coverage under the existing dsv4-fp4-gb300-dynamo-sglang-mtp key so a single top-level config covers both 8k/1k and 1k/1k on GB300. Bump config image from lmsysorg/sglang:nightly-dev-20260527-14f81a67 (expired on Docker Hub) to lmsysorg/sglang:nightly-dev-cu13-20260706-8673e85e; the 11 new 1k/1k recipes pin lmsysorg/sglang:v0.5.13.post1-cu130" + - "Add 1k/1k coverage under the existing dsv4-fp4-gb300-dynamo-sglang-mtp key so a single top-level config covers both 8k/1k and 1k/1k on GB300. Master image and all 19 (11 x 1k1k + 8 x 8k1k) recipe containers pinned to lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" - "Wire 11 disagg recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/: 3 high-throughput conc=8192 recipes (1p1d-dep8, 1p1d-dep16, 2p1d-dep16), 4 flat low-latency dep4 recipes (1p1d/1p2d/1p4d/1p6d-dep4-tp4 at conc=64), and 4 flat low-latency tp4 recipes (1p1d/1p2d/1p4d/1p6d-tp4-tp4 at conc=2)" - "Flat dep4 and tp4 splits replace the upstream disagg-low-latency-dep4-mtp.yaml and disagg-low-latency-tp4-mtp.yaml templates (each base + zip_override_1k1k_lowlat with decode_nodes [1,2,4,6]). The templated form silently broke under the launcher: srtctl fans the single recipe into 4 Slurm submissions per `srtctl apply`, but the launcher captures job IDs with `grep -oP '✅ Job \\K[0-9]+'` into a scalar JOB_ID, jamming 4 newline-separated IDs into one variable and corrupting downstream LOGS_DIR interpolation and squeue polling. Single-config files match the 8k1k convention (one recipe per decode-worker count) and force srtctl to emit exactly 1 job per launcher invocation" - "Decode side runs EAGLE (num-steps=3, eagle-topk=1, num-draft-tokens=4) on all 11 new 1k/1k recipes" From 1300a1c81257194d9a27682f8758e04603fb9592 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Fri, 10 Jul 2026 15:21:58 -0700 Subject: [PATCH 19/20] Pin srt-slurm to v1.0.17 for dsv4-fp4-gb300-dynamo-sglang-mtp The runners/launch_gb300-nv.sh dynamo-sglang + dsv4 branch clones NVIDIA/srt-slurm and previously ran 'git checkout main', so every sweep picked up whatever srtctl 'main' happened to be at that moment. Between the last green run (2026-06-16) and the current failing head (2026-07-06), main advanced meaningfully, which is a plausible driver of the recent 'decode worker exit 137 / Server did not become healthy' failures on the byte-identical 1k1k recipes. Pin to v1.0.17 so all future sweeps in this PR run against a fixed srtctl SHA and drift can be ruled in or out cleanly. --- runners/launch_gb300-nv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index c224dbabbb..d63352d161 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -167,7 +167,7 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" ]]; then elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout main + git checkout v1.0.17 mkdir -p recipes/sglang/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then From 46798d2337b7219bc5056df96f07f3fea36f03c5 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Thu, 16 Jul 2026 15:44:02 -0700 Subject: [PATCH 20/20] update dynamo version --- .../deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml | 3 +-- .../sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml | 3 +-- .../deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml | 3 +-- .../deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml | 3 +-- .../deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml | 3 +-- .../deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml | 3 +-- .../deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml | 3 +-- .../deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml | 3 +-- .../deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml | 3 +-- .../deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml | 3 +-- .../deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml | 3 +-- 11 files changed, 11 insertions(+), 22 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml index bc2334a959..ab1484c4a4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192-mtp.yaml @@ -5,8 +5,7 @@ sbatch_directives: mem: '0' dynamo: - hash: "34d55a596fb8d3d44daefe425ec1e303131f4d2c" - install: true + version: 1.2.1 frontend: type: dynamo diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml index 31667bddd4..4818b96044 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192-mtp.yaml @@ -5,8 +5,7 @@ sbatch_directives: mem: '0' dynamo: - hash: "34d55a596fb8d3d44daefe425ec1e303131f4d2c" - install: true + version: 1.2.1 frontend: type: dynamo diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml index 9f64729d79..159022c579 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc8192-mtp.yaml @@ -5,8 +5,7 @@ sbatch_directives: mem: '0' dynamo: - hash: "34d55a596fb8d3d44daefe425ec1e303131f4d2c" - install: true + version: 1.2.1 frontend: type: dynamo diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml index 830da5fdf9..84eedde6e3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml @@ -6,8 +6,7 @@ frontend: num_additional_frontends: 8 dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true + version: 1.2.1 model: path: "deepseek-v4-pro" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml index f812dfc826..574c01f2af 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml @@ -6,8 +6,7 @@ frontend: num_additional_frontends: 8 dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true + version: 1.2.1 model: path: "deepseek-v4-pro" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml index 6ea464be7f..c095028aff 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -6,8 +6,7 @@ frontend: num_additional_frontends: 8 dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true + version: 1.2.1 model: path: "deepseek-v4-pro" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml index 4c5bc6c4dd..ccf0e942db 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -6,8 +6,7 @@ frontend: num_additional_frontends: 8 dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true + version: 1.2.1 model: path: "deepseek-v4-pro" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml index b5114dd6e7..9a66895242 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml @@ -6,8 +6,7 @@ frontend: num_additional_frontends: 8 dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true + version: 1.2.1 model: path: "deepseek-v4-pro" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml index b92407a88b..263339a2c7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml @@ -6,8 +6,7 @@ frontend: num_additional_frontends: 8 dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true + version: 1.2.1 model: path: "deepseek-v4-pro" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml index fe06bedf10..16141c6771 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml @@ -6,8 +6,7 @@ frontend: num_additional_frontends: 8 dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true + version: 1.2.1 model: path: "deepseek-v4-pro" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml index 1c1bdec6f1..636049cba4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml @@ -6,8 +6,7 @@ frontend: num_additional_frontends: 8 dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true + version: 1.2.1 model: path: "deepseek-v4-pro"