From 3ca4ba9a043b5474ae20c9bcb0c67844ffa9abba Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 16 Jul 2026 15:33:22 -0700 Subject: [PATCH 1/2] feat: add DSV4 FP4 B200 Dynamo SGLang MTP config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add five recipe-backed 8k/1k MTP topologies and config-scoped B200 launcher handling. 中文：新增 DSV4 FP4 B200 分离式 Dynamo SGLang MTP 配置，加入五个由 recipe 驱动的 8k/1k MTP 拓扑，并将 B200 启动器处理限定到该配置。 --- ...agg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml | 126 +++++++++++++++ ...gg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml | 133 ++++++++++++++++ ...agg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml | 132 ++++++++++++++++ ...agg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml | 145 ++++++++++++++++++ ...agg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml | 132 ++++++++++++++++ configs/nvidia-master.yaml | 87 +++++++++++ perf-changelog.yaml | 8 + runners/launch_b200-dgxc.sh | 40 +++-- 8 files changed, 789 insertions(+), 14 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml new file mode 100644 index 0000000000..17d8501270 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml @@ -0,0 +1,126 @@ +name: "disagg-b200-8k1k-low-latency-1p1d-tp8-tp8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 16 + cuda-graph-max-bs: 8 + chunked-prefill-size: 65536 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml new file mode 100644 index 0000000000..e4233a4cb4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml @@ -0,0 +1,133 @@ +name: "disagg-b200-8k1k-low-latency-1p6d-dep8-tp8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 6 + decode_workers: 6 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "32x64x128" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..3b198e99da --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,132 @@ +name: "disagg-b200-8k1k-mid-curve-1p1d-dep8-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + mem-fraction-static: 0.9 + max-running-requests: 1536 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256x1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..36d2fc58ad --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml @@ -0,0 +1,145 @@ +name: "disagg-b200-8k1k-mid-curve-1p2d-dep8-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 2 + decode_workers: 2 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.8 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.85 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..f478b057f8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml @@ -0,0 +1,132 @@ +name: "disagg-b200-8k1k-mid-curve-5p3d-dep8-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 5 + prefill_workers: 5 + gpus_per_prefill: 8 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + mem-fraction-static: 0.9 + max-running-requests: 2560 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "6144" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ff922946c5..099774ee41 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9381,6 +9381,93 @@ dsv4-fp4-b200-dynamo-vllm: ep: 8 dp-attn: true +dsv4-fp4-b200-dynamo-sglang-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: b200-multinode + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [32, 64, 128] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml" + decode: + num-worker: 6 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [256, 1024] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [6144] + prefill: + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + dsv4-fp4-gb200-dynamo-vllm: image: vllm/vllm-openai:v0.20.0-ubuntu2404 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c620aceaa5..e1e98ca7df 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4918,3 +4918,11 @@ - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.92); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2241 + +- config-keys: + - dsv4-fp4-b200-dynamo-sglang-mtp + description: + - "Add DSV4 NVFP4 B200 disaggregated dynamo-sglang MTP configuration for 8k1k" + - "Register five recipe-backed topologies and B200 launcher support" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 58051e6405..b0790d4304 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -13,6 +13,7 @@ set -x # launches. # NOTE: per-node /raid/models/* would be faster but is only populated on a # subset of dgxc nodes today, so we use Lustre for reliability. +PREFLIGHT_ARGS=() if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/fsw/models/DeepSeek-R1-0528-NVFP4-v2" export SRT_SLURM_MODEL_PREFIX="dsr1" @@ -20,18 +21,23 @@ elif [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/dsr1-0528-fp8" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - SELECTED_MODEL_PATH="" - if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then - SELECTED_MODEL_PATH="$MODEL_PATH" + if [[ $FRAMEWORK == "dynamo-sglang" && $SPEC_DECODING == "mtp" ]]; then + export MODEL_PATH="/scratch/fsw/models/DeepSeek-V4-Pro-NVFP4" + PREFLIGHT_ARGS=(--no-preflight) else - for candidate in /lustre/fsw/models/deepseek-v4-pro /lustre/fsw/models/dsv4-pro /lustre/fsw/models/DeepSeek-V4-Pro; do - if [[ -d "$candidate" ]]; then - SELECTED_MODEL_PATH="$candidate" - break - fi - done + SELECTED_MODEL_PATH="" + if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then + SELECTED_MODEL_PATH="$MODEL_PATH" + else + for candidate in /lustre/fsw/models/deepseek-v4-pro /lustre/fsw/models/dsv4-pro /lustre/fsw/models/DeepSeek-V4-Pro; do + if [[ -d "$candidate" ]]; then + SELECTED_MODEL_PATH="$candidate" + break + fi + done + fi + export MODEL_PATH="${SELECTED_MODEL_PATH:-/lustre/fsw/models/deepseek-v4-pro}" fi - export MODEL_PATH="${SELECTED_MODEL_PATH:-/lustre/fsw/models/deepseek-v4-pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "bf16" ]]; then export MODEL_PATH="/lustre/fsw/models/Qwen3.5-397B-A17B" @@ -88,9 +94,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then exit 1 fi - # Multinode dsv4 currently only ships with the dynamo-vllm recipe - if [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK != "dynamo-vllm" ]]; then - echo "Unsupported framework for multinode dsv4: $FRAMEWORK (only dynamo-vllm)" + if [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK != "dynamo-vllm" ]] && + [[ $FRAMEWORK != "dynamo-sglang" || $SPEC_DECODING != "mtp" ]]; then + echo "Unsupported framework for multinode dsv4: $FRAMEWORK" exit 1 fi @@ -113,6 +119,12 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git checkout aflowers/vllm-gb200-v0.20.0 mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $SPEC_DECODING == "mtp" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 + git checkout main + mkdir -p recipes/sglang/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -255,7 +267,7 @@ EOF # so large-model loads (e.g. DSR1-FP8 ~680GB off shared FS) finish in time. # Uses ${CONFIG_FILE%%:*} because CONFIG_FILE may carry an :override[N] suffix. sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}" - SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) + SRTCTL_OUTPUT=$(srtctl apply "${PREFLIGHT_ARGS[@]}" -f "$CONFIG_FILE" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output From de72cd67c6864ac7caba75019efa4bbf1fa9f4d7 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 16 Jul 2026 15:33:49 -0700 Subject: [PATCH 2/2] chore: update changelog link for PR 2252 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record the public pull request URL in the final changelog entry. 中文：在性能变更日志的最后一个条目中记录公开拉取请求链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e1e98ca7df..0d377e675b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4925,4 +4925,4 @@ - "Add DSV4 NVFP4 B200 disaggregated dynamo-sglang MTP configuration for 8k1k" - "Register five recipe-backed topologies and B200 launcher support" - "Image: lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2252