diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index f8cc486b2..30c657da1 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -9428,3 +9428,213 @@ glm5-fp4-gb300-dynamo-sglang: tp: 4 ep: 1 dp-attn: false + +glm5-fp8-gb300-dynamo-sglang: + image: lmsysorg/sglang:v0.5.11-cu130 + model: zai-org/GLM-5-FP8 + model-prefix: glm5 + runner: gb300-nv + precision: fp8 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + # ---------- 8k1k high-throughput (wide-EP decode) ---------- + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [2800] + prefill: + num-worker: 14 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [1700] + prefill: + num-worker: 12 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml" + decode: + num-worker: 1 + tp: 24 + ep: 24 + dp-attn: true + - conc-list: [1300] + prefill: + num-worker: 10 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [900] + prefill: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml" + decode: + num-worker: 1 + tp: 40 + ep: 40 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers) ---------- + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [150] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + decode: + num-worker: 9 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [128, 64, 32] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [24] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + # ---------- 1k1k high-throughput (wide-EP decode) ---------- + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [8192] + prefill: + num-worker: 12 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_0.yaml" + decode: + num-worker: 1 + tp: 24 + ep: 24 + dp-attn: true + - conc-list: [7500] + prefill: + num-worker: 10 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_1.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [7300] + prefill: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_2.yaml" + decode: + num-worker: 1 + tp: 40 + ep: 40 + dp-attn: true + - conc-list: [6500] + prefill: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_3.yaml" + decode: + num-worker: 1 + tp: 48 + ep: 48 + dp-attn: true + - conc-list: [5700] + prefill: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_4.yaml" + decode: + num-worker: 1 + tp: 56 + ep: 56 + dp-attn: true + # ---------- 1k1k low-latency (per-node TP=4 decode workers) ---------- + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [512, 256, 128, 64] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [32] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_0.yaml new file mode 100644 index 000000000..33da57e94 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_0.yaml @@ -0,0 +1,148 @@ +name: gb300-fp8-glm5_1k1k_hightpt_0 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 12 + prefill_workers: 12 + decode_nodes: 6 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 24 + expert-parallel-size: 24 + data-parallel-size: 24 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 8192 + cuda-graph-max-bs: 512 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '8192' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_1.yaml new file mode 100644 index 000000000..03cb7e671 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_1.yaml @@ -0,0 +1,148 @@ +name: gb300-fp8-glm5_1k1k_hightpt_1 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 10 + prefill_workers: 10 + decode_nodes: 8 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 32 + expert-parallel-size: 32 + data-parallel-size: 32 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 8192 + cuda-graph-max-bs: 256 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '7500' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_2.yaml new file mode 100644 index 000000000..c06206c81 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_2.yaml @@ -0,0 +1,148 @@ +name: gb300-fp8-glm5_1k1k_hightpt_2 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 8 + prefill_workers: 8 + decode_nodes: 10 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 40 + expert-parallel-size: 40 + data-parallel-size: 40 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 24 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 7200 + cuda-graph-max-bs: 180 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '7300' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_3.yaml new file mode 100644 index 000000000..951772479 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_3.yaml @@ -0,0 +1,148 @@ +name: gb300-fp8-glm5_1k1k_hightpt_3 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 6 + prefill_workers: 6 + decode_nodes: 12 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 48 + expert-parallel-size: 48 + data-parallel-size: 48 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 6144 + cuda-graph-max-bs: 128 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '6500' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_4.yaml new file mode 100644 index 000000000..9a1f320a5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_4.yaml @@ -0,0 +1,148 @@ +name: gb300-fp8-glm5_1k1k_hightpt_4 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 4 + decode_nodes: 14 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 56 + expert-parallel-size: 56 + data-parallel-size: 56 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 24 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 5600 + cuda-graph-max-bs: 100 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '5700' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml new file mode 100644 index 000000000..3ace5647c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml @@ -0,0 +1,140 @@ +name: gb300-fp8-glm5_1k1k_lowlat_0 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 17 + decode_workers: 17 +frontend: + type: dynamo +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 32 + cuda-graph-max-bs: 32 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: 512x256x128x64 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml new file mode 100644 index 000000000..965b6f148 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml @@ -0,0 +1,140 @@ +name: gb300-fp8-glm5_1k1k_lowlat_1 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 17 + decode_workers: 17 +frontend: + type: dynamo +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 1 + cuda-graph-max-bs: 1 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '32' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml new file mode 100644 index 000000000..150e62233 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml @@ -0,0 +1,148 @@ +name: gb300-fp8-glm5_8k1k_hightpt_0 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 14 + prefill_workers: 14 + decode_nodes: 4 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 2800 + cuda-graph-max-bs: 175 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '2800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml new file mode 100644 index 000000000..6393069c8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml @@ -0,0 +1,148 @@ +name: gb300-fp8-glm5_8k1k_hightpt_1 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 12 + prefill_workers: 12 + decode_nodes: 6 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 24 + expert-parallel-size: 24 + data-parallel-size: 24 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 1680 + cuda-graph-max-bs: 70 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '1700' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml new file mode 100644 index 000000000..56b11ed8a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml @@ -0,0 +1,148 @@ +name: gb300-fp8-glm5_8k1k_hightpt_2 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 10 + prefill_workers: 10 + decode_nodes: 8 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 32 + expert-parallel-size: 32 + data-parallel-size: 32 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 1280 + cuda-graph-max-bs: 40 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '1300' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml new file mode 100644 index 000000000..13fb0f326 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml @@ -0,0 +1,148 @@ +name: gb300-fp8-glm5_8k1k_hightpt_3 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 8 + prefill_workers: 8 + decode_nodes: 10 + decode_workers: 1 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 40 + expert-parallel-size: 40 + data-parallel-size: 40 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 24 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 880 + cuda-graph-max-bs: 22 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '900' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml new file mode 100644 index 000000000..8065160bd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml @@ -0,0 +1,140 @@ +name: gb300-fp8-glm5_8k1k_lowlat_0 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 9 + decode_workers: 9 +frontend: + type: dynamo +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 15 + cuda-graph-max-bs: 15 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '150' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml new file mode 100644 index 000000000..33f0324b6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml @@ -0,0 +1,140 @@ +name: gb300-fp8-glm5_8k1k_lowlat_1 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 17 + decode_workers: 17 +frontend: + type: dynamo +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 8 + cuda-graph-max-bs: 8 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: 128x64x32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml new file mode 100644 index 000000000..64d4c701a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml @@ -0,0 +1,140 @@ +name: gb300-fp8-glm5_8k1k_lowlat_2 + +model: + path: glm-5-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 17 + decode_workers: 17 +frontend: + type: dynamo +dynamo: + version: 1.1.0 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + sglang_config: + prefill: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: GLM-5-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 1 + cuda-graph-max-bs: 1 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '24' diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 935cded22..1b194b52f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3200,3 +3200,11 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523, 1P1D TP8/EP1, dp-attn false, conc [8..512]" - "MoRI conn.py overlay (48e459bd) via job.slurm; launcher qwen3.5_fp4_mi355x_sglang-disagg.sh" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1579 + +- config-keys: + - glm5-fp8-gb300-dynamo-sglang + description: + - "Add GLM-5 FP8 GB300 Dynamo SGLang disaggregated multi-node coverage using lmsysorg/sglang:v0.5.11-cu130" + - "1k1k and 8k1k STP hightpt and lowlat srt-slurm recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp8/ (resolved from upstream srt-slurm PR #160 via srtctl resolve-override)" + - "Wire glm5/fp8 model + dynamo-sglang framework branches into runners/launch_gb300-nv.sh with SA upstream defaults (SLURM_PARTITION=batch_1, SLURM_ACCOUNT=benchmark, SQUASH_FILE under /home/sa-shared/gharunners/squash/)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1557 diff --git a/runners/launch_gb300-cw.sh b/runners/launch_gb300-cw.sh index 25e7f4db5..676362c01 100644 --- a/runners/launch_gb300-cw.sh +++ b/runners/launch_gb300-cw.sh @@ -25,8 +25,20 @@ if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then echo "Unsupported framework on gb300-cw for dsv4/fp4: $FRAMEWORK. Currently supported: dynamo-sglang, dynamo-vllm" exit 1 fi +elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then + export MODEL_PATH="/mnt/vast/models/GLM-5-FP8" + + if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + SRT_SLURM_RECIPES_REPO="https://github.com/NVIDIA/srt-slurm.git" + SRT_SLURM_RECIPES_REF="sa-submission-q2-2026" + SRT_RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5" + SRT_RECIPE_DST="recipes/sglang/glm5" + else + echo "Unsupported framework on gb300-cw for glm5/fp8: $FRAMEWORK. Currently supported: dynamo-sglang" + exit 1 + fi else - echo "Unsupported model prefix/precision combination on gb300-cw: $MODEL_PREFIX/$PRECISION. Currently supported: dsv4/fp4" + echo "Unsupported model prefix/precision combination on gb300-cw: $MODEL_PREFIX/$PRECISION. Currently supported: dsv4/fp4, glm5/fp8" exit 1 fi @@ -183,6 +195,8 @@ model_paths: # is not a local model path and is not defined in srtslurm.yaml # model_paths". deepseek-v4-pro: "${MODEL_PATH}" + # GLM-5 FP8 sglang recipes use `model.path: glm-5-fp8`. + glm-5-fp8: "${MODEL_PATH}" containers: dynamo-trtllm: ${SQUASH_FILE} dynamo-sglang: ${SQUASH_FILE} diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 5248e63ed..56ee8033d 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -24,8 +24,11 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then export MODEL_PATH=/scratch/models/GLM-5-NVFP4 export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" +elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then + export MODEL_PATH=/scratch/models/GLM-5-FP8 + export SRT_SLURM_MODEL_PREFIX="glm-5-fp8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8" exit 1 fi @@ -72,11 +75,6 @@ if [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" ]]; then - # Hand-rolled GLM-5 sglang recipes (one flat yaml per topology, - # derived from upstream srt-slurm PR #152's combined glm5.yaml). - # Directory layout mirrors SemiAnalysisAI/InferenceX PR #1372: - # sglang/glm5/-//disagg/stp/_stp__.yaml - # so multiple precisions (fp4/fp8) and hardware targets can coexist. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout sa-submission-q2-2026