File tree Expand file tree Collapse file tree
benchmarks/single_node/fixed_seq_len Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -21,6 +21,7 @@ export SGLANG_USE_AITER=1
2121export SGLANG_USE_AITER_UNIFIED_ATTN=1
2222export AITER_FLYDSL_FORCE=1
2323export SGLANG_MAMBA_SSM_DTYPE=bfloat16
24+ export ROCM_QUICK_REDUCE_QUANTIZATION=INT8
2425
2526SERVER_LOG=/workspace/server.log
2627MEM_FRAC_STATIC=${MEM_FRAC_STATIC:- 0.8}
@@ -41,7 +42,7 @@ python3 -m sglang.launch_server --model-path=$MODEL --trust-remote-code \
4142--model-loader-extra-config ' {"enable_multithread_load": true}' \
4243--watchdog-timeout 1200 \
4344--disable-radix-cache \
44- --enable-aiter-allreduce-fusion -- max-running-requests $CONC \
45+ --max-running-requests $CONC \
4546--page-size 16 \
4647> $SERVER_LOG 2>&1 &
4748
Original file line number Diff line number Diff line change @@ -21,6 +21,7 @@ export SGLANG_USE_AITER=1
2121export SGLANG_USE_AITER_UNIFIED_ATTN=1
2222export AITER_FLYDSL_FORCE=1
2323export SGLANG_MAMBA_SSM_DTYPE=bfloat16
24+ export ROCM_QUICK_REDUCE_QUANTIZATION=INT8
2425
2526SERVER_LOG=/workspace/server.log
2627MEM_FRAC_STATIC=${MEM_FRAC_STATIC:- 0.8}
@@ -41,7 +42,7 @@ python3 -m sglang.launch_server --model-path=$MODEL --trust-remote-code \
4142--model-loader-extra-config ' {"enable_multithread_load": true}' \
4243--watchdog-timeout 1200 \
4344--disable-radix-cache \
44- --enable-aiter-allreduce-fusion -- max-running-requests $CONC \
45+ --max-running-requests $CONC \
4546--page-size 16 \
4647--speculative-algorithm EAGLE \
4748--speculative-num-steps 3 \
Original file line number Diff line number Diff line change 48874887 - " Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts"
48884888 pr-link : https://github.com/SemiAnalysisAI/InferenceX/pull/2201
48894889
4890+ - config-keys :
4891+ - qwen3.5-fp4-mi355x-sglang
4892+ - qwen3.5-fp4-mi355x-sglang-mtp
4893+ description :
4894+ - " Add ROCM_QUICK_REDUCE_QUANTIZATION=INT8 in both non-MTP and MTP benchmark scripts"
4895+ - " Remove --enable-aiter-allreduce-fusion from both non-MTP and MTP launch commands"
4896+ pr-link : https://github.com/SemiAnalysisAI/InferenceX/pull/2265
4897+
48904898- config-keys :
48914899 - dsv4-fp4-b200-vllm-agentic
48924900 description :
You can’t perform that action at this time.
0 commit comments