diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh index c38a30f17..96f7aeba1 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh @@ -123,6 +123,17 @@ export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 METRICS_ARGS=(--enable-metrics) SPEC_ARGS=() +# ---- Temp patch (TODO:remove) ---------------------------------------------------------- +GIST_BASE="https://gist.githubusercontent.com/seungrokj/9911b69be4c6b0f70ebca5dc03b5fc5d/raw" +curl -LsSf "$GIST_BASE/df54b1879567807bbfc92e5ad004b74c90d030cb/dsv4_fp8fp4_tuned_fmoe.csv" \ + -o /sgl-workspace/aiter/aiter/configs/model_configs/dsv4_fp8fp4_tuned_fmoe.csv +curl -LsSf "$GIST_BASE/4a79f11bca8607b6b9438e9fd37af16a2534e3fb/dsv4_fp8fp4_untuned_fmoe.csv" \ + -o /sgl-workspace/aiter/aiter/configs/model_configs/dsv4_fp8fp4_untuned_fmoe.csv +curl -LsSf "$GIST_BASE/bc4ba01751f2576d6ff099ffd2167d2a76fe85b6/shuffle.py" \ + -o /sgl-workspace/aiter/aiter/ops/shuffle.py +curl -LsSf "$GIST_BASE/741d6e90dceac4ed21284c8bfc6f2c25d9442649/fp8.py" \ + -o /sgl-workspace/sglang/python/sglang/srt/layers/quantization/fp8.py + SGLANG_CMD=( python3 -m sglang.launch_server --model-path "$MODEL_PATH" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ac4e27690..1c19f580c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1536,7 +1536,7 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: - "DECODE_MTP_SIZE=1" dsv4-fp4-mi355x-sglang-agentic-hicache: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm700-mi35x-20260721 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1547,9 +1547,9 @@ dsv4-fp4-mi355x-sglang-agentic-hicache: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8] } + #- { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8] } - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 64] } - - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } + #- { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [64] } # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1822ffcb7..e94e9505a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5019,3 +5019,11 @@ - "dsv4-fp4-b200-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms -- TP8 GPU-resident, TP8 SimpleCPU, DEP8 SimpleCPU, DEP8 Mooncake, each mirroring its non-MTP conc-list; routed via spec-decoding=mtp to dsv4_fp4_b200_vllm_mtp.sh." - "MTP capture sizes are in TOKENS: --max-cudagraph-capture-size scaled to MAX_NUM_SEQS*(1+num_speculative_tokens) (= MAX_NUM_SEQS*4) so FULL_DECODE_ONLY covers the largest decode batches." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2259 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-hicache + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.15.post1-rocm700-mi35x-20260721" + - "Patch aiter tuned/untuned FMoE CSV configs, shuffle.py, and sglang fp8.py at runtime from gist" + - "Comment out non-DP-attention and HiCache sweep rows to focus on DP-attention conc 16-64" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2298