[AMD] Update AMD MI300X, MI325X, MI355X GPT-OSS vLLM images to v0.16.0 (#806)

* Update AMD MI300X, MI325X, MI355X GPT-OSS vLLM images to v0.16.0

MI300X & MI325X: Bump image from v0.15.1 to v0.16.0 (scripts compatible)
MI355X: Major rewrite from custom ROCm v0.10.1 to upstream v0.16.0:
  - Fix env var names (VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION)
  - Add missing VLLM_ROCM_USE_AITER=1 master toggle
  - Remove non-existent VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4
  - Remove deprecated --max-seq-len-to-capture and --async-scheduling
  - Simplify compilation config to cudagraph_mode FULL_AND_PIECEWISE
  - Add HIP_VISIBLE_DEVICES Ray compatibility and MEC firmware check

Closes #803

Co-authored-by: Cameron Quilici <cquil11@users.noreply.github.com>

* Update perf-changelog.yaml

* Update perf-changelog PR links to #806

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Update gptoss_fp4_mi355x.sh

* Consolidate perf-changelog entries for MI300X/MI325X/MI355X into one

Co-authored-by: Cameron Quilici <cquil11@users.noreply.github.com>

* update HIP VISIBLE DEVICES

---------

Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com>
Co-authored-by: Cameron Quilici <cquil11@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: 3 people

.github/configs/amd-master.yaml

@@ -230,7 +230,7 @@ minimaxm2.5-fp8-mi355x-vllm:
     - { tp: 4, conc-start: 4, conc-end: 64 }
 
 gptoss-fp4-mi300x-vllm:
-  image: vllm/vllm-openai-rocm:v0.15.1
+  image: vllm/vllm-openai-rocm:v0.16.0
   model: openai/gpt-oss-120b
   model-prefix: gptoss
   runner: mi300x
@@ -261,7 +261,7 @@ gptoss-fp4-mi300x-vllm:
     - { tp: 8, conc-start: 4, conc-end: 16 }
 
 gptoss-fp4-mi325x-vllm:
-  image: vllm/vllm-openai-rocm:v0.15.1
+  image: vllm/vllm-openai-rocm:v0.16.0
   model: openai/gpt-oss-120b
   model-prefix: gptoss
   runner: mi325x
@@ -292,7 +292,7 @@ gptoss-fp4-mi325x-vllm:
     - { tp: 8, conc-start: 4, conc-end: 16 }
 
 gptoss-fp4-mi355x-vllm:
-  image: rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1
+  image: vllm/vllm-openai-rocm:v0.16.0
   model: openai/gpt-oss-120b
   model-prefix: gptoss
   runner: mi355x

benchmarks/single_node/gptoss_fp4_mi355x.sh

@@ -18,16 +18,24 @@ fi
 
 hf download "$MODEL"
 
-cat > config.yaml << EOF
-compilation-config: '{"compile_sizes":[1,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,256,512,1024,2048,8192] , "cudagraph_capture_sizes":[1,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,264,272,280,288,296,304,312,320,328,336,344,352,360,368,376,384,392,400,408,416,424,432,440,448,456,464,472,480,488,496,504,512,520,528,536,544,552,560,568,576,584,592,600,608,616,624,632,640,648,656,664,672,680,688,696,704,712,720,728,736,744,752,760,768,776,784,792,800,808,816,824,832,840,848,856,864,872,880,888,896,904,912,920,928,936,944,952,960,968,976,984,992,1000,1008,1016,1024,2048,4096,8192] , "cudagraph_mode": "FULL_AND_PIECEWISE"}' 
-EOF
+# If the machine runs a MEC FW older than 177, RCCL
+# cannot reclaim some memory.
+# Disable that features to avoid crashes.
+# This is related to the changes in the driver at:
+# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates
+version=`rocm-smi --showfw | grep MEC | head -n 1 |  awk '{print $NF}'`
+if [[ "$version" == "" || $version -lt 177 ]]; then
+  export HSA_NO_SCRATCH_RECLAIM=1
+fi
 
-sleep 5
-cat config.yaml
+# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+
+if [ -n "$ROCR_VISIBLE_DEVICES" ]; then
+    export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
+fi
 
-export VLLM_USE_AITER_UNIFIED_ATTENTION=1
+export VLLM_ROCM_USE_AITER=1
+export VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION=1
 export VLLM_ROCM_USE_AITER_MHA=0
-export VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4=1
 
 SERVER_LOG=/workspace/server.log
 PORT=${PORT:-8888}
@@ -37,12 +45,10 @@ vllm serve $MODEL --port $PORT \
 --tensor-parallel-size=$TP \
 --gpu-memory-utilization 0.95 \
 --max-model-len $MAX_MODEL_LEN \
---max-seq-len-to-capture $MAX_MODEL_LEN \
---config config.yaml \
+--compilation-config  '{"cudagraph_mode": "FULL_AND_PIECEWISE"}' \
 --block-size=64 \
 --no-enable-prefix-caching \
---disable-log-requests \
---async-scheduling > $SERVER_LOG 2>&1 &
+--disable-log-requests > $SERVER_LOG 2>&1 &
 
 SERVER_PID=$!
 

perf-changelog.yaml

@@ -733,4 +733,15 @@
     - "Extend concurrency range to conc-end: 256 across all sequence lengths (1k1k, 1k8k, 8k1k)"
     - "Fix MTP 1k8k conc-start from 256 to 4 to enable full concurrency sweep"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/699
-  
+
+- config-keys:
+    - gptoss-fp4-mi300x-vllm
+    - gptoss-fp4-mi325x-vllm
+    - gptoss-fp4-mi355x-vllm
+  description:
+    - "Update AMD GPT-OSS vLLM images to v0.16.0 (MI300X/MI325X from v0.15.1, MI355X from custom v0.10.1)"
+    - "MI355X: Fix env vars (VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION), add VLLM_ROCM_USE_AITER=1, remove deprecated flags"
+    - "MI355X: Simplify compilation config to cudagraph_mode FULL_AND_PIECEWISE, add HIP_VISIBLE_DEVICES Ray fix"
+    - "Gains: fused add+rmsnorm+pad for GPT-OSS (automatic via PassManager), AITER attention block size fix"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/806
+