Update dpskv4 GB300 MTP SGLang image to nightly-20260518 and clean env vars

Author: yhyang201

.github/configs/amd-master.yaml

@@ -162,7 +162,7 @@ qwen3.5-bf16-mi355x-sglang-mtp:
       - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp }
 
 qwen3.5-bf16-mi300x-sglang:
-  image: lmsysorg/sglang:v0.5.12-rocm720-mi30x
+  image: lmsysorg/sglang:v0.5.10-rocm720-mi30x
   model: Qwen/Qwen3.5-397B-A17B
   model-prefix: qwen3.5
   runner: mi300x
@@ -527,7 +527,7 @@ kimik2.5-int4-mi355x-vllm:
       - { tp: 8, conc-start: 4, conc-end: 64 }
 
 kimik2.5-int4-mi325x-vllm:
-  image: vllm/vllm-openai-rocm:v0.21.0
+  image: vllm/vllm-openai-rocm:v0.18.0
   model: moonshotai/Kimi-K2.5
   model-prefix: kimik2.5
   runner: mi325x
@@ -802,7 +802,7 @@ minimaxm2.5-fp8-mi300x-vllm-agentic:
       - { tp: 4, offloading: cpu,  conc-list: [16, 20, 24, 28, 32] }
 
 minimaxm2.5-fp8-mi325x-vllm:
-  image: vllm/vllm-openai-rocm:v0.21.0
+  image: vllm/vllm-openai-rocm:v0.18.0
   model: MiniMaxAI/MiniMax-M2.5
   model-prefix: minimaxm2.5
   runner: mi325x
@@ -872,7 +872,7 @@ gptoss-fp4-mi300x-vllm:
       - { tp: 8, conc-start: 1, conc-end: 16 }
 
 gptoss-fp4-mi325x-vllm:
-  image: vllm/vllm-openai-rocm:v0.21.0
+  image: vllm/vllm-openai-rocm:v0.17.0
   model: openai/gpt-oss-120b
   model-prefix: gptoss
   runner: mi325x

.github/configs/nvidia-master.yaml

@@ -2052,7 +2052,7 @@ dsv4-fp4-b300-sglang-mtp:
       - { tp: 4, ep: 1, conc-start: 4, conc-end: 32, spec-decoding: mtp }
 
 qwen3.5-bf16-b200-sglang:
-  image: lmsysorg/sglang:v0.5.12-cu130
+  image: lmsysorg/sglang:nightly-dev-20260216-d3bae71e
   model: Qwen/Qwen3.5-397B-A17B
   model-prefix: qwen3.5
   runner: b200
@@ -2071,7 +2071,7 @@ qwen3.5-bf16-b200-sglang:
       - { tp: 8, ep: 1, conc-start: 4, conc-end: 64 }
 
 qwen3.5-bf16-b200-sglang-mtp:
-  image: lmsysorg/sglang:v0.5.12-cu130
+  image: lmsysorg/sglang:nightly-dev-20260216-d3bae71e
   model: Qwen/Qwen3.5-397B-A17B
   model-prefix: qwen3.5
   runner: b200
@@ -2089,22 +2089,24 @@ qwen3.5-bf16-b200-sglang-mtp:
       search-space:
       - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp }
 
-# agentic-coding sibling — temporarily disabled, blocked by e2e-tests.yml
-# artifact-name mismatch (downloads `agentic_*` but benchmark-tmpl.yml uploads
-# as `bmk_agentic_*`). Re-enable once that workflow is aligned.
-# qwen3.5-bf16-b200-sglang-agentic:
-#   image: lmsysorg/sglang:v0.5.12-cu130
-#   model: Qwen/Qwen3.5-397B-A17B
-#   model-prefix: qwen3.5
-#   runner: b200
-#   precision: bf16
-#   framework: sglang
-#   multinode: false
-#   scenarios:
-#     agentic-coding:
-#     - duration: 1800
-#       search-space:
-#       - { tp: 8, ep: 1, offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
+# Diverged from qwen3.5-bf16-b200-sglang (agentic-coding sibling). Metadata is
+# identical to origin/main's qwen3.5-bf16-b200-sglang; the split exists because this
+# PR adds an agentic-coding scenarios block that differs from main
+# (either main had none or had a different conc/offload sweep).
+# The original qwen3.5-bf16-b200-sglang entry stays byte-identical to origin/main.
+qwen3.5-bf16-b200-sglang-agentic:
+  image: lmsysorg/sglang:nightly-dev-20260216-d3bae71e
+  model: Qwen/Qwen3.5-397B-A17B
+  model-prefix: qwen3.5
+  runner: b200
+  precision: bf16
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - duration: 1800
+      search-space:
+      - { tp: 8, ep: 1, offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
 
 qwen3.5-fp8-b200-sglang:
   image: lmsysorg/sglang:nightly-dev-20260422-de962f32
@@ -2632,7 +2634,7 @@ kimik2.5-int4-h200-vllm-agentic:
       - { tp: 8, offloading: cpu,  conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] }
 
 kimik2.5-fp4-b200-vllm:
-  image: vllm/vllm-openai:v0.20.2
+  image: vllm/vllm-openai:v0.17.0
   model: nvidia/Kimi-K2.5-NVFP4
   model-prefix: kimik2.5
   runner: b200
@@ -4252,7 +4254,7 @@ gptoss-fp4-b200-vllm-agentic:
       - { tp: 8, offloading: cpu,  conc-list: [64, 96, 128, 192, 256] }
 
 minimaxm2.5-fp8-b200-vllm:
-  image: vllm/vllm-openai:v0.21.0
+  image: vllm/vllm-openai:v0.19.0-cu130
   model: MiniMaxAI/MiniMax-M2.5
   model-prefix: minimaxm2.5
   runner: b200
@@ -4353,7 +4355,7 @@ minimaxm2.5-fp8-b300-vllm-agentic:
       - { tp: 4, offloading: cpu,  conc-list: [48, 64, 96, 100, 104, 108, 112, 116, 120, 124, 128, 192] }
 
 minimaxm2.5-fp4-b200-vllm:
-  image: vllm/vllm-openai:v0.21.0
+  image: vllm/vllm-openai:v0.19.0-cu130
   model: nvidia/MiniMax-M2.5-NVFP4
   model-prefix: minimaxm2.5
   runner: b200

KLAUD_DEBUG.md

@@ -66,7 +66,7 @@ Seen on: #1460 (dsv4-fp8-h200-sglang+mtp).
 
 ## 4. Upstream sglang v0.5.12 B300 regressions
 
-Three distinct upstream regressions on NVIDIA B300 (Blackwell Ultra, `sm_103` — compute capability 10.3) shipped in `lmsysorg/sglang:v0.5.12-cu130`. (sm_120 is for *consumer* Blackwell / RTX 50 series, not B300 — don't propagate that.)
+Two distinct upstream regressions on NVIDIA B300 (Blackwell, `sm_120`) shipped in `lmsysorg/sglang:v0.5.12-cu130`:
 
 ### 4a. DeepGemm TMA-descriptor crash (GLM-5-FP8)
 **Symptom:** CUDA graph capture aborts with `CUDA_ERROR_ILLEGAL_ADDRESS (700)` at `/deepgemm/csrc/.../runtime_utils.hpp:143` on the **first batch size** for **every TP rank**. Server never serves a prompt.
@@ -86,17 +86,17 @@ Filed upstream: sgl-project/sglang#25551. Seen on #1421.
 2. Comment out the MTP/EAGLE scenarios on B300 in the recipe.
 3. Pin to v0.5.11-cu130.
 
-Filed upstream: sgl-project/sglang#25563. Seen on #1420.
+Seen on #1420.
 
 ### 4c. flash_attn SM-arch assertion (qwen3.5-bf16)
 **Symptom:** All 4 TP workers AssertionError on first forward pass:
 ```
 File "/opt/venv/.../sglang/srt/layers/attention/flashattention_backend.py:..."
   assert sm_100 <= arch <= sm_110f
 ```
-B300 is `sm_103` (compute capability 10.3, Blackwell Ultra) — which is *nominally inside* the asserted `sm_100..sm_110f` range, yet the assertion still fires. Best guess is the cute kernel's `Arch.sm_110f` set only matches the architecture-specific feature-flag variants it was compiled for (e.g. `sm_100`, `sm_100f`, `sm_110`, `sm_110f`) and `sm_103` / `sm_103a` isn't in that explicit list. Server never becomes healthy; warmup times out at 600s.
+B300 is `sm_120`, outside the asserted range. Server never becomes healthy; warmup times out at 600s.
 
-**Fix:** Needs an sglang image with `flash_attn` that recognises `sm_103` / `sm_103a` — no local workaround. Pin to `v0.5.11-cu130` in the meantime.
+**Fix:** Needs sglang image with flash_attn supporting `sm_120` — no local workaround. Pin to v0.5.11-cu130 in the meantime.
 
 Seen on #1422.
 

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml

@@ -11,7 +11,7 @@ dynamo:
 
 model:
   path: "deepseek-v4-pro"
-  container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e"
+  container: "lmsysorg/sglang:nightly-dev-cu13-20260518-c67b2870"
   precision: "mxfp4"
 
 sbatch_directives:
@@ -31,14 +31,12 @@ backend:
 
   prefill_environment:
     PYTHONUNBUFFERED: "1"
-    SGLANG_RADIX_DISABLE_REUSE: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
     SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
     SGLANG_DEFAULT_THINKING: "1"
     SGLANG_DSV4_REASONING_EFFORT: "max"
     SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
-    SGLANG_OPT_USE_JIT_NORM: "1"
-    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
-    SGLANG_OPT_USE_TOPK_V2: "1"
+    SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
     NCCL_MNNVL_ENABLE: "1"
     NCCL_CUMEM_ENABLE: "1"
     SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
@@ -49,14 +47,12 @@ backend:
 
   decode_environment:
     PYTHONUNBUFFERED: "1"
-    SGLANG_RADIX_DISABLE_REUSE: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
     SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
     SGLANG_DEFAULT_THINKING: "1"
     SGLANG_DSV4_REASONING_EFFORT: "max"
     SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
-    SGLANG_OPT_USE_JIT_NORM: "1"
-    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
-    SGLANG_OPT_USE_TOPK_V2: "1"
+    SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
     NCCL_MNNVL_ENABLE: "1"
     NCCL_CUMEM_ENABLE: "1"
     SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml

@@ -11,7 +11,7 @@ dynamo:
 
 model:
   path: "deepseek-v4-pro"
-  container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e"
+  container: "lmsysorg/sglang:nightly-dev-cu13-20260518-c67b2870"
   precision: "mxfp4"
 
 sbatch_directives:
@@ -31,23 +31,16 @@ backend:
 
   prefill_environment:
     PYTHONUNBUFFERED: "1"
-    SGLANG_RADIX_DISABLE_REUSE: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
     SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
     SGLANG_DEFAULT_THINKING: "1"
     SGLANG_DSV4_REASONING_EFFORT: "max"
     SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
-    SGLANG_OPT_USE_JIT_NORM: "1"
-    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
-    SGLANG_OPT_USE_TOPK_V2: "1"
-
     SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
-    SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
-    SGLANG_OPT_USE_FAST_MASK_EP: "1"
+
     SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
-    SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
     SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216"
     SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
-    SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
     SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
 
     NCCL_MNNVL_ENABLE: "1"
@@ -60,14 +53,11 @@ backend:
 
   decode_environment:
     PYTHONUNBUFFERED: "1"
-    SGLANG_RADIX_DISABLE_REUSE: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
     SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
     SGLANG_DEFAULT_THINKING: "1"
     SGLANG_DSV4_REASONING_EFFORT: "max"
     SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
-    SGLANG_OPT_USE_JIT_NORM: "1"
-    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
-    SGLANG_OPT_USE_TOPK_V2: "1"
     NCCL_MNNVL_ENABLE: "1"
     NCCL_CUMEM_ENABLE: "1"
     SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml

@@ -11,7 +11,7 @@ dynamo:
 
 model:
   path: "deepseek-v4-pro"
-  container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e"
+  container: "lmsysorg/sglang:nightly-dev-cu13-20260518-c67b2870"
   precision: "mxfp4"
 
 sbatch_directives:
@@ -33,23 +33,16 @@ backend:
 
   prefill_environment:
     PYTHONUNBUFFERED: "1"
-    SGLANG_RADIX_DISABLE_REUSE: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
     SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
     SGLANG_DEFAULT_THINKING: "1"
     SGLANG_DSV4_REASONING_EFFORT: "max"
     SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
-    SGLANG_OPT_USE_JIT_NORM: "1"
-    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
-    SGLANG_OPT_USE_TOPK_V2: "1"
-
     SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
-    SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
-    SGLANG_OPT_USE_FAST_MASK_EP: "1"
+
     SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
-    SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
     SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216"
     SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
-    SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
     SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
 
     NCCL_MNNVL_ENABLE: "1"
@@ -62,22 +55,16 @@ backend:
 
   decode_environment:
     PYTHONUNBUFFERED: "1"
-    SGLANG_RADIX_DISABLE_REUSE: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
     SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
     SGLANG_DEFAULT_THINKING: "1"
     SGLANG_DSV4_REASONING_EFFORT: "max"
     SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
-    SGLANG_OPT_USE_JIT_NORM: "1"
-    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
-    SGLANG_OPT_USE_TOPK_V2: "1"
-
     SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
-    SGLANG_OPT_USE_FAST_MASK_EP: "1"
+
     SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
-    SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
     SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048"
     SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
-    SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
     SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
 
     NCCL_MNNVL_ENABLE: "1"

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml

@@ -11,7 +11,7 @@ dynamo:
 
 model:
   path: "deepseek-v4-pro"
-  container: "lmsysorg/sglang:nightly-dev-cu13-20260510-2473659e"
+  container: "lmsysorg/sglang:nightly-dev-cu13-20260518-c67b2870"
   precision: "mxfp4"
 
 sbatch_directives:
@@ -33,23 +33,16 @@ backend:
 
   prefill_environment:
     PYTHONUNBUFFERED: "1"
-    SGLANG_RADIX_DISABLE_REUSE: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
     SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
     SGLANG_DEFAULT_THINKING: "1"
     SGLANG_DSV4_REASONING_EFFORT: "max"
     SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
-    SGLANG_OPT_USE_JIT_NORM: "1"
-    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
-    SGLANG_OPT_USE_TOPK_V2: "1"
-
     SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
-    SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
-    SGLANG_OPT_USE_FAST_MASK_EP: "1"
+
     SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
-    SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
     SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216"
     SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
-    SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
     SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
 
     NCCL_MNNVL_ENABLE: "1"
@@ -62,22 +55,16 @@ backend:
 
   decode_environment:
     PYTHONUNBUFFERED: "1"
-    SGLANG_RADIX_DISABLE_REUSE: "1"
+    SGLANG_RADIX_FORCE_MISS: "1"
     SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
     SGLANG_DEFAULT_THINKING: "1"
     SGLANG_DSV4_REASONING_EFFORT: "max"
     SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
-    SGLANG_OPT_USE_JIT_NORM: "1"
-    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
-    SGLANG_OPT_USE_TOPK_V2: "1"
-
     SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
-    SGLANG_OPT_USE_FAST_MASK_EP: "1"
+
     SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: "1"
-    SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
     SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048"
     SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
-    SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
     SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "0"
 
     NCCL_MNNVL_ENABLE: "1"