Merge branch 'main' into diff-only-runs

Author: cquil11

.github/configs/nvidia-master.yaml

@@ -209,7 +209,7 @@ gptoss-fp4-b200-trt:
     - { tp: 8, conc-start: 4, conc-end: 8 }
 
 gptoss-fp4-b200-vllm:
-  image: vllm/vllm-openai:v0.11.0
+  image: vllm/vllm-openai:v0.11.2
   model: openai/gpt-oss-120b
   model-prefix: gptoss
   runner: b200
@@ -240,7 +240,7 @@ gptoss-fp4-b200-vllm:
     - { tp: 8, conc-start: 4, conc-end: 4 }
 
 gptoss-fp4-h100-vllm:
-  image: vllm/vllm-openai:v0.11.0
+  image: vllm/vllm-openai:v0.11.2
   model: openai/gpt-oss-120b
   model-prefix: gptoss
   runner: h100
@@ -300,7 +300,7 @@ gptoss-fp4-h200-trt:
     - { tp: 8, ep: 8, dp-attn: false, conc-start: 4, conc-end: 8 }
 
 gptoss-fp4-h200-vllm:
-  image: vllm/vllm-openai:v0.11.0
+  image: vllm/vllm-openai:v0.11.2
   model: openai/gpt-oss-120b
   model-prefix: gptoss
   runner: h200

.github/workflows/full-sweep-1k1k-scheduler.yml

@@ -2,6 +2,8 @@ name: "Full Sweep Scheduler - 1k1k"
 
 on:
     workflow_dispatch:
+    schedule:
+        - cron: "0 0 * * *"
 
 jobs:
     get-dsr1-configs:
@@ -49,6 +51,7 @@ jobs:
                 config: ${{ fromJson(needs.get-dsr1-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "dsr1_1k1k"
             isl: 1024
             osl: 1024
             max-model-len: 2248
@@ -58,7 +61,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_1k1k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}
@@ -114,6 +116,7 @@ jobs:
                 config: ${{ fromJson(needs.get-gptoss-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "gptoss_1k1k"
             isl: 1024
             osl: 1024
             max-model-len: 2248
@@ -123,7 +126,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_1k1k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}

.github/workflows/full-sweep-1k8k-scheduler.yml

@@ -2,6 +2,8 @@ name: "Full Sweep Scheduler - 1k8k"
 
 on:
     workflow_dispatch:
+    schedule:
+        - cron: "0 0 * * *"
 
 jobs:
     get-dsr1-configs:
@@ -49,6 +51,7 @@ jobs:
                 config: ${{ fromJson(needs.get-dsr1-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "dsr1_1k8k"
             isl: 1024
             osl: 8192
             max-model-len: 9416
@@ -58,7 +61,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_1k8k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}
@@ -114,6 +116,7 @@ jobs:
                 config: ${{ fromJson(needs.get-gptoss-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "gptoss_1k8k"
             isl: 1024
             osl: 8192
             max-model-len: 9416
@@ -123,7 +126,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_1k8k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}

.github/workflows/full-sweep-8k1k-scheduler.yml

@@ -2,6 +2,8 @@ name: "Full Sweep Scheduler - 8k1k"
 
 on:
     workflow_dispatch:
+    schedule:
+        - cron: "0 0 * * *"
 
 jobs:
     get-dsr1-configs:
@@ -49,6 +51,7 @@ jobs:
                 config: ${{ fromJson(needs.get-dsr1-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "dsr1_8k1k"
             isl: 8192
             osl: 1024
             max-model-len: 9416
@@ -58,7 +61,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_8k1k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}
@@ -114,6 +116,7 @@ jobs:
                 config: ${{ fromJson(needs.get-gptoss-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "gptoss_8k1k"
             isl: 8192
             osl: 1024
             max-model-len: 9416
@@ -123,7 +126,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_8k1k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}

benchmarks/gptoss_fp4_b200_docker.sh

@@ -29,16 +29,16 @@ else
 fi
 
 cat > config.yaml << EOF
-compilation-config: '{"pass_config":{"enable_fi_allreduce_fusion":true,"enable_attn_fusion":true,"enable_noop":true},"custom_ops":["+rms_norm"],"cudagraph_mode":"FULL_AND_PIECEWISE"}'
+kv-cache-dtype: fp8
+compilation-config: '{"pass_config":{"enable_fi_allreduce_fusion":true,"enable_noop":true}}'
 async-scheduling: true
 no-enable-prefix-caching: true
-cuda-graph-sizes: 2048
+max-cudagraph-capture-size: 2048
 max-num-batched-tokens: 8192
 max-model-len: $CALCULATED_MAX_MODEL_LEN
 EOF
 
 export TORCH_CUDA_ARCH_LIST="10.0"
-export VLLM_FLASHINFER_ALLREDUCE_FUSION_THRESHOLDS_MB='{"2":32,"4":32,"8":8}'
 export PYTHONNOUSERSITE=1
 export VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1
 
@@ -47,7 +47,7 @@ SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log)
 set -x
 vllm serve $MODEL --host 0.0.0.0 --port $PORT --config config.yaml \
 --gpu-memory-utilization 0.9 --tensor-parallel-size $TP --max-num-seqs 512 \
---disable-log-requests > $SERVER_LOG 2>&1 &
+> $SERVER_LOG 2>&1 &
 
 SERVER_PID=$!
 
@@ -69,4 +69,4 @@ run_benchmark_serving \
     --num-prompts "$NUM_PROMPTS" \
     --max-concurrency "$CONC" \
     --result-filename "$RESULT_FILENAME" \
-    --result-dir /workspace/
+    --result-dir /workspace/

benchmarks/gptoss_fp4_h100_docker.sh

@@ -12,10 +12,9 @@
 
 
 cat > config.yaml << EOF
-compilation-config: '{"cudagraph_mode":"PIECEWISE"}'
 async-scheduling: true
 no-enable-prefix-caching: true
-cuda-graph-sizes: 2048
+max-cudagraph-capture-size: 2048
 max-num-batched-tokens: 8192
 max-model-len: 10240
 EOF
@@ -29,7 +28,7 @@ vllm serve $MODEL --host=0.0.0.0 --port=$PORT \
 --gpu-memory-utilization=0.9 \
 --tensor-parallel-size=$TP \
 --max-num-seqs=$CONC  \
---disable-log-requests > $SERVER_LOG 2>&1 &
+> $SERVER_LOG 2>&1 &
 
 SERVER_PID=$!
 
@@ -51,4 +50,4 @@ run_benchmark_serving \
     --num-prompts $(( $CONC * 10 )) \
     --max-concurrency 512 \
     --result-filename "$RESULT_FILENAME" \
-    --result-dir /workspace/
+    --result-dir /workspace/

benchmarks/gptoss_fp4_h100_slurm.sh

@@ -13,10 +13,9 @@
 echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
 
 cat > config.yaml << EOF
-compilation-config: '{"cudagraph_mode":"PIECEWISE"}'
 async-scheduling: true
 no-enable-prefix-caching: true
-cuda-graph-sizes: 2048
+max-cudagraph-capture-size: 2048
 max-num-batched-tokens: 8192
 max-model-len: 10240
 EOF
@@ -30,7 +29,7 @@ PYTHONNOUSERSITE=1 vllm serve $MODEL --host=0.0.0.0 --port=$PORT \
 --gpu-memory-utilization=0.9 \
 --tensor-parallel-size=$TP \
 --max-num-seqs=$CONC  \
---disable-log-requests > $SERVER_LOG 2>&1 &
+ > $SERVER_LOG 2>&1 &
 
 SERVER_PID=$!
 

benchmarks/gptoss_fp4_h200_slurm.sh

@@ -27,10 +27,9 @@ fi
 
 # Create config.yaml
 cat > config.yaml << EOF
-compilation-config: '{"cudagraph_mode":"PIECEWISE"}'
 async-scheduling: true
 no-enable-prefix-caching: true
-cuda-graph-sizes: 2048
+max-cudagraph-capture-size: 2048
 max-num-batched-tokens: 8192
 max-model-len: $CALCULATED_MAX_MODEL_LEN
 EOF
@@ -42,7 +41,7 @@ export TORCH_CUDA_ARCH_LIST="9.0"
 
 PYTHONNOUSERSITE=1 vllm serve $MODEL --host 0.0.0.0 --port $PORT --config config.yaml \
  --gpu-memory-utilization 0.9 --tensor-parallel-size $TP --max-num-seqs $CONC  \
- --disable-log-requests > $SERVER_LOG 2>&1 &
+ > $SERVER_LOG 2>&1 &
 
 SERVER_PID=$!
 

runners/launch_b200-nb.sh

@@ -1,15 +1,20 @@
 #!/usr/bin/bash
 
-HF_HUB_CACHE_MOUNT="/root/hf_hub_cache-${USER: -1}/"
+HF_HUB_CACHE_MOUNT="/mnt/data/hf-hub-cache-${USER: -1}/"
 PARTITION="main"
 FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '')
 
+UCX_NET_DEVICES=eth0
+
+# Cleanup any stale enroot locks from previous runs
+find /var/cache/enroot-container-images/$UID -type f -name "*.lock" | xargs rm
+
 set -x
 srun --partition=$PARTITION --gres=gpu:$TP --exclusive \
 --container-image=$IMAGE \
 --container-name=$(echo "$IMAGE" | sed 's/[\/:@#]/_/g')-${USER: -1} \
 --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
 --no-container-mount-home --container-writable \
 --container-workdir=/workspace/ \
---no-container-entrypoint --export=ALL,PORT_OFFSET=${USER: -1} \
+--no-container-entrypoint --export=ALL,PORT_OFFSET=${USER: -1},UCX_NET_DEVICES=$UCX_NET_DEVICES \
 bash benchmarks/${EXP_NAME%%_*}_${PRECISION}_b200${FRAMEWORK_SUFFIX}_slurm.sh

runners/launch_h200-cw.sh

@@ -21,12 +21,14 @@ else
     CONTAINER_IMAGE=$(realpath $SQUASH_FILE)
 fi
 
+# The 'rm -rf /dev/shm/sagemaker_sessions' is to clean up shared memory used by sagemaker sessions inside the container
+# This seems to have been introduced in vLLM 0.11.2, but the issue is specific to CoreWeave runners.
 srun --jobid=$JOB_ID \
 --container-image=$CONTAINER_IMAGE \
 --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
 --container-mount-home \
 --container-workdir=/workspace/ \
 --no-container-entrypoint --export=ALL \
-bash benchmarks/${MODEL_CODE}_${PRECISION}_h200${FRAMEWORK_SUFFIX}_slurm.sh
+bash -c "bash benchmarks/${MODEL_CODE}_${PRECISION}_h200${FRAMEWORK_SUFFIX}_slurm.sh; rm -rf /dev/shm/sagemaker_sessions"
 
 scancel $JOB_ID