Merge branch 'main' into gptoss-trt-docker

Author: cquil11

.github/configs/nvidia-master.yaml

@@ -167,34 +167,38 @@ dsr1-fp8-h200-trt:
     - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 64 }
 
 gptoss-fp4-b200-trt:
-  image: nvcr.io#nvidia/tensorrt-llm/release:1.2.0rc0.post1
+  image: nvcr.io#nvidia/tensorrt-llm/release:1.2.0rc2
   model: openai/gpt-oss-120b
   model-prefix: gptoss
   runner: b200-trt
   precision: fp4
   framework: trt
-  # For all sequence lengths, if CONC >= 256, then EP=TP and DP_ATTN=true
+  # Enable DP_ATTENTION for conc >= 32
   seq-len-configs:
   - isl: 1024
     osl: 1024
     search-space:
+    - { tp: 2, dp-attn: true, conc-start: 32, conc-end: 128 }
+    - { tp: 4, dp-attn: true, conc-start: 32, conc-end: 64 }
     - { tp: 1, conc-start: 64, conc-end: 128 }
-    - { tp: 2, conc-start: 4, conc-end: 128 }
-    - { tp: 4, conc-start: 4, conc-end: 128 }
+    - { tp: 2, conc-start: 4, conc-end: 32 }
+    - { tp: 4, conc-start: 4, conc-end: 64 }
     - { tp: 8, conc-start: 4, conc-end: 8 }
   - isl: 1024
     osl: 8192
     search-space:
     - { tp: 1, conc-start: 64, conc-end: 128 }
+    - { tp: 2, dp-attn: true, conc-start: 64, conc-end: 128 }
     - { tp: 2, conc-start: 4, conc-end: 128 }
     - { tp: 4, conc-start: 4, conc-end: 128 }
     - { tp: 8, conc-start: 4, conc-end: 16 }
   - isl: 8192
     osl: 1024
     search-space:
     - { tp: 1, conc-start: 64, conc-end: 128 }
+    - { tp: 2, dp-attn: true, conc-start: 64, conc-end: 128 }
     - { tp: 2, conc-start: 4, conc-end: 128 }
-    - { tp: 4, conc-start: 4, conc-end: 128 }
+    - { tp: 4, conc-start: 4, conc-end: 32 }
     - { tp: 8, conc-start: 4, conc-end: 8 }
 
 gptoss-fp4-b200-vllm:

.github/configs/runners.yaml

@@ -24,6 +24,8 @@ b200:
 - 'b200-nvd_1'
 - 'b200-nvd_2'
 - 'b200-nvd_3'
+- 'b200-dgxc_1'
+- 'b200-dgxc_2'
 mi300x:
 - 'mi300x-amd_0'
 - 'mi300x-amd_1'

.github/workflows/label-validation.yml

@@ -6,17 +6,20 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  pull_request:
-    types: [labeled, synchronize]
+  pull_request_target:
+    types: labeled
 
 jobs:
   get-jobs:
     runs-on: ubuntu-latest
+    environment: fork-pr-validation
     outputs:
       search-space-config: ${{ steps.get-jobs.outputs.search-space-config }}
     steps:
       - name: Checkout code
         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - id: get-jobs
         shell: python
@@ -80,12 +83,15 @@ jobs:
 
   validate:
     needs: get-jobs
-    if: ${{ needs.get-jobs.outputs.search-space-config != '[]' }}
+    if: ${{ always() && needs.get-jobs.result == 'success' && needs.get-jobs.outputs.search-space-config != '[]' }}
     uses: ./.github/workflows/benchmark-tmpl.yml
     strategy:
       fail-fast: false
       matrix:
-        config: ${{ fromJson(needs.get-jobs.outputs.search-space-config) }}
+        # The '|| '[]'' fallback is required because GitHub evaluates the matrix expression
+        # before the 'if' condition. Without it, fromJson() throws a parse error when the
+        # output is empty/undefined (e.g., when get-jobs is pending approval or was skipped).
+        config: ${{ fromJson(needs.get-jobs.outputs.search-space-config || '[]') }}
     secrets: inherit
     name: validate ${{ matrix.config.runner }}
     with:
@@ -104,14 +110,14 @@ jobs:
       conc: ${{ matrix.config.conc }}
 
   collect-results:
-    needs: validate
-    if: ${{ always() }}
+    needs: [get-jobs, validate]
+    if: ${{ always() && needs.get-jobs.result == 'success' }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
 
   calc-success-rate:
-    needs: collect-results
-    if: ${{ always() }}
+    needs: [get-jobs, collect-results]
+    if: ${{ always() && needs.get-jobs.result == 'success' }}
     runs-on: ubuntu-latest
 
     env:

.github/workflows/test-matrix-logic.yml

@@ -20,7 +20,7 @@ jobs:
         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
 
       - name: Set up Python
-        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
+        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
         with:
           python-version: '3.12'
 

.github/workflows/test-process-result.yml

@@ -21,7 +21,7 @@ jobs:
         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
 
       - name: Set up Python
-        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
+        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
         with:
           python-version: '3.12'
 

benchmarks/gptoss_fp4_b200_trt_slurm.sh

@@ -31,7 +31,6 @@ echo "MOE_BACKEND set to '$MOE_BACKEND'"
 
 EXTRA_CONFIG_FILE="gptoss-fp4.yml"
 export TRTLLM_ENABLE_PDL=1
-export NCCL_GRAPH_REGISTER=0
 
 cat > $EXTRA_CONFIG_FILE << EOF
 cuda_graph_config:
@@ -50,6 +49,9 @@ moe_config:
 EOF
 
 if [[ "$DP_ATTENTION" == "true" ]]; then
+    export TRTLLM_MOE_ALLTOALL_BACKEND="mnnvlthroughput"
+    export TRTLLM_FORCE_ALLTOALL_METHOD="MNNVL"
+    export TRTLLM_MOE_A2A_WORKSPACE_MB="2048"
     cat << EOF >> $EXTRA_CONFIG_FILE
 attention_dp_config:
     enable_balance: true

runners/launch_b200-dgxc.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/bash
+
+HF_HUB_CACHE_MOUNT="/raid/hf_hub_cache/"
+FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '')
+PORT=8888
+
+# Create unique cache directory based on model parameters
+MODEL_NAME=$(basename "$MODEL")
+
+server_name="bmk-server"
+
+nvidia-smi
+
+# GPUs must be idle
+if nvidia-smi --query-compute-apps=pid --format=csv,noheader | grep -q '[0-9]'; then
+  echo "[ERROR] GPU busy from previous run"; nvidia-smi; exit 1
+fi
+
+set -x
+# Use --init flag to run an init process (PID 1) inside container for better signal handling and zombie process cleanup
+# Ref: https://www.paolomainardi.com/posts/docker-run-init/
+
+# NCCL_GRAPH_REGISTER tries to automatically enable user buffer registration with CUDA Graphs. 
+# Disabling it can reduce perf but will improve CI stability. i.e. we won't see vLLM/Sglang crashes.
+# Ref: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html#nccl-graph-register
+
+if [[ "$MODEL" == "nvidia/DeepSeek-R1-0528-FP4" || "$MODEL" == "deepseek-ai/DeepSeek-R1-0528" ]]; then
+  if [[ "$OSL" == "8192" ]]; then
+    export NUM_PROMPTS=$(( CONC * 20 ))
+  else
+    export NUM_PROMPTS=$(( CONC * 50 ))
+  fi
+else
+  export NUM_PROMPTS=$(( CONC * 10 ))
+fi
+
+docker run --rm --init --network host --name $server_name \
+--runtime nvidia --gpus all --ipc host --privileged --shm-size=16g --ulimit memlock=-1 --ulimit stack=67108864 \
+-v $HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
+-v $GITHUB_WORKSPACE:/workspace/ -w /workspace/ \
+-e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e PORT=$PORT -e EP_SIZE -e DP_ATTENTION \
+-e NCCL_GRAPH_REGISTER=0 \
+-e TORCH_CUDA_ARCH_LIST="10.0" -e CUDA_DEVICE_ORDER=PCI_BUS_ID -e CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" \
+-e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e NUM_PROMPTS \
+--entrypoint=/bin/bash \
+$(echo "$IMAGE" | sed 's/#/\//') \
+benchmarks/"${EXP_NAME%%_*}_${PRECISION}_b200${FRAMEWORK_SUFFIX}_docker.sh"
+
+# Try graceful first
+docker stop -t 90 "$server_name" || true
+# Wait until it's really dead
+docker wait "$server_name" >/dev/null 2>&1 || true
+# Force remove if anything lingers
+docker rm -f "$server_name" >/dev/null 2>&1 || true
+
+# Give a moment for GPU processes to fully terminate
+sleep 2
+# Verify GPUs are now idle; if not, print diag and (optionally) reset
+if nvidia-smi --query-compute-apps=pid --format=csv,noheader | grep -q '[0-9]'; then
+  echo "[WARN] After stop, GPU still busy:"; nvidia-smi
+  # Last resort if driver allows and GPUs appear idle otherwise:
+  #nvidia-smi --gpu-reset -i 0,1,2,3,4,5,6,7 2>/dev/null || true
+fi
+
+nvidia-smi

runners/launch_gb200-nv.sh

@@ -8,9 +8,17 @@ export SLURM_PARTITION="batch"
 export SLURM_ACCOUNT="benchmark"
 export SLURM_JOB_NAME="benchmark-dynamo.job"
 
+# For SGLang - we are working on updating the 8k1k configs 
+# For now we add conditionals to this script to use newer code for the 1k1k configs
+
 ### FRAMEWORK_DIFF_IF_STATEMENT #1 - difference in setting up envvars
 if [[ $FRAMEWORK == "dynamo-sglang" ]]; then
-    export IMAGE="/mnt/lustre01/artifacts/containers/dynamo-sglang.sqsh"
+    # Set IMAGE based on ISL/OSL
+    if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then
+        export IMAGE="/mnt/lustre01/artifacts/containers/lmsysorg+sglang+v0.5.5.post2.sqsh"
+    else
+        export IMAGE="/mnt/lustre01/artifacts/containers/dynamo-sglang.sqsh"
+    fi
     export MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528"
     export CONFIG_DIR="/mnt/lustre01/artifacts/sglang-configs/1k1k"
 else
@@ -157,13 +165,24 @@ if [[ $FRAMEWORK == "dynamo-trtllm" ]]; then
 
 else # if statement at the top - search for "FRAMEWORK_DIFF_IF_STATEMENT #2"
     # Set up Dynamo repository path
+    set -x
     DYNAMO_PATH="/mnt/lustre01/users/sa-shared/benchmarks/dynamo"
-    SGL_SLURM_JOBS_PATH="$DYNAMO_PATH/components/backends/sglang/slurm_jobs"
+    if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then
+        SGL_SLURM_JOBS_PATH="$DYNAMO_PATH/examples/backends/sglang/slurm_jobs"
+    else
+        SGL_SLURM_JOBS_PATH="$DYNAMO_PATH/components/backends/sglang/slurm_jobs"
+    fi
 
     # Always clone and setup Dynamo
     echo "Cloning Dynamo repository..."
     rm -rf "$DYNAMO_PATH"
-    git clone --branch update-result-file-name https://github.com/Elnifio/dynamo.git $DYNAMO_PATH
+    if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then
+        # TODO: before merge this will be a different branch off of main
+        git clone --branch ishan/sa-1.1-sgl-dsr1-fp8 https://github.com/ai-dynamo/dynamo.git $DYNAMO_PATH
+    else
+        git clone --branch update-result-file-name https://github.com/Elnifio/dynamo.git $DYNAMO_PATH
+    fi
+
     cd "$DYNAMO_PATH"
 
     # Navigate to corresponding directory
@@ -179,15 +198,32 @@ else # if statement at the top - search for "FRAMEWORK_DIFF_IF_STATEMENT #2"
 
     # Launch jobs based on ISL/OSL
     if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then
-        concurrency_list="1024x2048x4096x4608x4864x4992x5120x5376x5632x6144x8192"
-        bash ./submit_disagg.sh 6 3 12 1 8 $ISL $OSL $concurrency_list inf
+        NUMBER_OF_EXPERIMENTS=3
+
+        top_of_curve_concurrency_list="4096"
+	    middle_of_curve_concurrency_list="1024x2048x4096"
+        bottom_of_curve_concurrency_list="2x4x8x16x64x128"
+
+        # Top of curve (2 prefill workers each at DEP8 and 1 decode worker at DEP32)
+        bash ./submit_disagg.sh 4 2 8 1 9 $ISL $OSL $top_of_curve_concurrency_list inf
+
+        # Bottom of curve (1 prefill worker at DEP4 and 4 decode workers at DEP4)
+        bash ./submit_disagg.sh 1 1 4 4 9 $ISL $OSL $bottom_of_curve_concurrency_list inf 1p_4d
+
+        # Middle of curve (3 prefill workers each at DEP8 and 1 decode worker at DEP48)
+        bash ./submit_disagg.sh 6 3 12 1 9 $ISL $OSL $middle_of_curve_concurrency_list inf
+
     elif [ "$ISL" = "8192" ] && [ "$OSL" = "1024" ]; then
+        NUMBER_OF_EXPERIMENTS=1
+
         concurrency_list="128x256x384x448x512x576x1024x2048x4096"
         bash ./submit_disagg.sh 12 6 6 1 8 $ISL $OSL $concurrency_list inf
     else
         echo "Unsupported ISL/OSL combination: $ISL/$OSL"
         exit 1
     fi
+
+    set +x
 fi
 
 # Wait for all jobs to complete
@@ -259,9 +295,14 @@ if [[ $FRAMEWORK == "dynamo-trtllm" ]]; then
     done
 
 else # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement
-    # Find the latest log directory
-    # we do "tail -1" here since only the latest job will yield the result
-    LOGS_DIR=$(find logs/*/vllm_isl_${ISL}_osl_${OSL} -type d | sort -V | tail -1)
+    # Find the latest log directory that contains the data
+    cat > collect_latest_results.py <<'PY'
+import os, sys
+isl, osl, nexp = [int(x) for x in sys.argv[1:]]
+for path in sorted([f"logs/{name}/vllm_isl_{isl}_osl_{osl}" for name in os.listdir("logs/") if os.path.isdir(f"logs/{name}/vllm_isl_{isl}_osl_{osl}")], key=os.path.getmtime, reverse=True)[:nexp]:
+    print(path)
+PY
+    LOGS_DIR=$(python3 collect_latest_results.py $ISL $OSL $NUMBER_OF_EXPERIMENTS)
     if [ -z "$LOGS_DIR" ]; then
         echo "No logs directory found for ISL=${ISL}, OSL=${OSL}"
         exit 1