fix: various follow-up bug fixes (#302)

* various bug fixes

* various bug fixes pt 2

* various bug fixes pt 3

* various bug fixes pt 4 -- use bash expansion in cw

Author: cquil11

.github/workflows/full-sweep-1k1k-scheduler.yml

@@ -51,6 +51,7 @@ jobs:
                 config: ${{ fromJson(needs.get-dsr1-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "dsr1_1k1k"
             isl: 1024
             osl: 1024
             max-model-len: 2248
@@ -60,7 +61,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_1k1k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}
@@ -116,6 +116,7 @@ jobs:
                 config: ${{ fromJson(needs.get-gptoss-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "gptoss_1k1k"
             isl: 1024
             osl: 1024
             max-model-len: 2248
@@ -125,7 +126,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_1k1k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}

.github/workflows/full-sweep-1k8k-scheduler.yml

@@ -51,6 +51,7 @@ jobs:
                 config: ${{ fromJson(needs.get-dsr1-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "dsr1_1k8k"
             isl: 1024
             osl: 8192
             max-model-len: 9416
@@ -60,7 +61,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_1k8k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}
@@ -116,6 +116,7 @@ jobs:
                 config: ${{ fromJson(needs.get-gptoss-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "gptoss_1k8k"
             isl: 1024
             osl: 8192
             max-model-len: 9416
@@ -125,7 +126,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_1k8k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}

.github/workflows/full-sweep-8k1k-scheduler.yml

@@ -51,6 +51,7 @@ jobs:
                 config: ${{ fromJson(needs.get-dsr1-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "dsr1_8k1k"
             isl: 8192
             osl: 1024
             max-model-len: 9416
@@ -60,7 +61,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_8k1k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}
@@ -116,6 +116,7 @@ jobs:
                 config: ${{ fromJson(needs.get-gptoss-configs.outputs.multi-node-search-space-config) }}
         secrets: inherit
         with:
+            exp-name: "gptoss_8k1k"
             isl: 8192
             osl: 1024
             max-model-len: 9416
@@ -125,7 +126,6 @@ jobs:
             model-prefix: ${{ matrix.config.model-prefix }}
             framework: ${{ matrix.config.framework }}
             precision: ${{ matrix.config.precision }}
-            exp-name: "dsr1_8k1k"
             conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}

runners/launch_b200-nb.sh

@@ -4,7 +4,7 @@ HF_HUB_CACHE_MOUNT="/mnt/data/hf-hub-cache-${USER: -1}/"
 PARTITION="main"
 FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '')
 
-UCX_NET_DEVICES=mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1,mlx5_8:1,mlx5_9:1,mlx5_10:1,mlx5_11:1
+UCX_NET_DEVICES=eth0
 
 # Cleanup any stale enroot locks from previous runs
 find /var/cache/enroot-container-images/$UID -type f -name "*.lock" | xargs rm

runners/launch_h200-cw.sh

@@ -21,12 +21,14 @@ else
     CONTAINER_IMAGE=$(realpath $SQUASH_FILE)
 fi
 
+# The 'rm -rf /dev/shm/sagemaker_sessions' is to clean up shared memory used by sagemaker sessions inside the container
+# This seems to have been introduced in vLLM 0.11.2, but the issue is specific to CoreWeave runners.
 srun --jobid=$JOB_ID \
 --container-image=$CONTAINER_IMAGE \
 --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
 --container-mount-home \
 --container-workdir=/workspace/ \
 --no-container-entrypoint --export=ALL \
-bash -c 'bash benchmarks/'"${EXP_NAME%%_*}_${PRECISION}"'_h200_slurm.sh; rm -rf /dev/shm/sagemaker_sessions'
+bash -c "bash benchmarks/${MODEL_CODE}_${PRECISION}_h200${FRAMEWORK_SUFFIX}_slurm.sh; rm -rf /dev/shm/sagemaker_sessions"
 
 scancel $JOB_ID