runners: add new mi300x SLURM cluster (#239)

* add new mi300x stuff

* nfs

* nfs 2

Author: cquil11

.github/configs/runners.yaml

@@ -32,6 +32,10 @@ mi300x:
 - 'mi300x-amd_4'
 - 'mi300x-cr_0'
 - 'mi300x-oci_0'
+- 'mi300x-amds_0'
+- 'mi300x-amds_1'
+- 'mi300x-amds_2'
+- 'mi300x-amds_3'
 mi325x:
 - 'mi325x-amd_0'
 - 'mi325x-amd_1'

.github/workflows/benchmark-tmpl.yml

@@ -133,9 +133,18 @@ jobs:
           RESULT_FILENAME: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}_ep${{ env.EP_SIZE }}_dpa_${{ env.DP_ATTENTION }}_conc${{ env.CONC }}_${{ runner.name }}
         run: |
           bash ./runners/launch_${RUNNER_NAME%%_*}.sh
-          if [ -f "$RESULT_FILENAME.json" ]; then
-            echo "RESULT_FILENAME=${RESULT_FILENAME}" >> $GITHUB_ENV
-          else
+          FOUND_RESULT_FILE=
+          for i in {1..10}; do
+            if [ -f "$RESULT_FILENAME.json" ]; then
+              echo "RESULT_FILENAME=${RESULT_FILENAME}" >> $GITHUB_ENV
+              FOUND_RESULT_FILE=true
+              break
+            fi
+            echo "Waiting for result file... (attempt $i)"
+            sleep 1
+          done
+   
+          if [ -z "$FOUND_RESULT_FILE" ]; then
             echo "Run failed: Benchmark result $RESULT_FILENAME.json not found." >&2
             exit 1
           fi

runners/launch_mi300x-amds.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+export HF_HUB_CACHE_MOUNT="/nfsdata/sa/hf_hub_cache-${USER: -1}/"
+export PORT_OFFSET=${USER: -1}
+
+PARTITION="compute"
+SQUASH_FILE="/nfsdata/sa/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
+
+set -x
+salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=256 --time=180 --no-shell
+JOB_ID=$(squeue -u $USER -h -o %A | head -n1)
+
+srun --jobid=$JOB_ID bash -c "sudo enroot import -o $SQUASH_FILE docker://$IMAGE"
+srun --jobid=$JOB_ID \
+--container-image=$SQUASH_FILE \
+--container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
+--container-mount-home \
+--container-writable \
+--container-remap-root \
+--container-workdir=/workspace/ \
+--no-container-entrypoint --export=ALL \
+bash benchmarks/${EXP_NAME%%_*}_${PRECISION}_mi300x_slurm.sh
+
+scancel $JOB_ID