adding prop of failure for server logs

Author: cquil11

benchmarks/dsr1_fp4_mi355x_slurm.sh

@@ -34,6 +34,15 @@ python3 -m sglang.launch_server --model-path=$MODEL --trust-remote-code \
 --cuda-graph-max-bs=128 \
 > $SERVER_LOG 2>&1 &
 
+# Show logs until server is ready
+tail -f $SERVER_LOG &
+TAIL_PID=$!
+set +x
+until curl --output /dev/null --silent --fail http://0.0.0.0:$PORT/health; do
+    sleep 5
+done
+kill $TAIL_PID
+
 set -x
 BENCH_SERVING_DIR=$(mktemp -d /tmp/bmk-XXXXXX)
 git clone https://github.com/kimbochen/bench_serving.git $BENCH_SERVING_DIR

benchmarks/dsr1_fp8_h200_trt_slurm.sh

@@ -69,9 +69,9 @@ PYTHONNOUSERSITE=1 mpirun -n 1 --oversubscribe --allow-run-as-root \
     --tp_size=$TP --ep_size=$EP_SIZE \
     --extra_llm_api_options=$EXTRA_CONFIG_FILE \
     > $SERVER_LOG 2>&1 &
+    
 SERVER_PID=$!
 
-
 # Show logs until server is ready
 tail -f $SERVER_LOG &
 TAIL_PID=$!

benchmarks/gptoss_fp4_h100_docker.sh

@@ -32,11 +32,17 @@ vllm serve $MODEL --host=0.0.0.0 --port=$PORT \
 --max-num-seqs=$CONC  \
 --disable-log-requests > $SERVER_LOG 2>&1 &
 
+SERVER_PID=$!
+
 # Show logs until server is ready
 tail -f $SERVER_LOG &
 TAIL_PID=$!
 set +x
 until curl --output /dev/null --silent --fail http://0.0.0.0:$PORT/health; do
+    if ! kill -0 $SERVER_PID 2>/dev/null; then
+        echo "Server died before becoming healthy. Exiting."
+        exit 1
+    fi
     sleep 5
 done
 kill $TAIL_PID

runners/launch_b200-nb.sh

@@ -13,3 +13,5 @@ srun --partition=$PARTITION --gres=gpu:$TP --exclusive \
 --container-workdir=/workspace/ \
 --no-container-entrypoint --export=ALL,PORT_OFFSET=${USER: -1} \
 bash benchmarks/${EXP_NAME%%_*}_${PRECISION}_b200${FRAMEWORK_SUFFIX}_slurm.sh
+
+scancel $JOB_ID