cleanup

Author: cquil11

benchmarks/dsr1_fp8_h200_trt_slurm.sh

@@ -71,17 +71,19 @@ PYTHONNOUSERSITE=1 mpirun -n 1 --oversubscribe --allow-run-as-root \
     > $SERVER_LOG 2>&1 &
 
 
+# Show logs until server is ready
+tail -f $SERVER_LOG &
+TAIL_PID=$!
 set +x
-while IFS= read -r line; do
-    printf '%s\n' "$line"
-    if [[ "$line" == *"Application startup complete"* ]]; then
-        break
-    fi
-done < <(tail -F -n0 "$SERVER_LOG")
+until curl --output /dev/null --silent --fail http://0.0.0.0:$PORT/health; do
+    sleep 5
+done
+kill $TAIL_PID
 
-git clone https://github.com/kimbochen/bench_serving.git
 set -x
-python3 bench_serving/benchmark_serving.py \
+BENCH_SERVING_DIR=$(mktemp -d /tmp/bmk-XXXXXX)
+git clone https://github.com/kimbochen/bench_serving.git $BENCH_SERVING_DIR
+python3 $BENCH_SERVING_DIR/benchmark_serving.py \
 --model $MODEL --backend openai \
 --base-url http://0.0.0.0:$PORT \
 --dataset-name random \

benchmarks/gptoss_fp4_h100_docker.sh

@@ -22,26 +22,29 @@ max-model-len: 10240
 EOF
 
 export PYTHONNOUSERSITE=1
+SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log)
 
 set -x
 vllm serve $MODEL --host=0.0.0.0 --port=$PORT \
 --config config.yaml \
 --gpu-memory-utilization=0.9 \
 --tensor-parallel-size=$TP \
 --max-num-seqs=$CONC  \
---disable-log-requests 2>&1 | tee $(mktemp /tmp/server-XXXXXX.log) &
+--disable-log-requests > $SERVER_LOG 2>&1 &
 
-# Show server logs til' it is up, then stop showing
+# Show logs until server is ready
+tail -f $SERVER_LOG &
+TAIL_PID=$!
 set +x
-until curl --output /dev/null --silent --fail http://localhost:$PORT/health; do
+until curl --output /dev/null --silent --fail http://0.0.0.0:$PORT/health; do
     sleep 5
 done
-pkill -P $$ tee 2>/dev/null
+kill $TAIL_PID
 
 pip install -q datasets pandas
+set -x
 BENCH_SERVING_DIR=$(mktemp -d /tmp/bmk-XXXXXX)
 git clone https://github.com/kimbochen/bench_serving.git $BENCH_SERVING_DIR
-set -x
 python3 $BENCH_SERVING_DIR/benchmark_serving.py \
 --model=$MODEL \
 --backend=vllm \

benchmarks/gptoss_fp4_h100_slurm.sh

@@ -35,18 +35,20 @@ PYTHONNOUSERSITE=1 vllm serve $MODEL --host=0.0.0.0 --port=$PORT \
 --max-num-seqs=$CONC  \
 --disable-log-requests > $SERVER_LOG 2>&1 &
 
+# Show logs until server is ready
+tail -f $SERVER_LOG &
+TAIL_PID=$!
 set +x
-while IFS= read -r line; do
-    printf '%s\n' "$line"
-    if [[ "$line" == *"Application startup complete"* ]]; then
-        break
-    fi
-done < <(tail -F -n0 "$SERVER_LOG")
+until curl --output /dev/null --silent --fail http://0.0.0.0:$PORT/health; do
+    sleep 5
+done
+kill $TAIL_PID
 
 pip install -q datasets pandas
-git clone https://github.com/kimbochen/bench_serving.git
 set -x
-python3 bench_serving/benchmark_serving.py \
+BENCH_SERVING_DIR=$(mktemp -d /tmp/bmk-XXXXXX)
+git clone https://github.com/kimbochen/bench_serving.git $BENCH_SERVING_DIR
+python3 $BENCH_SERVING_DIR/benchmark_serving.py \
 --model=$MODEL \
 --backend=vllm \
 --base-url="http://0.0.0.0:$PORT" \

benchmarks/gptoss_fp4_h200_trt_slurm.sh

@@ -19,7 +19,7 @@
 echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
 
 hf download $MODEL
-# SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log)
+SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log)
 PORT=$(( 8888 + $PORT_OFFSET ))
 
 
@@ -44,9 +44,6 @@ print_iter_log: true
 stream_interval: 20 
 EOF
 
-SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log)
-
-#mpirun -n 1 --oversubscribe --allow-run-as-root trtllm-serve $MODEL --tp_size $TP --trust_remote_code --max_seq_len $MAX_MODEL_LEN --max_num_tokens $MAX_MODEL_LEN --num_postprocess_workers 2 --extra_llm_api_options llama-config.yml --port $PORT > $SERVER_LOG 2>&1 &
 mpirun -n 1 --oversubscribe --allow-run-as-root \
 trtllm-serve $MODEL \
 --max_batch_size $CONC \
@@ -65,8 +62,6 @@ trtllm-serve $MODEL \
 # Show logs until server is ready
 tail -f $SERVER_LOG &
 TAIL_PID=$!
-
-# Show server logs til' it is up, then stop showing
 set +x
 until curl --output /dev/null --silent --fail http://0.0.0.0:$PORT/health; do
     sleep 5

benchmarks/gptoss_fp4_mi355x_slurm.sh

@@ -38,17 +38,19 @@ vllm serve $MODEL --port $PORT \
 --disable-log-requests \
 --async-scheduling > $SERVER_LOG 2>&1 &
 
+# Show logs until server is ready
+tail -f $SERVER_LOG &
+TAIL_PID=$!
 set +x
-while IFS= read -r line; do
-    printf '%s\n' "$line"
-    if [[ "$line" == *"Application startup complete"* ]]; then
-        break
-    fi
-done < <(tail -F -n0 "$SERVER_LOG")
+until curl --output /dev/null --silent --fail http://0.0.0.0:$PORT/health; do
+    sleep 5
+done
+kill $TAIL_PID
 
 set -x
-git clone https://github.com/kimbochen/bench_serving.git
-python3 bench_serving/benchmark_serving.py \
+BENCH_SERVING_DIR=$(mktemp -d /tmp/bmk-XXXXXX)
+git clone https://github.com/kimbochen/bench_serving.git $BENCH_SERVING_DIR
+python3 $BENCH_SERVING_DIR/benchmark_serving.py \
 --model $MODEL --backend vllm \
 --base-url "http://0.0.0.0:$PORT" \
 --dataset-name random \