Skip to content

Commit 71eb59d

Browse files
ipanfiloipanfilo
authored
CI test progress reporting (#612)
* Log sGPU tests progress * JAX CI: use persistent compilation cache * Increase test timeout * Add logging of runner devices
1 parent 72d01a0 commit 71eb59d

7 files changed

Lines changed: 272 additions & 115 deletions

File tree

‎.github/scripts/ci_sgpu_jobs.conf‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
2+
#
3+
# See LICENSE for license information.
4+
#
5+
# sGPU job definitions for CI test suites.
6+
# Format: <label> <logfile> <command> [args...]
7+
# GPUs are assigned sequentially by run_parallel_sgpu.sh starting from --first-gpu.
8+
9+
examples examples.log .github/scripts/run_examples.sh
10+
torch torch.log ci/pytorch.sh
11+
jax jax.log ci/jax.sh
12+
core core.log ci/core.sh

‎.github/scripts/run_examples.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/usr/bin/bash
2+
# Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
3+
#
4+
# See LICENSE for license information.
5+
#
6+
# Run TE examples on a single GPU.
7+
# HIP_VISIBLE_DEVICES must be set by the caller (run_parallel_sgpu.sh).
8+
9+
set -e
10+
11+
# Autodetect repo root from this script's location (.github/scripts/ -> ../..)
12+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
13+
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
14+
15+
python -c "import os; print('HF_TOKEN set:', bool(os.environ.get('HF_TOKEN')))"
16+
17+
JAX_CONSTRAINTS=/tmp/jax-constraints.txt
18+
pip freeze | grep -iE '^(jax|jaxlib|jax[_-]rocm|jax[_-]plugins)[=@]' > "$JAX_CONSTRAINTS" || true
19+
20+
cd "${REPO_ROOT}/examples/pytorch/mnist"
21+
python main.py
22+
python main.py --use-te
23+
python main.py --use-fp8
24+
25+
cd "${REPO_ROOT}/examples/jax/mnist"
26+
pip3 install -c "$JAX_CONSTRAINTS" -r requirements.txt
27+
python test_single_gpu_mnist.py
28+
python test_single_gpu_mnist.py --use-te
29+
python test_single_gpu_mnist.py --use-fp8
30+
31+
cd "${REPO_ROOT}/examples/jax/encoder"
32+
pip3 install -c "$JAX_CONSTRAINTS" -r requirements.txt
33+
python test_single_gpu_encoder.py
34+
python test_single_gpu_encoder.py --use-fp8

‎.github/scripts/run_parallel_sgpu.sh‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
#!/bin/bash
2+
# Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
3+
#
4+
# See LICENSE for license information.
5+
#
6+
# Run multiple sGPU jobs in parallel from one or more config files.
7+
#
8+
# Usage: run_parallel_sgpu.sh [-g|--first-gpu <n>] [-l|--log-dir <dir>] <config>...
9+
#
10+
# Config format (one job per line; # comments and blank lines are ignored):
11+
# <label> <logfile> <command> [args...]
12+
#
13+
# GPUs are assigned sequentially starting from --first-gpu across all config
14+
# entries in the order configs are passed on the command line.
15+
#
16+
# Each job's exit code is written to <log-dir>/<logfile>.rc by the job runner.
17+
18+
# Resolve repo root relative to this script's location (.github/scripts/ -> ../..)
19+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
20+
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
21+
22+
# How often (seconds) to poll child processes
23+
: "${POLL_INTERVAL:=5}"
24+
# Warn if a log file has not been updated in this many seconds
25+
: "${STALL_WARN_SECS:=180}"
26+
# Number of tail lines to show for stalled logs
27+
: "${STALL_TAIL_LINES:=3}"
28+
# Number of new lines to show for resumed logs
29+
: "${STALL_RESUME_CONTEXT_LINES:=2}"
30+
31+
# Associative arrays: _JOB_PIDS[name]=pid _JOB_LOGS[pid]=logfile
32+
declare -A _JOB_PIDS
33+
declare -A _JOB_LOGS
34+
_OVERALL_RC=0
35+
36+
# Launch a background job and register it.
37+
# Usage: launch_job <name> <logfile> <cmd> [args...]
38+
launch_job() {
39+
local name="$1"
40+
local logfile="$2"
41+
local rcfile="${logfile}.rc"
42+
shift 2
43+
rm -f "$rcfile"
44+
"$@" >"$logfile" 2>&1 &
45+
local pid=$!
46+
_JOB_PIDS["$name"]=$pid
47+
_JOB_LOGS[$pid]="$logfile"
48+
echo "Started '${name}' (pid ${pid}) -> ${logfile}"
49+
}
50+
51+
# Wait for all currently registered jobs, polling every POLL_INTERVAL seconds.
52+
# Writes <logfile>.rc for every finished job; warns about stalled logs.
53+
# Clears _JOB_PIDS/_JOB_LOGS when done.
54+
wait_for_jobs() {
55+
local -A remaining
56+
local -A stall_mtime
57+
local -A stall_lineno
58+
for name in "${!_JOB_PIDS[@]}"; do
59+
remaining["$name"]="${_JOB_PIDS[$name]}"
60+
done
61+
62+
while [ ${#remaining[@]} -gt 0 ]; do
63+
sleep "$POLL_INTERVAL"
64+
65+
for name in "${!remaining[@]}"; do
66+
local pid="${remaining[$name]}"
67+
68+
if ! kill -0 "$pid" 2>/dev/null; then
69+
# Process has exited — capture its return code
70+
wait "$pid"
71+
local rc=$?
72+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] '${name}' (pid ${pid}) finished with rc=${rc}"
73+
if [ $rc -ne 0 ]; then
74+
_OVERALL_RC=$rc
75+
fi
76+
echo "$rc" > "${_JOB_LOGS[$pid]}.rc"
77+
unset "remaining[$name]"
78+
else
79+
# Process still running — check for log staleness
80+
local logfile="${_JOB_LOGS[$pid]}"
81+
if [ -f "$logfile" ]; then
82+
local now mtime age
83+
now=$(date +%s)
84+
mtime=$(stat -c '%Y' "$logfile" 2>/dev/null || echo "$now")
85+
age=$(( now - mtime ))
86+
if [ -n "${stall_mtime[$pid]+set}" ]; then
87+
if [ "$mtime" -gt "${stall_mtime[$pid]}" ]; then
88+
local frozen_secs=$(( mtime - stall_mtime[$pid] ))
89+
local freeze_line="${stall_lineno[$pid]}"
90+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: '${name}' (pid ${pid}) log '${logfile}' resumed updating after ${frozen_secs}s"
91+
echo "--- first ${STALL_RESUME_CONTEXT_LINES} lines of ${logfile} starting ${freeze_line} ---"
92+
tail -n "+${freeze_line}" "$logfile" | head -n ${STALL_RESUME_CONTEXT_LINES}
93+
echo "---"
94+
unset "stall_mtime[$pid]"
95+
unset "stall_lineno[$pid]"
96+
fi
97+
# else: still stalled but already warned — do nothing
98+
elif [ "$age" -ge "$STALL_WARN_SECS" ]; then
99+
# don't use wc here because it does not count the last line if it doesn't end with a newline
100+
local freeze_line=$(grep -c '' < "$logfile")
101+
stall_mtime[$pid]=$mtime
102+
stall_lineno[$pid]=$freeze_line
103+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING: '${name}' (pid ${pid}) log '${logfile}' has not been updated for ${age}s"
104+
echo "--- last ${STALL_TAIL_LINES} lines of ${logfile} up to ${freeze_line} ---"
105+
head -n "${freeze_line}" "$logfile" | tail -n "$STALL_TAIL_LINES"
106+
echo "--- end of ${logfile} ---"
107+
fi
108+
fi
109+
fi
110+
done
111+
done
112+
113+
# Reset for next batch
114+
unset _JOB_PIDS
115+
unset _JOB_LOGS
116+
declare -gA _JOB_PIDS
117+
declare -gA _JOB_LOGS
118+
}
119+
120+
FIRST_GPU=${TEST_FIRST_GPU:-0}
121+
LOG_DIR=${LOG_DIR:-/tmp/te_ci_logs}
122+
123+
# ---------------------------------------------------------------------------
124+
# Parse arguments
125+
while [[ $# -gt 0 ]]; do
126+
case "$1" in
127+
-g|--first-gpu)
128+
FIRST_GPU="$2"; shift 2 ;;
129+
--first-gpu=*)
130+
FIRST_GPU="${1#*=}"; shift ;;
131+
-l|--log-dir)
132+
LOG_DIR="$2"; shift 2 ;;
133+
--log-dir=*)
134+
LOG_DIR="${1#*=}"; shift ;;
135+
-*)
136+
echo "Unknown option: $1" >&2
137+
echo "Usage: $0 [-g|--first-gpu <n>] [-l|--log-dir <dir>] <config>..." >&2
138+
exit 1 ;;
139+
*)
140+
break ;;
141+
esac
142+
done
143+
144+
if [[ $# -eq 0 ]]; then
145+
echo "Error: at least one config file is required." >&2
146+
echo "Usage: $0 [-g|--first-gpu <n>] [-l|--log-dir <dir>] <config>..." >&2
147+
exit 1
148+
fi
149+
150+
# Resolve config paths and LOG_DIR to absolute against the caller's CWD
151+
# *before* changing directory, so relative paths passed by the caller remain valid.
152+
resolved_configs=()
153+
for c in "$@"; do
154+
resolved_configs+=( "$(realpath -m "$c")" )
155+
done
156+
[[ "$LOG_DIR" != /* ]] && LOG_DIR="$(realpath -m "$LOG_DIR")"
157+
158+
mkdir -p "$LOG_DIR"
159+
160+
# cd to repo root so that commands in configs (e.g. ci/pytorch.sh) resolve correctly
161+
cd "$REPO_ROOT" || { echo "Error: cannot cd to '${REPO_ROOT}'" >&2; exit 1; }
162+
163+
# ---------------------------------------------------------------------------
164+
# Launch all jobs, assigning GPUs sequentially across all config files
165+
gpu=$FIRST_GPU
166+
for config in "${resolved_configs[@]}"; do
167+
while IFS= read -r line || [[ -n "$line" ]]; do
168+
# Skip blank lines and comments
169+
[[ "$line" =~ ^[[:space:]]*# ]] && continue
170+
[[ -z "${line//[[:space:]]/}" ]] && continue
171+
read -r label logfile rest <<< "$line"
172+
# Each suite appends its own subdir to the inherited base prefix so the base path is defined once.
173+
if [ -n "${JUNITXML_PREFIX}${JUNITXML_SUFFIX}" ]; then
174+
junitxml_dir="${JUNITXML_PREFIX}${label}/"
175+
mkdir -p "${junitxml_dir}"
176+
else
177+
junitxml_dir=""
178+
fi
179+
# shellcheck disable=SC2086 # $rest is intentionally word-split
180+
HIP_VISIBLE_DEVICES=$gpu JUNITXML_PREFIX="${junitxml_dir}" launch_job "$label" "${LOG_DIR}/$logfile" $rest
181+
(( gpu++ ))
182+
done < "$config"
183+
done
184+
185+
wait_for_jobs
186+
exit $_OVERALL_RC

0 commit comments

Comments
 (0)