Skip to content

Commit 1627311

Browse files
sbryngelsonclaude
sbryngelson
and
claude
committed
Add build progress heartbeat and collapsible GHA log groups
Prints a periodic heartbeat (every 2 min) during parallel Frontier builds showing which configs are still running with log sizes and last output line. Build logs are printed in collapsible ::group:: sections for passed builds and in full for failed builds so failures are immediately visible. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent d852af3 commit 1627311

2 files changed

Lines changed: 88 additions & 26 deletions

File tree

‎.github/scripts/run_frontier_all_benchmarks.sh‎

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -53,37 +53,68 @@ for cfg in "${configs[@]}"; do
5353
build_pids+=($!)
5454
done
5555

56+
# Periodic heartbeat while builds run
57+
(
58+
while true; do
59+
sleep 120
60+
alive=0
61+
for pid in "${build_pids[@]}"; do
62+
kill -0 "$pid" 2>/dev/null && alive=$((alive + 1))
63+
done
64+
[ "$alive" -eq 0 ] && break
65+
echo "--- Build heartbeat ($(date +%H:%M:%S)): $alive/${#build_pids[@]} running ---"
66+
for i in "${!configs[@]}"; do
67+
read -r version cluster device interface <<< "${configs[$i]}"
68+
log="build-${version}-${cluster}-${device}-${interface}.log"
69+
if kill -0 "${build_pids[$i]}" 2>/dev/null; then
70+
size=$(stat -c%s "$log" 2>/dev/null || echo 0)
71+
last=$(tail -n 1 "$log" 2>/dev/null | head -c 120 || echo "")
72+
echo " $version $cluster $device $interface: running (${size} bytes) $last"
73+
fi
74+
done
75+
done
76+
) &
77+
heartbeat_pid=$!
78+
5679
# Wait for all builds and report results
5780
build_failed=0
5881
build_exits=()
5982
for i in "${!build_pids[@]}"; do
6083
read -r version cluster device interface <<< "${configs[$i]}"
6184
if wait "${build_pids[$i]}"; then
62-
echo " Build PASSED: $version $cluster $device $interface"
6385
build_exits+=(0)
6486
else
6587
code=$?
66-
echo " Build FAILED: $version $cluster $device $interface (exit $code)"
6788
build_exits+=($code)
6889
build_failed=1
6990
fi
7091
done
7192

72-
# On failure, print logs for failed builds and abort
93+
# Stop heartbeat
94+
kill "$heartbeat_pid" 2>/dev/null; wait "$heartbeat_pid" 2>/dev/null || true
95+
96+
# Print build logs: passed builds collapsed, failed builds in full
97+
for i in "${!configs[@]}"; do
98+
read -r version cluster device interface <<< "${configs[$i]}"
99+
log="build-${version}-${cluster}-${device}-${interface}.log"
100+
if [ "${build_exits[$i]}" -eq 0 ]; then
101+
echo "::group::Build PASSED: $version $cluster $device $interface"
102+
cat "$log"
103+
echo "::endgroup::"
104+
else
105+
echo "=========================================="
106+
echo "Build FAILED: $version $cluster $device $interface (exit ${build_exits[$i]})"
107+
echo "=========================================="
108+
cat "$log"
109+
fi
110+
done
111+
112+
# Abort on failure
73113
if [ "$build_failed" -ne 0 ]; then
74114
echo ""
75115
echo "=========================================="
76-
echo "Build failures detected. Printing failed logs:"
116+
echo "Build failures detected — see logs above."
77117
echo "=========================================="
78-
for i in "${!configs[@]}"; do
79-
if [ "${build_exits[$i]}" -ne 0 ]; then
80-
read -r version cluster device interface <<< "${configs[$i]}"
81-
log="build-${version}-${cluster}-${device}-${interface}.log"
82-
echo "--- $log ---"
83-
cat "$log"
84-
echo "--- end $log ---"
85-
fi
86-
done
87118
exit 1
88119
fi
89120

‎.github/scripts/run_frontier_all_tests.sh‎

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -58,37 +58,68 @@ for cfg in "${configs[@]}"; do
5858
build_pids+=($!)
5959
done
6060

61+
# Periodic heartbeat while builds run
62+
(
63+
while true; do
64+
sleep 120
65+
alive=0
66+
for pid in "${build_pids[@]}"; do
67+
kill -0 "$pid" 2>/dev/null && alive=$((alive + 1))
68+
done
69+
[ "$alive" -eq 0 ] && break
70+
echo "--- Build heartbeat ($(date +%H:%M:%S)): $alive/${#build_pids[@]} running ---"
71+
for i in "${!configs[@]}"; do
72+
read -r cluster device interface <<< "${configs[$i]}"
73+
log="build-${cluster}-${device}-${interface}.log"
74+
if kill -0 "${build_pids[$i]}" 2>/dev/null; then
75+
size=$(stat -c%s "$log" 2>/dev/null || echo 0)
76+
last=$(tail -n 1 "$log" 2>/dev/null | head -c 120 || echo "")
77+
echo " $cluster $device $interface: running (${size} bytes) $last"
78+
fi
79+
done
80+
done
81+
) &
82+
heartbeat_pid=$!
83+
6184
# Wait for all builds and report results
6285
build_failed=0
6386
build_exits=()
6487
for i in "${!build_pids[@]}"; do
6588
read -r cluster device interface <<< "${configs[$i]}"
6689
if wait "${build_pids[$i]}"; then
67-
echo " Build PASSED: $cluster $device $interface"
6890
build_exits+=(0)
6991
else
7092
code=$?
71-
echo " Build FAILED: $cluster $device $interface (exit $code)"
7293
build_exits+=($code)
7394
build_failed=1
7495
fi
7596
done
7697

77-
# On failure, print logs for failed builds and abort
98+
# Stop heartbeat
99+
kill "$heartbeat_pid" 2>/dev/null; wait "$heartbeat_pid" 2>/dev/null || true
100+
101+
# Print build logs: passed builds collapsed, failed builds in full
102+
for i in "${!configs[@]}"; do
103+
read -r cluster device interface <<< "${configs[$i]}"
104+
log="build-${cluster}-${device}-${interface}.log"
105+
if [ "${build_exits[$i]}" -eq 0 ]; then
106+
echo "::group::Build PASSED: $cluster $device $interface"
107+
cat "$log"
108+
echo "::endgroup::"
109+
else
110+
echo "=========================================="
111+
echo "Build FAILED: $cluster $device $interface (exit ${build_exits[$i]})"
112+
echo "=========================================="
113+
cat "$log"
114+
fi
115+
done
116+
117+
# Abort on failure
78118
if [ "$build_failed" -ne 0 ]; then
79119
echo ""
80120
echo "=========================================="
81-
echo "Build failures detected. Printing failed logs:"
121+
echo "Build failures detected — see logs above."
82122
echo "=========================================="
83-
for i in "${!configs[@]}"; do
84-
if [ "${build_exits[$i]}" -ne 0 ]; then
85-
read -r cluster device interface <<< "${configs[$i]}"
86-
log="build-${cluster}-${device}-${interface}.log"
87-
echo "--- $log ---"
88-
cat "$log"
89-
echo "--- end $log ---"
90-
fi
91-
done
92123
exit 1
93124
fi
94125

0 commit comments

Comments
 (0)