Skip to content

Commit 0a5a0f6

Browse files
sbryngelsonclaude
andcommitted
Add build heartbeat during throttled build phase
The throttle loop blocks the main process while waiting for build slots, so the post-loop heartbeat subprocess never starts until late. Add inline heartbeat every 120s during throttle waits showing the last log line of each active build. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 1347a54 commit 0a5a0f6

2 files changed

Lines changed: 28 additions & 2 deletions

File tree

.github/scripts/run_frontier_all_benchmarks.sh

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,26 @@ build_pids=()
5959
running=()
6060
for i in "${!configs[@]}"; do
6161
# Wait until a build slot is available
62+
throttle_timer=0
6263
while [ ${#running[@]} -ge $MAX_PARALLEL ]; do
63-
sleep 2
64+
sleep 5
65+
throttle_timer=$((throttle_timer + 5))
6466
still_running=()
6567
for pid in "${running[@]}"; do
6668
kill -0 "$pid" 2>/dev/null && still_running+=("$pid")
6769
done
6870
running=("${still_running[@]}")
71+
if [ $throttle_timer -ge 120 ]; then
72+
throttle_timer=0
73+
echo "--- Build heartbeat ($(date +%H:%M:%S)) ---"
74+
for j in $(seq 0 $((i - 1))); do
75+
read -r v c d iface <<< "${configs[$j]}"
76+
if kill -0 "${build_pids[$j]}" 2>/dev/null; then
77+
last=$(tail -n 1 "build-${v}-${c}-${d}-${iface}.log" 2>/dev/null | head -c 120 || echo "")
78+
echo " $v $c $d $iface: $last"
79+
fi
80+
done
81+
fi
6982
done
7083

7184
read -r version cluster device interface <<< "${configs[$i]}"

.github/scripts/run_frontier_all_tests.sh

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,13 +64,26 @@ build_pids=()
6464
running=()
6565
for i in "${!configs[@]}"; do
6666
# Wait until a build slot is available
67+
throttle_timer=0
6768
while [ ${#running[@]} -ge $MAX_PARALLEL ]; do
68-
sleep 2
69+
sleep 5
70+
throttle_timer=$((throttle_timer + 5))
6971
still_running=()
7072
for pid in "${running[@]}"; do
7173
kill -0 "$pid" 2>/dev/null && still_running+=("$pid")
7274
done
7375
running=("${still_running[@]}")
76+
if [ $throttle_timer -ge 120 ]; then
77+
throttle_timer=0
78+
echo "--- Build heartbeat ($(date +%H:%M:%S)) ---"
79+
for j in $(seq 0 $((i - 1))); do
80+
read -r c d iface <<< "${configs[$j]}"
81+
if kill -0 "${build_pids[$j]}" 2>/dev/null; then
82+
last=$(tail -n 1 "build-${c}-${d}-${iface}.log" 2>/dev/null | head -c 120 || echo "")
83+
echo " $c $d $iface: $last"
84+
fi
85+
done
86+
fi
7487
done
7588

7689
read -r cluster device interface <<< "${configs[$i]}"

0 commit comments

Comments
 (0)