Add build heartbeat during throttled build phase

The throttle loop blocks the main process while waiting for build slots,
so the post-loop heartbeat subprocess never starts until late. Add
inline heartbeat every 120s during throttle waits showing the last log
line of each active build.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sbryngelson

.github/scripts/run_frontier_all_benchmarks.sh

@@ -59,13 +59,26 @@ build_pids=()
 running=()
 for i in "${!configs[@]}"; do
     # Wait until a build slot is available
+    throttle_timer=0
     while [ ${#running[@]} -ge $MAX_PARALLEL ]; do
-        sleep 2
+        sleep 5
+        throttle_timer=$((throttle_timer + 5))
         still_running=()
         for pid in "${running[@]}"; do
             kill -0 "$pid" 2>/dev/null && still_running+=("$pid")
         done
         running=("${still_running[@]}")
+        if [ $throttle_timer -ge 120 ]; then
+            throttle_timer=0
+            echo "--- Build heartbeat ($(date +%H:%M:%S)) ---"
+            for j in $(seq 0 $((i - 1))); do
+                read -r v c d iface <<< "${configs[$j]}"
+                if kill -0 "${build_pids[$j]}" 2>/dev/null; then
+                    last=$(tail -n 1 "build-${v}-${c}-${d}-${iface}.log" 2>/dev/null | head -c 120 || echo "")
+                    echo "  $v $c $d $iface: $last"
+                fi
+            done
+        fi
     done
 
     read -r version cluster device interface <<< "${configs[$i]}"

.github/scripts/run_frontier_all_tests.sh

@@ -64,13 +64,26 @@ build_pids=()
 running=()
 for i in "${!configs[@]}"; do
     # Wait until a build slot is available
+    throttle_timer=0
     while [ ${#running[@]} -ge $MAX_PARALLEL ]; do
-        sleep 2
+        sleep 5
+        throttle_timer=$((throttle_timer + 5))
         still_running=()
         for pid in "${running[@]}"; do
             kill -0 "$pid" 2>/dev/null && still_running+=("$pid")
         done
         running=("${still_running[@]}")
+        if [ $throttle_timer -ge 120 ]; then
+            throttle_timer=0
+            echo "--- Build heartbeat ($(date +%H:%M:%S)) ---"
+            for j in $(seq 0 $((i - 1))); do
+                read -r c d iface <<< "${configs[$j]}"
+                if kill -0 "${build_pids[$j]}" 2>/dev/null; then
+                    last=$(tail -n 1 "build-${c}-${d}-${iface}.log" 2>/dev/null | head -c 120 || echo "")
+                    echo "  $c $d $iface: $last"
+                fi
+            done
+        fi
     done
 
     read -r cluster device interface <<< "${configs[$i]}"