Limit parallel builds to 2 concurrent on login node

Running all 5-6 builds simultaneously on a Frontier login node causes
memory pressure that crashes the Cray compiler (optcg segfault). Cap
concurrent builds at 2 using a polling semaphore to stay within login
node resource limits.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sbryngelson

.github/scripts/run_frontier_all_benchmarks.sh

@@ -46,22 +46,35 @@ for cfg in "${configs[@]}"; do
 done
 
 # --- Phase 2: Build all configs on login node in parallel ---
+MAX_PARALLEL=2
+
 echo ""
 echo "=========================================="
-echo "Starting parallel builds (${num_nodes} configs)..."
+echo "Starting parallel builds (${num_nodes} configs, max $MAX_PARALLEL concurrent)..."
 echo "=========================================="
-
 build_pids=()
-for cfg in "${configs[@]}"; do
-    read -r version cluster device interface <<< "$cfg"
+running=()
+for i in "${!configs[@]}"; do
+    # Wait until a build slot is available
+    while [ ${#running[@]} -ge $MAX_PARALLEL ]; do
+        sleep 2
+        still_running=()
+        for pid in "${running[@]}"; do
+            kill -0 "$pid" 2>/dev/null && still_running+=("$pid")
+        done
+        running=("${still_running[@]}")
+    done
+
+    read -r version cluster device interface <<< "${configs[$i]}"
     dir="${version}-${cluster}-${device}-${interface}"
     log="build-${version}-${cluster}-${device}-${interface}.log"
     echo "  Starting: $version $cluster $device $interface"
     (
         cd "$dir"
         bash .github/workflows/${cluster}/build.sh "$device" "$interface" bench
     ) > "$log" 2>&1 &
-    build_pids+=($!)
+    build_pids[$i]=$!
+    running+=($!)
 done
 
 # Periodic heartbeat while builds run

.github/scripts/run_frontier_all_tests.sh

@@ -51,22 +51,35 @@ for cfg in "${configs[@]}"; do
 done
 
 # --- Phase 2: Build all configs on login node in parallel ---
+MAX_PARALLEL=2
+
 echo ""
 echo "=========================================="
-echo "Starting parallel builds (${num_nodes} configs)..."
+echo "Starting parallel builds (${num_nodes} configs, max $MAX_PARALLEL concurrent)..."
 echo "=========================================="
-
 build_pids=()
-for cfg in "${configs[@]}"; do
-    read -r cluster device interface <<< "$cfg"
+running=()
+for i in "${!configs[@]}"; do
+    # Wait until a build slot is available
+    while [ ${#running[@]} -ge $MAX_PARALLEL ]; do
+        sleep 2
+        still_running=()
+        for pid in "${running[@]}"; do
+            kill -0 "$pid" 2>/dev/null && still_running+=("$pid")
+        done
+        running=("${still_running[@]}")
+    done
+
+    read -r cluster device interface <<< "${configs[$i]}"
     dir="test-${cluster}-${device}-${interface}"
     log="build-${cluster}-${device}-${interface}.log"
     echo "  Starting: $cluster $device $interface"
     (
         cd "$dir"
         bash .github/workflows/${cluster}/build.sh "$device" "$interface"
     ) > "$log" 2>&1 &
-    build_pids+=($!)
+    build_pids[$i]=$!
+    running+=($!)
 done
 
 # Periodic heartbeat while builds run