fix: address all PR review findings in runner scripts

- frontier/deploy-runners.sh: pre-download tarball once before spawning
  parallel make-runner.sh instances; use atomic tmp+mv to prevent
  concurrent curl writes corrupting the tarball
- frontier/make-runner.sh: same atomic tmp+mv for solo-invocation safety
- common/rerun-failed.sh: guard gh run view with || continue so an
  expired/deleted run skips rather than exits the whole script; switch
  pipe loops to process substitution so continue works correctly and
  remove the dead rerun_count variable (incremented in a subshell, never
  read)
- frontier/list-runners.sh, phoenix/list-runners.sh, restart-all.sh,
  rebalance-runners.sh: replace for dir in $(find_runner_dirs) with
  while IFS= read -r dir; do ... done < <(find_runner_dirs) to eliminate
  word-splitting and glob expansion on runner paths
- phoenix/check-runners.sh, phoenix/list-runners.sh: guard rss value
  before arithmetic expansion — if SSH fails and rss="?", the expression
  $(( CGROUP_LIMIT - rss )) is a syntax error that exits under
  set -euo pipefail; default to 0 instead

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: Spencer Bryngelson

misc/common/rerun-failed.sh

@@ -22,7 +22,6 @@ if [ -z "$prs" ]; then
     exit 0
 fi
 
-rerun_count=0
 for pr in $prs; do
     title=$(gh pr view --repo "$REPO" "$pr" --json title --jq .title)
     branch=$(gh pr view --repo "$REPO" "$pr" --json headRefName --jq .headRefName)
@@ -35,21 +34,21 @@ for pr in $prs; do
     if [ -n "$failed_runs" ]; then
         echo ""
         echo "=== PR #$pr: $title ==="
-        echo "$failed_runs" | while read -r run_id run_name; do
-            # Check which jobs failed
+        while read -r run_id run_name; do
+            # Check which jobs failed; skip if run has expired or been deleted
             failed_jobs=$(gh run view --repo "$REPO" "$run_id" \
-                --json jobs --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "cancelled") | .name')
+                --json jobs --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "cancelled") | .name' \
+                2>/dev/null) || { echo "  WARNING: could not fetch jobs for run $run_id, skipping"; continue; }
             echo "  Run $run_id ($run_name):"
-            echo "$failed_jobs" | while read -r job; do
+            while read -r job; do
                 echo "    - $job"
-            done
+            done <<< "$failed_jobs"
 
             if [ "${APPLY:-0}" = "1" ]; then
                 echo "  Rerunning failed jobs..."
                 gh run rerun --repo "$REPO" "$run_id" --failed || echo "  WARNING: rerun failed (may already be rerunning)"
-                rerun_count=$((rerun_count + 1))
             fi
-        done
+        done < <(echo "$failed_runs")
     fi
 done
 
@@ -60,13 +59,13 @@ master_failed=$(gh run list --repo "$REPO" --branch master --limit 5 \
     --json databaseId,status,conclusion,name \
     --jq '.[] | select(.conclusion == "failure") | "\(.databaseId) \(.name)"')
 if [ -n "$master_failed" ]; then
-    echo "$master_failed" | while read -r run_id run_name; do
+    while read -r run_id run_name; do
         echo "  Run $run_id ($run_name)"
         if [ "${APPLY:-0}" = "1" ]; then
             echo "  Rerunning failed jobs..."
             gh run rerun --repo "$REPO" "$run_id" --failed || echo "  WARNING: rerun failed"
         fi
-    done
+    done < <(echo "$master_failed")
 else
     echo "  All passing"
 fi

misc/frontier/deploy-runners.sh

@@ -18,6 +18,21 @@ if [ ${#TARGET_NODES[@]} -eq 0 ]; then
     exit 1
 fi
 
+# Pre-download the runner tarball once before spawning parallel make-runner.sh
+# instances. Without this, all instances race to download the same file
+# concurrently and corrupt it. The tmp+mv ensures an atomic final placement.
+RUNNER_VERSION="${RUNNER_VERSION:-$(gh_latest_runner_version 2>/dev/null || echo "2.332.0")}"
+TARBALL="actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz"
+if [ ! -f "${SHARED_DIR}/${TARBALL}" ]; then
+    echo "==> Downloading runner v${RUNNER_VERSION}..."
+    tmp="${SHARED_DIR}/${TARBALL}.tmp.$$"
+    curl -fsSL \
+        "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${TARBALL}" \
+        -o "$tmp"
+    mv "$tmp" "${SHARED_DIR}/${TARBALL}"
+fi
+export RUNNER_VERSION
+
 for i in "${!TARGET_NODES[@]}"; do
     NODE="${TARGET_NODES[$i]}"
     NUM=$((START_NUM + i))

misc/frontier/list-runners.sh

@@ -54,7 +54,7 @@ done < <(gh_list_runners)
 printf "%-25s %-8s %-14s %s\n" "NAME" "GITHUB" "NODE" "RSS"
 printf "%s\n" "$(printf '%.0s-' {1..60})"
 
-for dir in $(find_runner_dirs); do
+while IFS= read -r dir; do
     name=$(get_runner_name "$dir")
     [ -z "$name" ] && continue
 
@@ -86,4 +86,4 @@ for dir in $(find_runner_dirs); do
     fi
 
     printf "%-25s %-8s %-14s %sMB\n" "$name" "$gh_col" "$node_col" "$rss"
-done
+done < <(find_runner_dirs)

misc/frontier/make-runner.sh

@@ -24,9 +24,11 @@ echo "==> Setting up runner: ${RUNNER_NAME} on ${TARGET_NODE}"
 # --- Download tarball once to shared dir ---
 if [ ! -f "${SHARED_DIR}/${TARBALL}" ]; then
     echo "==> Downloading runner v${RUNNER_VERSION}..."
+    tmp="${SHARED_DIR}/${TARBALL}.tmp.$$"
     curl -fsSL \
         "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${TARBALL}" \
-        -o "${SHARED_DIR}/${TARBALL}"
+        -o "$tmp"
+    mv "$tmp" "${SHARED_DIR}/${TARBALL}"
 fi
 
 # --- Extract (filesystem is shared across all nodes) ---

misc/phoenix/check-runners.sh

@@ -28,6 +28,7 @@ for node in "${NODES[@]}"; do
     ' 2>/dev/null || echo "  (unreachable)"
 
     rss=$(ssh -o ConnectTimeout=5 "$node" "ps -u \$(whoami) -o rss= 2>/dev/null | awk '{sum+=\$1} END {printf \"%.0f\", sum/1024}'" 2>/dev/null || echo "?")
+    [[ "$rss" =~ ^[0-9]+$ ]] || rss=0
     echo "  --- Total: ${rss} MB / ${CGROUP_LIMIT} MB ($(( CGROUP_LIMIT - rss )) MB free) ---"
     echo ""
 done

misc/phoenix/list-runners.sh

@@ -22,7 +22,7 @@ while read -r id name status busy; do
 done <<< "$(gh_list_runners)"
 
 # Walk local runner directories and cross-reference
-for dir in $(find_runner_dirs); do
+while IFS= read -r dir; do
     name=$(get_runner_name "$dir")
     [ -z "$name" ] && continue
 
@@ -61,12 +61,13 @@ for dir in $(find_runner_dirs); do
 
     printf "%-25s %-8s %-22s %-8s %5sMB  %s\n" \
         "$name" "$gh_col" "$node" "$slurm" "$rss" "$dir"
-done
+done < <(find_runner_dirs)
 
 echo ""
 echo "=== Per-node memory ==="
 for node in "${NODES[@]}"; do
     count=$(ssh -o ConnectTimeout=5 "$node" "ps aux | grep Runner.Listener | grep -v grep | wc -l" 2>/dev/null || echo 0)
     rss=$(ssh -o ConnectTimeout=5 "$node" "ps -u \$(whoami) -o rss= 2>/dev/null | awk '{sum+=\$1} END {printf \"%.0f\", sum/1024}'" 2>/dev/null || echo "?")
+    [[ "$rss" =~ ^[0-9]+$ ]] || rss=0
     echo "  $node: $count runners, ${rss} MB / ${CGROUP_LIMIT} MB ($(( CGROUP_LIMIT - rss )) MB free)"
 done

misc/phoenix/rebalance-runners.sh

@@ -15,12 +15,12 @@ source "$SCRIPT_DIR/config.sh"
 
 # Discover runners
 declare -a dirs=() names=()
-for dir in $(find_runner_dirs); do
+while IFS= read -r dir; do
     name=$(get_runner_name "$dir")
     [ -z "$name" ] && continue
     dirs+=("$dir")
     names+=("$name")
-done
+done < <(find_runner_dirs)
 
 num_nodes=${#NODES[@]}
 num_runners=${#dirs[@]}

misc/phoenix/restart-all.sh

@@ -15,7 +15,7 @@ source "$SCRIPT_DIR/config.sh"
 echo "=== Discovering runners ==="
 declare -a restart_list=()
 
-for dir in $(find_runner_dirs); do
+while IFS= read -r dir; do
     name=$(get_runner_name "$dir")
     [ -z "$name" ] && continue
     node=$(find_node "$dir")
@@ -37,7 +37,7 @@ for dir in $(find_runner_dirs); do
     fi
 
     restart_list+=("$node $dir $name")
-done
+done < <(find_runner_dirs)
 
 if [ ${#restart_list[@]} -eq 0 ]; then
     echo "Nothing to restart."