refactor: deduplicate runner scripts into misc/common/

Extract shared logic from frontier/phoenix into common scripts:
- runner-lib.sh: add gh_list_runners(), has_slurm() (portable, grep PATH
  for slurm keyword), sweep_all_nodes() (exe-based, parallel SSH), CGROUP_LIMIT default
- check-runners.sh: new common script (exe-based discovery, slurm column,
  conditional cgroup footer); both sites now show slurm status
- list-runners.sh: new common script (parallel sweep, slurm column,
  stale runner.node detection, conditional cgroup footer)
- rebalance-runners.sh: new common script (optional sync_runner_nodes hook,
  writes runner.node after start, checks slurm after start)

All site scripts (frontier/, phoenix/) reduced to thin wrappers that source
config.sh then the common implementation. phoenix/create-runner.sh fixed to
write runner.node after successful start.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: Spencer Bryngelson

misc/common/check-runners.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Check runner health across all login nodes.
+#
+# Sourced by site wrappers (frontier/check-runners.sh, phoenix/check-runners.sh)
+# after config.sh is loaded. Shows Runner.Listener processes per node with
+# name, busy/idle status, slurm availability, and RSS memory.
+# If CGROUP_LIMIT > 0, also shows per-node total memory vs the cgroup limit.
+#
+# Usage: bash check-runners.sh
+set -euo pipefail
+
+declare -f sync_runner_nodes > /dev/null 2>&1 && {
+    echo "==> Syncing runner node locations..."
+    sync_runner_nodes
+}
+
+for node in "${NODES[@]}"; do
+    echo "=== $node ==="
+    ssh $SSH_OPTS "$node" '
+        found=0
+        for p in $(ps aux | grep Runner.Listener | grep -v grep | awk "{print \$2}"); do
+            found=1
+            exe=$(readlink -f /proc/$p/exe 2>/dev/null || echo "???")
+            dir=$(dirname "$(dirname "$exe")" 2>/dev/null || echo "???")
+            name=$(basename "$dir")
+            worker=$(ps aux | grep "Runner.Worker" | grep "$dir" | grep -v grep | awk "{print \$2}" | head -1)
+            [ -n "$worker" ] && status="BUSY" || status="idle"
+            rss=$(ps -p $p -o rss= 2>/dev/null | awk "{printf \"%.0f\", \$1/1024}" || echo "?")
+            slurm=$(tr "\0" "\n" < /proc/$p/environ 2>/dev/null | grep -c "^PATH=.*slurm" || echo 0)
+            [ "$slurm" -gt 0 ] && slurm_ok="ok" || slurm_ok="MISSING"
+            printf "  %-30s %5s  slurm=%-7s  %s MB\n" "$name" "$status" "$slurm_ok" "$rss"
+        done
+        [ "$found" -eq 0 ] && echo "  (no runners)"
+    ' 2>/dev/null || echo "  (unreachable)"
+
+    if [ "${CGROUP_LIMIT:-0}" -gt 0 ]; then
+        rss=$(ssh $SSH_OPTS "$node" \
+            "ps -u \$(whoami) -o rss= 2>/dev/null | awk '{sum+=\$1} END {printf \"%.0f\", sum/1024}'" \
+            2>/dev/null || echo "?")
+        [[ "$rss" =~ ^[0-9]+$ ]] || rss=0
+        echo "  --- Total: ${rss} MB / ${CGROUP_LIMIT} MB ($(( CGROUP_LIMIT - rss )) MB free) ---"
+    fi
+    echo ""
+done

misc/common/list-runners.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# List all runners combining GitHub API status with live node process info.
+#
+# Sourced by site wrappers (frontier/list-runners.sh, phoenix/list-runners.sh)
+# after config.sh is loaded. Uses a parallel SSH sweep across all nodes
+# simultaneously (one SSH per node regardless of runner count).
+# Shows name, GitHub status, node, slurm availability, and RSS.
+# If CGROUP_LIMIT > 0, also shows a per-node memory summary.
+#
+# Usage: bash list-runners.sh
+set -euo pipefail
+
+declare -f sync_runner_nodes > /dev/null 2>&1 && {
+    echo "==> Syncing runner node locations..."
+    sync_runner_nodes
+}
+
+tmpdir=$(mktemp -d)
+trap 'rm -rf "$tmpdir"' EXIT
+
+sweep_all_nodes "$tmpdir"
+
+# Parse sweep results into associative arrays
+declare -A runner_node runner_rss runner_slurm
+for node in "${NODES[@]}"; do
+    while IFS= read -r line; do
+        read -r _s sweep_node dir rss slurm_ok <<< "$line"
+        runner_node["$dir"]="$sweep_node"
+        runner_rss["$dir"]="$rss"
+        runner_slurm["$dir"]="$slurm_ok"
+    done < <(grep '^RUNNER ' "$tmpdir/$node.out" 2>/dev/null || true)
+done
+
+# Fetch GitHub API status
+declare -A gh_status gh_busy
+while read -r _id name status busy; do
+    gh_status["$name"]="$status"
+    gh_busy["$name"]="$busy"
+done < <(gh_list_runners)
+
+# Print table
+printf "%-25s %-8s %-20s %-8s %s\n" "NAME" "GITHUB" "NODE" "SLURM" "RSS"
+printf "%s\n" "$(printf '%.0s-' {1..70})"
+
+while IFS= read -r dir; do
+    name=$(get_runner_name "$dir")
+    [ -z "$name" ] && continue
+
+    [ "${gh_busy[$name]:-false}" = "true" ] && gh_col="BUSY" || gh_col="${gh_status[$name]:-unknown}"
+
+    actual_node="${runner_node[$dir]:-}"
+    rss="${runner_rss[$dir]:-—}"
+    slurm="${runner_slurm[$dir]:-—}"
+
+    if [ -z "$actual_node" ]; then
+        printf "%-25s %-8s %-20s %-8s %s\n" "$name" "$gh_col" "offline" "—" "—"
+        continue
+    fi
+
+    # Flag stale runner.node entries
+    node_col="$actual_node"
+    if [ -f "$dir/runner.node" ]; then
+        recorded=$(cat "$dir/runner.node")
+        [ "$actual_node" != "$recorded" ] && node_col="${actual_node} *(stale: ${recorded})"
+    fi
+
+    printf "%-25s %-8s %-20s %-8s %sMB\n" "$name" "$gh_col" "$node_col" "$slurm" "$rss"
+done < <(find_runner_dirs)
+
+# Per-node memory summary (only when site has a cgroup limit)
+if [ "${CGROUP_LIMIT:-0}" -gt 0 ]; then
+    echo ""
+    echo "=== Per-node memory ==="
+    for node in "${NODES[@]}"; do
+        count=$(ssh $SSH_OPTS "$node" \
+            "ps aux | grep Runner.Listener | grep -v grep | wc -l" 2>/dev/null || echo 0)
+        rss=$(ssh $SSH_OPTS "$node" \
+            "ps -u \$(whoami) -o rss= 2>/dev/null | awk '{sum+=\$1} END {printf \"%.0f\", sum/1024}'" \
+            2>/dev/null || echo "?")
+        [[ "$rss" =~ ^[0-9]+$ ]] || rss=0
+        echo "  $node: $count runners, ${rss} MB / ${CGROUP_LIMIT} MB ($(( CGROUP_LIMIT - rss )) MB free)"
+    done
+fi

misc/common/rebalance-runners.sh

@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+# Core rebalance algorithm for GitHub Actions runners.
+#
+# Sourced by site wrappers (frontier/rebalance-runners.sh,
+# phoenix/rebalance-runners.sh) after config.sh is loaded.
+# Discovers all runner directories, checks which node each is on,
+# computes the optimal distribution, and moves runners to balance.
+# Prefers moving idle runners over busy ones. Also places offline runners.
+#
+# Usage: bash rebalance-runners.sh              # dry run
+#        APPLY=1 bash rebalance-runners.sh      # execute
+#        APPLY=1 FORCE=1 bash rebalance-runners.sh  # move busy runners too
+set -euo pipefail
+
+declare -f sync_runner_nodes > /dev/null 2>&1 && {
+    echo "==> Syncing runner node locations..."
+    sync_runner_nodes
+}
+
+# Discover runners
+declare -a dirs=() names=()
+while IFS= read -r dir; do
+    name=$(get_runner_name "$dir")
+    [ -z "$name" ] && continue
+    dirs+=("$dir")
+    names+=("$name")
+done < <(find_runner_dirs)
+
+num_nodes=${#NODES[@]}
+num_runners=${#dirs[@]}
+target=$(( num_runners / num_nodes ))
+remainder=$(( num_runners % num_nodes ))
+
+echo "=== Current state ==="
+echo "Runners: $num_runners across $num_nodes nodes"
+echo "Target:  $target per node (+1 on first $remainder nodes)"
+echo ""
+
+# Map runners to nodes and check busy status
+declare -A node_runners
+declare -A runner_node runner_busy
+
+for node in "${NODES[@]}"; do node_runners[$node]=""; done
+
+for i in "${!dirs[@]}"; do
+    node=$(find_node "${dirs[$i]}")
+    runner_node[$i]="$node"
+    if [ "$node" != "offline" ]; then
+        node_runners[$node]="${node_runners[$node]:-} $i"
+        worker=$(ssh $SSH_OPTS "$node" "ps aux | grep Runner.Worker | grep '${dirs[$i]}' | grep -v grep" 2>/dev/null || true)
+        [ -n "$worker" ] && runner_busy[$i]=1 || runner_busy[$i]=0
+    else
+        runner_busy[$i]=0
+    fi
+done
+
+# Show current distribution
+for node in "${NODES[@]}"; do
+    indices=(${node_runners[$node]:-})
+    echo "$node: ${#indices[@]} runners"
+    for i in "${indices[@]}"; do
+        busy=""
+        [ "${runner_busy[$i]:-0}" = "1" ] && busy=" (BUSY)"
+        echo "  ${names[$i]}$busy"
+    done
+done
+
+offline=()
+for i in "${!dirs[@]}"; do
+    [ "${runner_node[$i]}" = "offline" ] && offline+=("$i")
+done
+if [ ${#offline[@]} -gt 0 ]; then
+    echo ""
+    echo "OFFLINE:"
+    for i in "${offline[@]}"; do echo "  ${names[$i]}"; done
+fi
+echo ""
+
+# Compute per-node targets
+declare -A node_target
+n=0
+for node in "${NODES[@]}"; do
+    node_target[$node]=$target
+    [ $n -lt $remainder ] && node_target[$node]=$(( target + 1 ))
+    n=$((n + 1))
+done
+
+# Plan moves: pull runners from overloaded nodes (idle first)
+to_place=()
+for node in "${NODES[@]}"; do
+    indices=(${node_runners[$node]:-})
+    excess=$(( ${#indices[@]} - ${node_target[$node]} ))
+    [ $excess -le 0 ] && continue
+    idle=() busy=()
+    for i in "${indices[@]}"; do
+        [ "${runner_busy[$i]:-0}" = "1" ] && busy+=("$i") || idle+=("$i")
+    done
+    moved=0
+    for i in "${idle[@]}" "${busy[@]}"; do
+        [ $moved -ge $excess ] && break
+        to_place+=("$node $i")
+        moved=$((moved + 1))
+    done
+done
+
+# Add offline runners to be placed
+for i in "${offline[@]}"; do to_place+=("offline $i"); done
+
+# Assign to underloaded nodes
+moves=()
+for entry in "${to_place[@]}"; do
+    read -r src idx <<< "$entry"
+    best="" best_deficit=-999
+    for node in "${NODES[@]}"; do
+        cur=(${node_runners[$node]:-})
+        deficit=$(( ${node_target[$node]} - ${#cur[@]} ))
+        [ $deficit -gt $best_deficit ] && best_deficit=$deficit && best=$node
+    done
+    [ -z "$best" ] || [ "$best_deficit" -le 0 ] && continue
+    moves+=("$src $best $idx")
+    # Update bookkeeping so subsequent assignments reflect this move
+    if [ "$src" != "offline" ]; then
+        new=""
+        for j in ${node_runners[$src]}; do [ "$j" != "$idx" ] && new="$new $j"; done
+        node_runners[$src]="$new"
+    fi
+    node_runners[$best]="${node_runners[$best]:-} $idx"
+done
+
+if [ ${#moves[@]} -eq 0 ]; then
+    echo "Already balanced."
+    exit 0
+fi
+
+echo "=== Planned moves ==="
+has_busy=false
+for move in "${moves[@]}"; do
+    read -r src dst idx <<< "$move"
+    busy=""
+    [ "${runner_busy[$idx]:-0}" = "1" ] && busy=" (BUSY!)" && has_busy=true
+    echo "  ${names[$idx]}: $src -> $dst$busy"
+done
+echo ""
+echo "=== Target ==="
+for node in "${NODES[@]}"; do
+    cur=(${node_runners[$node]:-})
+    echo "  $node: ${#cur[@]} runners"
+done
+
+[ "$has_busy" = true ] && [ "${FORCE:-0}" != "1" ] && echo "" && echo "Set FORCE=1 to move busy runners." && exit 1
+[ "${APPLY:-0}" != "1" ] && echo "" && echo "Dry run — set APPLY=1 to execute." && exit 0
+
+echo ""
+echo "=== Executing ==="
+for move in "${moves[@]}"; do
+    read -r src dst idx <<< "$move"
+    echo "Moving ${names[$idx]}: $src -> $dst"
+    [ "$src" != "offline" ] && stop_runner "$src" "${dirs[$idx]}"
+    if start_runner "$dst" "${dirs[$idx]}"; then
+        echo "$dst" > "${dirs[$idx]}/runner.node"
+        pids=$(find_pids "$dst" "${dirs[$idx]}")
+        pid=${pids%% *}
+        if has_slurm "$dst" "$pid"; then
+            echo "  OK: ${names[$idx]} started on $dst (slurm ok)"
+        else
+            echo "  WARNING: ${names[$idx]} started on $dst but slurm MISSING from PATH"
+        fi
+    else
+        echo "  ERROR: Failed to start ${names[$idx]} on $dst"
+    fi
+done
+
+echo ""
+bash "${SITE_SCRIPT_DIR}/check-runners.sh"

misc/common/runner-lib.sh

@@ -5,8 +5,20 @@
 # misc/phoenix/config.sh). Callers must define ORG, NODES, and SSH_OPTS
 # before sourcing this file.
 
+# Default: no cgroup memory limit displayed. Override in site config (e.g. CGROUP_LIMIT=4096).
+CGROUP_LIMIT=${CGROUP_LIMIT:-0}
+
 # --- GitHub API ---
 
+# List runners from the GitHub API, filtered to this site's label.
+# Prints: id name status busy (one runner per line)
+gh_list_runners() {
+    gh api "orgs/$ORG/actions/runners" --paginate \
+        --jq ".runners[]
+              | select(.labels | map(.name) | index(\"$RUNNER_LABEL\"))
+              | \"\(.id) \(.name) \(.status) \(.busy)\""
+}
+
 # Get a registration token for new runners.
 gh_registration_token() {
     gh api "orgs/$ORG/actions/runners/registration-token" --jq .token
@@ -62,6 +74,40 @@ find_node() {
     echo "offline"
 }
 
+# Check if a runner process has a slurm directory in its PATH.
+# Works across sites regardless of the specific slurm installation path.
+# Args: $1 = node, $2 = PID (or "PID rest..." — uses first token only)
+has_slurm() {
+    local node="$1" pid="${2%% *}"
+    ssh $SSH_OPTS "$node" \
+        "tr '\0' '\n' < /proc/$pid/environ 2>/dev/null | grep -q '^PATH=.*slurm'" \
+        2>/dev/null
+}
+
+# Sweep all nodes in parallel, writing per-node result files to tmpdir.
+# Each output line: RUNNER <node> <dir> <rss_mb> <slurm_ok>
+#   dir      = runner directory derived from the Runner.Listener exe path
+#   slurm_ok = "ok" if slurm appears in the process PATH, "MISSING" otherwise
+# Caller must create tmpdir and parse the output files.
+# Args: $1 = tmpdir
+sweep_all_nodes() {
+    local tmpdir="$1" node
+    for node in "${NODES[@]}"; do
+        ssh $SSH_OPTS "$node" '
+            for p in $(ps aux | grep Runner.Listener | grep -v grep | awk "{print \$2}"); do
+                exe=$(readlink -f /proc/$p/exe 2>/dev/null || true)
+                [ -z "$exe" ] && continue
+                dir=$(dirname "$(dirname "$exe")")
+                rss=$(ps -p $p -o rss= 2>/dev/null | awk "{printf \"%.0f\", \$1/1024}" || echo 0)
+                slurm=$(tr "\0" "\n" < /proc/$p/environ 2>/dev/null | grep -c "^PATH=.*slurm" || echo 0)
+                [ "$slurm" -gt 0 ] && slurm_ok="ok" || slurm_ok="MISSING"
+                echo "RUNNER '"$node"' $dir $rss $slurm_ok"
+            done
+        ' 2>/dev/null > "$tmpdir/$node.out" &
+    done
+    wait
+}
+
 # Start a runner on a node.
 # Uses a login shell (bash -lc) so site PATH (e.g. SLURM) is available.
 # Args: $1 = node, $2 = runner directory