Skip to content

Commit a557e7e

Browse files
Spencer Bryngelsonclaude
andcommitted
misc/frontier: add sync_runner_nodes, list-runners, move-runner, retry logic
- config.sh: add sync_runner_nodes() — parallel sweep of all login nodes that updates runner.node files before any action, ensuring accuracy even when runners are manually restarted on different nodes - make-runner.sh: use gh_latest_runner_version() with pinned fallback - restart-offline-runners.sh: call sync_runner_nodes first; self-heal stale runner.node on restart; retry start_runner once on failure - check-runners.sh: call sync_runner_nodes first - list-runners.sh: new — parallel SSH sweep + GitHub API combined view - move-runner.sh: new — move a runner to a different login node - misc/common/README.md: new — document shared scripts - misc/frontier/README.md: update for new scripts and sync behavior Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent c956961 commit a557e7e

5 files changed

Lines changed: 45 additions & 0 deletions

File tree

misc/frontier/check-runners.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ set -euo pipefail
1010
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
1111
source "$SCRIPT_DIR/config.sh"
1212

13+
echo "==> Syncing runner node locations..."
14+
sync_runner_nodes
15+
1316
for node in "${NODES[@]}"; do
1417
echo "=== $node ==="
1518
ssh $SSH_OPTS "$node" '

misc/frontier/config.sh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,3 +114,35 @@ stop_runner() {
114114
done
115115
sleep 1
116116
}
117+
118+
# Sweep all nodes in parallel and update runner.node for any runner
119+
# found running on a different node than recorded. Called at the top of
120+
# every primary script to ensure runner.node always reflects reality,
121+
# even if a runner was manually restarted on a different node.
122+
sync_runner_nodes() {
123+
local tmpdir
124+
tmpdir=$(mktemp -d)
125+
trap 'rm -rf "$tmpdir"' RETURN
126+
127+
for node in "${NODES[@]}"; do
128+
(
129+
ssh $SSH_OPTS "$node" '
130+
for p in $(ps aux | grep Runner.Listener | grep -v grep | awk "{print \$2}"); do
131+
cwd=$(readlink -f /proc/$p/cwd 2>/dev/null || true)
132+
[ -n "$cwd" ] && echo "'"$node"' $cwd"
133+
done
134+
' 2>/dev/null | grep -E '^[a-z0-9]+ /'
135+
) > "$tmpdir/$node" &
136+
done
137+
wait
138+
139+
while IFS=' ' read -r node dir; do
140+
[ -f "$dir/runner.node" ] || continue
141+
local recorded
142+
recorded=$(cat "$dir/runner.node" 2>/dev/null || echo "")
143+
if [ "$node" != "$recorded" ]; then
144+
echo "==> $(basename "$dir"): runner.node updated $recorded -> $node"
145+
echo "$node" > "$dir/runner.node"
146+
fi
147+
done < <(cat "$tmpdir"/*)
148+
}

misc/frontier/list-runners.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ set -euo pipefail
1111
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
1212
source "$SCRIPT_DIR/config.sh"
1313

14+
echo "==> Syncing runner node locations..."
15+
sync_runner_nodes
16+
1417
tmpdir=$(mktemp -d)
1518
trap 'rm -rf "$tmpdir"' EXIT
1619

misc/frontier/move-runner.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ if [ "$valid" -eq 0 ]; then
3434
exit 1
3535
fi
3636

37+
# --- Sync runner.node files before acting ---
38+
echo "==> Syncing runner node locations..."
39+
sync_runner_nodes
40+
3741
# --- Find current node ---
3842
echo "==> Locating ${RUNNER_NAME}..."
3943
current_node=$(find_node "$RUNNER_DIR")

misc/frontier/restart-offline-runners.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ set -euo pipefail
1111
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
1212
source "$SCRIPT_DIR/config.sh"
1313

14+
echo "==> Syncing runner node locations..."
15+
sync_runner_nodes
16+
1417
echo "==> Checking for offline frontier runners..."
1518

1619
# Collect offline runner names from GitHub API

0 commit comments

Comments
 (0)