Skip to content

Commit 536f433

Browse files
Spencer Bryngelsonclaude
Spencer Bryngelson
and
claude
committed
refactor: extract shared runner functions into misc/common/runner-lib.sh
Move functions identical across both sites into a common library: gh_registration_token, gh_latest_runner_version, gh_remove_runner, get_runner_name, find_pids, find_node, start_runner, stop_runner Both config.sh files now source runner-lib.sh after defining their site constants (ORG, NODES, SSH_OPTS), keeping only site-specific logic locally: - frontier/config.sh: gh_list_runners, find_runner_dirs, sync_runner_nodes - phoenix/config.sh: gh_list_runners, find_runner_dirs, has_slurm Harmonize start_runner() across both sites: - Use bash -lc on both (was frontier-only) for login shell PATH - Use timeout 15 + synchronous SSH (was phoenix's background SSH + poll loop) - cd into runner dir before run.sh so CWD-based discovery works - Standardize log file to runner.log (was runner-nohup.log on phoenix) - Use $SSH_OPTS variable (added to phoenix config) throughout all phoenix scripts instead of hardcoded -o ConnectTimeout=5 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 5733ee3 commit 536f433

8 files changed

Lines changed: 112 additions & 177 deletions

File tree

‎misc/common/runner-lib.sh‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#!/usr/bin/env bash
2+
# Shared GitHub Actions runner management library.
3+
#
4+
# Sourced by site-specific config.sh files (misc/frontier/config.sh,
5+
# misc/phoenix/config.sh). Callers must define ORG, NODES, and SSH_OPTS
6+
# before sourcing this file.
7+
8+
# --- GitHub API ---
9+
10+
# Get a registration token for new runners.
11+
gh_registration_token() {
12+
gh api "orgs/$ORG/actions/runners/registration-token" --jq .token
13+
}
14+
15+
# Get the latest runner binary version.
16+
gh_latest_runner_version() {
17+
gh api repos/actions/runner/releases/latest --jq '.tag_name | ltrimstr("v")'
18+
}
19+
20+
# Remove a runner registration from GitHub.
21+
# Args: $1 = runner ID (numeric, from API)
22+
gh_remove_runner() {
23+
gh api "orgs/$ORG/actions/runners/$1" -X DELETE
24+
}
25+
26+
# --- Local filesystem ---
27+
28+
# Get the GitHub runner name from a .runner config file.
29+
# Args: $1 = runner directory
30+
get_runner_name() {
31+
python3 -c "
32+
import json
33+
d = json.loads(open('$1/.runner').read().lstrip('\ufeff'))
34+
print(d.get('agentName', ''))
35+
" 2>/dev/null
36+
}
37+
38+
# --- Login-node process management ---
39+
40+
# Find PIDs of a runner on a node by matching its CWD.
41+
# (Runner.Listener's command line is just "Runner.Listener run" — no path.)
42+
# Output is filtered to numeric lines only to strip SSH MOTD noise.
43+
# Args: $1 = node, $2 = runner directory
44+
# Prints: space-separated PIDs, or empty.
45+
find_pids() {
46+
ssh $SSH_OPTS "$1" '
47+
for p in $(ps aux | grep Runner.Listener | grep -v grep | awk "{print \$2}"); do
48+
cwd=$(readlink -f /proc/$p/cwd 2>/dev/null || true)
49+
[ "$cwd" = "'"$2"'" ] && echo "$p"
50+
done
51+
' 2>/dev/null | grep -E '^[0-9]+$' | tr '\n' ' ' || true
52+
}
53+
54+
# Find which login node a runner is on.
55+
# Args: $1 = runner directory
56+
# Prints: node hostname, or "offline".
57+
find_node() {
58+
for node in "${NODES[@]}"; do
59+
[ -n "$(find_pids "$node" "$1")" ] && echo "$node" && return
60+
done
61+
echo "offline"
62+
}
63+
64+
# Start a runner on a node.
65+
# Uses a login shell (bash -lc) so site PATH (e.g. SLURM) is available.
66+
# Args: $1 = node, $2 = runner directory
67+
# Returns: 0 if running after start, 1 otherwise.
68+
start_runner() {
69+
local node="$1" dir="$2"
70+
timeout 15 ssh $SSH_OPTS "$node" \
71+
"cd $dir && setsid bash -lc 'nohup ./run.sh >> runner.log 2>&1 < /dev/null &'" \
72+
</dev/null 2>/dev/null || true
73+
sleep 3
74+
[ -n "$(find_pids "$node" "$dir")" ]
75+
}
76+
77+
# Stop a runner on a node (SIGTERM then SIGKILL).
78+
# Args: $1 = node, $2 = runner directory
79+
stop_runner() {
80+
local node="$1" dir="$2" pids
81+
pids=$(find_pids "$node" "$dir")
82+
[ -z "$pids" ] && return 0
83+
for pid in $pids; do
84+
ssh $SSH_OPTS "$node" "kill $pid" 2>/dev/null || true
85+
done
86+
sleep 3
87+
pids=$(find_pids "$node" "$dir")
88+
for pid in $pids; do
89+
ssh $SSH_OPTS "$node" "kill -9 $pid" 2>/dev/null || true
90+
done
91+
sleep 1
92+
}

‎misc/frontier/config.sh‎

Lines changed: 4 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
#!/usr/bin/env bash
22
# Shared configuration for Frontier GitHub Actions runner management.
33
#
4-
# Sourced by all other scripts. Provides constants, GitHub API helpers,
5-
# and login-node process management functions.
4+
# Sourced by all other scripts. Provides Frontier constants, GitHub API
5+
# helpers, and login-node process management functions.
66

77
# --- Frontier constants ---
88
ORG="MFlowCode"
@@ -13,6 +13,8 @@ SHARED_DIR="/lustre/orion/cfd154/proj-shared/runners"
1313

1414
SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes -o ServerAliveInterval=10 -o ServerAliveCountMax=3"
1515

16+
source "$(dirname "${BASH_SOURCE[0]}")/../common/runner-lib.sh"
17+
1618
# --- GitHub API ---
1719

1820
# List Frontier runners from the GitHub API.
@@ -24,22 +26,6 @@ gh_list_runners() {
2426
| \"\(.id) \(.name) \(.status) \(.busy)\""
2527
}
2628

27-
# Get a registration token for new runners.
28-
gh_registration_token() {
29-
gh api "orgs/$ORG/actions/runners/registration-token" --jq .token
30-
}
31-
32-
# Get the latest runner binary version.
33-
gh_latest_runner_version() {
34-
gh api repos/actions/runner/releases/latest --jq '.tag_name | ltrimstr("v")'
35-
}
36-
37-
# Remove a runner registration from GitHub.
38-
# Args: $1 = runner ID (numeric, from API)
39-
gh_remove_runner() {
40-
gh api "orgs/$ORG/actions/runners/$1" -X DELETE
41-
}
42-
4329
# --- Local filesystem ---
4430

4531
# Find all runner directories on shared storage.
@@ -50,71 +36,8 @@ find_runner_dirs() {
5036
done
5137
}
5238

53-
# Get the GitHub runner name from a .runner config file.
54-
# Args: $1 = runner directory
55-
get_runner_name() {
56-
python3 -c "
57-
import json
58-
d = json.loads(open('$1/.runner').read().lstrip('\ufeff'))
59-
print(d.get('agentName', ''))
60-
" 2>/dev/null
61-
}
62-
6339
# --- Login-node process management ---
6440

65-
# Find PIDs of a runner on a node by matching its CWD.
66-
# (Runner.Listener's command line is just "Runner.Listener run" — no path.)
67-
# Frontier SSH prints MOTD to stdout, so output is filtered to numeric lines only.
68-
# Args: $1 = node, $2 = runner directory
69-
# Prints: space-separated PIDs, or empty.
70-
find_pids() {
71-
ssh $SSH_OPTS "$1" '
72-
for p in $(ps aux | grep Runner.Listener | grep -v grep | awk "{print \$2}"); do
73-
cwd=$(readlink -f /proc/$p/cwd 2>/dev/null || true)
74-
[ "$cwd" = "'"$2"'" ] && echo "$p"
75-
done
76-
' 2>/dev/null | grep -E '^[0-9]+$' | tr '\n' ' ' || true
77-
}
78-
79-
# Find which login node a runner is on.
80-
# Args: $1 = runner directory
81-
# Prints: node hostname, or "offline".
82-
find_node() {
83-
for node in "${NODES[@]}"; do
84-
[ -n "$(find_pids "$node" "$1")" ] && echo "$node" && return
85-
done
86-
echo "offline"
87-
}
88-
89-
# Start a runner on a node.
90-
# Args: $1 = node, $2 = runner directory
91-
# Returns: 0 if running after start, 1 otherwise.
92-
start_runner() {
93-
local node="$1" dir="$2"
94-
timeout 15 ssh $SSH_OPTS "$node" \
95-
"cd $dir && setsid nohup ./run.sh >> runner.log 2>&1 < /dev/null &" \
96-
</dev/null 2>/dev/null || true
97-
sleep 3
98-
[ -n "$(find_pids "$node" "$dir")" ]
99-
}
100-
101-
# Stop a runner on a node (SIGTERM then SIGKILL).
102-
# Args: $1 = node, $2 = runner directory
103-
stop_runner() {
104-
local node="$1" dir="$2" pids
105-
pids=$(find_pids "$node" "$dir")
106-
[ -z "$pids" ] && return 0
107-
for pid in $pids; do
108-
ssh $SSH_OPTS "$node" "kill $pid" 2>/dev/null || true
109-
done
110-
sleep 3
111-
pids=$(find_pids "$node" "$dir")
112-
for pid in $pids; do
113-
ssh $SSH_OPTS "$node" "kill -9 $pid" 2>/dev/null || true
114-
done
115-
sleep 1
116-
}
117-
11841
# Sweep all nodes in parallel and update runner.node for any runner
11942
# found running on a different node than recorded. Called at the top of
12043
# every primary script to ensure runner.node always reflects reality,

‎misc/phoenix/check-runners.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ source "$SCRIPT_DIR/config.sh"
1212

1313
for node in "${NODES[@]}"; do
1414
echo "=== $node ==="
15-
ssh -o ConnectTimeout=5 "$node" '
15+
ssh $SSH_OPTS "$node" '
1616
for p in $(ps aux | grep Runner.Listener | grep -v grep | awk "{print \$2}"); do
1717
cwd=$(readlink -f /proc/$p/cwd 2>/dev/null || echo "???")
1818
has_slurm=$(cat /proc/$p/environ 2>/dev/null | tr "\0" "\n" | grep -c /opt/slurm || echo 0)
@@ -27,7 +27,7 @@ for node in "${NODES[@]}"; do
2727
done
2828
' 2>/dev/null || echo " (unreachable)"
2929

30-
rss=$(ssh -o ConnectTimeout=5 "$node" "ps -u \$(whoami) -o rss= 2>/dev/null | awk '{sum+=\$1} END {printf \"%.0f\", sum/1024}'" 2>/dev/null || echo "?")
30+
rss=$(ssh $SSH_OPTS "$node" "ps -u \$(whoami) -o rss= 2>/dev/null | awk '{sum+=\$1} END {printf \"%.0f\", sum/1024}'" 2>/dev/null || echo "?")
3131
[[ "$rss" =~ ^[0-9]+$ ]] || rss=0
3232
echo " --- Total: ${rss} MB / ${CGROUP_LIMIT} MB ($(( CGROUP_LIMIT - rss )) MB free) ---"
3333
echo ""

‎misc/phoenix/config.sh‎

Lines changed: 8 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
#!/bin/bash
22
# Shared configuration for Phoenix GitHub Actions runner management.
33
#
4-
# Sourced by all other scripts. Provides constants, GitHub API helpers,
5-
# and login-node process management functions.
4+
# Sourced by all other scripts. Provides Phoenix constants, GitHub API
5+
# helpers, and login-node process management functions.
66

77
# --- Phoenix constants ---
88
ORG="MFlowCode"
@@ -11,12 +11,16 @@ RUNNER_LABEL="gt"
1111
NODES=(login-phoenix-gnr-1 login-phoenix-gnr-2 login-phoenix-gnr-3)
1212
CGROUP_LIMIT=4096 # per-user memory limit in MB on login nodes
1313

14+
SSH_OPTS="-o ConnectTimeout=5"
15+
1416
# Parent directories containing actions-runner-*/ installations on shared storage.
1517
RUNNER_PARENT_DIRS=(
1618
/storage/scratch1/6/sbryngelson3/mfc-runners
1719
/storage/project/r-sbryngelson3-0/sbryngelson3/mfc-runners-2
1820
)
1921

22+
source "$(dirname "${BASH_SOURCE[0]}")/../common/runner-lib.sh"
23+
2024
# --- GitHub API ---
2125

2226
# List Phoenix runners from the GitHub API.
@@ -28,22 +32,6 @@ gh_list_runners() {
2832
| \"\(.id) \(.name) \(.status) \(.busy)\""
2933
}
3034

31-
# Get a registration token for new runners.
32-
gh_registration_token() {
33-
gh api "orgs/$ORG/actions/runners/registration-token" --jq .token
34-
}
35-
36-
# Get the latest runner binary version.
37-
gh_latest_runner_version() {
38-
gh api repos/actions/runner/releases/latest --jq '.tag_name | ltrimstr("v")'
39-
}
40-
41-
# Remove a runner registration from GitHub.
42-
# Args: $1 = runner ID (numeric, from API)
43-
gh_remove_runner() {
44-
gh api "orgs/$ORG/actions/runners/$1" -X DELETE
45-
}
46-
4735
# --- Local filesystem ---
4836

4937
# Find all runner directories on shared storage.
@@ -56,81 +44,13 @@ find_runner_dirs() {
5644
done
5745
}
5846

59-
# Get the GitHub runner name from a .runner config file.
60-
# Args: $1 = runner directory
61-
get_runner_name() {
62-
python3 -c "
63-
import json
64-
d = json.loads(open('$1/.runner').read().lstrip('\ufeff'))
65-
print(d.get('agentName', ''))
66-
" 2>/dev/null
67-
}
68-
69-
# --- Login-node process management ---
70-
71-
# Find PIDs of a runner on a node by matching its CWD.
72-
# (Runner.Listener's command line is just "Runner.Listener run" — no path.)
73-
# Args: $1 = node, $2 = runner directory
74-
# Prints: space-separated PIDs, or empty.
75-
find_pids() {
76-
ssh -o ConnectTimeout=5 "$1" '
77-
for p in $(ps aux | grep Runner.Listener | grep -v grep | awk "{print \$2}"); do
78-
cwd=$(readlink -f /proc/$p/cwd 2>/dev/null || true)
79-
[ "$cwd" = "'"$2"'" ] && echo -n "$p "
80-
done
81-
' 2>/dev/null || true
82-
}
83-
84-
# Find which login node a runner is on.
85-
# Args: $1 = runner directory
86-
# Prints: node hostname, or "offline".
87-
find_node() {
88-
for node in "${NODES[@]}"; do
89-
[ -n "$(find_pids "$node" "$1")" ] && echo "$node" && return
90-
done
91-
echo "offline"
92-
}
93-
94-
# Start a runner on a node with login shell (for /opt/slurm PATH).
95-
# Args: $1 = node, $2 = runner directory
96-
# Returns: 0 if running after start, 1 otherwise.
97-
start_runner() {
98-
local node="$1" dir="$2"
99-
ssh -o ConnectTimeout=5 "$node" \
100-
"setsid bash -lc 'cd $dir && nohup ./run.sh >> runner-nohup.log 2>&1 &'" \
101-
</dev/null 2>/dev/null &
102-
local ssh_pid=$!
103-
local i; for i in $(seq 1 10); do
104-
kill -0 $ssh_pid 2>/dev/null || break; sleep 1
105-
done
106-
kill $ssh_pid 2>/dev/null || true
107-
wait $ssh_pid 2>/dev/null || true
108-
sleep 3
109-
[ -n "$(find_pids "$node" "$dir")" ]
110-
}
111-
112-
# Stop a runner on a node (SIGTERM then SIGKILL).
113-
# Args: $1 = node, $2 = runner directory
114-
stop_runner() {
115-
local node="$1" dir="$2" pids
116-
pids=$(find_pids "$node" "$dir")
117-
[ -z "$pids" ] && return 0
118-
for pid in $pids; do
119-
ssh -o ConnectTimeout=5 "$node" "kill $pid" 2>/dev/null || true
120-
done
121-
sleep 3
122-
pids=$(find_pids "$node" "$dir")
123-
for pid in $pids; do
124-
ssh -o ConnectTimeout=5 "$node" "kill -9 $pid" 2>/dev/null || true
125-
done
126-
sleep 1
127-
}
47+
# --- Login-node process management (Phoenix-specific) ---
12848

12949
# Check if a runner process has /opt/slurm in PATH.
13050
# Args: $1 = node, $2 = PID
13151
has_slurm() {
13252
local count
133-
count=$(ssh -o ConnectTimeout=5 "$1" \
53+
count=$(ssh $SSH_OPTS "$1" \
13454
"cat /proc/${2%% *}/environ 2>/dev/null | tr '\0' '\n' | grep -c /opt/slurm" \
13555
2>/dev/null || echo 0)
13656
[ "$count" -gt 0 ]

‎misc/phoenix/create-runner.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ if start_runner "$node" "$runner_dir"; then
9292
fi
9393
else
9494
echo " ERROR: Failed to start."
95-
echo " Try: ssh $node 'cd $runner_dir && setsid bash -lc \"nohup ./run.sh >> runner-nohup.log 2>&1 &\"'"
95+
echo " Try: ssh $node 'cd $runner_dir && setsid bash -lc \"nohup ./run.sh >> runner.log 2>&1 < /dev/null &\"'"
9696
fi
9797

9898
echo ""

‎misc/phoenix/list-runners.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ while IFS= read -r dir; do
4444
fi
4545

4646
# Process details (one SSH call)
47-
info=$(ssh -o ConnectTimeout=5 "$node" '
47+
info=$(ssh $SSH_OPTS "$node" '
4848
for p in $(ps aux | grep Runner.Listener | grep -v grep | awk "{print \$2}"); do
4949
cwd=$(readlink -f /proc/$p/cwd 2>/dev/null || true)
5050
if [ "$cwd" = "'"$dir"'" ]; then
@@ -66,8 +66,8 @@ done < <(find_runner_dirs)
6666
echo ""
6767
echo "=== Per-node memory ==="
6868
for node in "${NODES[@]}"; do
69-
count=$(ssh -o ConnectTimeout=5 "$node" "ps aux | grep Runner.Listener | grep -v grep | wc -l" 2>/dev/null || echo 0)
70-
rss=$(ssh -o ConnectTimeout=5 "$node" "ps -u \$(whoami) -o rss= 2>/dev/null | awk '{sum+=\$1} END {printf \"%.0f\", sum/1024}'" 2>/dev/null || echo "?")
69+
count=$(ssh $SSH_OPTS "$node" "ps aux | grep Runner.Listener | grep -v grep | wc -l" 2>/dev/null || echo 0)
70+
rss=$(ssh $SSH_OPTS "$node" "ps -u \$(whoami) -o rss= 2>/dev/null | awk '{sum+=\$1} END {printf \"%.0f\", sum/1024}'" 2>/dev/null || echo "?")
7171
[[ "$rss" =~ ^[0-9]+$ ]] || rss=0
7272
echo " $node: $count runners, ${rss} MB / ${CGROUP_LIMIT} MB ($(( CGROUP_LIMIT - rss )) MB free)"
7373
done

‎misc/phoenix/rebalance-runners.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ for i in "${!dirs[@]}"; do
4343
runner_node[$i]="$node"
4444
if [ "$node" != "offline" ]; then
4545
node_runners[$node]="${node_runners[$node]:-} $i"
46-
worker=$(ssh -o ConnectTimeout=5 "$node" "ps aux | grep Runner.Worker | grep '${dirs[$i]}' | grep -v grep" 2>/dev/null || true)
46+
worker=$(ssh $SSH_OPTS "$node" "ps aux | grep Runner.Worker | grep '${dirs[$i]}' | grep -v grep" 2>/dev/null || true)
4747
[ -n "$worker" ] && runner_busy[$i]=1 || runner_busy[$i]=0
4848
else
4949
runner_busy[$i]=0

0 commit comments

Comments
 (0)