Skip to content

Commit 02da98f

Browse files
apstenku123apstenku123
committed
feat(lever3): committed gb10 cleanup+e2e runner scratch/gb10_cleanup_and_e2e.sh
Re-verifies-then-removes the three VERIFIED-SAFE stale items (venv_unsloth, cuda-13.0, cuda-13.1 + dangling gds-13-0/13-1 ldconfig confs) keeping cuda-13.2/cuda-13.3 + cppmega-venv; sudo -n for system toolkits, PRINT+SKIP (never half-delete) if sudo -n absent. Then runs the existing pr6 e2e train_step at the MEASURED-GO config (8L/2L, bs1 seq=4096) via the NVRTC self-healing harness and prints a machine-parseable RESULT_E2E line (step ms, tok/s, peak GB, ratio vs Megatron 3399). RULE #1: fail-loud, no silent fallback, no fabricated tok/s. Not executed here (no GPU on Mac).
1 parent aea9259 commit 02da98f

1 file changed

Lines changed: 307 additions & 0 deletions

File tree

‎scratch/gb10_cleanup_and_e2e.sh‎

Lines changed: 307 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,307 @@
1+
#!/usr/bin/env bash
2+
# ==============================================================================
3+
# gb10_cleanup_and_e2e.sh -- LEVER 3 (cleanup + e2e perf test), RUN ON gb10.
4+
# ==============================================================================
5+
# This script is the committed deliverable for LEVER 3. It is written to run
6+
# ON the GB10 / DGX-Spark box (gx10-9cd4, aarch64 sbsa, sm_121) by the GB10
7+
# phase agent, which is the SOLE GPU owner and runs the levers serially. It is
8+
# NOT executed on the Mac and performs NO destructive op anywhere but gb10.
9+
#
10+
# It does two things, in order:
11+
# (A) CLEANUP -- re-verify-then-remove the three VERIFIED-SAFE stale items
12+
# (/home/dave/venv_unsloth, /usr/local/cuda-13.0, /usr/local/cuda-13.1 +
13+
# their two dangling gds ldconfig confs), capturing freed space, while
14+
# KEEPING cuda-13.2 (pip cu132 toolkit) + cuda-13.3 (NVRTC builtins) +
15+
# the cppmega-venv. System toolkits use `sudo -n`; if `sudo -n` is not
16+
# available it PRINTS the exact sudo command and SKIPS that item -- it
17+
# NEVER half-deletes (RULE #1: fail loud, no silent partial state).
18+
# (B) E2E PERF TEST -- run the existing pr6 e2e train_step runner at the
19+
# MEASURED-GO config (Stage1=8 layers, Stage2=2 layers, bs1 seq=4096 =>
20+
# 4096 tok/step) and print a single machine-parseable RESULT line
21+
# (step ms, tok/s, peak GB, config) plus the Megatron 3399 tok/s compare.
22+
#
23+
# RULE #1 (NO SILENT FALLBACKS): every removal is gated on a fresh re-verify
24+
# (target must NOT be the live `cuda`/`cuda-13` alternative target, must have
25+
# no open handles via lsof, must have no running process referencing it). If a
26+
# re-verify FAILS, the item is SKIPPED with a loud reason and the script exits
27+
# non-zero at the end -- it never deletes a half-checked target, never falls
28+
# back to a degraded delete, and the e2e perf test RAISES (propagates the
29+
# runner's non-zero exit) rather than reporting a fabricated tok/s.
30+
#
31+
# USAGE (on gb10): bash scratch/gb10_cleanup_and_e2e.sh
32+
# env knobs:
33+
# SKIP_CLEANUP=1 -> run only the e2e perf test (no removals)
34+
# SKIP_E2E=1 -> run only the cleanup (no GPU work)
35+
# PR6_STAGE1_LAYERS (default 8), PR6_STAGE2_LAYERS (default 2)
36+
# ==============================================================================
37+
38+
set -u # nounset; we deliberately do NOT `set -e` so a SKIPPED item cannot
39+
# abort the whole script -- we want cleanup + e2e to both be attempted
40+
# and a single explicit non-zero exit at the end if anything was unsafe.
41+
42+
# ---- gb10-local authoritative paths (from LEVER-3 research, verified) --------
43+
CPPMEGA_DIR="/home/dave/source/cppmega_mlx"
44+
PR6_RUNNER="${CPPMEGA_DIR}/scratch/pr6_cuda_e2e_train_step_gb10.py"
45+
VENV_PY="/home/dave/cppmega-venv/bin/python"
46+
TVM_PY="/home/dave/source/tilelang/3rdparty/tvm/python"
47+
TVMFFI_PY="/home/dave/source/tilelang/3rdparty/tvm/3rdparty/tvm-ffi/python"
48+
TVM_LIBRARY_PATH_VAL="/home/dave/source/tilelang/build/lib"
49+
50+
# Cleanup targets (VERIFIED-SAFE in LEVER-3 research):
51+
VENV_UNSLOTH="/home/dave/venv_unsloth"
52+
CUDA_OLD_1="/usr/local/cuda-13.0"
53+
CUDA_OLD_2="/usr/local/cuda-13.1"
54+
GDS_CONF_1="/etc/ld.so.conf.d/gds-13-0.conf"
55+
GDS_CONF_2="/etc/ld.so.conf.d/gds-13-1.conf"
56+
57+
# Must-keep (NEVER touched):
58+
KEEP_CUDA_132="/usr/local/cuda-13.2"
59+
KEEP_CUDA_133="/usr/local/cuda-13.3"
60+
KEEP_VENV="/home/dave/cppmega-venv"
61+
62+
# Megatron baseline for the compare (MEASURED, bf16+FP8/Muon, 16384 tok/step):
63+
MEGATRON_TOKS=3399
64+
65+
CLEANUP_HAD_SKIP=0 # set to 1 if any cleanup item was skipped/unsafe
66+
67+
log() { printf '%s\n' "$*"; }
68+
hdr() { printf '\n========== %s ==========\n' "$*"; }
69+
70+
# du -sh that never aborts the script if the path is already gone.
71+
du_gb() {
72+
local p="$1"
73+
if [ -e "$p" ]; then
74+
du -sb "$p" 2>/dev/null | awk '{printf "%.2f", $1/1073741824.0}'
75+
else
76+
printf '0.00'
77+
fi
78+
}
79+
80+
# ------------------------------------------------------------------------------
81+
# (A) CLEANUP
82+
# ------------------------------------------------------------------------------
83+
cleanup() {
84+
hdr "LEVER 3 CLEANUP (gb10) -- re-verify-then-remove"
85+
86+
if [ "$(uname -m)" != "aarch64" ]; then
87+
log "REFUSE: uname -m = $(uname -m), expected aarch64 (gb10). Not on gb10 -- skipping ALL cleanup (RULE #1: never delete on the wrong host)."
88+
CLEANUP_HAD_SKIP=1
89+
return 0
90+
fi
91+
92+
# Resolve the LIVE cuda alternative target so we never delete the active one.
93+
local live_cuda live_cuda13
94+
live_cuda="$(readlink -f /usr/local/cuda 2>/dev/null || true)"
95+
live_cuda13="$(readlink -f /usr/local/cuda-13 2>/dev/null || true)"
96+
log "live /usr/local/cuda -> ${live_cuda:-<none>}"
97+
log "live /usr/local/cuda-13 -> ${live_cuda13:-<none>}"
98+
99+
# Confirm the must-keep toolkits + venv still exist BEFORE we remove anything.
100+
local k
101+
for k in "$KEEP_CUDA_132" "$KEEP_CUDA_133" "$KEEP_VENV"; do
102+
if [ ! -e "$k" ]; then
103+
log "REFUSE: must-keep '$k' is MISSING -- aborting cleanup (RULE #1: environment is not the expected one)."
104+
CLEANUP_HAD_SKIP=1
105+
return 0
106+
fi
107+
done
108+
# The NVRTC harness HARDCODES cuda-13.3 builtins; assert the SONAME is present.
109+
if [ ! -e "${KEEP_CUDA_133}/targets/sbsa-linux/lib/libnvrtc-builtins.so.13.3" ]; then
110+
log "REFUSE: ${KEEP_CUDA_133}/.../libnvrtc-builtins.so.13.3 MISSING -- aborting cleanup (NVRTC harness needs it)."
111+
CLEANUP_HAD_SKIP=1
112+
return 0
113+
fi
114+
log "must-keep OK: cuda-13.2, cuda-13.3 (+ libnvrtc-builtins.so.13.3), cppmega-venv all present."
115+
116+
# ---- (A1) user-owned venv_unsloth: re-verify no running proc + no open handle
117+
hdr "A1: ${VENV_UNSLOTH}"
118+
local freed_unsloth="0.00"
119+
if [ ! -e "$VENV_UNSLOTH" ]; then
120+
log "already absent -- nothing to do."
121+
elif [ "$live_cuda" = "$VENV_UNSLOTH" ] || [ "$live_cuda13" = "$VENV_UNSLOTH" ]; then
122+
log "SKIP: unexpectedly the live cuda target -- refusing to delete."
123+
CLEANUP_HAD_SKIP=1
124+
elif ps aux | grep -F "$VENV_UNSLOTH" | grep -vw grep | grep -q .; then
125+
log "SKIP: a running process references ${VENV_UNSLOTH} (RULE #1: do not delete an in-use env):"
126+
ps aux | grep -F "$VENV_UNSLOTH" | grep -vw grep
127+
CLEANUP_HAD_SKIP=1
128+
elif command -v lsof >/dev/null 2>&1 && lsof +D "$VENV_UNSLOTH" >/dev/null 2>&1; then
129+
log "SKIP: lsof reports open handles under ${VENV_UNSLOTH} -- refusing to delete."
130+
CLEANUP_HAD_SKIP=1
131+
else
132+
freed_unsloth="$(du_gb "$VENV_UNSLOTH")"
133+
log "re-verify OK (no proc, no open handle, not live-cuda). Removing (user-owned)..."
134+
if rm -rf "$VENV_UNSLOTH"; then
135+
log "REMOVED ${VENV_UNSLOTH} (freed ${freed_unsloth} GB)."
136+
else
137+
log "ERROR: rm -rf ${VENV_UNSLOTH} failed -- left as-is."
138+
CLEANUP_HAD_SKIP=1
139+
freed_unsloth="0.00"
140+
fi
141+
fi
142+
143+
# ---- (A2) system toolkits cuda-13.0 / cuda-13.1 + their gds confs (sudo -n)
144+
hdr "A2: ${CUDA_OLD_1} ${CUDA_OLD_2} (+ gds confs)"
145+
local have_sudo_n=0
146+
if sudo -n true 2>/dev/null; then
147+
have_sudo_n=1
148+
log "sudo -n: AVAILABLE (unattended system removal enabled)."
149+
else
150+
log "sudo -n: NOT available -- will PRINT exact sudo commands and SKIP (RULE #1: never half-delete)."
151+
fi
152+
153+
# Pre-measure sizes (for the freed-space report) before any removal.
154+
local sz_old1 sz_old2
155+
sz_old1="$(du_gb "$CUDA_OLD_1")"
156+
sz_old2="$(du_gb "$CUDA_OLD_2")"
157+
158+
# Per-toolkit re-verify: must NOT be the live cuda target, no proc ref, no open handle.
159+
local unsafe_sys=0 t
160+
for t in "$CUDA_OLD_1" "$CUDA_OLD_2"; do
161+
[ -e "$t" ] || continue
162+
if [ "$live_cuda" = "$t" ] || [ "$live_cuda13" = "$t" ]; then
163+
log "SKIP ALL system removal: ${t} IS the live cuda alternative target -- refusing."
164+
unsafe_sys=1
165+
fi
166+
if ps aux | grep -F "$t" | grep -vw grep | grep -q .; then
167+
log "SKIP ALL system removal: a running process references ${t}:"
168+
ps aux | grep -F "$t" | grep -vw grep
169+
unsafe_sys=1
170+
fi
171+
if command -v lsof >/dev/null 2>&1 && lsof +D "$t" >/dev/null 2>&1; then
172+
log "SKIP ALL system removal: lsof reports open handles under ${t} -- refusing."
173+
unsafe_sys=1
174+
fi
175+
done
176+
177+
local freed_sys="0.00"
178+
local sudo_cmd="sudo rm -rf ${CUDA_OLD_1} ${CUDA_OLD_2}; sudo rm -f ${GDS_CONF_1} ${GDS_CONF_2}; sudo ldconfig"
179+
if [ "$unsafe_sys" -ne 0 ]; then
180+
log "System toolkits NOT removed (a re-verify failed above). RULE #1: failing loud, not deleting."
181+
CLEANUP_HAD_SKIP=1
182+
elif [ "$have_sudo_n" -ne 1 ]; then
183+
log "ACTION REQUIRED (run with real sudo): ${sudo_cmd}"
184+
CLEANUP_HAD_SKIP=1
185+
else
186+
log "re-verify OK for system toolkits. Removing with sudo -n..."
187+
local ok=1
188+
sudo -n rm -rf "$CUDA_OLD_1" "$CUDA_OLD_2" || ok=0
189+
sudo -n rm -f "$GDS_CONF_1" "$GDS_CONF_2" || ok=0
190+
sudo -n ldconfig || ok=0
191+
if [ "$ok" -eq 1 ]; then
192+
freed_sys="$(awk -v a="$sz_old1" -v b="$sz_old2" 'BEGIN{printf "%.2f", a+b}')"
193+
log "REMOVED ${CUDA_OLD_1} (${sz_old1} GB) + ${CUDA_OLD_2} (${sz_old2} GB) + gds-13-0/13-1.conf; ldconfig refreshed."
194+
else
195+
log "ERROR: a sudo -n step failed. Re-run with real sudo: ${sudo_cmd}"
196+
CLEANUP_HAD_SKIP=1
197+
fi
198+
fi
199+
200+
# ---- cleanup summary
201+
hdr "CLEANUP SUMMARY"
202+
local total
203+
total="$(awk -v u="$freed_unsloth" -v s="$freed_sys" 'BEGIN{printf "%.2f", u+s}')"
204+
log "RESULT_CLEANUP freed_unsloth_gb=${freed_unsloth} freed_system_gb=${freed_sys} freed_total_gb=${total} kept=cuda-13.2,cuda-13.3,cppmega-venv had_skip=${CLEANUP_HAD_SKIP}"
205+
if command -v df >/dev/null 2>&1; then df -h / | tail -1; fi
206+
}
207+
208+
# ------------------------------------------------------------------------------
209+
# (B) E2E PERF TEST (GPU) -- run the existing pr6 runner, parse, print RESULT
210+
# ------------------------------------------------------------------------------
211+
e2e_perf_test() {
212+
hdr "LEVER 3 E2E PERF TEST (gb10 CUDA) -- pr6 train_step"
213+
214+
if [ ! -f "$PR6_RUNNER" ]; then
215+
log "FAIL-LOUD: pr6 runner not found at ${PR6_RUNNER} -- cannot run e2e perf test."
216+
return 3
217+
fi
218+
if [ ! -x "$VENV_PY" ]; then
219+
log "FAIL-LOUD: cppmega-venv python not found/executable at ${VENV_PY}."
220+
return 3
221+
fi
222+
223+
local s1="${PR6_STAGE1_LAYERS:-8}"
224+
local s2="${PR6_STAGE2_LAYERS:-2}"
225+
local ts logf
226+
ts="$(date +%Y%m%d_%H%M%S)"
227+
logf="/tmp/pr6_e2e_${ts}.log"
228+
229+
log "config: PR6_STAGE1_LAYERS=${s1} PR6_STAGE2_LAYERS=${s2} (bs1, seq=4096 => 4096 tok/step)"
230+
log "log -> ${logf}"
231+
232+
# The NVRTC harness must be applied BEFORE torch/TE import. The pr6 runner
233+
# imports tvm directly; we apply the builtins path via a one-line preload of
234+
# the committed harness so the in-process import order is correct (RULE #1:
235+
# use the real self-healing harness, not an ad-hoc LD_LIBRARY_PATH hack).
236+
PYTHONPATH="${CPPMEGA_DIR}:${TVM_PY}:${TVMFFI_PY}" \
237+
TVM_LIBRARY_PATH="${TVM_LIBRARY_PATH_VAL}" \
238+
PR6_STAGE1_LAYERS="${s1}" PR6_STAGE2_LAYERS="${s2}" \
239+
"$VENV_PY" -c "import cppmega_mlx._gb10_nvrtc_env as e; e.ensure_nvrtc_builtins_path(); import runpy, sys; sys.argv=['pr6']; runpy.run_path('${PR6_RUNNER}', run_name='__main__')" \
240+
2>&1 | tee "$logf"
241+
local rc="${PIPESTATUS[0]}"
242+
243+
if [ "$rc" -ne 0 ]; then
244+
log "FAIL-LOUD: pr6 e2e runner exited ${rc} (see ${logf}). NOT reporting a fabricated tok/s."
245+
return "$rc"
246+
fi
247+
248+
# ---- parse the runner's MEASURED output (do not fabricate). ----
249+
# Stage 1 line: " compile=...s run=<run_s>s" and the SUMMARY line:
250+
# "Stage 1 (...): RUNS=yes loss=... ... measured-free-delta=<peak> GB"
251+
local run_s peak_gb
252+
run_s="$(grep -Eo 'run=[0-9.]+s' "$logf" | head -1 | sed -E 's/run=([0-9.]+)s/\1/')"
253+
peak_gb="$(grep -E 'Stage 1 .*measured-free-delta=' "$logf" | head -1 \
254+
| grep -Eo 'measured-free-delta=[0-9.]+' | sed -E 's/.*=//')"
255+
if [ -z "$peak_gb" ]; then
256+
peak_gb="$(grep -Eo 'MEASURED peak delta=[0-9.]+ GB' "$logf" | head -1 \
257+
| grep -Eo '[0-9.]+' | head -1)"
258+
fi
259+
260+
if [ -z "$run_s" ] || [ -z "$peak_gb" ]; then
261+
log "FAIL-LOUD: could not parse run_s ('${run_s}') or peak_gb ('${peak_gb}') from ${logf}. NOT fabricating a result."
262+
return 4
263+
fi
264+
265+
# tok/s = 4096 tok/step (bs1, seq=4096) / mean-step-s ; step_ms = run_s*1000.
266+
local step_ms toks ratio
267+
step_ms="$(awk -v r="$run_s" 'BEGIN{printf "%.1f", r*1000.0}')"
268+
toks="$(awk -v r="$run_s" 'BEGIN{printf "%.1f", 4096.0/r}')"
269+
ratio="$(awk -v t="$toks" -v m="$MEGATRON_TOKS" 'BEGIN{printf "%.3f", t/m}')"
270+
271+
hdr "E2E RESULT (machine-parseable)"
272+
# ONE line, key=value, easy for the GB10 phase to grep.
273+
log "RESULT_E2E config=stage1_${s1}L_stage2_${s2}L_bs1_seq4096 tok_per_step=4096 step_ms=${step_ms} tok_s=${toks} peak_gb=${peak_gb} megatron_tok_s=${MEGATRON_TOKS} ratio_vs_megatron=${ratio} kind=MEASURED note=bs1@${s1}L_GO_config;bs4/28L/16384-tok-step_would_be_EXTRAPOLATION(tok/s_batch-invariant,SSD-scan_launch-bound)"
274+
return 0
275+
}
276+
277+
# ------------------------------------------------------------------------------
278+
# main
279+
# ------------------------------------------------------------------------------
280+
main() {
281+
hdr "gb10_cleanup_and_e2e.sh host=$(hostname 2>/dev/null) arch=$(uname -m) date=$(date)"
282+
283+
local e2e_rc=0
284+
if [ "${SKIP_CLEANUP:-0}" = "1" ]; then
285+
log "SKIP_CLEANUP=1 -> skipping cleanup."
286+
else
287+
cleanup
288+
fi
289+
290+
if [ "${SKIP_E2E:-0}" = "1" ]; then
291+
log "SKIP_E2E=1 -> skipping e2e perf test."
292+
else
293+
e2e_perf_test
294+
e2e_rc=$?
295+
fi
296+
297+
hdr "DONE"
298+
log "cleanup_had_skip=${CLEANUP_HAD_SKIP} e2e_rc=${e2e_rc}"
299+
# Non-zero if a cleanup item was unsafe/skipped OR the e2e perf test failed
300+
# (RULE #1: surface it; never exit 0 over a half-done/failed run).
301+
if [ "$CLEANUP_HAD_SKIP" -ne 0 ] || [ "$e2e_rc" -ne 0 ]; then
302+
return 1
303+
fi
304+
return 0
305+
}
306+
307+
main "$@"

0 commit comments

Comments
 (0)