ci: use rename trick to handle stale NFS file handles on build cleanup

Replace 'rm -rf build || true' with 'mv build build.stale.$$' followed
by a background best-effort delete. mv is a metadata-only operation that
succeeds even when files have stale NFS handles, guaranteeing build/ is
gone before the fresh build starts. Old stale trees are cleaned up
opportunistically in the background.

This also simplifies test.sh and bench.sh: since mv reliably removes
build/, the Phoenix-specific override in the build condition is no longer
needed — the plain '[ ! -d build ]' check is sufficient again.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: sbryngelson

.github/scripts/prebuild-case-optimization.sh

@@ -22,7 +22,8 @@ case "$cluster" in
     *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;;
 esac
 
-rm -rf build 2>/dev/null || true
+mv build build.stale.$$ 2>/dev/null || true
+rm -rf build.stale.* 2>/dev/null & disown
 
 . ./mfc.sh load -c "$flag" -m g
 

.github/workflows/common/bench.sh

@@ -25,11 +25,13 @@ fi
 # Phoenix builds inside SLURM; Frontier pre-builds via build.sh on the login node.
 # Phoenix: always nuke stale builds (heterogeneous compute nodes → ISA mismatch risk).
 if [ "$job_cluster" = "phoenix" ]; then
-    # Suppress stale NFS file handle errors — those files are inaccessible anyway.
-    rm -rf build 2>/dev/null || true
+    # Rename instead of rm: mv is a metadata-only op that succeeds even with stale
+    # NFS file handles. Delete the old tree in the background (best-effort).
+    mv build build.stale.$$ 2>/dev/null || true
+    rm -rf build.stale.* 2>/dev/null & disown
 fi
 
-if [ "$job_cluster" = "phoenix" ] || [ ! -d "build" ]; then
+if [ ! -d "build" ]; then
     source .github/scripts/retry-build.sh
     retry_build ./mfc.sh build -j $n_jobs $build_opts || exit 1
 fi

.github/workflows/common/test.sh

@@ -13,13 +13,13 @@ build_opts="$gpu_opts"
 # Phoenix builds inside SLURM on heterogeneous compute nodes — always start fresh
 # to avoid SIGILL from stale binaries compiled on a different microarchitecture.
 if [ "$job_cluster" = "phoenix" ]; then
-    # Suppress stale NFS file handle errors — those files are inaccessible anyway.
-    rm -rf build 2>/dev/null || true
+    # Rename instead of rm: mv is a metadata-only op that succeeds even with stale
+    # NFS file handles. Delete the old tree in the background (best-effort).
+    mv build build.stale.$$ 2>/dev/null || true
+    rm -rf build.stale.* 2>/dev/null & disown
 fi
 
-# Phoenix must always rebuild (heterogeneous compute nodes → ISA mismatch risk),
-# even if rm above left a partial build/ directory behind.
-if [ "$job_cluster" = "phoenix" ] || [ ! -d "build" ]; then
+if [ ! -d "build" ]; then
     source .github/scripts/retry-build.sh
 
     # Phoenix: smoke-test the syscheck binary to catch architecture mismatches

.github/workflows/frontier/build.sh

@@ -20,8 +20,8 @@ build_opts="$gpu_opts"
 
 . ./mfc.sh load -c $compiler_flag -m $([ "$job_device" = "gpu" ] && echo "g" || echo "c")
 
-# Suppress stale NFS file handle errors — those files are inaccessible anyway.
-rm -rf build 2>/dev/null || true
+mv build build.stale.$$ 2>/dev/null || true
+rm -rf build.stale.* 2>/dev/null & disown
 
 source .github/scripts/retry-build.sh
 if [ "$run_bench" == "bench" ]; then