Detect stale cached binaries and include install/ in retry cleanup

Phoenix compute nodes may have different CPU architectures, causing
SIGILL when running binaries cached from a different node. After a
successful build, smoke-test syscheck to detect stale installs and
trigger a full rebuild.

Also include build/install in retry cleanup for all clusters. With
per-runner caching there are no concurrent readers sharing the same
cache directory, so clearing install is safe.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sbryngelson

.github/workflows/frontier/build.sh

@@ -50,8 +50,8 @@ while [ $attempt -le $max_attempts ]; do
     fi
 
     if [ $attempt -lt $max_attempts ]; then
-        echo "Build failed on attempt $attempt. Clearing staging and retrying in 30s..."
-        rm -rf build/staging build/lock.yaml
+        echo "Build failed on attempt $attempt. Clearing cache and retrying in 30s..."
+        rm -rf build/staging build/install build/lock.yaml
         sleep 30
     fi
     attempt=$((attempt + 1))

.github/workflows/frontier_amd/build.sh

@@ -50,8 +50,8 @@ while [ $attempt -le $max_attempts ]; do
     fi
 
     if [ $attempt -lt $max_attempts ]; then
-        echo "Build failed on attempt $attempt. Clearing staging and retrying in 30s..."
-        rm -rf build/staging build/lock.yaml
+        echo "Build failed on attempt $attempt. Clearing cache and retrying in 30s..."
+        rm -rf build/staging build/install build/lock.yaml
         sleep 30
     fi
     attempt=$((attempt + 1))

.github/workflows/phoenix/test.sh

@@ -19,12 +19,30 @@ while [ $attempt -le $max_attempts ]; do
     echo "Build attempt $attempt of $max_attempts..."
     if ./mfc.sh test -v --dry-run -j 8 $build_opts; then
         echo "Build succeeded on attempt $attempt."
+
+        # Smoke-test the cached binaries to catch architecture mismatches
+        # (SIGILL from binaries compiled on a different compute node).
+        syscheck_bin=$(find build/install -name syscheck -type f 2>/dev/null | head -1)
+        if [ -n "$syscheck_bin" ] && ! "$syscheck_bin" > /dev/null 2>&1; then
+            echo "WARNING: syscheck binary crashed — cached install is stale."
+            if [ $attempt -lt $max_attempts ]; then
+                echo "Clearing cache and rebuilding..."
+                rm -rf build/staging build/install build/lock.yaml
+                sleep 5
+                attempt=$((attempt + 1))
+                continue
+            else
+                echo "ERROR: syscheck still failing after $max_attempts attempts."
+                exit 1
+            fi
+        fi
+
         break
     fi
 
     if [ $attempt -lt $max_attempts ]; then
-        echo "Build failed on attempt $attempt. Clearing staging and retrying in 30s..."
-        rm -rf build/staging build/lock.yaml
+        echo "Build failed on attempt $attempt. Clearing cache and retrying in 30s..."
+        rm -rf build/staging build/install build/lock.yaml
         sleep 30
     else
         echo "Build failed after $max_attempts attempts."
@@ -43,4 +61,3 @@ if [ "$job_device" = "gpu" ]; then
 fi
 
 ./mfc.sh test -v --max-attempts 3 -a -j $n_test_threads $device_opts -- -c phoenix
-