Use nick-fields/retry for Frontier builds; reduce -j to 4

Move build retry logic from shell scripts to GHA using nick-fields/retry
with 60s backoff between attempts. This gives better visibility into
retries and lets login node memory pressure subside between attempts.

Also reduce build parallelism from -j 8 to -j 4 to lower peak memory
on shared Frontier login nodes, and remove the outdated Node 16 version
overrides from self-hosted runner env.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sbryngelson

.github/workflows/bench.yml

@@ -86,9 +86,6 @@ jobs:
       group: ${{ matrix.group }}
       labels: ${{ matrix.labels }}
     timeout-minutes: 480
-    env:
-      ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     steps:
       - name: Clone - PR
         uses: actions/checkout@v4
@@ -104,10 +101,14 @@ jobs:
 
       - name: Setup & Build
         if: matrix.build_script != ''
-        run: |
-          (cd pr     && ${{ matrix.build_script }}) &
-          (cd master && ${{ matrix.build_script }}) &
-          wait %1 && wait %2
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 3
+          retry_wait_seconds: 60
+          command: |
+            (cd pr     && ${{ matrix.build_script }}) &
+            (cd master && ${{ matrix.build_script }}) &
+            wait %1 && wait %2
 
       - name: Bench (Master v. PR)
         run: bash pr/.github/scripts/run_parallel_benchmarks.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }}

.github/workflows/frontier/build.sh

@@ -18,39 +18,10 @@ fi
 
 . ./mfc.sh load -c f -m g
 
-max_attempts=3
-attempt=1
-while [ $attempt -le $max_attempts ]; do
-    echo "Build attempt $attempt of $max_attempts..."
-    if [ "$run_bench" == "bench" ]; then
-        build_cmd_ok=true
-        for dir in benchmarks/*/; do
-            dirname=$(basename "$dir")
-            if ! ./mfc.sh run -v "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts; then
-                build_cmd_ok=false
-                break
-            fi
-        done
-    else
-        if ./mfc.sh test -v -a --dry-run --rdma-mpi -j 8 $build_opts; then
-            build_cmd_ok=true
-        else
-            build_cmd_ok=false
-        fi
-    fi
-
-    if [ "$build_cmd_ok" = true ]; then
-        echo "Build succeeded on attempt $attempt."
-        exit 0
-    fi
-
-    if [ $attempt -lt $max_attempts ]; then
-        echo "Build failed on attempt $attempt. Cleaning and retrying in 30s..."
-        ./mfc.sh clean
-        sleep 30
-    fi
-    attempt=$((attempt + 1))
-done
-
-echo "Build failed after $max_attempts attempts."
-exit 1
+if [ "$run_bench" == "bench" ]; then
+    for dir in benchmarks/*/; do
+        ./mfc.sh run -v "$dir/case.py" --case-optimization -j 4 --dry-run $build_opts
+    done
+else
+    ./mfc.sh test -v -a --dry-run --rdma-mpi -j 4 $build_opts
+fi

.github/workflows/frontier_amd/build.sh

@@ -18,39 +18,10 @@ fi
 
 . ./mfc.sh load -c famd -m g
 
-max_attempts=3
-attempt=1
-while [ $attempt -le $max_attempts ]; do
-    echo "Build attempt $attempt of $max_attempts..."
-    if [ "$run_bench" == "bench" ]; then
-        build_cmd_ok=true
-        for dir in benchmarks/*/; do
-            dirname=$(basename "$dir")
-            if ! ./mfc.sh run -v "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts; then
-                build_cmd_ok=false
-                break
-            fi
-        done
-    else
-        if ./mfc.sh test -v -a --dry-run -j 8 $build_opts; then
-            build_cmd_ok=true
-        else
-            build_cmd_ok=false
-        fi
-    fi
-
-    if [ "$build_cmd_ok" = true ]; then
-        echo "Build succeeded on attempt $attempt."
-        exit 0
-    fi
-
-    if [ $attempt -lt $max_attempts ]; then
-        echo "Build failed on attempt $attempt. Cleaning and retrying in 30s..."
-        ./mfc.sh clean
-        sleep 30
-    fi
-    attempt=$((attempt + 1))
-done
-
-echo "Build failed after $max_attempts attempts."
-exit 1
+if [ "$run_bench" == "bench" ]; then
+    for dir in benchmarks/*/; do
+        ./mfc.sh run -v "$dir/case.py" --case-optimization -j 4 --dry-run $build_opts
+    done
+else
+    ./mfc.sh test -v -a --dry-run -j 4 $build_opts
+fi

.github/workflows/test.yml

@@ -245,15 +245,17 @@ jobs:
       labels: ${{ matrix.runner }}
     env:
       NODE_OPTIONS: ${{ matrix.cluster == 'phoenix' && '--max-old-space-size=2048' || '' }}
-      ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     steps:
       - name: Clone
         uses: actions/checkout@v4
 
       - name: Build
         if:   matrix.cluster != 'phoenix'
-        run:  bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 3
+          retry_wait_seconds: 60
+          command: bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Test
         run:  bash .github/workflows/${{ matrix.cluster }}/submit.sh .github/workflows/${{ matrix.cluster }}/test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.shard }}