ci: split frontier_amd case-opt into pre-build + run to avoid 2h wall limit

AMD flang case-opt compilation takes close to the 2h hackathon wall limit,
leaving no time for the run step. Split into two sequential hackathon GPU jobs:
1. Pre-Build: compiles all benchmarks via --dry-run (build only, no execution)
2. Run: skips build (binaries cached), runs and validates benchmarks

Also preserve dependency dirs in prebuild for non-Phoenix clusters (deps are
already built by the Fetch Dependencies step, so only clean staging dirs).

Author: sbryngelson

.github/scripts/prebuild-case-optimization.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
-# Pre-builds all benchmark cases with --case-optimization.
-# No GPU hardware needed — compilation only.
+# Pre-builds all benchmark cases with --case-optimization using --dry-run so
+# binaries are cached before the GPU run job. No simulation is executed.
 # Can run in two modes:
 #   1. Direct (Frontier login nodes): pass cluster/device/interface as args
-#   2. Inside SLURM (Phoenix): uses $job_device/$job_interface from submit-slurm-job.sh
+#   2. Inside SLURM (Phoenix/frontier_amd): uses $job_device/$job_interface
 # Usage: bash prebuild-case-optimization.sh [<cluster> <device> <interface>]
 
 set -e
@@ -22,14 +22,18 @@ case "$cluster" in
     *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;;
 esac
 
-source .github/scripts/clean-build.sh
-clean_build
+# Phoenix starts fresh (no prior dep build); other clusters pre-build deps via
+# build.sh first, so we must preserve them and only clean MFC target staging.
+if [ "$cluster" = "phoenix" ]; then
+    source .github/scripts/clean-build.sh
+    clean_build
+else
+    find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+    find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+fi
 
 . ./mfc.sh load -c "$flag" -m g
 
-# Set GPU build flags from interface — this is always a GPU build.
-# Don't use gpu-opts.sh since $job_device may be "cpu" when submitted
-# to a CPU SLURM partition (no GPU hardware needed for compilation).
 case "$job_interface" in
     acc) gpu_opts="--gpu acc" ;;
     omp) gpu_opts="--gpu mp" ;;
@@ -38,5 +42,5 @@ esac
 
 for case in benchmarks/*/case.py; do
     echo "=== Pre-building: $case ==="
-    ./mfc.sh build -i "$case" --case-optimization $gpu_opts -j 8
+    ./mfc.sh run "$case" --case-optimization $gpu_opts -j 8 --dry-run
 done

.github/scripts/run_case_optimization.sh

@@ -23,13 +23,14 @@ benchmarks=(
 
 # For Frontier/Frontier AMD: deps were fetched on the login node via --deps-only;
 # build case-optimized binaries here on the compute node before running.
-# For Phoenix: prebuild-case-optimization.sh already built everything in a prior SLURM job.
+# For Phoenix and frontier_amd: prebuild-case-optimization.sh already built
+# everything in a prior SLURM job (via --dry-run), so skip the build here.
 #
 # Clean stale MFC target staging before building. On self-hosted CI runners,
 # corrupted intermediate files from a prior failed build (e.g. CCE optcg crash)
 # can persist and poison subsequent builds. Each case-opt config gets its own
 # hash-named staging dir, but install dirs and other artifacts may be stale.
-if [ "$job_cluster" != "phoenix" ]; then
+if [ "$job_cluster" != "phoenix" ] && [ "$job_cluster" != "frontier_amd" ]; then
     # Clean stale MFC target dirs (hash-named) from prior builds, but
     # preserve dependency dirs (hipfort, fftw, etc.) since the compute
     # node has no internet to re-fetch them.

.github/workflows/test.yml

@@ -659,12 +659,16 @@ jobs:
         if:   matrix.cluster == 'phoenix'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh cpu ${{ matrix.interface }} ${{ matrix.cluster }}
 
+      - name: Pre-Build (SLURM)
+        if:   matrix.cluster == 'frontier_amd'
+        run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }}
+
       - name: Build & Run Case-Optimization Tests
-        if:   matrix.cluster != 'phoenix'
+        if:   matrix.cluster != 'phoenix' && matrix.cluster != 'frontier_amd'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }}
 
       - name: Run Case-Optimization Tests
-        if:   matrix.cluster == 'phoenix'
+        if:   matrix.cluster == 'phoenix' || matrix.cluster == 'frontier_amd'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }}
 
       - name: Cancel SLURM Jobs