Merge branch 'MFlowCode:master' into master

Author: danieljvickers

.ffmt.toml

@@ -0,0 +1,18 @@
+# MFC Fortran formatting configuration
+# These are the defaults — this file makes them explicit.
+
+indent-width = 4
+keyword-case = "lower"
+normalize-keywords = true
+indent-fypp = true
+
+[whitespace]
+relational = true
+logical = true
+plusminus = true
+multdiv = false
+power = false
+assignment = true
+declaration = true
+comma = true
+slice-colon = false

.github/scripts/run_case_optimization.sh

@@ -13,13 +13,6 @@ if [ "$job_device" = "gpu" ] && [ "$ngpus" -eq 0 ]; then
     ngpus=1
 fi
 
-# Verify the venv Python interpreter exists (created by ./mfc.sh build)
-if [ ! -x build/venv/bin/python3 ]; then
-    echo "ERROR: build/venv/bin/python3 not found."
-    echo "The MFC build venv may not have been created. Was the pre-build step successful?"
-    exit 1
-fi
-
 benchmarks=(
     benchmarks/5eq_rk3_weno3_hllc/case.py
     benchmarks/viscous_weno5_sgb_acoustic/case.py
@@ -28,6 +21,30 @@ benchmarks=(
     benchmarks/igr/case.py
 )
 
+# For Frontier/Frontier AMD: deps were fetched on the login node via --deps-only;
+# build case-optimized binaries here on the compute node before running.
+# For Phoenix: prebuild-case-optimization.sh already built everything in a prior SLURM job.
+#
+# Clean stale MFC target staging before building. On self-hosted CI runners,
+# corrupted intermediate files from a prior failed build (e.g. CCE optcg crash)
+# can persist and poison subsequent builds. Each case-opt config gets its own
+# hash-named staging dir, but install dirs and other artifacts may be stale.
+if [ "$job_cluster" != "phoenix" ]; then
+    # Clean stale MFC target dirs (hash-named) from prior builds, but
+    # preserve dependency dirs (hipfort, fftw, etc.) since the compute
+    # node has no internet to re-fetch them.
+    echo "=== Cleaning stale MFC target staging/install ==="
+    find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+    find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+
+    echo "=== Building case-optimized binaries on compute node ==="
+    for case in "${benchmarks[@]}"; do
+        echo "--- Building: $case ---"
+        ./mfc.sh build -i "$case" --case-optimization $gpu_opts -j 8
+    done
+    echo "=== All case-optimized binaries built ==="
+fi
+
 passed=0
 failed=0
 failed_cases=""
@@ -44,7 +61,7 @@ for case in "${benchmarks[@]}"; do
     rm -rf "$case_dir/D" "$case_dir/p_all" "$case_dir/restart_data"
 
     # Build + run with --case-optimization, small grid, 10 timesteps
-    if ./mfc.sh run "$case" --case-optimization $gpu_opts -n "$ngpus" -j 8 -- --gbpp 1 --steps 10; then
+    if ./mfc.sh run "$case" --case-optimization $gpu_opts -n "$ngpus" -j 8 -c "$job_cluster" -- --gbpp 1 --steps 10; then
         # Validate output
         if build/venv/bin/python3 .github/scripts/check_case_optimization_output.py "$case_dir"; then
             echo "PASS: $case_name"

.github/workflows/bench.yml

@@ -68,42 +68,47 @@ jobs:
             flag: f
             device: gpu
             interface: acc
-            build_script: "bash .github/workflows/frontier/build.sh gpu acc bench"
+            build_script: "bash .github/workflows/frontier/build.sh gpu acc"
           - cluster: frontier
             name: Oak Ridge | Frontier (CCE)
             group: phoenix
             labels: frontier
             flag: f
             device: gpu
             interface: omp
-            build_script: "bash .github/workflows/frontier/build.sh gpu omp bench"
+            build_script: "bash .github/workflows/frontier/build.sh gpu omp"
           - cluster: frontier_amd
             name: Oak Ridge | Frontier (AMD)
             group: phoenix
             labels: frontier
             flag: famd
             device: gpu
             interface: omp
-            build_script: "bash .github/workflows/frontier_amd/build.sh gpu omp bench"
+            build_script: "bash .github/workflows/frontier_amd/build.sh gpu omp"
     continue-on-error: ${{ matrix.cluster == 'frontier' || matrix.cluster == 'frontier_amd' }}
     runs-on:
       group: ${{ matrix.group }}
       labels: ${{ matrix.labels }}
     timeout-minutes: 480
     steps:
+      - name: Clean stale output files
+        run: rm -f *.out
+
       - name: Clone - PR
         uses: actions/checkout@v4
         with:
           path: pr
+          clean: false
 
       - name: Clone - Master
         uses: actions/checkout@v4
         with:
           repository: MFlowCode/MFC
           ref: master
           path: master
+          clean: false
 
-      - name: Setup & Build
+      - name: Fetch Dependencies
         if: matrix.build_script != ''
         timeout-minutes: 150
         run: |

.github/workflows/common/bench.sh

@@ -21,18 +21,18 @@ if [ "$job_cluster" = "phoenix" ]; then
     trap 'rm -rf "$currentdir" || true' EXIT
 fi
 
-# --- Build (if not pre-built on login node) ---
-# Phoenix builds inside SLURM; Frontier pre-builds via build.sh on the login node.
+# --- Build ---
+# Phoenix builds everything inside SLURM (no login-node build step).
+# Frontier/Frontier AMD: deps already fetched on login node via --deps-only;
+# source code is built here on the compute node.
 # Phoenix: always nuke stale builds (heterogeneous compute nodes → ISA mismatch risk).
 if [ "$job_cluster" = "phoenix" ]; then
     source .github/scripts/clean-build.sh
     clean_build
 fi
 
-if [ ! -d "build" ]; then
-    source .github/scripts/retry-build.sh
-    retry_build ./mfc.sh build -j $n_jobs $build_opts || exit 1
-fi
+source .github/scripts/retry-build.sh
+retry_build ./mfc.sh build -j $n_jobs $build_opts || exit 1
 
 # --- Bench cluster flag ---
 if [ "$job_cluster" = "phoenix" ]; then

.github/workflows/common/build.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Build-only script for all clusters.
+# Runs inside a SLURM job via submit-slurm-job.sh.
+# Builds MFC without running tests (--dry-run).
+# Expects env vars: $job_device, $job_interface, $job_shard, $job_cluster
+
+set -euo pipefail
+
+source .github/scripts/gpu-opts.sh
+build_opts="$gpu_opts"
+
+# --- Phoenix TMPDIR setup ---
+if [ "$job_cluster" = "phoenix" ]; then
+    tmpbuild=/storage/project/r-sbryngelson3-0/sbryngelson3/mytmp_build
+    currentdir=$tmpbuild/run-$(( RANDOM % 9000 ))
+    mkdir -p $tmpbuild
+    mkdir -p $currentdir
+    export TMPDIR=$currentdir
+    trap 'rm -rf "$currentdir" || true' EXIT
+fi
+
+# --- Build ---
+# Phoenix builds everything inside SLURM (no login-node build step).
+# Frontier/Frontier AMD: deps already fetched on login node via --deps-only;
+# source code is built here on the compute node.
+# Phoenix: always start fresh to avoid SIGILL from stale binaries compiled
+# on a different microarchitecture.
+if [ "$job_cluster" = "phoenix" ]; then
+    source .github/scripts/clean-build.sh
+    clean_build
+fi
+
+source .github/scripts/retry-build.sh
+
+# Phoenix: smoke-test the syscheck binary to catch architecture mismatches
+# (SIGILL from binaries compiled on a different compute node).
+validate_cmd=""
+if [ "$job_cluster" = "phoenix" ]; then
+    validate_cmd='syscheck_bin=$(find build/install -name syscheck -type f 2>/dev/null | head -1); [ -z "$syscheck_bin" ] || "$syscheck_bin" > /dev/null 2>&1'
+fi
+
+RETRY_VALIDATE_CMD="$validate_cmd" \
+    retry_build ./mfc.sh test -v --dry-run -j 8 $build_opts || exit 1

.github/workflows/common/test.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
-# Unified test script for all clusters.
+# Test-only script for all clusters.
 # Runs inside a SLURM job via submit-slurm-job.sh.
+# Assumes MFC is already built (by a prior build.sh SLURM job).
 # Expects env vars: $job_device, $job_interface, $job_shard, $job_cluster
 
 set -euo pipefail
@@ -9,9 +10,6 @@ source .github/scripts/gpu-opts.sh
 build_opts="$gpu_opts"
 
 # --- Phoenix TMPDIR setup ---
-# Phoenix compute nodes have a small /tmp. With 8 parallel test threads each
-# spawning MPI processes, it fills up and ORTE session dir creation fails.
-# Redirect TMPDIR to project storage, same as bench.sh.
 if [ "$job_cluster" = "phoenix" ]; then
     tmpbuild=/storage/project/r-sbryngelson3-0/sbryngelson3/mytmp_build
     currentdir=$tmpbuild/run-$(( RANDOM % 9000 ))
@@ -21,29 +19,6 @@ if [ "$job_cluster" = "phoenix" ]; then
     trap 'rm -rf "$currentdir" || true' EXIT
 fi
 
-# --- Build (if not pre-built on login node) ---
-# Phoenix builds inside SLURM; Frontier pre-builds via build.sh on the login node.
-# Phoenix builds inside SLURM on heterogeneous compute nodes — always start fresh
-# to avoid SIGILL from stale binaries compiled on a different microarchitecture.
-if [ "$job_cluster" = "phoenix" ]; then
-    source .github/scripts/clean-build.sh
-    clean_build
-fi
-
-if [ ! -d "build" ]; then
-    source .github/scripts/retry-build.sh
-
-    # Phoenix: smoke-test the syscheck binary to catch architecture mismatches
-    # (SIGILL from binaries compiled on a different compute node).
-    validate_cmd=""
-    if [ "$job_cluster" = "phoenix" ]; then
-        validate_cmd='syscheck_bin=$(find build/install -name syscheck -type f 2>/dev/null | head -1); [ -z "$syscheck_bin" ] || "$syscheck_bin" > /dev/null 2>&1'
-    fi
-
-    RETRY_VALIDATE_CMD="$validate_cmd" \
-        retry_build ./mfc.sh test -v --dry-run -j 8 $build_opts || exit 1
-fi
-
 # --- GPU detection and thread count ---
 device_opts=""
 rdma_opts=""
@@ -88,4 +63,4 @@ if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ]; then
     prune_flag="--only-changes"
 fi
 
-./mfc.sh test -v --max-attempts 3 $prune_flag -a -j $n_test_threads $rdma_opts $device_opts $build_opts $shard_opts -- -c $job_cluster
+./mfc.sh test -v --max-attempts 3 --no-build $prune_flag -a -j $n_test_threads $rdma_opts $device_opts $build_opts $shard_opts -- -c $job_cluster

.github/workflows/frontier/build.sh

@@ -14,7 +14,6 @@ esac
 
 job_device=$1
 job_interface=$2
-run_bench=$3
 source .github/scripts/gpu-opts.sh
 build_opts="$gpu_opts"
 
@@ -24,8 +23,4 @@ source .github/scripts/clean-build.sh
 clean_build
 
 source .github/scripts/retry-build.sh
-if [ "$run_bench" == "bench" ]; then
-    retry_build ./mfc.sh build -j 8 $build_opts || exit 1
-else
-    retry_build ./mfc.sh test -v -a --dry-run $([ "$cluster_name" = "frontier" ] && echo "--rdma-mpi") -j 8 $build_opts || exit 1
-fi
+retry_build ./mfc.sh build --deps-only -j 8 $build_opts || exit 1

.github/workflows/test.yml

@@ -162,7 +162,7 @@ jobs:
         os:    ['ubuntu', 'macos']
         mpi:   ['mpi']
         precision: ['']
-        debug: ['debug', 'no-debug']
+        debug: ['reldebug', 'no-debug']
         intel: [true, false]
         exclude:
           - os:    macos
@@ -225,7 +225,7 @@ jobs:
         if:   matrix.os == 'macos'
         run:  |
           brew update
-          brew upgrade
+          brew upgrade || true
           brew install coreutils python fftw hdf5 gcc@15 boost open-mpi lapack
           echo "FC=gfortran-15" >> $GITHUB_ENV
           echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV
@@ -245,14 +245,20 @@ jobs:
           sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
           sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
           sudo apt-get update
-          sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-mpi intel-oneapi-mpi-devel
+          sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel
           # Export only new/changed env vars from setvars.sh.
           # `printenv >> $GITHUB_ENV` dumps all vars including shell internals
           # with special characters that corrupt GITHUB_ENV parsing.
           printenv | sort > /tmp/env_before
           source /opt/intel/oneapi/setvars.sh
           printenv | sort > /tmp/env_after
           diff /tmp/env_before /tmp/env_after | grep '^>' | sed 's/^> //' >> $GITHUB_ENV
+          echo "FC=ifx" >> $GITHUB_ENV
+          echo "CC=icx" >> $GITHUB_ENV
+          echo "CXX=icpx" >> $GITHUB_ENV
+          echo "MPIFC=mpiifx" >> $GITHUB_ENV
+          echo "MPICC=mpiicx" >> $GITHUB_ENV
+          echo "MPICXX=mpiicpx" >> $GITHUB_ENV
 
       - name: Build
         run:  |
@@ -266,7 +272,7 @@ jobs:
           /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) $ONLY_CHANGES $TEST_ALL $TEST_PCT
         env:
           TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
-          TEST_PCT: ${{ matrix.debug == 'debug' && '-% 20' || '' }}
+          TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }}
           ONLY_CHANGES: ${{ github.event_name == 'pull_request' && '--only-changes' || '' }}
 
   self:
@@ -400,11 +406,14 @@ jobs:
             echo "Coverage cache: none available — full test suite will run"
           fi
 
-      - name: Build (login node)
+      - name: Fetch Dependencies
         if:   matrix.cluster != 'phoenix'
         timeout-minutes: 60
         run:  bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
 
+      - name: Build
+        run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
+
       - name: Test
         run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
 
@@ -421,23 +430,29 @@ jobs:
         if:   always()
         id:   log
         run:  |
-          SLUG="test-${{ matrix.device }}-${{ matrix.interface }}"
+          SHARD_SUFFIX=""
           SHARD="${{ matrix.shard }}"
           if [ -n "$SHARD" ]; then
-            SLUG="${SLUG}-$(echo "$SHARD" | sed 's|/|-of-|')"
+            SHARD_SUFFIX="-$(echo "$SHARD" | sed 's|/|-of-|')"
           fi
-          echo "slug=${SLUG}" >> "$GITHUB_OUTPUT"
+          echo "build_slug=build-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT"
+          echo "test_slug=test-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT"
 
       - name: Print Logs
         if:   always()
-        run:  cat ${{ steps.log.outputs.slug }}.out
+        run:  |
+          for f in ${{ steps.log.outputs.build_slug }}.out ${{ steps.log.outputs.test_slug }}.out; do
+            [ -f "$f" ] && echo "=== $f ===" && cat "$f"
+          done
 
       - name: Archive Logs
         uses: actions/upload-artifact@v4
         if:   matrix.cluster != 'phoenix'
         with:
-          name: logs-${{ strategy.job-index }}-${{ steps.log.outputs.slug }}
-          path: ${{ steps.log.outputs.slug }}.out
+          name: logs-${{ strategy.job-index }}-${{ steps.log.outputs.test_slug }}
+          path: |
+            ${{ steps.log.outputs.build_slug }}.out
+            ${{ steps.log.outputs.test_slug }}.out
 
   case-optimization:
     name: "Case Opt | ${{ matrix.cluster_name }} (${{ matrix.device }}-${{ matrix.interface }})"
@@ -486,15 +501,20 @@ jobs:
       - name: Clean stale output files
         run:  rm -f *.out
 
+      - name: Fetch Dependencies
+        if:   matrix.cluster != 'phoenix'
+        run:  bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
+
       - name: Pre-Build (SLURM)
         if:   matrix.cluster == 'phoenix'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh cpu ${{ matrix.interface }} ${{ matrix.cluster }}
 
-      - name: Pre-Build (login node)
+      - name: Build & Run Case-Optimization Tests
         if:   matrix.cluster != 'phoenix'
-        run:  bash .github/scripts/prebuild-case-optimization.sh ${{ matrix.cluster }} ${{ matrix.device }} ${{ matrix.interface }}
+        run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }}
 
       - name: Run Case-Optimization Tests
+        if:   matrix.cluster == 'phoenix'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }}
 
       - name: Cancel SLURM Jobs

.gitignore

@@ -113,3 +113,4 @@ benchmarks/*.png
 cce_*/
 cce_*.log
 run_cce_*.sh
+.ffmt_cache/