Merge upstream/master: CCE 19.0.0 workaround, cache/build improvements

Merged f95b2c4 (Work around CCE 19.0.0 compiler bugs for Cray+OpenACC
builds) with the following conflict resolutions:

- run_monitored_slurm_job.sh: kept ours (superset with id_file writing
  and UNKNOWN fallback)
- phoenix/submit.sh: kept ours (thin wrapper calling submit-job.sh)
- phoenix/bench.sh: kept our 64-job cap + took upstream RETRY_CLEAN_CMD
- frontier/submit.sh: took upstream (job_type detection block)
- CMakeLists.txt: took upstream (CCE IPA workaround for m_bubbles_EL
  and m_phase_change, comment fix)
- test.yml: took upstream github job changes (simplified cache key,
  Python 3.14, updated build/test commands); kept our clean: false and
  Submit/Monitor split steps for Phoenix self and case-optimization jobs

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: sbryngelson

.github/workflows/frontier/submit.sh

@@ -25,6 +25,13 @@ else
     exit 1
 fi
 
+# Detect job type from submitted script basename
+script_basename="$(basename "$1" .sh)"
+case "$script_basename" in
+    bench*) job_type="bench" ;;
+    *)      job_type="test"  ;;
+esac
+
 if [ "$2" = "cpu" ]; then
     sbatch_device_opts="\
 #SBATCH -n 32                       # Number of cores required"
@@ -37,10 +44,17 @@ else
 fi
 
 # Select SBATCH params based on job type
-sbatch_account="#SBATCH -A CFD154"
-sbatch_time="#SBATCH -t 01:59:00"
-sbatch_partition="#SBATCH -p batch"
-sbatch_extra="#SBATCH --qos=normal"
+if [ "$job_type" = "bench" ]; then
+    sbatch_account="#SBATCH -A CFD154"
+    sbatch_time="#SBATCH -t 01:59:00"
+    sbatch_partition="#SBATCH -p batch"
+    sbatch_extra="#SBATCH --qos=normal"
+else
+    sbatch_account="#SBATCH -A CFD154"
+    sbatch_time="#SBATCH -t 01:59:00"
+    sbatch_partition="#SBATCH -p batch"
+    sbatch_extra="#SBATCH --qos=normal"
+fi
 
 shard_suffix=""
 if [ -n "$4" ]; then

.github/workflows/phoenix/bench.sh

@@ -22,7 +22,7 @@ fi
 rm -rf build
 
 source .github/scripts/retry-build.sh
-retry_build ./mfc.sh build -j $n_jobs $build_opts || exit 1
+RETRY_CLEAN_CMD="./mfc.sh clean" retry_build ./mfc.sh build -j $n_jobs $build_opts || exit 1
 
 ./mfc.sh bench $bench_opts -j $n_jobs -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks
 

.github/workflows/test.yml

@@ -98,6 +98,12 @@ jobs:
       - name: Clone
         uses: actions/checkout@v4
 
+      - name: Restore Build Cache
+        uses: actions/cache@v4
+        with:
+          path: build
+          key: mfc-build-${{ matrix.os }}-${{ matrix.mpi }}-${{ matrix.debug }}-${{ matrix.precision }}-${{ matrix.intel }}-${{ hashFiles('CMakeLists.txt', 'toolchain/dependencies/**', 'toolchain/cmake/**', 'src/**/*.fpp', 'src/**/*.f90') }}
+
       - name: Setup MacOS
         if:   matrix.os == 'macos'
         run:  |
@@ -131,32 +137,20 @@ jobs:
           printenv | sort > /tmp/env_after
           diff /tmp/env_before /tmp/env_after | grep '^>' | sed 's/^> //' >> $GITHUB_ENV
 
-      - name: Get system info for cache key
-        id: sys-info
-        run: |
-          {
-            uname -m
-            cat /proc/cpuinfo 2>/dev/null | grep 'model name' | head -1 || sysctl -n machdep.cpu.brand_string 2>/dev/null || true
-            if command -v ifx &>/dev/null; then ifx --version 2>/dev/null | head -1; else ${FC:-gfortran} --version 2>/dev/null | head -1 || true; fi
-            ${CC:-gcc} --version 2>/dev/null | head -1 || true
-          } | (sha256sum 2>/dev/null || shasum -a 256) | cut -c1-16 > /tmp/sys-hash
-          echo "sys-hash=$(cat /tmp/sys-hash)" >> "$GITHUB_OUTPUT"
-
-      - name: Restore Build Cache
-        uses: actions/cache@v4
+      - name: Set up Python 3.14
+        uses: actions/setup-python@v5
         with:
-          path: build
-          key: mfc-build-${{ matrix.os }}-${{ matrix.mpi }}-${{ matrix.debug }}-${{ matrix.precision }}-${{ matrix.intel }}-${{ steps.sys-info.outputs.sys-hash }}-${{ hashFiles('CMakeLists.txt', 'toolchain/dependencies/**', 'toolchain/cmake/**', 'src/**/*.fpp', 'src/**/*.f90') }}
+          python-version: '3.14'
 
       - name: Build
         run:  |
-          /bin/bash mfc.sh test -v --dry-run -j "$(nproc)" --${{ matrix.debug }} --${{ matrix.mpi }} $PRECISION $TEST_ALL
+          /bin/bash mfc.sh test -v --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} --${{ matrix.precision }} $TEST_ALL
         env:
           TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
-          PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }}
 
       - name: Test
-        run: bash .github/scripts/run-tests-with-retry.sh -v --max-attempts 3 -j "$(nproc)" $TEST_ALL $TEST_PCT
+        run:  |
+          /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) $TEST_ALL $TEST_PCT
         env:
           TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
           TEST_PCT: ${{ matrix.debug == 'debug' && '-% 20' || '' }}
@@ -186,7 +180,7 @@ jobs:
             cluster_name: 'Georgia Tech | Phoenix'
             device: 'cpu'
             interface: 'none'
-          # Frontier (ORNL) — build on login node, GPU tests sharded for batch partition
+          # Frontier (ORNL) — CCE
           - runner:       'frontier'
             cluster:      'frontier'
             cluster_name: 'Oak Ridge | Frontier'

.gitignore

@@ -105,4 +105,8 @@ benchmarks/*.png
 *.avi
 
 **isolation_rules/
-**.supercode/
+**.supercode/
+# CCE stress-test log directories (local testing artifacts)
+cce_*/
+cce_*.log
+run_cce_*.sh

CMakeLists.txt

@@ -230,7 +230,7 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
         CHECK_FORTRAN_COMPILER_FLAG("-march=native" SUPPORTS_MARCH_NATIVE)
         if (SUPPORTS_MARCH_NATIVE)
             add_compile_options($<$<COMPILE_LANGUAGE:Fortran>:-march=native>)
-            # Disable AVX-512 FP16: gfortran ≥12 emits vmovw instructions on
+            # Disable AVX-512 FP16: gfortran >=12 emits vmovw instructions on
             # Granite Rapids CPUs, but binutils <2.38 cannot assemble them.
             # FP16 is unused in MFC's double-precision computations.
             CHECK_FORTRAN_COMPILER_FLAG("-mno-avx512fp16" SUPPORTS_MNO_AVX512FP16)
@@ -408,6 +408,7 @@ HANDLE_SOURCES(simulation ON)
 HANDLE_SOURCES(post_process ON)
 HANDLE_SOURCES(syscheck OFF)
 
+
 # MFC_SETUP_TARGET: Given a target (herein <target>), this macro creates a new
 # executable <target> with the appropriate sources, compiler definitions, and
 # linked libraries (assuming HANDLE_SOURCES was called on <target>).
@@ -644,6 +645,23 @@ if (MFC_SIMULATION)
     MFC_SETUP_TARGET(TARGET  simulation
                      SOURCES "${simulation_SRCs}"
                      MPI FFTW OpenACC OpenMP)
+    # CCE 19.0.0 IPA workaround: two files trigger IPA crashes:
+    #   m_bubbles_EL: castIsValid assertion (InstCombine/foldIntegerTypedPHI)
+    #   m_phase_change: bring_routine_resident SIGSEGV
+    # Disabling IPA per-file avoids the crashes while preserving IPA for
+    # the rest of simulation (needed for thermochem INLINEALWAYS inlining).
+    # Applied to Cray+OpenACC and Cray CPU, but NOT Cray+OpenMP: on OpenMP,
+    # m_thermochem uses !DIR$ INLINEALWAYS (requires IPA), so disabling IPA
+    # for these files breaks thermochem on-device calls. On OpenACC the
+    # pyrometheus patch emits !$acc routine seq instead (no IPA needed).
+    # See PR #1286.
+    if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray" AND NOT MFC_OpenMP)
+        set_source_files_properties(
+            "${CMAKE_BINARY_DIR}/fypp/simulation/m_bubbles_EL.fpp.f90"
+            "${CMAKE_BINARY_DIR}/fypp/simulation/m_phase_change.fpp.f90"
+            PROPERTIES COMPILE_OPTIONS "-Oipa0"
+        )
+    endif()
 endif()
 
 if (MFC_POST_PROCESS)

src/common/include/parallel_macros.fpp

@@ -48,18 +48,46 @@
 
 #:enddef
 
-#:def GPU_ROUTINE(function_name=None, parallelism=None, nohost=False, cray_inline=False, extraAccArgs=None, extraOmpArgs=None)
+#:def GPU_ROUTINE(function_name=None, parallelism=None, nohost=False, cray_inline=False, cray_noinline=False, extraAccArgs=None, extraOmpArgs=None)
     #:assert isinstance(cray_inline, bool)
+    #:assert isinstance(cray_noinline, bool)
+    #:assert not (cray_inline and cray_noinline), "cray_inline and cray_noinline are mutually exclusive"
     #:set acc_directive = ACC_ROUTINE(function_name=function_name, parallelism=parallelism, nohost=nohost, extraAccArgs=extraAccArgs)
     #:set omp_directive = OMP_ROUTINE(function_name=function_name, nohost=nohost, extraOmpArgs=extraOmpArgs)
 
-    #:if cray_inline == True
+    #:if cray_noinline == True
+        #:if not isinstance(function_name, str)
+            #:stop "When using cray_noinline, function name must be given and given as a string"
+        #:endif
+        #:set cray_noinline_directive = ('!DIR$ NOINLINE ' + function_name).strip('\n')
+#ifdef _CRAYFTN
+#if MFC_OpenACC
+        $:acc_directive
+#elif MFC_OpenMP
+        $:omp_directive
+#else
+        $:cray_noinline_directive
+#endif
+        #! On non-Cray CPU builds (no _CRAYFTN, no MFC_OpenACC, no MFC_OpenMP), nothing is
+        #! emitted — intentional, since !DIR$ NOINLINE is a Cray-specific directive.
+#elif MFC_OpenACC
+        $:acc_directive
+#elif MFC_OpenMP
+        $:omp_directive
+#endif
+    #:elif cray_inline == True
         #:if not isinstance(function_name, str)
             #:stop "When inlining for Cray Compiler, function name must be given and given as a string"
         #:endif
         #:set cray_directive = ('!DIR$ INLINEALWAYS ' + function_name).strip('\n')
 #ifdef _CRAYFTN
+#if MFC_OpenACC
+        $:acc_directive
+#elif MFC_OpenMP
+        $:omp_directive
+#else
         $:cray_directive
+#endif
 #elif MFC_OpenACC
         $:acc_directive
 #elif MFC_OpenMP

src/common/m_phase_change.fpp

@@ -104,6 +104,16 @@ contains
         !< Generic loop iterators
         integer :: i, j, k, l
 
+#ifdef _CRAYFTN
+#ifdef MFC_OpenACC
+        ! CCE 19 IPA workaround: prevent bring_routine_resident SIGSEGV
+        !DIR$ NOINLINE s_infinite_pt_relaxation_k
+        !DIR$ NOINLINE s_infinite_ptg_relaxation_k
+        !DIR$ NOINLINE s_correct_partial_densities
+        !DIR$ NOINLINE s_TSat
+#endif
+#endif
+
         ! starting equilibrium solver
         $:GPU_PARALLEL_LOOP(collapse=3, private='[i,j,k,l,p_infOV, p_infpT, p_infSL, sk, hk, gk, ek, rhok,pS, pSOV, pSSL, TS, TSOV, TSatOV, TSatSL, TSSL, rhoe, dynE, rhos, rho, rM, m1, m2, MCT, TvF]')
         do j = 0, m
@@ -296,7 +306,7 @@ contains
         !!  @param TS equilibrium temperature at the interface
     subroutine s_infinite_pt_relaxation_k(j, k, l, MFL, pS, p_infpT, q_cons_vf, rhoe, TS)
         $:GPU_ROUTINE(function_name='s_infinite_pt_relaxation_k', &
-            & parallelism='[seq]', cray_inline=True)
+            & parallelism='[seq]', cray_noinline=True)
 
         ! initializing variables
         integer, intent(in) :: j, k, l, MFL
@@ -411,7 +421,7 @@ contains
         !!  @param TS equilibrium temperature at the interface
     subroutine s_infinite_ptg_relaxation_k(j, k, l, pS, p_infpT, rhoe, q_cons_vf, TS)
         $:GPU_ROUTINE(function_name='s_infinite_ptg_relaxation_k', &
-            & parallelism='[seq]', cray_inline=True)
+            & parallelism='[seq]', cray_noinline=True)
 
         integer, intent(in) :: j, k, l
         real(wp), intent(inout) :: pS
@@ -579,7 +589,8 @@ contains
             InvJac = InvJac/(Jac(1, 1)*Jac(2, 2) - Jac(1, 2)*Jac(2, 1))
 
             ! calculating correction array for Newton's method
-            DeltamP = -1.0_wp*(matmul(InvJac, R2D))
+            DeltamP(1) = -1.0_wp*(InvJac(1, 1)*R2D(1) + InvJac(1, 2)*R2D(2))
+            DeltamP(2) = -1.0_wp*(InvJac(2, 1)*R2D(1) + InvJac(2, 2)*R2D(2))
 
             ! updating two reacting 'masses'. Recall that inert 'masses' do not change during the phase change
             ! liquid
@@ -638,7 +649,7 @@ contains
         !!  @param l generic loop iterator for z direction
     subroutine s_correct_partial_densities(MCT, q_cons_vf, rM, j, k, l)
         $:GPU_ROUTINE(function_name='s_correct_partial_densities', &
-            & parallelism='[seq]', cray_inline=True)
+            & parallelism='[seq]', cray_noinline=True)
 
         !> @name variables for the correction of the reacting partial densities
         !> @{
@@ -689,7 +700,7 @@ contains
         !!  @param TSIn equilibrium Temperature
     elemental subroutine s_TSat(pSat, TSat, TSIn)
         $:GPU_ROUTINE(function_name='s_TSat',parallelism='[seq]', &
-            & cray_inline=True)
+            & cray_noinline=True)
 
         real(wp), intent(in) :: pSat
         real(wp), intent(out) :: TSat
@@ -716,6 +727,12 @@ contains
 
             ! underrelaxation factor
             Om = 1.0e-3_wp
+
+            ! FT must be initialized before the do while condition is evaluated.
+            ! Fortran .or. is not short-circuit: abs(FT) is always evaluated even
+            ! when ns == 0, so FT must have a defined value here.
+            FT = huge(1.0_wp)
+
             do while ((abs(FT) > ptgalpha_eps) .or. (ns == 0))
                 ! increasing counter
                 ns = ns + 1

src/common/m_variables_conversion.fpp

@@ -116,7 +116,7 @@ contains
         !! @param pres_mag Magnetic pressure (optional)
     subroutine s_compute_pressure(energy, alf, dyn_p, pi_inf, gamma, rho, qv, rhoYks, pres, T, stress, mom, G, pres_mag)
         $:GPU_ROUTINE(function_name='s_compute_pressure',parallelism='[seq]', &
-            & cray_inline=True)
+            & cray_noinline=True)
 
         real(stp), intent(in) :: energy, alf
         real(wp), intent(in) :: dyn_p
@@ -326,7 +326,7 @@ contains
                                                           alpha_K, alpha_rho_K, Re_K, &
                                                           G_K, G)
         $:GPU_ROUTINE(function_name='s_convert_species_to_mixture_variables_acc', &
-            & parallelism='[seq]', cray_inline=True)
+            & parallelism='[seq]', cray_noinline=True)
 
         real(wp), intent(out) :: rho_K, gamma_K, pi_inf_K, qv_K
         #:if not MFC_CASE_OPTIMIZATION and USING_AMD
@@ -1335,7 +1335,7 @@ contains
     !>  This subroutine computes partial densities and volume fractions
     subroutine s_compute_species_fraction(q_vf, k, l, r, alpha_rho_K, alpha_K)
         $:GPU_ROUTINE(function_name='s_compute_species_fraction', &
-            & parallelism='[seq]', cray_inline=True)
+            & parallelism='[seq]', cray_noinline=True)
         type(scalar_field), dimension(sys_size), intent(in) :: q_vf
         integer, intent(in) :: k, l, r
         #:if not MFC_CASE_OPTIMIZATION and USING_AMD
@@ -1480,7 +1480,7 @@ contains
     !> @brief Computes the fast magnetosonic wave speed from the sound speed, density, and magnetic field components.
     subroutine s_compute_fast_magnetosonic_speed(rho, c, B, norm, c_fast, h)
         $:GPU_ROUTINE(function_name='s_compute_fast_magnetosonic_speed', &
-            & parallelism='[seq]', cray_inline=True)
+            & parallelism='[seq]', cray_noinline=True)
 
         real(wp), intent(in) :: B(3), rho, c
         real(wp), intent(in) :: h ! only used for relativity

src/simulation/m_bubbles_EL.fpp

@@ -1532,8 +1532,7 @@ contains
         integer(KIND=MPI_OFFSET_KIND) :: disp
         integer :: view
         integer, dimension(2) :: gsizes, lsizes, start_idx_part
-        integer, dimension(num_procs) :: part_order, part_ord_mpi
-        integer, dimension(num_procs) :: proc_bubble_counts
+        integer, allocatable :: proc_bubble_counts(:)
         real(wp), dimension(1:1, 1:lag_io_vars) :: dummy
         dummy = 0._wp
 
@@ -1548,6 +1547,8 @@ contains
 
         if (.not. parallel_io) return
 
+        allocate (proc_bubble_counts(num_procs))
+
         lsizes(1) = bub_id
         lsizes(2) = lag_io_vars
 
@@ -1659,6 +1660,8 @@ contains
             call MPI_FILE_CLOSE(ifile, ierr)
         end if
 
+        deallocate (proc_bubble_counts)
+
 #endif
 
     end subroutine s_write_restart_lag_bubbles

toolchain/mfc/run/input.py

@@ -90,13 +90,44 @@ def generate_fpp(self, target) -> None:
             directive_str = None
 
         # Write the generated Fortran code to the m_thermochem.f90 file with the chosen precision
+        sol = self.get_cantera_solution()
+
+        thermochem_code = pyro.FortranCodeGenerator().generate(
+            "m_thermochem",
+            sol,
+            pyro.CodeGenerationOptions(scalar_type = real_type, directive_offload = directive_str)
+        )
+
+        # CCE 19.0.0 workaround: pyrometheus generates !DIR$ INLINEALWAYS for Cray+ACC
+        # but omits !$acc routine seq, so thermochem routines are not registered as
+        # OpenACC device routines. Replace with plain !$acc routine seq (no INLINEALWAYS).
+        # This patch can be removed once pyrometheus upstream correctly emits !$acc routine seq
+        # for Cray+OpenACC (the broken macro originates in pyrometheus's code generator).
+        if directive_str == 'acc':
+            old_macro = (
+                "#ifdef _CRAYFTN\n#define GPU_ROUTINE(name) !DIR$ INLINEALWAYS name\n"
+                "#else\n#define GPU_ROUTINE(name) !$acc routine seq\n#endif"
+            )
+            new_macro = "#define GPU_ROUTINE(name) !$acc routine seq"
+            patched = thermochem_code.replace(old_macro, new_macro)
+            if patched == thermochem_code:
+                if new_macro in thermochem_code:
+                    pass  # pyrometheus already emits the correct form; no patch needed
+                else:
+                    raise common.MFCException(
+                        "CCE 19.0.0 workaround: pyrometheus output format changed — "
+                        "Cray+ACC GPU_ROUTINE macro patch did not apply. "
+                        "Update the pattern in toolchain/mfc/run/input.py."
+                    )
+            else:
+                cons.print("[yellow]Warning: Applied CCE 19.0.0 workaround patch to pyrometheus-generated "
+                           "m_thermochem.f90 (replaced _CRAYFTN GPU_ROUTINE macro with !$acc routine seq). "
+                           "Remove this patch once pyrometheus emits correct Cray+ACC directives upstream.[/yellow]")
+            thermochem_code = patched
+
         common.file_write(
             os.path.join(modules_dir, "m_thermochem.f90"),
-            pyro.FortranCodeGenerator().generate(
-                "m_thermochem",
-                self.get_cantera_solution(),
-                pyro.CodeGenerationOptions(scalar_type = real_type, directive_offload = directive_str)
-            ),
+            thermochem_code,
             True
         )