Merge branch 'master' into feature/non-newtonian-viscosity

Author: sbryngelson

.github/workflows/common/build.sh

@@ -25,9 +25,23 @@ fi
 # source code is built here on the compute node.
 # Phoenix: always start fresh to avoid SIGILL from stale binaries compiled
 # on a different microarchitecture.
+# Frontier: wipe only compiled Fortran staging/install slugs; preserve dep
+# dirs (silo, hdf5, lapack, fftw, hipfort) which were built on the login node
+# and cannot be re-fetched from a compute node (no internet).
 if [ "$job_cluster" = "phoenix" ]; then
     source .github/scripts/clean-build.sh
     clean_build
+elif [ "$job_cluster" = "frontier" ] || [ "$job_cluster" = "frontier_amd" ]; then
+    for _dir in build/staging/ build/install/; do
+        if [ -d "$_dir" ]; then
+            for _sub in "$_dir"*/; do
+                _name=$(basename "$_sub")
+                case "$_name" in silo|hdf5|lapack|fftw|hipfort) continue ;; esac
+                rm -rf "$_sub"
+            done
+        fi
+    done
+    unset _dir _sub _name
 fi
 
 source .github/scripts/retry-build.sh

CMakeLists.txt

@@ -734,26 +734,39 @@ if (MFC_PRE_PROCESS)
                      MPI)
     if(CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
         target_compile_options(pre_process PRIVATE -hfp0)
+        # CCE 19.0.0 IPA ICE: bring_routine_resident SIGSEGV in optcg during
+        # ipa_processing. Nondeterministic — any file can become the crash site
+        # via cross-file inlining. Safe to disable IPA for the whole target
+        # (CPU-only, no GPU device-call requirements). See PR #1286.
+        target_compile_options(pre_process PRIVATE -Oipa0)
     endif()
 endif()
 
 if (MFC_SIMULATION)
     MFC_SETUP_TARGET(TARGET  simulation
                      SOURCES "${simulation_SRCs}"
                      MPI FFTW OpenACC OpenMP)
-    # CCE 19.0.0 IPA workaround: disable interprocedural analysis for files
-    # that trigger compiler crashes during IPA:
-    #   m_bubbles_EL: castIsValid assertion (InstCombine/foldIntegerTypedPHI)
-    #   m_phase_change: bring_routine_resident SIGSEGV
-    # Not applied to Cray+OpenMP because thermochem uses !DIR$ INLINEALWAYS,
-    # which requires IPA to inline device calls. On OpenACC the pyrometheus
-    # patch emits !$acc routine seq instead (no IPA needed). See PR #1286.
-    if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray" AND NOT MFC_OpenMP)
-        set_source_files_properties(
-            "${CMAKE_BINARY_DIR}/fypp/simulation/m_bubbles_EL.fpp.f90"
-            "${CMAKE_BINARY_DIR}/fypp/simulation/m_phase_change.fpp.f90"
-            PROPERTIES COMPILE_OPTIONS "-Oipa0"
-        )
+    # CCE 19.0.0 optcg ICE: IPA inlining triggers two distinct crashes:
+    #   m_bubbles_EL:  castIsValid assertion (InstCombine/foldIntegerTypedPHI)
+    #   m_phase_change, m_mpi_common, m_start_up: bring_routine_resident SIGSEGV
+    # GPU builds: apply -Oipa0 per-file to known crash sites. Whole-target
+    # -Oipa0 is not safe for GPU builds — it breaks cross-file !$acc routine /
+    # !DIR$ INLINEALWAYS device-call registration, causing runtime aborts.
+    # CPU builds: disable IPA for the whole target (safe — no GPU requirements,
+    # and the crash is nondeterministic so any file can become the crash site).
+    # See PR #1286.
+    if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
+        if (MFC_OpenACC OR MFC_OpenMP)
+            set_source_files_properties(
+                "${CMAKE_BINARY_DIR}/fypp/simulation/m_bubbles_EL.fpp.f90"
+                "${CMAKE_BINARY_DIR}/fypp/simulation/m_phase_change.fpp.f90"
+                "${CMAKE_BINARY_DIR}/fypp/simulation/m_mpi_common.fpp.f90"
+                "${CMAKE_BINARY_DIR}/fypp/simulation/m_start_up.fpp.f90"
+                PROPERTIES COMPILE_OPTIONS "-Oipa0"
+            )
+        else()
+            target_compile_options(simulation PRIVATE -Oipa0)
+        endif()
     endif()
 endif()
 

src/pre_process/m_perturbation.fpp

@@ -63,7 +63,7 @@ contains
 
         type(scalar_field), dimension(sys_size), intent(inout) :: q_prim_vf
         integer                                                :: i, j, k
-        real(wp)                                               :: rand_real
+        real(wp)                                               :: rand_real, v_beg
 
         call random_seed()
 
@@ -72,8 +72,9 @@ contains
                 do i = 0, m
                     call random_number(rand_real)
                     rand_real = rand_real*perturb_flow_mag
-                    q_prim_vf(eqn_idx%mom%end)%sf(i, j, k) = rand_real*q_prim_vf(eqn_idx%mom%beg)%sf(i, j, k)
-                    q_prim_vf(eqn_idx%mom%beg)%sf(i, j, k) = (1._wp + rand_real)*q_prim_vf(eqn_idx%mom%beg)%sf(i, j, k)
+                    v_beg = q_prim_vf(eqn_idx%mom%beg)%sf(i, j, k)
+                    q_prim_vf(eqn_idx%mom%beg)%sf(i, j, k) = (1._wp + rand_real)*v_beg
+                    if (num_vels > 1) q_prim_vf(eqn_idx%mom%end)%sf(i, j, k) = rand_real*v_beg
                     if (bubbles_euler) then
                         q_prim_vf(eqn_idx%alf)%sf(i, j, k) = (1._wp + rand_real)*q_prim_vf(eqn_idx%alf)%sf(i, j, k)
                     end if

src/simulation/m_rhs.fpp

@@ -1827,13 +1827,11 @@ contains
                     end if
                 end if
 
-                if (cyl_coord) then
-                    do i = 1, num_dims
-                        @:DEALLOCATE(tau_re_vf(eqn_idx%cont%end + i)%sf)
-                    end do
-                    @:DEALLOCATE(tau_re_vf(eqn_idx%E)%sf)
-                    @:DEALLOCATE(tau_re_vf)
-                end if
+                do i = 1, num_dims
+                    @:DEALLOCATE(tau_Re_vf(eqn_idx%cont%end + i)%sf)
+                end do
+                @:DEALLOCATE(tau_Re_vf(eqn_idx%E)%sf)
+                @:DEALLOCATE(tau_Re_vf)
             end if
             @:DEALLOCATE(dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n)
             @:DEALLOCATE(dqR_prim_dx_n, dqR_prim_dy_n, dqR_prim_dz_n)

toolchain/bootstrap/modules.sh

@@ -145,7 +145,7 @@ if [ "$u_c" '==' 'famd' ]; then
     export CRAY_MPICH_LIB="-L${CRAY_MPICH_PREFIX}/lib \
                         ${CRAY_PMI_POST_LINK_OPTS} \
                         -lmpifort_amd -lmpi_amd -lmpi -lpmi -lpmi2"
-    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${CRAY_LD_LIBRARY_PATH}" 
+    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${CRAY_LD_LIBRARY_PATH}"
     export CMAKE_PREFIX_PATH="${OLCF_AFAR_ROOT}:${CMAKE_PREFIX_PATH}"
     export FC="${OLCF_AFAR_ROOT}/bin/amdflang"