merge: resolve conflicts with master (PR #1433 workaround removal)

- m_thinc.fpp: take master's extended Fypp for-loop tuple (STENCIL_VAR,
  COORDS, X_BND/Y_BND/Z_BND), update CC_PRI x_cc/y_cc/z_cc -> x%cc/y%cc/z%cc
- m_rhs.fpp: take master's drop of 'dummy' workaround condition, keep bc%y%beg naming
- m_riemann_solvers.fpp: take master's unified Re_avg_rsx_vf indexing (j,k,l)
  for all cylindrical faces, update y_cb/y_cc -> y%cb/y%cc

Author: sbryngelson

.github/scripts/prebuild-case-optimization.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
-# Pre-builds all benchmark cases with --case-optimization.
-# No GPU hardware needed — compilation only.
+# Pre-builds all benchmark cases with --case-optimization using --dry-run so
+# binaries are cached before the GPU run job. No simulation is executed.
 # Can run in two modes:
 #   1. Direct (Frontier login nodes): pass cluster/device/interface as args
-#   2. Inside SLURM (Phoenix): uses $job_device/$job_interface from submit-slurm-job.sh
+#   2. Inside SLURM (Phoenix/frontier_amd): uses $job_device/$job_interface
 # Usage: bash prebuild-case-optimization.sh [<cluster> <device> <interface>]
 
 set -e
@@ -22,14 +22,18 @@ case "$cluster" in
     *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;;
 esac
 
-source .github/scripts/clean-build.sh
-clean_build
+# Phoenix starts fresh (no prior dep build); other clusters pre-build deps via
+# build.sh first, so we must preserve them and only clean MFC target staging.
+if [ "$cluster" = "phoenix" ]; then
+    source .github/scripts/clean-build.sh
+    clean_build
+else
+    find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+    find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+fi
 
 . ./mfc.sh load -c "$flag" -m g
 
-# Set GPU build flags from interface — this is always a GPU build.
-# Don't use gpu-opts.sh since $job_device may be "cpu" when submitted
-# to a CPU SLURM partition (no GPU hardware needed for compilation).
 case "$job_interface" in
     acc) gpu_opts="--gpu acc" ;;
     omp) gpu_opts="--gpu mp" ;;
@@ -38,5 +42,5 @@ esac
 
 for case in benchmarks/*/case.py; do
     echo "=== Pre-building: $case ==="
-    ./mfc.sh build -i "$case" --case-optimization $gpu_opts -j 8
+    ./mfc.sh run "$case" --case-optimization $gpu_opts -j 8 --dry-run
 done

.github/scripts/run_case_optimization.sh

@@ -23,13 +23,14 @@ benchmarks=(
 
 # For Frontier/Frontier AMD: deps were fetched on the login node via --deps-only;
 # build case-optimized binaries here on the compute node before running.
-# For Phoenix: prebuild-case-optimization.sh already built everything in a prior SLURM job.
+# For Phoenix and frontier_amd: prebuild-case-optimization.sh already built
+# everything in a prior SLURM job (via --dry-run), so skip the build here.
 #
 # Clean stale MFC target staging before building. On self-hosted CI runners,
 # corrupted intermediate files from a prior failed build (e.g. CCE optcg crash)
 # can persist and poison subsequent builds. Each case-opt config gets its own
 # hash-named staging dir, but install dirs and other artifacts may be stale.
-if [ "$job_cluster" != "phoenix" ]; then
+if [ "$job_cluster" != "phoenix" ] && [ "$job_cluster" != "frontier_amd" ]; then
     # Clean stale MFC target dirs (hash-named) from prior builds, but
     # preserve dependency dirs (hipfort, fftw, etc.) since the compute
     # node has no internet to re-fetch them.

.github/workflows/test.yml

@@ -659,12 +659,16 @@ jobs:
         if:   matrix.cluster == 'phoenix'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh cpu ${{ matrix.interface }} ${{ matrix.cluster }}
 
+      - name: Pre-Build (SLURM)
+        if:   matrix.cluster == 'frontier_amd'
+        run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }}
+
       - name: Build & Run Case-Optimization Tests
-        if:   matrix.cluster != 'phoenix'
+        if:   matrix.cluster != 'phoenix' && matrix.cluster != 'frontier_amd'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }}
 
       - name: Run Case-Optimization Tests
-        if:   matrix.cluster == 'phoenix'
+        if:   matrix.cluster == 'phoenix' || matrix.cluster == 'frontier_amd'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }}
 
       - name: Cancel SLURM Jobs

CMakeLists.txt

@@ -644,7 +644,7 @@ exit 0
                     target_link_options(${a_target} PRIVATE -fopenmp)
                 elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
                     target_compile_options(${a_target} PRIVATE -fopenmp --offload-arch=gfx90a -fopenmp-target-fast -fopenmp-assume-threads-oversubscription -fopenmp-assume-teams-oversubscription)
-                    target_link_options(${a_target} PRIVATE -fopenmp --offload-arch=gfx90a)
+                    target_link_options(${a_target} PRIVATE -fopenmp --offload-arch=gfx90a -flto-partitions=${MFC_BUILD_JOBS})
                 endif()
             endif()
 
@@ -710,14 +710,15 @@ exit 0
                         PRIVATE -DFRONTIER_UNIFIED)
                 endif()
 
-                find_library(HIP_LIB amdhip64
+		        find_library(HIP_LIB amdhip64
                     HINTS "$ENV{OLCF_AFAR_ROOT}/lib" REQUIRED)
                 find_library(HIPFORT_AMDGCN_LIB hipfort-amdgcn
                     HINTS "$ENV{OLCF_AFAR_ROOT}/lib" REQUIRED)
                 target_include_directories(${a_target} PRIVATE
                     "$ENV{OLCF_AFAR_ROOT}/include/hipfort/amdgcn")
                 target_link_libraries(${a_target} PRIVATE
-                    ${HIP_LIB} ${HIPFORT_AMDGCN_LIB} flang_rt.hostdevice)
+                    ${HIP_LIB} ${HIPFORT_AMDGCN_LIB})
+
             endif()
         elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
             target_compile_options(${a_target} PRIVATE "SHELL:-h noacc" "SHELL:-x acc")
@@ -790,6 +791,12 @@ if (MFC_POST_PROCESS)
 
     # -O0 is in response to https://github.com/MFlowCode/MFC-develop/issues/95
     target_compile_options(post_process PRIVATE -O0)
+
+    # flang-23/LLD defaults to PIE; SILO and LAPACK static libs on Frontier are
+    # non-PIC, producing R_X86_64_32 relocations that LLD rejects in PIE mode.
+    if (CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+        target_link_options(post_process PRIVATE -no-pie)
+    endif()
 endif()
 
 if (MFC_SYSCHECK)

examples/3D_performance_test/case.py

@@ -6,7 +6,7 @@
     json.dumps(
         {
             # Logistics
-            "run_time_info": "T",
+            "run_time_info": "F",
             # Computational Domain Parameters
             "x_domain%beg": 0.0e00,
             "x_domain%end": 4.0e-03 / 1.0e-03,

src/common/m_chemistry.fpp

@@ -194,7 +194,7 @@ contains
 
         $:GPU_UPDATE(device='[isc1, isc2, isc3]')
 
-        if (chemistry .or. dummy) then
+        if (chemistry) then
             ! Set offsets based on direction using array indexing
             offsets = 0
             offsets(idir) = 1

src/post_process/m_global_parameters.fpp

@@ -177,7 +177,6 @@ module m_global_parameters
     logical                            :: E_wrt
     logical, dimension(num_fluids_max) :: alpha_rho_e_wrt
     logical                            :: fft_wrt
-    logical                            :: dummy  !< AMDFlang workaround for case-optimization + GPU-kernel bug
     logical                            :: pres_wrt
     logical, dimension(num_fluids_max) :: alpha_wrt
     logical                            :: gamma_wrt
@@ -388,7 +387,6 @@ contains
         file_per_process = .false.
         E_wrt = .false.
         fft_wrt = .false.
-        dummy = .false.
         pres_wrt = .false.
         alpha_wrt = .false.
         gamma_wrt = .false.

src/pre_process/m_global_parameters.fpp

@@ -182,7 +182,6 @@ module m_global_parameters
     real(wp)                               :: Bx0        !< Constant magnetic field in the x-direction (1D)
     integer                                :: buff_size  !< Number of ghost cells for boundary condition storage
     logical                                :: fft_wrt
-    logical                                :: dummy      !< AMDFlang workaround for case-optimization + GPU-kernel bug
 
 contains
 
@@ -299,7 +298,6 @@ contains
         elliptic_smoothing = .false.
 
         fft_wrt = .false.
-        dummy = .false.
 
         simplex_perturb = .false.
         simplex_params%perturb_vel(:) = .false.

src/simulation/m_acoustic_src.fpp

@@ -454,14 +454,16 @@ contains
                 call s_mpi_abort('Fatal Error: Inconsistent allocation of source_spatials')
             end if
 
-            $:GPU_UPDATE(device='[source_spatials(ai)%coord]')
-            $:GPU_UPDATE(device='[source_spatials(ai)%val]')
-            if (support(ai) >= 5) then
-                if (dim == 2) then
-                    $:GPU_UPDATE(device='[source_spatials(ai)%angle]')
-                end if
-                if (dim == 3) then
-                    $:GPU_UPDATE(device='[source_spatials(ai)%xyz_to_r_ratios]')
+            if (count > 0) then
+                $:GPU_UPDATE(device='[source_spatials(ai)%coord]')
+                $:GPU_UPDATE(device='[source_spatials(ai)%val]')
+                if (support(ai) >= 5) then
+                    if (dim == 2) then
+                        $:GPU_UPDATE(device='[source_spatials(ai)%angle]')
+                    end if
+                    if (dim == 3) then
+                        $:GPU_UPDATE(device='[source_spatials(ai)%xyz_to_r_ratios]')
+                    end if
                 end if
             end if
         end do

src/simulation/m_cbc.fpp

@@ -529,7 +529,7 @@ contains
         #:for CBC_DIR, XYZ in [(1, 'x'), (2, 'y'), (3, 'z')]
             if (cbc_dir == ${CBC_DIR}$ .and. recon_type == WENO_TYPE) then
                 ! PI2 of flux_rs_vf and flux_src_rs_vf at j = 1/2
-                if (weno_order == 3 .or. dummy) then
+                if (weno_order == 3) then
                     call s_convert_primitive_to_flux_variables(q_prim_rs${XYZ}$_vf, F_rs${XYZ}$_vf, F_src_rs${XYZ}$_vf, is1, is2, &
                         & is3, idwbuff(2)%beg, idwbuff(3)%beg)
 
@@ -557,7 +557,7 @@ contains
                 end if
 
                 ! PI4 of flux_rs_vf and flux_src_rs_vf at j = 1/2, 3/2
-                if (weno_order == 5 .or. dummy) then
+                if (weno_order == 5) then
                     call s_convert_primitive_to_flux_variables(q_prim_rs${XYZ}$_vf, F_rs${XYZ}$_vf, F_src_rs${XYZ}$_vf, is1, is2, &
                         & is3, idwbuff(2)%beg, idwbuff(3)%beg)