Skip to content

Commit 7da6212

Browse files
sbryngelsonsbryngelson
authored
add nvhpc to test suite github runners (MFlowCode#1317)
1 parent c76be93 commit 7da6212

5 files changed

Lines changed: 141 additions & 22 deletions

File tree

‎.github/workflows/test.yml‎

Lines changed: 100 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ jobs:
149149
continue-on-error: true
150150

151151
github:
152-
name: Github
152+
name: ${{ matrix.nvhpc && format('NVHPC {0} ({1})', matrix.nvhpc, matrix.target) || format('Github ({0}, {1}, {2}, intel={3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel) }}
153153
needs: [lint-gate, file-changes, rebuild-cache]
154154
if: >-
155155
!cancelled() &&
@@ -164,6 +164,8 @@ jobs:
164164
precision: ['']
165165
debug: ['reldebug', 'no-debug']
166166
intel: [true, false]
167+
nvhpc: ['']
168+
target: ['']
167169
exclude:
168170
- os: macos
169171
intel: true
@@ -175,11 +177,62 @@ jobs:
175177
debug: no-debug
176178
intel: false
177179

180+
# NVHPC compiler matrix: cpu (build+test), gpu (build-only, acc then omp)
181+
# Every release from 23.11 through 26.3 (current)
182+
- { nvhpc: '23.11', target: cpu }
183+
- { nvhpc: '23.11', target: gpu }
184+
- { nvhpc: '24.1', target: cpu }
185+
- { nvhpc: '24.1', target: gpu }
186+
- { nvhpc: '24.3', target: cpu }
187+
- { nvhpc: '24.3', target: gpu }
188+
- { nvhpc: '24.5', target: cpu }
189+
- { nvhpc: '24.5', target: gpu }
190+
- { nvhpc: '24.7', target: cpu }
191+
- { nvhpc: '24.7', target: gpu }
192+
- { nvhpc: '24.9', target: cpu }
193+
- { nvhpc: '24.9', target: gpu }
194+
- { nvhpc: '24.11', target: cpu }
195+
- { nvhpc: '24.11', target: gpu }
196+
- { nvhpc: '25.1', target: cpu }
197+
- { nvhpc: '25.1', target: gpu }
198+
- { nvhpc: '25.3', target: cpu }
199+
- { nvhpc: '25.3', target: gpu }
200+
- { nvhpc: '25.5', target: cpu }
201+
- { nvhpc: '25.5', target: gpu }
202+
- { nvhpc: '25.7', target: cpu }
203+
- { nvhpc: '25.7', target: gpu }
204+
- { nvhpc: '25.9', target: cpu }
205+
- { nvhpc: '25.9', target: gpu }
206+
- { nvhpc: '25.11', target: cpu }
207+
- { nvhpc: '25.11', target: gpu }
208+
- { nvhpc: '26.1', target: cpu }
209+
- { nvhpc: '26.1', target: gpu }
210+
- { nvhpc: '26.3', target: cpu }
211+
- { nvhpc: '26.3', target: gpu }
212+
178213
fail-fast: false
179214
continue-on-error: true
180-
runs-on: ${{ matrix.os }}-latest
215+
runs-on: ${{ matrix.nvhpc && 'ubuntu-22.04' || format('{0}-latest', matrix.os) }}
216+
container:
217+
image: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }}
218+
options: ${{ matrix.nvhpc && '--security-opt seccomp=unconfined' || '' }}
219+
env:
220+
CC: ${{ matrix.nvhpc && 'nvc' || '' }}
221+
CXX: ${{ matrix.nvhpc && 'nvc++' || '' }}
222+
FC: ${{ matrix.nvhpc && 'nvfortran' || '' }}
223+
OMPI_ALLOW_RUN_AS_ROOT: ${{ matrix.nvhpc && '1' || '' }}
224+
OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: ${{ matrix.nvhpc && '1' || '' }}
225+
PMIX_MCA_gds: ${{ matrix.nvhpc && 'hash' || '' }}
226+
OMPI_MCA_hwloc_base_binding_policy: ${{ matrix.nvhpc && 'none' || '' }}
227+
FFLAGS: ${{ matrix.nvhpc && '-tp=px -Kieee -noswitcherror' || '' }}
228+
CFLAGS: ${{ matrix.nvhpc && '-tp=px' || '' }}
229+
CXXFLAGS: ${{ matrix.nvhpc && '-tp=px' || '' }}
181230

182231
steps:
232+
- name: Git safe directory
233+
if: matrix.nvhpc
234+
run: git config --global --add safe.directory /__w/MFC/MFC
235+
183236
- name: Clone
184237
uses: actions/checkout@v4
185238

@@ -222,7 +275,7 @@ jobs:
222275
fi
223276
224277
- name: Setup MacOS
225-
if: matrix.os == 'macos'
278+
if: matrix.os == 'macos' && !matrix.nvhpc
226279
run: |
227280
brew update
228281
brew upgrade || true
@@ -231,7 +284,7 @@ jobs:
231284
echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV
232285
233286
- name: Setup Ubuntu
234-
if: matrix.os == 'ubuntu' && matrix.intel == false
287+
if: matrix.os == 'ubuntu' && matrix.intel == false && !matrix.nvhpc
235288
run: |
236289
sudo apt update -y
237290
sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \
@@ -260,21 +313,64 @@ jobs:
260313
echo "MPICC=mpiicx" >> $GITHUB_ENV
261314
echo "MPICXX=mpiicpx" >> $GITHUB_ENV
262315
316+
# --- NVHPC container setup ---
317+
- name: Setup NVHPC
318+
if: matrix.nvhpc
319+
run: |
320+
apt-get update -y
321+
apt-get install -y cmake python3 python3-venv python3-pip \
322+
libfftw3-dev libhdf5-dev hdf5-tools git
323+
# Set up NVHPC HPC-X MPI runtime paths
324+
HPCX_DIR=$(dirname "$(find /opt/nvidia/hpc_sdk -path "*/hpcx/hpcx-*/ompi/bin/mpirun" | head -1)")/../..
325+
MPI_LIB=$(mpifort --showme:link | grep -oP '(?<=-L)\S+' | head -1)
326+
echo "LD_LIBRARY_PATH=${MPI_LIB}:${HPCX_DIR}/ucx/lib:${HPCX_DIR}/ucc/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
327+
# Container MPI fixes: PMIx shared-memory, hwloc binding
328+
echo "PMIX_MCA_gds=hash" >> $GITHUB_ENV
329+
echo "OMPI_MCA_hwloc_base_binding_policy=none" >> $GITHUB_ENV
330+
echo "OMPI_MCA_rmaps_base_oversubscribe=1" >> $GITHUB_ENV
331+
# Debug: confirm compiler flags are set
332+
echo "=== NVHPC Environment ==="
333+
echo "FFLAGS=$FFLAGS"
334+
echo "CFLAGS=$CFLAGS"
335+
echo "CXXFLAGS=$CXXFLAGS"
336+
nvfortran --version
337+
cat /proc/cpuinfo | grep "model name" | head -1
338+
339+
# --- Standard build + test ---
263340
- name: Build
341+
if: '!matrix.nvhpc'
264342
run: |
265343
/bin/bash mfc.sh test -v --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} $PRECISION $TEST_ALL
266344
env:
267345
TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
268346
PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }}
269347

270348
- name: Test
349+
if: '!matrix.nvhpc'
271350
run: |
272351
/bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) $ONLY_CHANGES $TEST_ALL $TEST_PCT
273352
env:
274353
TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
275354
TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }}
276355
ONLY_CHANGES: ${{ github.event_name == 'pull_request' && '--only-changes' || '' }}
277356

357+
# --- NVHPC build + test ---
358+
- name: Build (NVHPC)
359+
if: matrix.nvhpc && matrix.target == 'cpu'
360+
run: /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all
361+
362+
- name: Build (NVHPC GPU)
363+
if: matrix.nvhpc && matrix.target == 'gpu'
364+
run: |
365+
/bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu acc
366+
/bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu mp
367+
368+
- name: Test (NVHPC)
369+
if: matrix.nvhpc && matrix.target == 'cpu'
370+
run: |
371+
ulimit -s unlimited || ulimit -s 65536 || true
372+
/bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all
373+
278374
self:
279375
name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }}${{ matrix.shard != '' && format(' [{0}]', matrix.shard) || '' }})"
280376
needs: [lint-gate, file-changes, rebuild-cache]

‎CMakeLists.txt‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -311,11 +311,6 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
311311
elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
312312
if (MFC_Unified)
313313
message(STATUS "LTO/IPO is not available with NVHPC using Unified Memory")
314-
elseif (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER "24.11" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "25.9")
315-
message(STATUS "LTO/IPO is not supported in NVHPC Version 24.11 to 25.9. Use >=25.9 or (<=24.11 && > 23.11) Performance will be degraded.")
316-
set(NVHPC_USE_TWO_PASS_IPO FALSE)
317-
elseif(CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "23.11")
318-
message(STATUS "LTO/IPO is not supported in NVHPC Version < 23.11. Use a newer version of NVHPC for best performance.")
319314
else()
320315
message(STATUS "Performing IPO using -Mextract followed by -Minline")
321316
set(NVHPC_USE_TWO_PASS_IPO TRUE)
@@ -502,7 +497,20 @@ function(MFC_SETUP_TARGET)
502497
$<$<COMPILE_LANGUAGE:Fortran>:-Minline>
503498
)
504499
add_dependencies(${ARGS_TARGET} ${ARGS_TARGET}_lib)
505-
target_compile_options(${ARGS_TARGET} PRIVATE -Minline=lib:${ARGS_TARGET}_lib)
500+
target_compile_options(${ARGS_TARGET} PRIVATE -Minline=lib:${ARGS_TARGET}_lib,except:f_is_default,except:s_compute_dt,except:my_inquire,except:s_mpi_abort,except:s_mpi_barrier,except:s_prohibit_abort,except:s_int_to_str,except:s_associate_cbc_coefficients_pointers)
501+
502+
# Exclude m_start_up and m_cbc from cross-file inlining: these are
503+
# initialization/boundary code that trigger NVHPC 25.x fort2 ICE when
504+
# too many functions are cross-inlined into them. GPU hot-path files
505+
# (m_rhs, m_riemann_solvers, m_viscous, m_weno, etc.) keep full IPO.
506+
foreach(_no_inline_file m_start_up m_cbc)
507+
set_source_files_properties(
508+
"${CMAKE_BINARY_DIR}/fypp/${ARGS_TARGET}/${_no_inline_file}.fpp.f90"
509+
TARGET_DIRECTORY ${ARGS_TARGET}
510+
PROPERTIES COMPILE_OPTIONS "-Mnoinline"
511+
)
512+
endforeach()
513+
506514
list(PREPEND IPO_TARGETS ${ARGS_TARGET}_lib)
507515
endif()
508516

@@ -708,15 +716,13 @@ if (MFC_SIMULATION)
708716
MFC_SETUP_TARGET(TARGET simulation
709717
SOURCES "${simulation_SRCs}"
710718
MPI FFTW OpenACC OpenMP)
711-
# CCE 19.0.0 IPA workaround: two files trigger IPA crashes:
719+
# CCE 19.0.0 IPA workaround: disable interprocedural analysis for files
720+
# that trigger compiler crashes during IPA:
712721
# m_bubbles_EL: castIsValid assertion (InstCombine/foldIntegerTypedPHI)
713722
# m_phase_change: bring_routine_resident SIGSEGV
714-
# Disabling IPA per-file avoids the crashes while preserving IPA for
715-
# the rest of simulation (needed for thermochem INLINEALWAYS inlining).
716-
# Applied to Cray+OpenACC and Cray CPU, but NOT Cray+OpenMP: on OpenMP,
717-
# CCE 19.0.0 IPA workaround: disable interprocedural analysis for files
718-
# that trigger compiler SIGSEGV during IPA (Bug 3: m_phase_change,
719-
# Bug 4: m_bubbles_EL). See PR #1286.
723+
# Not applied to Cray+OpenMP because thermochem uses !DIR$ INLINEALWAYS,
724+
# which requires IPA to inline device calls. On OpenACC the pyrometheus
725+
# patch emits !$acc routine seq instead (no IPA needed). See PR #1286.
720726
if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray" AND NOT MFC_OpenMP)
721727
set_source_files_properties(
722728
"${CMAKE_BINARY_DIR}/fypp/simulation/m_bubbles_EL.fpp.f90"

‎src/simulation/m_riemann_solvers.fpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ module m_riemann_solvers
8484
contains
8585

8686
!> Dispatch to the subroutines that are utilized to compute the Riemann problem solution. For additional information please
87-
!! reference: 1) s_hll_riemann_solver 2) s_hllc_riemann_solver 3) s_exact_riemann_solver 4) s_hlld_riemann_solver
87+
!! reference: 1) s_hll_riemann_solver 2) s_hllc_riemann_solver 3) s_lf_riemann_solver 4) s_hlld_riemann_solver
8888
subroutine s_riemann_solver(qL_prim_rsx_vf, qL_prim_rsy_vf, qL_prim_rsz_vf, dqL_prim_dx_vf, dqL_prim_dy_vf, dqL_prim_dz_vf, &
8989

9090
& qL_prim_vf, qR_prim_rsx_vf, qR_prim_rsy_vf, qR_prim_rsz_vf, dqR_prim_dx_vf, dqR_prim_dy_vf, dqR_prim_dz_vf, qR_prim_vf, &
@@ -2205,7 +2205,8 @@ contains
22052205
& G_L, G_R, rho_avg, H_avg, c_avg, gamma_avg, ptilde_L, ptilde_R, vel_L_rms, vel_R_rms, &
22062206
& vel_avg_rms, vel_L_tmp, vel_R_tmp, Ms_L, Ms_R, pres_SL, pres_SR, alpha_L_sum, &
22072207
& alpha_R_sum, rho_Star, E_Star, p_Star, p_K_Star, vel_K_star, s_L, s_R, s_M, s_P, s_S, &
2208-
& xi_M, xi_P, xi_L, xi_R, xi_MP, xi_PP]')
2208+
& xi_M, xi_P, xi_L, xi_R, xi_MP, xi_PP, Ys_L, Ys_R, Cp_iL, Cp_iR, Xs_L, Xs_R, Gamma_iL, &
2209+
& Gamma_iR, Yi_avg, Phi_avg, h_iL, h_iR, h_avg_2]')
22092210
do l = is3%beg, is3%end
22102211
do k = is2%beg, is2%end
22112212
do j = is1%beg, is1%end
@@ -2423,7 +2424,8 @@ contains
24232424
& qv_R, qv_avg, c_L, c_R, c_avg, vel_L_rms, vel_R_rms, vel_avg_rms, vel_L_tmp, vel_R_tmp, &
24242425
& Ms_L, Ms_R, pres_SL, pres_SR, alpha_L_sum, alpha_R_sum, s_L, s_R, s_M, s_P, s_S, xi_M, &
24252426
& xi_P, xi_L, xi_R, xi_MP, xi_PP, nbub_L, nbub_R, PbwR3Lbar, PbwR3Rbar, R3Lbar, R3Rbar, &
2426-
& R3V2Lbar, R3V2Rbar]')
2427+
& R3V2Lbar, R3V2Rbar, Ys_L, Ys_R, Cp_iL, Cp_iR, Xs_L, Xs_R, Gamma_iL, Gamma_iR, Yi_avg, &
2428+
& Phi_avg, h_iL, h_iR, h_avg_2]')
24272429
do l = is3%beg, is3%end
24282430
do k = is2%beg, is2%end
24292431
do j = is1%beg, is1%end

‎toolchain/dependencies/CMakeLists.txt‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,13 @@ if (MFC_HDF5)
6161
GIT_PROGRESS ON
6262
CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
6363
-DBUILD_SHARED_LIBS=OFF
64-
-DFORTRAN_LIBRARIES=ON
6564
-DBUILD_TESTING=OFF
6665
-DHDF5_BUILD_UTILS=OFF
6766
-DHDF5_BUILD_TOOLS=ON
6867
-DHDF5_BUILD_EXAMPLES=OFF
68+
-DHDF5_BUILD_CPP_LIB=OFF
69+
-DHDF5_BUILD_FORTRAN=OFF
70+
-DHDF5_ENABLE_PARALLEL=OFF
6971
)
7072
endif()
7173
endif()
@@ -82,13 +84,15 @@ if (MFC_SILO)
8284

8385
ExternalProject_Add(silo
8486
GIT_REPOSITORY "https://github.com/LLNL/Silo"
85-
GIT_TAG 0cddaa6865fb49ae7b2110fa1b6f2709592abe0b
87+
GIT_TAG 4.12.0
88+
GIT_SHALLOW ON
8689
GIT_PROGRESS ON
8790
PATCH_COMMAND "${GIT_EXECUTABLE}" stash
8891
&& "${GIT_EXECUTABLE}" apply "${CMAKE_SOURCE_DIR}/Silo.patch"
8992
CMAKE_ARGS -DSILO_ENABLE_SHARED=OFF
9093
-DSILO_ENABLE_SILOCK=OFF
9194
-DSILO_ENABLE_BROWSER=OFF
95+
-DSILO_ENABLE_ZFP=OFF
9296
-DFIND_LIBRARY_USE_LIB64_PATHS=ON
9397
"-DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}"
9498
"-DCMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH}"

‎toolchain/mfc/test/test.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,17 @@ def __filter(cases_) -> typing.Tuple[typing.List[TestCase], typing.List[TestCase
187187
if any(label in case.trace for label in skip):
188188
cases.remove(case)
189189

190+
# Skip tests that fail under nvfortran in Docker (pass natively/Apptainer):
191+
# - Lagrange Bubbles: NaN on 24.1/24.3 (other versions not verified in Docker)
192+
# - 3D_rayleigh_taylor_muscl: segfaults with nvfortran+MPI (seccomp/mprotect)
193+
if os.environ.get("FC") == "nvfortran" and os.path.exists("/.dockerenv"):
194+
nvhpc_skip_uuids = {"B9553426", "4A1BD9B8", "0D1FA5C5", "2122A4F6"}
195+
nvhpc_skip_traces = {"rayleigh_taylor_muscl"}
196+
for case in cases[:]:
197+
if case.get_uuid() in nvhpc_skip_uuids or any(t in case.trace for t in nvhpc_skip_traces):
198+
cases.remove(case)
199+
skipped_cases.append(case)
200+
190201
if ARG("no_examples"):
191202
example_cases = [case for case in cases if "Example" in case.trace]
192203
skipped_cases += example_cases

0 commit comments

Comments
 (0)