Skip to content

Commit f4ab1aa

Browse files
sbryngelsonsbryngelson
authored
Add AMD HPCFund (amdfund) amdflang OpenMP-offload build/run support (#1632)
1 parent 4ce6a2a commit f4ab1aa

4 files changed

Lines changed: 123 additions & 9 deletions

File tree

‎cmake/MFCTargets.cmake‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,22 @@ exit 0
9696
find_package(MPI COMPONENTS Fortran REQUIRED)
9797

9898
target_compile_definitions(${a_target} PRIVATE MFC_MPI)
99-
if(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang" AND
100-
DEFINED ENV{CRAY_MPICH_INC} AND NOT "$ENV{CRAY_MPICH_INC}" STREQUAL "")
101-
target_compile_options(${a_target} PRIVATE "$ENV{CRAY_MPICH_INC}")
102-
target_link_libraries(${a_target} PRIVATE $ENV{CRAY_MPICH_LIB})
99+
# amdflang (LLVMFlang) needs MPI include/link paths passed explicitly, as
100+
# its mpi.mod must match the compiler ABI. Prefer the compiler-neutral
101+
# MFC_FLANG_MPI_* vars; fall back to the legacy CRAY_MPICH_* names (Frontier).
102+
set(_flang_mpi_inc "$ENV{MFC_FLANG_MPI_INC}")
103+
set(_flang_mpi_lib "$ENV{MFC_FLANG_MPI_LIB}")
104+
if("${_flang_mpi_inc}" STREQUAL "")
105+
set(_flang_mpi_inc "$ENV{CRAY_MPICH_INC}")
106+
set(_flang_mpi_lib "$ENV{CRAY_MPICH_LIB}")
107+
endif()
108+
if(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang" AND NOT "${_flang_mpi_inc}" STREQUAL "")
109+
if("${_flang_mpi_lib}" STREQUAL "")
110+
message(FATAL_ERROR "amdflang MPI include path is set but its link libraries are not. "
111+
"Set both MFC_FLANG_MPI_INC and MFC_FLANG_MPI_LIB (or both CRAY_MPICH_INC and CRAY_MPICH_LIB).")
112+
endif()
113+
target_compile_options(${a_target} PRIVATE "${_flang_mpi_inc}")
114+
target_link_libraries(${a_target} PRIVATE ${_flang_mpi_lib})
103115
else()
104116
target_link_libraries(${a_target} PRIVATE MPI::MPI_Fortran)
105117
endif()
@@ -121,8 +133,13 @@ exit 0
121133
find_package(CUDAToolkit REQUIRED)
122134
target_link_libraries(${a_target} PRIVATE CUDA::cudart CUDA::cufft)
123135
elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
124-
if(DEFINED ENV{CRAY_HIPFORT_LIB} AND NOT "$ENV{CRAY_HIPFORT_LIB}" STREQUAL "")
125-
target_link_libraries(${a_target} PRIVATE $ENV{CRAY_HIPFORT_LIB})
136+
# Prefer the neutral MFC_FLANG_HIPFORT_LIB; fall back to CRAY_HIPFORT_LIB (Frontier).
137+
set(_flang_hipfort_lib "$ENV{MFC_FLANG_HIPFORT_LIB}")
138+
if("${_flang_hipfort_lib}" STREQUAL "")
139+
set(_flang_hipfort_lib "$ENV{CRAY_HIPFORT_LIB}")
140+
endif()
141+
if(NOT "${_flang_hipfort_lib}" STREQUAL "")
142+
target_link_libraries(${a_target} PRIVATE ${_flang_hipfort_lib})
126143
else()
127144
find_library(HIPFFT_LIB hipfft
128145
HINTS "$ENV{OLCF_AFAR_ROOT}/lib" REQUIRED)
@@ -244,11 +261,14 @@ exit 0
244261
endif()
245262

246263
find_library(HIP_LIB amdhip64
247-
HINTS "$ENV{OLCF_AFAR_ROOT}/lib" REQUIRED)
264+
HINTS "$ENV{OLCF_AFAR_ROOT}/lib" "$ENV{OLCF_AFAR_ROOT}/lib/llvm/lib" REQUIRED)
248265
find_library(HIPFORT_AMDGCN_LIB hipfort-amdgcn
249-
HINTS "$ENV{OLCF_AFAR_ROOT}/lib" REQUIRED)
266+
HINTS "$ENV{OLCF_AFAR_ROOT}/lib" "$ENV{OLCF_AFAR_ROOT}/lib/llvm/lib" REQUIRED)
267+
# The hipfort module dir moved to lib/llvm/include in newer AFAR
268+
# (therock) drops; keep the classic path for Frontier's layout.
250269
target_include_directories(${a_target} PRIVATE
251-
"$ENV{OLCF_AFAR_ROOT}/include/hipfort/amdgcn")
270+
"$ENV{OLCF_AFAR_ROOT}/include/hipfort/amdgcn"
271+
"$ENV{OLCF_AFAR_ROOT}/lib/llvm/include/hipfort/amdgcn")
252272
target_link_libraries(${a_target} PRIVATE
253273
${HIP_LIB} ${HIPFORT_AMDGCN_LIB})
254274

‎toolchain/bootstrap/modules.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ show_help() {
99
echo " -h, --help Display this help message and exit."
1010
echo " -c, --computer COMPUTER Configures for COMPUTER environment."
1111
echo " Options: Ascent (a) | Frontier (f) | Frontier_amd (famd) | Summit (s) | Wombat (w)"
12+
echo " AMD HPCFund (amdfund)"
1213
echo " Bridges2 (b) | Expanse (e) | Delta (d) | DeltaAI (dai)"
1314
echo " Phoenix (p) | Richardson (r) | Oscar (o)"
1415
echo " Carpenter Cray (cc) | Carpenter GNU (c) | Nautilus (n)"
@@ -48,6 +49,7 @@ if [ -v $u_c ]; then
4849
log "$B""DoD$W: Carpenter Cray (cc) | Carpenter GNU (c) | Nautilus (n)"
4950
log "$OR""Florida$W: HiPerGator (h)"
5051
log "$C""WPI $W: Turing (t)"
52+
log "$R""AMD$W: HPCFund (amdfund)"
5153
log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$B""tuo$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""cc$CR/$B""c$CR/$B""n$CR/$BR""o$CR/$BR""pa"$CR"/$OR""h"$CR/$C""t""$CR"): "
5254
read u_c
5355
log
@@ -159,6 +161,39 @@ if [ "$u_c" '==' 'famd' ]; then
159161
unset MPICH_GPU_SUPPORT_ENABLED
160162
fi
161163

164+
# AMD HPCFund: AFAR LLVM Flang (amdflang) + system GNU-built OpenMPI, gfx90a.
165+
# The stock mpi.mod is GNU-built and binary-incompatible with amdflang, so we
166+
# point at a pre-generated flang-compatible one under $OLCF_AFAR_ROOT/include/mpi.
167+
# amdflang is only supported for OpenMP-offload GPU builds; CPU-mode (-m c) builds
168+
# use the gfortran + OpenMPI provided by the amdfund-all modules, so gate on GPU.
169+
if [ "$u_c" '==' 'amdfund' ] && [ "$cg" '==' 'gpu' ]; then
170+
# Direct WORK-filesystem path (mounted at the same /work1 point on login and
171+
# compute nodes); override OLCF_AFAR_ROOT before loading to use another drop.
172+
export OLCF_AFAR_ROOT="${OLCF_AFAR_ROOT:-/work1/spencerbryngelson/sbryngelson/software/therock-afar-23.2.1-gfx90a-7.13.0-7357b5084b}"
173+
# Track the loaded openmpi4 module's prefix (OpenHPC sets MPI_DIR) instead of
174+
# pinning a path that can drift from the module.
175+
_mfc_mpi_lib="${MPI_DIR:-/opt/ohpc/pub/mpi/openmpi4-gnu12/4.1.8}/lib"
176+
177+
export PATH="${OLCF_AFAR_ROOT}/lib/llvm/bin:${OLCF_AFAR_ROOT}/bin:${PATH}"
178+
export LD_LIBRARY_PATH="${OLCF_AFAR_ROOT}/lib:${OLCF_AFAR_ROOT}/lib/llvm/lib:${_mfc_mpi_lib}:${LD_LIBRARY_PATH}"
179+
export CMAKE_PREFIX_PATH="${OLCF_AFAR_ROOT}:${CMAKE_PREFIX_PATH}"
180+
181+
export MFC_FLANG_MPI_INC="-I${OLCF_AFAR_ROOT}/include/mpi"
182+
export MFC_FLANG_MPI_LIB="${_mfc_mpi_lib}/libmpi.so;${_mfc_mpi_lib}/libmpi_mpifh.so"
183+
export MFC_FLANG_HIPFORT_LIB="${OLCF_AFAR_ROOT}/lib/llvm/lib/libhipfort-amdgcn.a;${OLCF_AFAR_ROOT}/lib/libhipfft.so"
184+
185+
export FC="${OLCF_AFAR_ROOT}/bin/amdflang"
186+
187+
unset _mfc_mpi_lib
188+
189+
if [ ! -x "$FC" ]; then
190+
error "amdfund: amdflang not found at $M$FC$CR."
191+
error "Set $M\$OLCF_AFAR_ROOT$CR to your AFAR drop, then reload, e.g.:"
192+
error " export OLCF_AFAR_ROOT=/path/to/therock-afar-<ver>-gfx90a-...; source ./mfc.sh load -c amdfund -m g"
193+
return
194+
fi
195+
fi
196+
162197
ok 'All modules and environment variables have been loaded.'
163198

164199
return

‎toolchain/modules‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ famd-all cpe/25.09
5252
famd-all PrgEnv-amd
5353
famd-all amd/7.2.0 rocm/7.2.0
5454

55+
amdfund AMD HPCFund
56+
amdfund-all gnu12 openmpi4/4.1.8 cmake rocm/7.2.0
57+
5558
tuo OLCF Tuolumne
5659
tuo-all cpe/25.03 rocm/6.3.1
5760
tuo-all cray-fftw/3.3.10.9 cray-hdf5 python/3.12.2 cmake

‎toolchain/templates/amdfund.mako‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/usr/bin/env bash
2+
3+
<%namespace name="helpers" file="helpers.mako"/>
4+
5+
% if engine == 'batch':
6+
#SBATCH --nodes=${nodes}
7+
#SBATCH --ntasks-per-node=${tasks_per_node}
8+
#SBATCH --job-name="${name}"
9+
#SBATCH --output="${name}.out"
10+
#SBATCH --error="${name}.err"
11+
#SBATCH --time=${walltime}
12+
## hpcfund does not track GPUs through Slurm (GresTypes=null): the MI250X nodes
13+
## hand out every GCD with the node, so there is no --gres / --gpu-bind to set.
14+
#SBATCH --partition=${partition or 'mi2508x'}
15+
% if account:
16+
#SBATCH --account="${account}"
17+
% endif
18+
% if quality_of_service:
19+
#SBATCH --qos=${quality_of_service}
20+
% endif
21+
% if email:
22+
#SBATCH --mail-user=${email}
23+
#SBATCH --mail-type="BEGIN, END, FAIL"
24+
% endif
25+
% endif
26+
27+
${helpers.template_prologue()}
28+
29+
ok ":) Loading modules:\n"
30+
cd "${MFC_ROOT_DIR}"
31+
. ./mfc.sh load -c amdfund -m ${'g' if gpu_enabled else 'c'}
32+
cd - > /dev/null
33+
echo
34+
35+
% if gpu_enabled:
36+
# Abort loudly if OpenMP offload cannot reach a GPU (e.g. run on a login node).
37+
export OMP_TARGET_OFFLOAD=MANDATORY
38+
% endif
39+
40+
% for target in targets:
41+
${helpers.run_prologue(target)}
42+
43+
% if not mpi:
44+
(set -x; ${profiler} "${target.get_install_binpath(case)}")
45+
% else:
46+
(set -x; ${profiler} \
47+
mpirun -np ${nodes*tasks_per_node} \
48+
"${target.get_install_binpath(case)}")
49+
% endif
50+
51+
${helpers.run_epilogue(target)}
52+
53+
echo
54+
% endfor
55+
56+
${helpers.template_epilogue()}

0 commit comments

Comments
 (0)