Skip to content

Commit f3e1e57

Browse files
fluidnumericsJoefluidnumericsJoe
authored
[DRAFT] Add buildkite pipeline (#116)
* Draft buildkite pipeline We still need base images posted to higherordermethods registry * Run `srun` directly, assuming no pre-command * Fix missing "\" * Change gpu type to mi210 * Fix tag for mi210 base image * Adjust to use new pre-command hook for wrapping commands in srun * Make build env and step more verbose * Move slurm vars to env * Enable spack environment, specify fortran compiler in cmake stage * Add workspace env variable This is needed to load the mesh files in tests * Update pipeline assuming buildkite is running as batch job in container * Report commands to stdout * Hard-code cmake build options in command * Clean up cmake and add codecov upload stage to pipeline * Add partyparrot emoji in pipeline * Add call to rocminfo * Add container remap root flag * Fix buildkite pipeline typo * set container remap root to true to assert boolean flag for sbatch * Remove container-remap-root * Fix secrets syntax for codecov
1 parent e9632e9 commit f3e1e57

5 files changed

Lines changed: 143 additions & 55 deletions

File tree

‎.buildkite/pipeline.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
steps:
2+
- label: ":partyparrot: Build & Test AMD MI210 (Coverage)"
3+
key: "build-test-mi210"
4+
command: |
5+
source /opt/spack-environment/activate.sh
6+
export WORKSPACE=/workspace
7+
mkdir -p /workspace/build
8+
rocminfo
9+
cd /workspace/build
10+
FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
11+
-DCMAKE_BUILD_TYPE="coverage" \
12+
-DSELF_ENABLE_GPU=ON \
13+
-DSELF_GPU_BACKEND=HIP \
14+
-DSELF_ENABLE_TESTING=ON \
15+
-DCMAKE_HIP_ARCHITECTURES="gfx90a" \
16+
-DGPU_TARGETS="gfx90a" \
17+
-DSELF_ENABLE_EXAMPLES=ON \
18+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
19+
/workspace/
20+
make -j
21+
lcov --capture --initial \
22+
--directory /workspace/build/src/ \
23+
--output-file /workspace/initial.info
24+
ctest --test-dir /workspace/build --output-on-failure
25+
lcov --capture \
26+
--directory /workspace/build/src/ \
27+
--output-file /workspace/ctest-capture.info
28+
lcov --add-tracefile /workspace/initial.info \
29+
--add-tracefile /workspace/ctest-capture.info \
30+
--output-file /workspace/coverage.info
31+
env:
32+
slurm_partition: "main"
33+
slurm_gres: "gpu:mi210:2"
34+
slurm_time: "01:00:00"
35+
slurm_nodes: 1
36+
slurm_ntasks: 2
37+
slurm_cpus_per_task: 8
38+
slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
39+
agents:
40+
queue: "galapagos"
41+
42+
- label: ":codecov: Upload Coverage"
43+
key: "upload-coverage"
44+
depends_on: "build-test-mi210"
45+
command: |
46+
curl -Os https://uploader.codecov.io/latest/linux/codecov
47+
chmod +x codecov
48+
./codecov -t "$${CODECOV_TOKEN}" \
49+
-f "$${BUILDKITE_BUILD_CHECKOUT_PATH}/coverage.info" \
50+
-F buildkite-rocm-mi210
51+
secrets:
52+
- CODECOV_TOKEN
53+
agents:
54+
queue: "galapagos"

‎CMakeLists.txt‎

Lines changed: 82 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,12 @@ option(SELF_ENABLE_MULTITHREADING "Option to enable CPU multithreading for `do c
3636
option(SELF_ENABLE_TESTING "Option to enable build of tests. (Default On)" ON)
3737
option(SELF_ENABLE_EXAMPLES "Option to enable build of examples. (Default On)" ON)
3838
option(SELF_ENABLE_GPU "Option to enable GPU backend. Requires either CUDA or HIP. (Default Off)" OFF)
39-
option(SELF_ENABLE_APU "Option to enable APU backend. Requires either CUDA or HIP. (Default Off)" OFF)
39+
option(SELF_ENABLE_APU "Option to enable APU backend. (Default Off)" OFF)
40+
41+
set(SELF_GPU_BACKEND "AUTO" CACHE STRING "GPU backend to use when SELF_ENABLE_GPU is ON: AUTO, CUDA, or HIP. AUTO prefers HIP when available. (Default AUTO)")
42+
set_property(CACHE SELF_GPU_BACKEND PROPERTY STRINGS AUTO CUDA HIP)
43+
44+
option(SELF_REQUIRE_GPU_AWARE_MPI "Require GPU-aware MPI when building with GPU support. Set OFF to downgrade the check to a warning. (Default ON)" ON)
4045
option(SELF_ENABLE_DOUBLE_PRECISION "Option to enable double precision for floating point arithmetic. (Default On)" ON)
4146

4247
set(SELF_MPIEXEC_NUMPROCS "2" CACHE STRING "The number of MPI ranks to use to launch MPI tests. Only used when launching test programs via ctest.")
@@ -151,76 +156,103 @@ if(SELF_ENABLE_DOUBLE_PRECISION)
151156

152157
endif()
153158

159+
if(SELF_ENABLE_GPU AND SELF_ENABLE_APU)
160+
message(FATAL_ERROR "SELF_ENABLE_GPU and SELF_ENABLE_APU are mutually exclusive. Enable only one.")
161+
endif()
162+
154163
if(SELF_ENABLE_GPU)
155-
message("-- SELF Build System : Enabling GPU Support")
164+
message("-- SELF Build System : Enabling GPU Support (backend: ${SELF_GPU_BACKEND})")
156165
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DENABLE_GPU" )
157166
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DENABLE_GPU" )
158167
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DENABLE_GPU")
159168
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DENABLE_GPU")
160169
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DENABLE_GPU" )
161170

162-
# Check MPI for GPU awareness
163171
# Add SELF's cmake module directory to the search path
164172
set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake")
165-
166-
# Include the custom module
167173
include(CheckMPIFeatures)
168174
CheckMPIFeatures()
169175

170-
# HIP (Optional)
171-
# IF HIP is found, we prefer HIP. IF HIP is not found, then we fall back to look for CUDA
172-
# only. If neither HIP nor CUDA are found, Cmake will error out.
173-
find_package(hip)
174-
if(hip_FOUND)
175-
if(MPI_HAS_QUERY_HIP_SUPPORT)
176-
find_package(hipblas REQUIRED)
177-
message("-- HIP found. Enabling HIP language.")
178-
enable_language(HIP)
179-
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" )
180-
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" )
181-
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP")
182-
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP")
183-
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" )
184-
185-
if(SELF_ENABLE_DOUBLE_PRECISION)
186-
set( CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -DDOUBLE_PRECISION" )
187-
set( CMAKE_HIP_FLAGS_DEBUG "${CMAKE_HIP_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
188-
set( CMAKE_HIP_FLAGS_COVERAGE "${CMAKE_HIP_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
189-
set( CMAKE_HIP_FLAGS_PROFILE "${CMAKE_HIP_FLAGS_PROFILE} -DDOUBLE_PRECISION")
190-
set( CMAKE_HIP_FLAGS_RELEASE "${CMAKE_HIP_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
191-
endif()
192-
set( BACKEND_LIBRARIES hip::device roc::hipblas)
176+
# Resolve which backend to activate based on SELF_GPU_BACKEND
177+
if(SELF_GPU_BACKEND STREQUAL "HIP")
178+
find_package(hip REQUIRED)
179+
set(_self_use_hip TRUE)
180+
elseif(SELF_GPU_BACKEND STREQUAL "CUDA")
181+
find_package(CUDAToolkit REQUIRED)
182+
set(_self_use_hip FALSE)
183+
else() # AUTO: prefer HIP, fall back to CUDA
184+
find_package(hip)
185+
if(hip_FOUND)
186+
set(_self_use_hip TRUE)
193187
else()
194-
message( FATAL_ERROR "MPI installation is not GPU-aware" )
188+
find_package(CUDAToolkit REQUIRED)
189+
set(_self_use_hip FALSE)
195190
endif()
196-
else()
197-
# CUDA
198-
find_package(CUDAToolkit REQUIRED)
191+
endif()
192+
193+
if(_self_use_hip)
194+
find_package(hipblas REQUIRED)
195+
message("-- HIP found. Enabling HIP language.")
196+
enable_language(HIP)
197+
if(NOT MPI_HAS_QUERY_HIP_SUPPORT)
198+
if(SELF_REQUIRE_GPU_AWARE_MPI)
199+
message(FATAL_ERROR "MPI installation is not HIP/ROCm-aware. Set SELF_REQUIRE_GPU_AWARE_MPI=OFF to downgrade this to a warning.")
200+
else()
201+
message(WARNING "MPI installation is not HIP/ROCm-aware. GPU-direct communication will not be available.")
202+
endif()
203+
endif()
204+
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_HIP" )
205+
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_HIP" )
206+
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_HIP")
207+
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_HIP")
208+
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_HIP" )
209+
if(SELF_ENABLE_DOUBLE_PRECISION)
210+
set( CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -DDOUBLE_PRECISION" )
211+
set( CMAKE_HIP_FLAGS_DEBUG "${CMAKE_HIP_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
212+
set( CMAKE_HIP_FLAGS_COVERAGE "${CMAKE_HIP_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
213+
set( CMAKE_HIP_FLAGS_PROFILE "${CMAKE_HIP_FLAGS_PROFILE} -DDOUBLE_PRECISION")
214+
set( CMAKE_HIP_FLAGS_RELEASE "${CMAKE_HIP_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
215+
endif()
216+
set( BACKEND_LIBRARIES hip::device roc::hipblas)
217+
set( SELF_ACTIVE_GPU_BACKEND "HIP")
218+
219+
else() # CUDA
199220
message("-- CUDA found. Enabling CUDA language.")
200221
enable_language(CUDA)
201-
if(MPI_HAS_QUERY_CUDA_SUPPORT)
202-
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" )
203-
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" )
204-
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA")
205-
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA")
206-
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" )
207-
208-
if(SELF_ENABLE_DOUBLE_PRECISION)
209-
set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DDOUBLE_PRECISION" )
210-
set( CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
211-
set( CMAKE_CUDA_FLAGS_COVERAGE "${CMAKE_CUDA_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
212-
set( CMAKE_CUDA_FLAGS_PROFILE "${CMAKE_CUDA_FLAGS_PROFILE} -DDOUBLE_PRECISION")
213-
set( CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
222+
if(NOT MPI_HAS_QUERY_CUDA_SUPPORT)
223+
if(SELF_REQUIRE_GPU_AWARE_MPI)
224+
message(FATAL_ERROR "MPI installation is not CUDA-aware. Set SELF_REQUIRE_GPU_AWARE_MPI=OFF to downgrade this to a warning.")
225+
else()
226+
message(WARNING "MPI installation is not CUDA-aware. GPU-direct communication will not be available.")
214227
endif()
215-
216-
set( BACKEND_LIBRARIES CUDA::cuda_driver CUDA::cudart CUDA::cublas)
217-
218-
else()
219-
message( FATAL_ERROR "MPI installation is not GPU-aware" )
220228
endif()
229+
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DHAVE_CUDA" )
230+
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DHAVE_CUDA" )
231+
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DHAVE_CUDA")
232+
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DHAVE_CUDA")
233+
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DHAVE_CUDA" )
234+
if(SELF_ENABLE_DOUBLE_PRECISION)
235+
set( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DDOUBLE_PRECISION" )
236+
set( CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -DDOUBLE_PRECISION" )
237+
set( CMAKE_CUDA_FLAGS_COVERAGE "${CMAKE_CUDA_FLAGS_COVERAGE} -DDOUBLE_PRECISION")
238+
set( CMAKE_CUDA_FLAGS_PROFILE "${CMAKE_CUDA_FLAGS_PROFILE} -DDOUBLE_PRECISION")
239+
set( CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS_RELEASE} -DDOUBLE_PRECISION" )
240+
endif()
241+
set( BACKEND_LIBRARIES CUDA::cuda_driver CUDA::cudart CUDA::cublas)
242+
set( SELF_ACTIVE_GPU_BACKEND "CUDA")
243+
221244
endif()
222245
endif()
223246

247+
if(SELF_ENABLE_APU)
248+
message("-- SELF Build System : Enabling APU Support")
249+
set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -DENABLE_APU" )
250+
set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -DENABLE_APU" )
251+
set( CMAKE_Fortran_FLAGS_COVERAGE "${CMAKE_Fortran_FLAGS_COVERAGE} -DENABLE_APU")
252+
set( CMAKE_Fortran_FLAGS_PROFILE "${CMAKE_Fortran_FLAGS_PROFILE} -DENABLE_APU")
253+
set( CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -DENABLE_APU" )
254+
endif()
255+
224256
# ----- end dependencies ----- #
225257

226258
# Libraries

‎cmake/CheckMPIFeatures.cmake‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,4 @@ function(CheckMPIFeatures)
9898

9999
list(REMOVE_ITEM CMAKE_REQUIRED_DEFINITIONS -DHAVE_MPI_EXT)
100100
endif()
101-
endfunction()
102-
103-
CheckMPIFeatures()
101+
endfunction()

‎src/CMakeLists.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,13 @@ if(SELF_ENABLE_GPU)
3030
file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.f*")
3131
file(GLOB SELF_BACKEND_CPPSRC "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.cpp*")
3232

33-
if(hip_FOUND)
33+
if(SELF_ACTIVE_GPU_BACKEND STREQUAL "HIP")
3434
set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE HIP)
3535
else()
3636
set_source_files_properties(${SELF_BACKEND_CPPSRC} PROPERTIES LANGUAGE CUDA)
3737
endif()
38+
elseif(SELF_ENABLE_APU)
39+
file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/apu/*.f*")
3840
else()
3941
file(GLOB SELF_BACKEND_FSRC "${CMAKE_CURRENT_SOURCE_DIR}/cpu/*.f*")
4042
endif()

‎test/CMakeLists.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ function (add_fortran_tests)
3838
target_link_libraries(${TEST_NAME} self)
3939
target_include_directories(${TEST_NAME} PUBLIC ${CMAKE_BINARY_DIR}/include)
4040
add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME})
41+
set_tests_properties(${TEST_NAME} PROPERTIES LABELS "serial")
4142
install(TARGETS ${TEST_NAME} DESTINATION test)
4243
endforeach ()
4344

@@ -52,6 +53,7 @@ function (add_mpi_fortran_tests)
5253
target_link_libraries(${TEST_NAME} self)
5354
target_include_directories(${TEST_NAME} PUBLIC ${CMAKE_BINARY_DIR}/include)
5455
add_test(NAME ${TEST_NAME} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SELF_MPIEXEC_NUMPROCS} ${SELF_MPIEXEC_OPTIONS} ${TEST_NAME})
56+
set_tests_properties(${TEST_NAME} PROPERTIES LABELS "parallel")
5557

5658
install(TARGETS ${TEST_NAME} DESTINATION test)
5759
endforeach ()
@@ -146,4 +148,4 @@ add_mpi_fortran_tests( "mappedvectordgdivergence_2d_linear_mpi.f90"
146148
"advection_diffusion_2d_rk3_mpi.f90"
147149
"advection_diffusion_2d_rk3_pickup_mpi.f90"
148150
"advection_diffusion_3d_rk3_mpi.f90"
149-
"advection_diffusion_3d_rk3_pickup_mpi.f90" )
151+
"advection_diffusion_3d_rk3_pickup_mpi.f90" )

0 commit comments

Comments
 (0)