Skip to content

Commit a5b96ef

Browse files
danielpeterdanielpeter
committed
updates cuda-aware mpi initialization
1 parent 4f40a57 commit a5b96ef

4 files changed

Lines changed: 172 additions & 49 deletions

File tree

‎src/gpu/initialize_gpu.c‎

Lines changed: 110 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1487,6 +1487,27 @@ This simulation will continue using the HIP runtime...\n", runtime_type, HIP, CO
14871487
// CUDA-aware MPI
14881488
// we need to call cudaSetDevice before MPI_Init to ensure that the same GPU is chosen by MPI and your application
14891489

1490+
// CUDA-aware support compile-time information
1491+
#ifdef WITH_CUDA_AWARE_MPI
1492+
// rank info
1493+
#if defined(OPEN_MPI) && OPEN_MPI
1494+
// OpenMPI
1495+
#pragma message ("\n\nCompiling with: WITH_CUDA_AWARE_MPI uses OPEN_MPI CUDA-aware local rank\n")
1496+
#elif defined(MVAPICH2_NUMVERSION) && (MVAPICH2_NUMVERSION >= 20205300)
1497+
// MVAPICH
1498+
#pragma message ("\n\nCompiling with: WITH_CUDA_AWARE_MPI uses MVAPICH2 CUDA-aware local rank\n")
1499+
#else
1500+
// unknown
1501+
#pragma message ("\n\nCompiling with: unknown CUDA-aware local rank environment, use -DENV_LOCAL_RANK \"<MY_LOCAL_RANK>\" setting\n")
1502+
#endif
1503+
// MPIX query
1504+
#if defined(MPIX_CUDA_AWARE_SUPPORT)
1505+
#pragma message ("\n\nCompiling with: WITH_CUDA_AWARE_MPI has MPIX_CUDA_AWARE_SUPPORT\n")
1506+
#else
1507+
#pragma message ("\n\nCompiling with: WITH_CUDA_AWARE_MPI has no MPIX_CUDA_AWARE_SUPPORT, please check MPI installation\n")
1508+
#endif // MPIX_CUDA_AWARE_SUPPORT
1509+
#endif // WITH_CUDA_AWARE_MPI
1510+
14901511
extern EXTERN_LANG
14911512
void FC_FUNC_ (check_cuda_aware_mpi,
14921513
CHECK_CUDA_AWARE_MPI) (int* has_cuda_aware_mpi_f) {
@@ -1507,23 +1528,17 @@ void FC_FUNC_ (check_cuda_aware_mpi,
15071528
// for Open MPI, use "OMPI_COMM_WORLD_LOCAL_RANK".
15081529
#if defined(OPEN_MPI) && OPEN_MPI
15091530
// OpenMPI
1510-
#pragma message ("\n\nCompiling with: WITH_CUDA_AWARE_MPI uses OPEN_MPI CUDA-aware local rank\n")
15111531
#define ENV_LOCAL_RANK "OMPI_COMM_WORLD_LOCAL_RANK"
1512-
15131532
#elif defined(MVAPICH2_NUMVERSION) && (MVAPICH2_NUMVERSION >= 20205300)
15141533
// MVAPICH
1515-
#pragma message ("\n\nCompiling with: WITH_CUDA_AWARE_MPI uses MVAPICH2 CUDA-aware local rank\n")
15161534
#define ENV_LOCAL_RANK "MV2_COMM_WORLD_LOCAL_RANK"
1517-
15181535
#else
15191536
// unknown
1520-
#pragma message ("\n\nCompiling with: unknown CUDA-aware local rank environment, use -DENV_LOCAL_RANK \"<MY_LOCAL_RANK>\" setting\n")
15211537
// defines local rank environment variables as unknown if not set by compilation flag, mostly to be able to run getenv() command
15221538
#ifndef ENV_LOCAL_RANK
15231539
#define ENV_LOCAL_RANK "UNKNOWN_LOCAL_RANK"
15241540
#endif
1525-
1526-
#endif
1541+
#endif // OPEN_MPI
15271542

15281543
// sets GPU device before MPI initialization
15291544
// MPI will then recognize the setting and take over the GPU device setup
@@ -1536,6 +1551,9 @@ void FC_FUNC_ (check_cuda_aware_mpi,
15361551
int rank = 0;
15371552
char * localRankStr = NULL;
15381553

1554+
// info
1555+
const char* msg;
1556+
15391557
// local rank info from environment
15401558
if ((localRankStr = getenv(ENV_LOCAL_RANK)) != NULL) {
15411559
// catching OpenMPI environment rank
@@ -1559,28 +1577,27 @@ void FC_FUNC_ (check_cuda_aware_mpi,
15591577
//printf("debug: compile time check for CUDA-aware MPI - rank %d\n",rank);
15601578

15611579
#if defined(MPIX_CUDA_AWARE_SUPPORT)
1562-
#pragma message ("\n\nCompiling with: WITH_CUDA_AWARE_MPI has MPIX_CUDA_AWARE_SUPPORT\n")
1563-
int ret = MPIX_Query_cuda_support();
1564-
if (ret == 1) {
1565-
// MPI library has CUDA-aware support
1566-
has_cuda_aware_mpi = 1;
1567-
} else {
1568-
// MPI library does not have CUDA-aware support
1569-
has_cuda_aware_mpi = 0;
1570-
}
1580+
// assume CUDA-aware support, will run query later to double-check
1581+
has_cuda_aware_mpi = 1;
1582+
msg = "\
1583+
This version has been compiled with flag WITH_CUDA_AWARE_MPI and MPI library has MPIX query cuda support.\n \
1584+
We assume CUDA-aware MPI.\n\n";
15711585
#else
1572-
#pragma message ("\n\nCompiling with: WITH_CUDA_AWARE_MPI has no MPIX_CUDA_AWARE_SUPPORT, please check MPI installation\n")
1573-
// user info
1574-
if (rank == 0){
1575-
printf("\
1576-
This version has been compiled with flag WITH_CUDA_AWARE_MPI, but MPI library cannot determine if there is CUDA-aware support.\n \
1577-
Please check MPI installation.\n\n");
1578-
}
1586+
// no MPIX query support
15791587
has_cuda_aware_mpi = 0;
1588+
msg = "\
1589+
This version has been compiled with flag WITH_CUDA_AWARE_MPI, but MPI library cannot determine if there is CUDA-aware support.\n \
1590+
Please check MPI installation.\n\n";
1591+
//debug
1592+
//printf("[check_cuda_aware_mpi] rank = %d - no MPIX has cuda aware MPI = %d\n",rank,has_cuda_aware_mpi);
15801593
#endif // MPIX_CUDA_AWARE_SUPPORT
15811594

1595+
// user info
1596+
if (rank == 0){
1597+
printf("%s",msg);
1598+
}
15821599
// debug
1583-
//printf("debug: query cuda support: MPI library CUDA-aware support - rank %d has support %d\n\n",rank,has_cuda_aware_mpi);
1600+
//printf("[check_cuda_aware_mpi] query cuda support: MPI library CUDA-aware support - rank %d has support %d\n\n",rank,has_cuda_aware_mpi);
15841601

15851602
// sets local rank's GPU association
15861603
if (has_cuda_aware_mpi){
@@ -1606,3 +1623,72 @@ Please check MPI installation.\n\n");
16061623
// return value
16071624
*has_cuda_aware_mpi_f = has_cuda_aware_mpi;
16081625
}
1626+
1627+
/* ----------------------------------------------------------------------------------------------- */
1628+
1629+
// note: MPIX_Query_cuda_support() can only be called after MPI_init().
1630+
// However, for CUDA-aware MPI, we must set the device with cudaSetDevice() before MPI_init().
1631+
// Thus, we use a compile-time check for MPIX_CUDA_AWARE_SUPPORT to see if there is CUDA-aware MPI support.
1632+
// And then use the query here after setting device and MPI initialization as a validity check of the loaded MPI system.
1633+
1634+
extern EXTERN_LANG
1635+
void FC_FUNC_ (query_cuda_aware_mpi,
1636+
QUERY_CUDA_AWARE_MPI) (int* myrank_f, int* has_cuda_aware_mpi_f) {
1637+
1638+
TRACE ("query_cuda_aware_mpi");
1639+
1640+
// rank
1641+
int myrank = *myrank_f;
1642+
1643+
// flags
1644+
int has_cuda_aware_mpi = 0;
1645+
1646+
#ifdef WITH_CUDA_AWARE_MPI
1647+
// debug output to file
1648+
char filename[BUFSIZ];
1649+
FILE* fp;
1650+
const char* msg;
1651+
1652+
sprintf(filename,"./OUTPUT_FILES/gpu_aware_info.txt");
1653+
1654+
#if defined(MPIX_CUDA_AWARE_SUPPORT)
1655+
int ret = MPIX_Query_cuda_support();
1656+
if (ret == 1) {
1657+
// MPI library loaded has CUDA-aware support
1658+
has_cuda_aware_mpi = 1;
1659+
msg = "CUDA-aware support is available. MPIX query returned CUDA support.\n";
1660+
} else {
1661+
// MPI library does not have CUDA-aware support
1662+
has_cuda_aware_mpi = 0;
1663+
msg = "CUDA-aware support is disabled or unavailable. MPIX query returned no CUDA support.\n";
1664+
}
1665+
//debug
1666+
//printf("[query_cuda_aware_mpi] myrank = %d - MPIX query: has cuda aware MPI = %d\n",myrank,has_cuda_aware_mpi);
1667+
#else
1668+
// no MPIX query support
1669+
has_cuda_aware_mpi = 0;
1670+
msg = "\
1671+
This version has been compiled with flag WITH_CUDA_AWARE_MPI, but MPI library cannot determine if there is CUDA-aware support.\n \
1672+
Please check MPI installation.\n";
1673+
//debug
1674+
//printf("[query_cuda_aware_mpi] myrank = %d - no MPIX has cuda aware MPI = %d\n",myrank,has_cuda_aware_mpi);
1675+
#endif // MPIX_CUDA_AWARE_SUPPORT
1676+
1677+
// user info
1678+
if (myrank == 0){
1679+
printf("%s",msg);
1680+
}
1681+
1682+
// file output
1683+
if (myrank == 0){
1684+
fp = fopen(filename,"w");
1685+
if (fp != NULL){
1686+
fprintf (fp, "%s", msg);
1687+
fclose(fp);
1688+
}
1689+
}
1690+
#endif // WITH_CUDA_AWARE_MPI
1691+
1692+
// return value
1693+
*has_cuda_aware_mpi_f = has_cuda_aware_mpi;
1694+
}

‎src/gpu/mesh_constants_gpu.h‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,17 @@
5353

5454
#include "config.h"
5555

56+
#ifdef WITH_MPI
57+
#include <mpi.h>
58+
// CUDA-aware support
59+
#ifdef WITH_CUDA_AWARE_MPI
60+
// extension
61+
#if defined(OPEN_MPI)
62+
#include <mpi-ext.h> /* extensions */
63+
#endif
64+
#endif // WITH_CUDA_AWARE_MPI
65+
#endif // WITH_MPI
66+
5667
#ifdef USE_CUDA
5768
#include <cuda.h>
5869
#include <cuda_runtime.h>
@@ -62,13 +73,6 @@
6273
#include <hip/hip_runtime.h>
6374
#endif
6475

65-
#ifdef WITH_MPI
66-
#include <mpi.h>
67-
#endif
68-
69-
#ifdef WITH_CUDA_AWARE_MPI
70-
#include <mpi-ext.h>
71-
#endif
7276

7377
// type of "working" variables: see also CUSTOM_REAL in constants.h
7478
//
@@ -91,7 +95,9 @@ typedef double realw;
9195
#endif
9296

9397
/*----------------------------------------------------------------------------------------------- */
98+
9499
// for debugging and benchmarking
100+
95101
/*----------------------------------------------------------------------------------------------- */
96102

97103
// debug: outputs traces
@@ -182,7 +188,9 @@ typedef double realw;
182188
#define VAR_NAME_VALUE(var) #var " = " VALUE(var)
183189

184190
/*----------------------------------------------------------------------------------------------- */
191+
185192
// GPU constant arrays
193+
186194
/*----------------------------------------------------------------------------------------------- */
187195
// (must match constants.h definitions)
188196

@@ -557,7 +565,9 @@ typedef union {
557565
#endif
558566

559567
/*----------------------------------------------------------------------------------------------- */
568+
560569
// mesh pointer wrapper structure
570+
561571
/*----------------------------------------------------------------------------------------------- */
562572

563573
typedef struct mesh_ {
@@ -1287,7 +1297,9 @@ typedef struct mesh_ {
12871297

12881298

12891299
/*----------------------------------------------------------------------------------------------- */
1300+
12901301
// utility functions
1302+
12911303
/*----------------------------------------------------------------------------------------------- */
12921304

12931305
// defined in helper_functions_gpu.c
@@ -1410,7 +1422,9 @@ realw get_device_array_maximum_value (gpu_realw_mem d_array, int size);
14101422

14111423

14121424
/* ----------------------------------------------------------------------------------------------- */
1425+
14131426
// kernel setup function
1427+
14141428
/* ----------------------------------------------------------------------------------------------- */
14151429
// moved here into header to inline function calls if possible
14161430

‎src/gpu/write_seismograms_gpu.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -395,8 +395,8 @@ void FC_FUNC_(transfer_seismo_from_device_async,
395395
Mesh *mp = (Mesh *) *Mesh_pointer_f;
396396

397397
int irec,ispec,iglob,i;
398-
realw* h_field;
399-
int* h_ispec_selected;
398+
realw* h_field = NULL;
399+
int* h_ispec_selected = NULL;
400400

401401
// checks if anything to do
402402
if (mp->nrec_local == 0) return;

‎src/specfem3D/initialize_simulation.F90‎

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -662,10 +662,30 @@ subroutine initialize_GPU()
662662

663663
!----------------------------------------------------------------
664664

665-
if (GPU_MODE .and. USE_HYBRID_CPU_GPU) then
665+
! checks if GPU simulation turned on
666+
if (.not. GPU_MODE) return
667+
668+
! GPU_MODE now defined in Par_file
669+
if (myrank == 0) then
670+
write(IMAIN,*)
671+
write(IMAIN,*) "GPU_MODE Active."
672+
call flush_IMAIN()
673+
endif
674+
675+
! check for GPU runs
676+
if (NGLLX /= 5 .or. NGLLY /= 5 .or. NGLLZ /= 5 ) &
677+
stop 'GPU mode can only be used if NGLLX == NGLLY == NGLLZ == 5'
678+
if (CUSTOM_REAL /= SIZE_REAL) &
679+
stop 'GPU mode runs only with CUSTOM_REAL == SIZE_REAL'
680+
if (ATTENUATION_VAL) then
681+
if (N_SLS /= 3 ) &
682+
stop 'GPU mode does not support N_SLS /= 3 yet'
683+
endif
684+
685+
if (USE_HYBRID_CPU_GPU) then
666686
! distributes processes on GPU and CPU
667687
if (mod(myrank,TOTAL_PROCESSES_PER_NODE) < PROCESSES_PER_CPU) then
668-
! turns of GPU mode for this process
688+
! turns off GPU mode for this process
669689
GPU_MODE = .false.
670690
else
671691
!leaves GPU_MODE == .true.
@@ -691,35 +711,38 @@ subroutine initialize_GPU()
691711
call any_all_l(USE_CUDA_AWARE_MPI,USE_CUDA_AWARE_MPI_all)
692712
USE_CUDA_AWARE_MPI = USE_CUDA_AWARE_MPI_all
693713

714+
#ifdef WITH_CUDA_AWARE_MPI
715+
if (USE_CUDA_AWARE_MPI) then
716+
! double check if loaded MPI system has CUDA support
717+
! query MPI for cuda support
718+
call query_cuda_aware_mpi(myrank,USE_CUDA_AWARE_MPI)
719+
720+
! all processes need support
721+
call any_all_l(USE_CUDA_AWARE_MPI,USE_CUDA_AWARE_MPI_all)
722+
USE_CUDA_AWARE_MPI = USE_CUDA_AWARE_MPI_all
723+
724+
if (myrank == 0) then
725+
write(IMAIN,*) "check CUDA-aware MPI returned : ",USE_CUDA_AWARE_MPI
726+
call flush_IMAIN()
727+
endif
728+
endif
729+
#endif
730+
694731
! GPU_MODE now defined in Par_file
695732
if (GPU_MODE) then
696733
! user output
697734
if (myrank == 0) then
698-
write(IMAIN,*)
699-
write(IMAIN,*) "GPU_MODE Active."
700735
write(IMAIN,*) " runtime : ",GPU_RUNTIME
701736
write(IMAIN,*) " platform: ",trim(GPU_PLATFORM)
702737
write(IMAIN,*) " device : ",trim(GPU_DEVICE)
703738
call flush_IMAIN()
704739
endif
705740

706-
! check for GPU runs
707-
if (NGLLX /= 5 .or. NGLLY /= 5 .or. NGLLZ /= 5 ) &
708-
stop 'GPU mode can only be used if NGLLX == NGLLY == NGLLZ == 5'
709-
if (CUSTOM_REAL /= SIZE_REAL) &
710-
stop 'GPU mode runs only with CUSTOM_REAL == SIZE_REAL'
711-
if (ATTENUATION_VAL) then
712-
if (N_SLS /= 3 ) &
713-
stop 'GPU mode does not support N_SLS /= 3 yet'
714-
endif
715-
716741
! initializes GPU and outputs info to files for all processes
717742
if (USE_CUDA_AWARE_MPI) then
718743
! devices have already been set before MPI_init()
719744
if (myrank == 0) then
720-
write(IMAIN,*)
721745
write(IMAIN,*) " using CUDA-aware MPI"
722-
write(IMAIN,*)
723746
call flush_IMAIN()
724747
endif
725748
! just to get number of devices and device info output

0 commit comments

Comments
 (0)