From 6e33a13c60b12f126327161dfa475910a4ee632b Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 29 Jun 2023 16:31:39 -0500 Subject: [PATCH 01/22] enable subfiling with h5fuse --- examples/dam_break.cpp | 15 ++++++++++ src/ExaMPM_Solver.hpp | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/examples/dam_break.cpp b/examples/dam_break.cpp index c60b30a..3a53c0c 100644 --- a/examples/dam_break.cpp +++ b/examples/dam_break.cpp @@ -118,6 +118,21 @@ void damBreak( const double cell_size, const int ppc, const int halo_size, //---------------------------------------------------------------------------// int main( int argc, char* argv[] ) { + + const char* env_val = std::getenv("H5FD_SUBFILING"); + if(env_val != NULL) { + int mpi_thread_required = MPI_THREAD_MULTIPLE; + int mpi_thread_provided = 0; + + // HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE + + MPI_Init_thread(&argc, &argv, mpi_thread_required, &mpi_thread_provided); + if (mpi_thread_provided < mpi_thread_required) { + printf("MPI_THREAD_MULTIPLE not supported\n"); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + else MPI_Init( &argc, &argv ); Kokkos::initialize( argc, argv ); diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index 5e0108f..e26b859 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -25,6 +25,10 @@ #include #include +#include +#include +#include +#include namespace ExaMPM { @@ -106,12 +110,72 @@ class Solver : public SolverBase // Prefer HDF5 output over Silo. Only output if one is enabled. #ifdef Cabana_ENABLE_HDF5 Cabana::Experimental::HDF5ParticleOutput::HDF5Config h5_config; + const char* env_val = std::getenv("H5FD_SUBFILING"); + if(env_val != NULL) + h5_config.subfiling = true; Cabana::Experimental::HDF5ParticleOutput::writeTimeStep( h5_config, "particles", _mesh->localGrid()->globalGrid().comm(), _step, _time, _pm->numParticle(), _pm->get( Location::Particle(), Field::Position() ), _pm->get( Location::Particle(), Field::Velocity() ), _pm->get( Location::Particle(), Field::J() ) ); + + if(h5_config.subfiling) { + + if ( _rank == 0) { + pid_t pid = 0; + pid_t tmppid; + int status; + + pid = fork(); + + if (pid == 0) { + + // stub HDF5 filename + std::stringstream filename_hdf5; + filename_hdf5 << "particles" << "_" << _step << ".h5"; + + // Directory containing the subfiling configuration file + std::stringstream config_dir; + if(const char* env_value = std::getenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX)) + config_dir << env_value; + else + config_dir << "."; + + // Find the name of the subfiling configuration file + struct stat file_info; + stat(filename_hdf5.str().c_str(), &file_info); + + char config_filename[PATH_MAX]; + snprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir.str().c_str(), + filename_hdf5.str().c_str(), (uint64_t)file_info.st_ino); + + // Call the h5fuse utility + char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; + + execvp(args[0], args); + + } + else { + tmppid = waitpid(pid, &status, 0); + + if (WIFEXITED(status)) { + int ret; + + if ((ret = WEXITSTATUS(status)) != 0) { + printf("h5fuse process exited with error code %d\n", ret); + fflush(stdout); + MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); + } + } + else { + printf("h5fuse process terminated abnormally\n"); + fflush(stdout); + MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); + } + } + } + } #else #ifdef Cabana_ENABLE_SILO Cajita::Experimental::SiloParticleOutput::writeTimeStep( From 9fd94bb29bec0b0415f7916c5bf6892570f44cc6 Mon Sep 17 00:00:00 2001 From: Michael Breitenfeld Date: Wed, 2 Aug 2023 10:03:08 -0400 Subject: [PATCH 02/22] new fork schema --- src/ExaMPM_Solver.hpp | 163 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 152 insertions(+), 11 deletions(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index e26b859..a3ba785 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -32,6 +32,7 @@ namespace ExaMPM { +int nfork; //---------------------------------------------------------------------------// class SolverBase { @@ -84,7 +85,7 @@ class Solver : public SolverBase while ( _time < t_final ) { if ( 0 == _rank && 0 == _step % write_freq ) - printf( "Time %f / %f\n", _time, t_final ); + printf( "Time %12.5e / %12.5e\n", _time, t_final ); // Fixed timestep is guaranteed only when sufficently low dt // does not violate the CFL condition (otherwise user-set dt is @@ -103,6 +104,48 @@ class Solver : public SolverBase if ( 0 == ( _step ) % write_freq ) outputParticles(); } + Cabana::Experimental::HDF5ParticleOutput::HDF5Config h5_config; + const char* env_val = std::getenv("H5FUSE"); + if(env_val != NULL) { + + if(h5_config.subfiling) { + + MPI_Comm shmcomm; + MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, + MPI_INFO_NULL, &shmcomm); + + int shmrank; + MPI_Comm_rank(shmcomm, &shmrank); + int status; + if ( shmrank == 0) { + //pid_t pid; + for (int i = 0; i < nfork; i++) { + waitpid(-1, NULL, 0); + } +#if 0 + while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { + if (WIFEXITED(status)) { + int ret; + + if ((ret = WEXITSTATUS(status)) != 0) { + printf("h5fuse process exited with error code %d\n", ret); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + else { + printf("h5fuse process terminated abnormally\n"); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } +#endif + } + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_free(&shmcomm); + } + } + } void outputParticles() @@ -113,24 +156,122 @@ class Solver : public SolverBase const char* env_val = std::getenv("H5FD_SUBFILING"); if(env_val != NULL) h5_config.subfiling = true; + + // Sets the HDF5 alignment equal to the SUBFILING STRIPE SIZE + env_val = std::getenv("SUBFILING STRIPE SIZE"); + if(env_val != NULL) { + h5_config.align = true; + h5_config.threshold = 0; + h5_config.alignment = std::atoi(env_val); + } + Cabana::Experimental::HDF5ParticleOutput::writeTimeStep( h5_config, "particles", _mesh->localGrid()->globalGrid().comm(), _step, _time, _pm->numParticle(), _pm->get( Location::Particle(), Field::Position() ), _pm->get( Location::Particle(), Field::Velocity() ), _pm->get( Location::Particle(), Field::J() ) ); - +#if 1 + env_val = std::getenv("H5FUSE"); + if(env_val != NULL) { + if(h5_config.subfiling) { - if ( _rank == 0) { - pid_t pid = 0; + MPI_Comm shmcomm; + MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, + MPI_INFO_NULL, &shmcomm); + + int shmrank; + MPI_Comm_rank(shmcomm, &shmrank); + + if ( shmrank == 0) { + pid_t pid = 0; + //pid_t tmppid; + int status; + //std::cout << "DOING OUTPUT " << _rank << std::endl; + //signal(SIGHUP, SIG_IGN); + //not interested in its childs, prevent zombies + //signal(SIGCHLD, SIG_IGN); + //signal(SIGQUIT, SIG_IGN); + + pid = fork(); + nfork++; + if (pid == 0) { + std::stringstream filename_hdf5; + filename_hdf5 << "particles" << "_" << _step << ".h5"; + + // Directory containing the subfiling configuration file + std::stringstream config_dir; + if(const char* env_value = std::getenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX)) + config_dir << env_value; + else + config_dir << "."; + // Find the name of the subfiling configuration file + struct stat file_info; + stat(filename_hdf5.str().c_str(), &file_info); + + char config_filename[PATH_MAX]; + snprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir.str().c_str(), + filename_hdf5.str().c_str(), (uint64_t)file_info.st_ino); + + // Call the h5fuse utility + char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-f"), config_filename, NULL}; + //char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; + + execvp(args[0], args); + //exit(0); + +#if 0 + // char *tmp_filename; + char *args[7]; + + //tmp_filename = (char *)malloc(PATH_MAX); + + /* Generate name for configuration file */ + //HDsnprintf(tmp_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir, + // SUBF_FILENAME, file_inode); + args[0] = strdup("ls"); + args[1] = strdup("-l"); + args[2] = NULL; + execvp("ls", args); +#endif + } +#if 0 + else { + //tmppid = waitpid(pid, &status, WNOHANG); + waitpid(pid, &status, 0); + if (WIFEXITED(status)) { + int ret; + + if ((ret = WEXITSTATUS(status)) != 0) { + printf("h5fuse process exited with error code %d\n", ret); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + else { + printf("h5fuse process terminated abnormally\n"); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } +#endif + } +} +} +#endif +#if 0 pid_t tmppid; int status; - pid = fork(); - + pid_t pid = fork(); + if (pid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } +#endif +#if 0 if (pid == 0) { - // stub HDF5 filename std::stringstream filename_hdf5; filename_hdf5 << "particles" << "_" << _step << ".h5"; @@ -141,7 +282,6 @@ class Solver : public SolverBase config_dir << env_value; else config_dir << "."; - // Find the name of the subfiling configuration file struct stat file_info; stat(filename_hdf5.str().c_str(), &file_info); @@ -154,7 +294,6 @@ class Solver : public SolverBase char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; execvp(args[0], args); - } else { tmppid = waitpid(pid, &status, 0); @@ -174,8 +313,10 @@ class Solver : public SolverBase MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); } } - } - } +#endif +// } +// } +//#endif #else #ifdef Cabana_ENABLE_SILO Cajita::Experimental::SiloParticleOutput::writeTimeStep( From 1b64d6834ded0bd3e920e31f51055ec0978a1b8e Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Tue, 15 Aug 2023 08:36:19 -0500 Subject: [PATCH 03/22] fixed env. var. --- src/ExaMPM_Solver.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index a3ba785..cf481ec 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -141,7 +141,6 @@ class Solver : public SolverBase } #endif } - MPI_Barrier(MPI_COMM_WORLD); MPI_Comm_free(&shmcomm); } } @@ -157,8 +156,8 @@ class Solver : public SolverBase if(env_val != NULL) h5_config.subfiling = true; - // Sets the HDF5 alignment equal to the SUBFILING STRIPE SIZE - env_val = std::getenv("SUBFILING STRIPE SIZE"); + // Sets the HDF5 alignment equal subfiling's stripe size + env_val = std::getenv("H5FD_SUBFILING_STRIPE_SIZE"); if(env_val != NULL) { h5_config.align = true; h5_config.threshold = 0; From 395fb2a4a21fd888716936c294101f9cea104a15 Mon Sep 17 00:00:00 2001 From: Michael Breitenfeld Date: Tue, 15 Aug 2023 16:11:14 -0400 Subject: [PATCH 04/22] subfiling h5fuse fix --- src/ExaMPM_Solver.hpp | 82 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index cf481ec..e17d10f 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -33,6 +33,52 @@ namespace ExaMPM { int nfork; + +struct timer_statsinfo { + double min; + double max; + double mean; + double std; +}; + + timer_statsinfo io_stats; + + // Collect statistics of timers on all ranks + // timer - elapsed time for rank + // comm - communicator for collecting stats + // destrank - the rank to which to collect stats + // stats - pointer to timer stats + // + + void timer_stats(double timer, MPI_Comm comm, int destrank, timer_statsinfo *stats) + { + int rank, nprocs, i; + double *rtimers=NULL; /* All timers from ranks */ + + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &nprocs); + if(rank == destrank) { + rtimers = (double *) malloc(nprocs*sizeof(double)); + stats->mean = 0.; + stats->min = timer; + stats->max = timer; + stats->std = 0.f; + } + MPI_Gather(&timer, 1, MPI_DOUBLE, rtimers, 1, MPI_DOUBLE, destrank, comm); + if(rank == destrank) { + for(i = 0; i < nprocs; i++) { + if(rtimers[i] > stats->max) stats->max = rtimers[i]; + if(rtimers[i] < stats->min) stats->min = rtimers[i]; + stats->mean += rtimers[i]; + } + stats->mean /= nprocs; + for(i = 0; i < nprocs; i++) + stats->std += (rtimers[i]-stats->mean)*(rtimers[i]-stats->mean); + stats->std = sqrt(stats->std / nprocs); + free(rtimers); + } + } + //---------------------------------------------------------------------------// class SolverBase { @@ -81,11 +127,12 @@ class Solver : public SolverBase { // Output initial state. outputParticles(); + while ( _time < t_final ) { if ( 0 == _rank && 0 == _step % write_freq ) - printf( "Time %12.5e / %12.5e\n", _time, t_final ); + printf( "Time %12.5e / %12.5e [iostats (s), mean min max: %12.5e %12.5e %12.5e] \n", _time, t_final, io_stats.mean, io_stats.min, io_stats.max ); // Fixed timestep is guaranteed only when sufficently low dt // does not violate the CFL condition (otherwise user-set dt is @@ -104,12 +151,10 @@ class Solver : public SolverBase if ( 0 == ( _step ) % write_freq ) outputParticles(); } - Cabana::Experimental::HDF5ParticleOutput::HDF5Config h5_config; + const char* env_val = std::getenv("H5FUSE"); if(env_val != NULL) { - - if(h5_config.subfiling) { - + MPI_Comm shmcomm; MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shmcomm); @@ -118,9 +163,24 @@ class Solver : public SolverBase MPI_Comm_rank(shmcomm, &shmrank); int status; if ( shmrank == 0) { - //pid_t pid; + for (int i = 0; i < nfork; i++) { - waitpid(-1, NULL, 0); + waitpid(-1, &status, 0); + if (WIFEXITED(status)) { + int ret; + + if ((ret = WEXITSTATUS(status)) != 0) { + printf("h5fuse process exited with error code %d\n", ret); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + else { + printf("h5fuse process terminated abnormally\n"); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } #if 0 while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { @@ -142,7 +202,6 @@ class Solver : public SolverBase #endif } MPI_Comm_free(&shmcomm); - } } } @@ -163,13 +222,17 @@ class Solver : public SolverBase h5_config.threshold = 0; h5_config.alignment = std::atoi(env_val); } - + + double t1, t2; + t1 = MPI_Wtime(); Cabana::Experimental::HDF5ParticleOutput::writeTimeStep( h5_config, "particles", _mesh->localGrid()->globalGrid().comm(), _step, _time, _pm->numParticle(), _pm->get( Location::Particle(), Field::Position() ), _pm->get( Location::Particle(), Field::Velocity() ), _pm->get( Location::Particle(), Field::J() ) ); + t2 = MPI_Wtime(); + timer_stats(t2-t1, MPI_COMM_WORLD, 0, &io_stats); #if 1 env_val = std::getenv("H5FUSE"); if(env_val != NULL) { @@ -342,6 +405,7 @@ class Solver : public SolverBase std::shared_ptr> _mesh; std::shared_ptr> _pm; int _rank; + }; //---------------------------------------------------------------------------// From c10ae55bfb731f58c08012315bc22393255ddb69 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Tue, 15 Aug 2023 16:13:25 -0500 Subject: [PATCH 05/22] code clean-up --- src/ExaMPM_Solver.hpp | 404 ++++++++++++++++-------------------------- 1 file changed, 151 insertions(+), 253 deletions(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index e17d10f..c4abdd4 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -25,60 +25,15 @@ #include #include -#include -#include -#include #include +#include +#include +#include namespace ExaMPM { int nfork; -struct timer_statsinfo { - double min; - double max; - double mean; - double std; -}; - - timer_statsinfo io_stats; - - // Collect statistics of timers on all ranks - // timer - elapsed time for rank - // comm - communicator for collecting stats - // destrank - the rank to which to collect stats - // stats - pointer to timer stats - // - - void timer_stats(double timer, MPI_Comm comm, int destrank, timer_statsinfo *stats) - { - int rank, nprocs, i; - double *rtimers=NULL; /* All timers from ranks */ - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &nprocs); - if(rank == destrank) { - rtimers = (double *) malloc(nprocs*sizeof(double)); - stats->mean = 0.; - stats->min = timer; - stats->max = timer; - stats->std = 0.f; - } - MPI_Gather(&timer, 1, MPI_DOUBLE, rtimers, 1, MPI_DOUBLE, destrank, comm); - if(rank == destrank) { - for(i = 0; i < nprocs; i++) { - if(rtimers[i] > stats->max) stats->max = rtimers[i]; - if(rtimers[i] < stats->min) stats->min = rtimers[i]; - stats->mean += rtimers[i]; - } - stats->mean /= nprocs; - for(i = 0; i < nprocs; i++) - stats->std += (rtimers[i]-stats->mean)*(rtimers[i]-stats->mean); - stats->std = sqrt(stats->std / nprocs); - free(rtimers); - } - } - //---------------------------------------------------------------------------// class SolverBase { @@ -127,12 +82,14 @@ class Solver : public SolverBase { // Output initial state. outputParticles(); - while ( _time < t_final ) { if ( 0 == _rank && 0 == _step % write_freq ) - printf( "Time %12.5e / %12.5e [iostats (s), mean min max: %12.5e %12.5e %12.5e] \n", _time, t_final, io_stats.mean, io_stats.min, io_stats.max ); + printf( "Time %12.5e / %12.5e [iostats, mean min max (s): " + "%12.5e %12.5e %12.5e] \n", + _time, t_final, io_stats.mean, io_stats.min, + io_stats.max ); // Fixed timestep is guaranteed only when sufficently low dt // does not violate the CFL condition (otherwise user-set dt is @@ -151,59 +108,33 @@ class Solver : public SolverBase if ( 0 == ( _step ) % write_freq ) outputParticles(); } - - const char* env_val = std::getenv("H5FUSE"); - if(env_val != NULL) { - - MPI_Comm shmcomm; - MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, - MPI_INFO_NULL, &shmcomm); - - int shmrank; - MPI_Comm_rank(shmcomm, &shmrank); - int status; - if ( shmrank == 0) { - for (int i = 0; i < nfork; i++) { - waitpid(-1, &status, 0); - if (WIFEXITED(status)) { + // Wait for all the h5fuse processes to complete + if ( shmrank == 0 ) + { + int status; + for ( int i = 0; i < nfork; i++ ) + { + waitpid( -1, &status, 0 ); + if ( WIFEXITED( status ) ) + { int ret; - - if ((ret = WEXITSTATUS(status)) != 0) { - printf("h5fuse process exited with error code %d\n", ret); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); + if ( ( ret = WEXITSTATUS( status ) ) != 0 ) + { + printf( "h5fuse process exited with error code %d\n", + ret ); + fflush( stdout ); + MPI_Abort( MPI_COMM_WORLD, -1 ); } } - else { - printf("h5fuse process terminated abnormally\n"); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - } -#if 0 - while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { - if (WIFEXITED(status)) { - int ret; - - if ((ret = WEXITSTATUS(status)) != 0) { - printf("h5fuse process exited with error code %d\n", ret); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } + else + { + printf( "h5fuse process terminated abnormally\n" ); + fflush( stdout ); + MPI_Abort( MPI_COMM_WORLD, -1 ); } - else { - printf("h5fuse process terminated abnormally\n"); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } - } -#endif } - MPI_Comm_free(&shmcomm); - } - + } } void outputParticles() @@ -211,20 +142,21 @@ class Solver : public SolverBase // Prefer HDF5 output over Silo. Only output if one is enabled. #ifdef Cabana_ENABLE_HDF5 Cabana::Experimental::HDF5ParticleOutput::HDF5Config h5_config; - const char* env_val = std::getenv("H5FD_SUBFILING"); - if(env_val != NULL) - h5_config.subfiling = true; + const char* env_val = std::getenv( "H5FD_SUBFILING" ); + if ( env_val != NULL ) + h5_config.subfiling = true; // Sets the HDF5 alignment equal subfiling's stripe size - env_val = std::getenv("H5FD_SUBFILING_STRIPE_SIZE"); - if(env_val != NULL) { - h5_config.align = true; - h5_config.threshold = 0; - h5_config.alignment = std::atoi(env_val); - } - - double t1, t2; - t1 = MPI_Wtime(); + env_val = std::getenv( "H5FD_SUBFILING_STRIPE_SIZE" ); + if ( env_val != NULL ) + { + h5_config.align = true; + h5_config.threshold = 0; + h5_config.alignment = std::atoi( env_val ); + } + + double t1, t2; + t1 = MPI_Wtime(); Cabana::Experimental::HDF5ParticleOutput::writeTimeStep( h5_config, "particles", _mesh->localGrid()->globalGrid().comm(), _step, _time, _pm->numParticle(), @@ -232,153 +164,65 @@ class Solver : public SolverBase _pm->get( Location::Particle(), Field::Velocity() ), _pm->get( Location::Particle(), Field::J() ) ); t2 = MPI_Wtime(); - timer_stats(t2-t1, MPI_COMM_WORLD, 0, &io_stats); -#if 1 - env_val = std::getenv("H5FUSE"); - if(env_val != NULL) { - - if(h5_config.subfiling) { - - MPI_Comm shmcomm; - MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, - MPI_INFO_NULL, &shmcomm); - - int shmrank; - MPI_Comm_rank(shmcomm, &shmrank); - - if ( shmrank == 0) { - pid_t pid = 0; - //pid_t tmppid; - int status; - //std::cout << "DOING OUTPUT " << _rank << std::endl; - //signal(SIGHUP, SIG_IGN); - //not interested in its childs, prevent zombies - //signal(SIGCHLD, SIG_IGN); - //signal(SIGQUIT, SIG_IGN); - - pid = fork(); - nfork++; - if (pid == 0) { - std::stringstream filename_hdf5; - filename_hdf5 << "particles" << "_" << _step << ".h5"; - - // Directory containing the subfiling configuration file - std::stringstream config_dir; - if(const char* env_value = std::getenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX)) - config_dir << env_value; - else - config_dir << "."; - // Find the name of the subfiling configuration file - struct stat file_info; - stat(filename_hdf5.str().c_str(), &file_info); - - char config_filename[PATH_MAX]; - snprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir.str().c_str(), - filename_hdf5.str().c_str(), (uint64_t)file_info.st_ino); - - // Call the h5fuse utility - char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-f"), config_filename, NULL}; - //char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; - - execvp(args[0], args); - //exit(0); - -#if 0 - // char *tmp_filename; - char *args[7]; - - //tmp_filename = (char *)malloc(PATH_MAX); - - /* Generate name for configuration file */ - //HDsnprintf(tmp_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir, - // SUBF_FILENAME, file_inode); - args[0] = strdup("ls"); - args[1] = strdup("-l"); - args[2] = NULL; - execvp("ls", args); -#endif - } -#if 0 - else { - //tmppid = waitpid(pid, &status, WNOHANG); - waitpid(pid, &status, 0); - if (WIFEXITED(status)) { - int ret; - - if ((ret = WEXITSTATUS(status)) != 0) { - printf("h5fuse process exited with error code %d\n", ret); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } - } - else { - printf("h5fuse process terminated abnormally\n"); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } - } -#endif - } -} -} -#endif -#if 0 - pid_t tmppid; - int status; - - pid_t pid = fork(); - if (pid == -1) { - perror("fork"); - exit(EXIT_FAILURE); - } -#endif -#if 0 - if (pid == 0) { - // stub HDF5 filename - std::stringstream filename_hdf5; - filename_hdf5 << "particles" << "_" << _step << ".h5"; - - // Directory containing the subfiling configuration file - std::stringstream config_dir; - if(const char* env_value = std::getenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX)) - config_dir << env_value; - else - config_dir << "."; - // Find the name of the subfiling configuration file - struct stat file_info; - stat(filename_hdf5.str().c_str(), &file_info); - - char config_filename[PATH_MAX]; - snprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir.str().c_str(), - filename_hdf5.str().c_str(), (uint64_t)file_info.st_ino); - - // Call the h5fuse utility - char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; - - execvp(args[0], args); - } - else { - tmppid = waitpid(pid, &status, 0); - - if (WIFEXITED(status)) { - int ret; + timer_stats( t2 - t1, MPI_COMM_WORLD, 0, &io_stats ); - if ((ret = WEXITSTATUS(status)) != 0) { - printf("h5fuse process exited with error code %d\n", ret); - fflush(stdout); - MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); + env_val = std::getenv( "H5FUSE" ); + if ( env_val != NULL ) + { + if ( h5_config.subfiling ) + { + + MPI_Comm shmcomm; + MPI_Comm_split_type( MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, + MPI_INFO_NULL, &shmcomm ); + + MPI_Comm_rank( shmcomm, &shmrank ); + + // One rank from each node executes h5fuse.sh + + if ( shmrank == 0 ) + { + pid_t pid = 0; + int status; + + pid = fork(); + nfork++; + if ( pid == 0 ) + { + std::stringstream filename_hdf5; + filename_hdf5 << "particles" + << "_" << _step << ".h5"; + + // Directory containing the subfiling configuration file + std::stringstream config_dir; + if ( const char* env_value = std::getenv( + H5FD_SUBFILING_CONFIG_FILE_PREFIX ) ) + config_dir << env_value; + else + config_dir << "."; + // Find the name of the subfiling configuration file + struct stat file_info; + stat( filename_hdf5.str().c_str(), &file_info ); + + char config_filename[PATH_MAX]; + snprintf( config_filename, PATH_MAX, + "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, + config_dir.str().c_str(), + filename_hdf5.str().c_str(), + (uint64_t)file_info.st_ino ); + + // Call the h5fuse utility + // Removes the subfiles in the process + char* args[] = { strdup( "./h5fuse.sh" ), + strdup( "-r" ), strdup( "-f" ), + config_filename, NULL }; + + execvp( args[0], args ); + } } - } - else { - printf("h5fuse process terminated abnormally\n"); - fflush(stdout); - MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); - } + MPI_Comm_free( &shmcomm ); } -#endif -// } -// } -//#endif + } #else #ifdef Cabana_ENABLE_SILO Cajita::Experimental::SiloParticleOutput::writeTimeStep( @@ -405,7 +249,61 @@ class Solver : public SolverBase std::shared_ptr> _mesh; std::shared_ptr> _pm; int _rank; + int shmrank; + + struct timer_statsinfo + { + double min; + double max; + double mean; + double std; + }; + timer_statsinfo io_stats; + + // Collect statistics of timers on all ranks + // timer - elapsed time for rank + // comm - communicator for collecting stats + // destrank - the rank to which to collect stats + // stats - pointer to timer stats + // + + void timer_stats( double timer, MPI_Comm comm, int destrank, + timer_statsinfo* stats ) + { + int rank, nprocs, i; + double* rtimers = NULL; /* All timers from ranks */ + + MPI_Comm_rank( comm, &rank ); + MPI_Comm_size( comm, &nprocs ); + if ( rank == destrank ) + { + rtimers = (double*)malloc( nprocs * sizeof( double ) ); + stats->mean = 0.; + stats->min = timer; + stats->max = timer; + stats->std = 0.f; + } + MPI_Gather( &timer, 1, MPI_DOUBLE, rtimers, 1, MPI_DOUBLE, destrank, + comm ); + if ( rank == destrank ) + { + for ( i = 0; i < nprocs; i++ ) + { + if ( rtimers[i] > stats->max ) + stats->max = rtimers[i]; + if ( rtimers[i] < stats->min ) + stats->min = rtimers[i]; + stats->mean += rtimers[i]; + } + stats->mean /= nprocs; + for ( i = 0; i < nprocs; i++ ) + stats->std += + ( rtimers[i] - stats->mean ) * ( rtimers[i] - stats->mean ); + stats->std = sqrt( stats->std / nprocs ); + free( rtimers ); + } + } }; //---------------------------------------------------------------------------// From 73998bf85210ba2344af0608efa6b2ef6de9a537 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Tue, 15 Aug 2023 16:41:56 -0500 Subject: [PATCH 06/22] clean-up --- examples/dam_break.cpp | 28 ++++++++++++++++------------ src/ExaMPM_Solver.hpp | 3 +++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/examples/dam_break.cpp b/examples/dam_break.cpp index 3a53c0c..c4cd531 100644 --- a/examples/dam_break.cpp +++ b/examples/dam_break.cpp @@ -119,21 +119,25 @@ void damBreak( const double cell_size, const int ppc, const int halo_size, int main( int argc, char* argv[] ) { - const char* env_val = std::getenv("H5FD_SUBFILING"); - if(env_val != NULL) { - int mpi_thread_required = MPI_THREAD_MULTIPLE; - int mpi_thread_provided = 0; + // enable the use of subfiling by setting enviroment H5FD_SUBFILING + const char* env_val = std::getenv( "H5FD_SUBFILING" ); + if ( env_val != NULL ) + { + int mpi_thread_required = MPI_THREAD_MULTIPLE; + int mpi_thread_provided = 0; - // HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE + // HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE - MPI_Init_thread(&argc, &argv, mpi_thread_required, &mpi_thread_provided); - if (mpi_thread_provided < mpi_thread_required) { - printf("MPI_THREAD_MULTIPLE not supported\n"); - MPI_Abort(MPI_COMM_WORLD, -1); + MPI_Init_thread( &argc, &argv, mpi_thread_required, + &mpi_thread_provided ); + if ( mpi_thread_provided < mpi_thread_required ) + { + printf( "MPI_THREAD_MULTIPLE not supported\n" ); + MPI_Abort( MPI_COMM_WORLD, -1 ); + } } - } - else - MPI_Init( &argc, &argv ); + else + MPI_Init( &argc, &argv ); Kokkos::initialize( argc, argv ); diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index c4abdd4..bfee6fb 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -166,6 +166,9 @@ class Solver : public SolverBase t2 = MPI_Wtime(); timer_stats( t2 - t1, MPI_COMM_WORLD, 0, &io_stats ); + // Setting enviroment H5FUSE enables fusing the subfiles into + // an HDF5 file. Assumes h5fuse.sh is in the same directory + // as the executable. env_val = std::getenv( "H5FUSE" ); if ( env_val != NULL ) { From 98836ad039b8375f4e39bf2ba5b6a776416f3028 Mon Sep 17 00:00:00 2001 From: Ioannis Vardas Date: Thu, 14 Sep 2023 19:18:08 +0200 Subject: [PATCH 07/22] Semicolon missing ExaMPM_Solver.hpp --- src/ExaMPM_Solver.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index 5e0108f..a1d9763 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -123,7 +123,7 @@ class Solver : public SolverBase if ( _rank == 0 ) std::cout << "No particle output enabled in Cabana. Add " "Cabana_REQUIRE_HDF5=ON or Cabana_REQUIRE_SILO=ON to " - "the Cabana build if needed." + "the Cabana build if needed."; #endif #endif } From 038f948749cd2c0670bbf147eb1125dea5e8d392 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Thu, 14 Sep 2023 13:26:28 -0400 Subject: [PATCH 08/22] Expand CI to catch issues with output --- .github/workflows/CI.yml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f4c336f..2a80a68 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -15,6 +15,26 @@ jobs: matrix: distro: ['ubuntu:latest'] backend: ["SERIAL", "OPENMP"] + output: ['HDF5'] + include: + - distro: 'ubuntu:latest' + cxx: 'g++' + backend: 'SERIAL' + cmake_build_type: 'Debug' + kokkos_ver: '3.6.01' + output: 'SILO' + - distro: 'ubuntu:latest' + cxx: 'g++' + backend: 'SERIAL' + cmake_build_type: 'Debug' + kokkos_ver: '3.6.01' + output: 'NONE' + - distro: 'ubuntu:latest' + cxx: 'g++' + backend: 'SERIAL' + cmake_build_type: 'Debug' + kokkos_ver: '3.6.01' + output: 'BOTH' runs-on: ubuntu-20.04 container: image: ghcr.io/ecp-copa/ci-containers/${{ matrix.distro }} @@ -42,7 +62,20 @@ jobs: - name: Build Cabana working-directory: Cabana run: | - cmake -B build -DCMAKE_INSTALL_PREFIX=$HOME/Cabana -DCMAKE_PREFIX_PATH="$HOME/kokkos" -DCabana_REQUIRE_${{ matrix.backend }}=ON + if [[ ${{ matrix.output }} == 'HDF5' ]]; then + cabana_cmake_opts+=( -DCabana_REQUIRE_HDF5=ON -DCMAKE_DISABLE_FIND_PACKAGE_SILO=ON ) + elif [[ ${{ matrix.output }} == 'SILO' ]]; then + cabana_cmake_opts+=( -DCabana_REQUIRE_SILO=ON -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON ) + elif [[ ${{ matrix.output }} == 'BOTH' ]]; then + cabana_cmake_opts+=( -DCabana_REQUIRE_SILO=ON -DCabana_REQUIRE_HDF5=ON ) + else + cabana_cmake_opts+=( -DCMAKE_DISABLE_FIND_PACKAGE_SILO=ON -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON ) + fi + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \ + -DCMAKE_PREFIX_PATH="$HOME/kokkos" \ + -DCabana_REQUIRE_${{ matrix.backend }}=ON \ + ${cabana_cmake_opts[@]} cmake --build build --parallel 2 cmake --install build - name: Checkout code From 27fe8015db232e301676fd3c83d9bd118ab8a644 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Thu, 14 Sep 2023 13:32:18 -0400 Subject: [PATCH 09/22] fixup: CI with bash --- .github/workflows/CI.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2a80a68..f7df30c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -11,6 +11,9 @@ on: jobs: CI: + defaults: + run: + shell: bash strategy: matrix: distro: ['ubuntu:latest'] From 28a9878d1efd835fbf5a0f435cf16e1055851911 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Thu, 14 Sep 2023 13:39:25 -0400 Subject: [PATCH 10/22] fixup: do not use Cabana master in CI --- .github/workflows/CI.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f7df30c..1d0d7c2 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -60,7 +60,8 @@ jobs: uses: actions/checkout@v2.2.0 with: repository: ECP-copa/Cabana - ref: master + # This version is post-release 0.5 + ref: a592efaadadeae2475c7e72efe3f62bf4f1fd3bd path: Cabana - name: Build Cabana working-directory: Cabana From dbaa8599169e085a8cff8f8951906a632107bd5e Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Mon, 25 Sep 2023 09:17:44 -0400 Subject: [PATCH 11/22] Remove explicit error for Cajita (still fails at link if not found) --- CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9333037..1be615f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,9 +12,6 @@ find_package(Cabana REQUIRED COMPONENTS Cabana::Cajita Cabana::cabanacore) if( NOT Cabana_ENABLE_MPI ) message( FATAL_ERROR "Cabana must be compiled with MPI" ) endif() -if( NOT Cabana_ENABLE_CAJITA ) - message( FATAL_ERROR "Cabana must be compiled with Cajita" ) -endif() # find Clang Format find_package( CLANG_FORMAT 14 ) From 3033b5f20b169fef849ac4cc8fdfe12b80921514 Mon Sep 17 00:00:00 2001 From: "Chong, Kwitae" Date: Fri, 20 Oct 2023 17:11:43 -0400 Subject: [PATCH 12/22] replace Cajita with Cabana::Grid --- CMakeLists.txt | 2 +- examples/dam_break.cpp | 4 +- examples/free_fall.cpp | 4 +- src/CMakeLists.txt | 4 +- src/ExaMPM_Mesh.hpp | 18 ++++--- src/ExaMPM_ParticleInit.hpp | 16 +++--- src/ExaMPM_ProblemManager.hpp | 80 +++++++++++++++------------- src/ExaMPM_Solver.hpp | 6 +-- src/ExaMPM_TimeIntegrator.hpp | 76 +++++++++++++------------- src/ExaMPM_TimeStepControl.hpp | 2 +- src/ExaMPM_Types.hpp | 4 +- src/ExaMPM_VelocityInterpolation.hpp | 12 ++--- 12 files changed, 118 insertions(+), 110 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1be615f..00a255e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ include(GNUInstallDirs) # find dependencies set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) -find_package(Cabana REQUIRED COMPONENTS Cabana::Cajita Cabana::cabanacore) +find_package(Cabana REQUIRED COMPONENTS Cabana::Grid Cabana::Core) if( NOT Cabana_ENABLE_MPI ) message( FATAL_ERROR "Cabana must be compiled with MPI" ) endif() diff --git a/examples/dam_break.cpp b/examples/dam_break.cpp index c60b30a..e394f10 100644 --- a/examples/dam_break.cpp +++ b/examples/dam_break.cpp @@ -3,7 +3,7 @@ #include -#include +#include #include @@ -87,7 +87,7 @@ void damBreak( const double cell_size, const int ppc, const int halo_size, int comm_size; MPI_Comm_size( MPI_COMM_WORLD, &comm_size ); std::array ranks_per_dim = { 1, comm_size, 1 }; - Cajita::ManualBlockPartitioner<3> partitioner( ranks_per_dim ); + Cabana::Grid::ManualBlockPartitioner<3> partitioner( ranks_per_dim ); // Material properties. double bulk_modulus = 1.0e5; diff --git a/examples/free_fall.cpp b/examples/free_fall.cpp index 24b26c9..35d2e38 100644 --- a/examples/free_fall.cpp +++ b/examples/free_fall.cpp @@ -3,7 +3,7 @@ #include -#include +#include #include @@ -84,7 +84,7 @@ void freeFall( const double cell_size, const int ppc, const int halo_size, int comm_size; MPI_Comm_size( MPI_COMM_WORLD, &comm_size ); std::array ranks_per_dim = { 1, comm_size, 1 }; - Cajita::ManualBlockPartitioner<3> partitioner( ranks_per_dim ); + Cabana::Grid::ManualBlockPartitioner<3> partitioner( ranks_per_dim ); // Material properties. double bulk_modulus = 5.0e5; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5c723f2..9a2f2c2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,8 +20,8 @@ install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) add_library(exampm ${SOURCES}) target_link_libraries(exampm - Cabana::cabanacore - Cabana::Cajita + Cabana::Core + Cabana::Grid ) target_include_directories(exampm diff --git a/src/ExaMPM_Mesh.hpp b/src/ExaMPM_Mesh.hpp index 0ec2941..16b39a2 100644 --- a/src/ExaMPM_Mesh.hpp +++ b/src/ExaMPM_Mesh.hpp @@ -12,7 +12,7 @@ #ifndef EXAMPM_MESH_HPP #define EXAMPM_MESH_HPP -#include +#include #include @@ -39,7 +39,7 @@ class Mesh Mesh( const Kokkos::Array& global_bounding_box, const std::array& global_num_cell, const std::array& periodic, - const Cajita::BlockPartitioner<3>& partitioner, + const Cabana::Grid::BlockPartitioner<3>& partitioner, const int halo_cell_width, const int minimum_halo_cell_width, MPI_Comm comm ) { @@ -91,20 +91,21 @@ class Mesh } // Create the global mesh. - auto global_mesh = Cajita::createUniformGlobalMesh( + auto global_mesh = Cabana::Grid::createUniformGlobalMesh( global_low_corner, global_high_corner, num_cell ); // Build the global grid. - auto global_grid = Cajita::createGlobalGrid( comm, global_mesh, - periodic, partitioner ); + auto global_grid = Cabana::Grid::createGlobalGrid( + comm, global_mesh, periodic, partitioner ); // Build the local grid. int halo_width = std::max( minimum_halo_cell_width, halo_cell_width ); - _local_grid = Cajita::createLocalGrid( global_grid, halo_width ); + _local_grid = Cabana::Grid::createLocalGrid( global_grid, halo_width ); } // Get the local grid. - const std::shared_ptr>>& + const std::shared_ptr< + Cabana::Grid::LocalGrid>>& localGrid() const { return _local_grid; @@ -129,7 +130,8 @@ class Mesh } public: - std::shared_ptr>> _local_grid; + std::shared_ptr>> + _local_grid; Kokkos::Array _min_domain_global_node_index; Kokkos::Array _max_domain_global_node_index; diff --git a/src/ExaMPM_ParticleInit.hpp b/src/ExaMPM_ParticleInit.hpp index 8b29b61..80a7343 100644 --- a/src/ExaMPM_ParticleInit.hpp +++ b/src/ExaMPM_ParticleInit.hpp @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include @@ -114,11 +114,11 @@ void initializeParticles( const ExecSpace& exec_space, using particle_type = typename ParticleList::tuple_type; // Create a local mesh. - auto local_mesh = Cajita::createLocalMesh( local_grid ); + auto local_mesh = Cabana::Grid::createLocalMesh( local_grid ); // Get the local set of owned cell indices. - auto owned_cells = - local_grid.indexSpace( Cajita::Own(), Cajita::Cell(), Cajita::Local() ); + auto owned_cells = local_grid.indexSpace( + Cabana::Grid::Own(), Cabana::Grid::Cell(), Cabana::Grid::Local() ); // Allocate enough space for the case the particles consume the entire // local grid. @@ -136,7 +136,7 @@ void initializeParticles( const ExecSpace& exec_space, int local_num_create = 0; Kokkos::parallel_reduce( "init_particles_uniform", - Cajita::createExecutionPolicy( owned_cells, exec_space ), + Cabana::Grid::createExecutionPolicy( owned_cells, exec_space ), KOKKOS_LAMBDA( const int i, const int j, const int k, int& create_count ) { // Compute the owned local cell id. @@ -150,12 +150,14 @@ void initializeParticles( const ExecSpace& exec_space, // Get the coordinates of the low cell node. int low_node[3] = { i, j, k }; double low_coords[3]; - local_mesh.coordinates( Cajita::Node(), low_node, low_coords ); + local_mesh.coordinates( Cabana::Grid::Node(), low_node, + low_coords ); // Get the coordinates of the high cell node. int high_node[3] = { i + 1, j + 1, k + 1 }; double high_coords[3]; - local_mesh.coordinates( Cajita::Node(), high_node, high_coords ); + local_mesh.coordinates( Cabana::Grid::Node(), high_node, + high_coords ); // Compute the particle spacing in each dimension. double spacing[3] = { ( high_coords[Dim::I] - low_coords[Dim::I] ) / diff --git a/src/ExaMPM_ProblemManager.hpp b/src/ExaMPM_ProblemManager.hpp index adee139..3659b89 100644 --- a/src/ExaMPM_ProblemManager.hpp +++ b/src/ExaMPM_ProblemManager.hpp @@ -17,7 +17,7 @@ #include -#include +#include #include @@ -91,13 +91,15 @@ class ProblemManager using particle_list = Cabana::AoSoA; using particle_type = typename particle_list::tuple_type; - using node_array = Cajita::Array, MemorySpace>; + using node_array = + Cabana::Grid::Array, MemorySpace>; - using cell_array = Cajita::Array, MemorySpace>; + using cell_array = + Cabana::Grid::Array, MemorySpace>; - using halo = Cajita::Halo; + using halo = Cabana::Grid::Halo; using mesh_type = Mesh; @@ -117,37 +119,38 @@ class ProblemManager initializeParticles( exec_space, *( _mesh->localGrid() ), particles_per_cell, create_functor, _particles ); - auto node_vector_layout = - Cajita::createArrayLayout( _mesh->localGrid(), 3, Cajita::Node() ); - auto node_scalar_layout = - Cajita::createArrayLayout( _mesh->localGrid(), 1, Cajita::Node() ); - auto cell_scalar_layout = - Cajita::createArrayLayout( _mesh->localGrid(), 1, Cajita::Cell() ); + auto node_vector_layout = Cabana::Grid::createArrayLayout( + _mesh->localGrid(), 3, Cabana::Grid::Node() ); + auto node_scalar_layout = Cabana::Grid::createArrayLayout( + _mesh->localGrid(), 1, Cabana::Grid::Node() ); + auto cell_scalar_layout = Cabana::Grid::createArrayLayout( + _mesh->localGrid(), 1, Cabana::Grid::Cell() ); - _momentum = Cajita::createArray( + _momentum = Cabana::Grid::createArray( "momentum", node_vector_layout ); - _mass = Cajita::createArray( "mass", - node_scalar_layout ); - _force = Cajita::createArray( "force", - node_vector_layout ); - _velocity = Cajita::createArray( + _mass = Cabana::Grid::createArray( + "mass", node_scalar_layout ); + _force = Cabana::Grid::createArray( + "force", node_vector_layout ); + _velocity = Cabana::Grid::createArray( "velocity", node_vector_layout ); - _position_correction = Cajita::createArray( + _position_correction = Cabana::Grid::createArray( "position_correction", node_vector_layout ); - _density = Cajita::createArray( + _density = Cabana::Grid::createArray( "density", cell_scalar_layout ); - _mark = Cajita::createArray( "mark", - cell_scalar_layout ); - - _node_scatter_halo = Cajita::createHalo( - Cajita::NodeHaloPattern<3>(), -1, *_momentum, *_mass, *_force ); - _node_gather_halo = - Cajita::createHalo( Cajita::NodeHaloPattern<3>(), -1, *_velocity ); - _node_correction_halo = Cajita::createHalo( - Cajita::NodeHaloPattern<3>(), -1, *_position_correction ); - _cell_halo = Cajita::createHalo( Cajita::NodeHaloPattern<3>(), -1, - *_density, *_mark ); + _mark = Cabana::Grid::createArray( + "mark", cell_scalar_layout ); + + _node_scatter_halo = + Cabana::Grid::createHalo( Cabana::Grid::NodeHaloPattern<3>(), -1, + *_momentum, *_mass, *_force ); + _node_gather_halo = Cabana::Grid::createHalo( + Cabana::Grid::NodeHaloPattern<3>(), -1, *_velocity ); + _node_correction_halo = Cabana::Grid::createHalo( + Cabana::Grid::NodeHaloPattern<3>(), -1, *_position_correction ); + _cell_halo = Cabana::Grid::createHalo( + Cabana::Grid::NodeHaloPattern<3>(), -1, *_density, *_mark ); } std::size_t numParticle() const { return _particles.size(); } @@ -237,21 +240,22 @@ class ProblemManager void scatter( Location::Node ) const { _node_scatter_halo->scatter( execution_space(), - Cajita::ScatterReduce::Sum(), *_momentum, - *_mass, *_force ); + Cabana::Grid::ScatterReduce::Sum(), + *_momentum, *_mass, *_force ); } void scatter( Location::Node, Field::PositionCorrection ) const { _node_correction_halo->scatter( execution_space(), - Cajita::ScatterReduce::Sum(), + Cabana::Grid::ScatterReduce::Sum(), *_position_correction ); } void scatter( Location::Cell ) const { - _cell_halo->scatter( execution_space(), Cajita::ScatterReduce::Sum(), - *_density, *_mark ); + _cell_halo->scatter( execution_space(), + Cabana::Grid::ScatterReduce::Sum(), *_density, + *_mark ); } void gather( Location::Node ) const @@ -268,8 +272,8 @@ class ProblemManager void communicateParticles( const int minimum_halo_width ) { auto positions = get( Location::Particle(), Field::Position() ); - Cajita::particleGridMigrate( *( _mesh->localGrid() ), positions, - _particles, minimum_halo_width ); + Cabana::Grid::particleGridMigrate( *( _mesh->localGrid() ), positions, + _particles, minimum_halo_width ); } private: diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index a1d9763..ac2cfc6 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -45,7 +45,7 @@ class Solver : public SolverBase Solver( MPI_Comm comm, const Kokkos::Array& global_bounding_box, const std::array& global_num_cell, const std::array& periodic, - const Cajita::BlockPartitioner<3>& partitioner, + const Cabana::Grid::BlockPartitioner<3>& partitioner, const int halo_cell_width, const InitFunc& create_functor, const int particles_per_cell, const double bulk_modulus, const double density, const double gamma, const double kappa, @@ -114,7 +114,7 @@ class Solver : public SolverBase _pm->get( Location::Particle(), Field::J() ) ); #else #ifdef Cabana_ENABLE_SILO - Cajita::Experimental::SiloParticleOutput::writeTimeStep( + Cabana::Grid::Experimental::SiloParticleOutput::writeTimeStep( "particles", _mesh->localGrid()->globalGrid(), _step, _time, _pm->get( Location::Particle(), Field::Position() ), _pm->get( Location::Particle(), Field::Velocity() ), @@ -148,7 +148,7 @@ createSolver( const std::string& device, MPI_Comm comm, const Kokkos::Array& global_bounding_box, const std::array& global_num_cell, const std::array& periodic, - const Cajita::BlockPartitioner<3>& partitioner, + const Cabana::Grid::BlockPartitioner<3>& partitioner, const int halo_cell_width, const InitFunc& create_functor, const int particles_per_cell, const double bulk_modulus, const double density, const double gamma, const double kappa, diff --git a/src/ExaMPM_TimeIntegrator.hpp b/src/ExaMPM_TimeIntegrator.hpp index d67a545..d0a90b2 100644 --- a/src/ExaMPM_TimeIntegrator.hpp +++ b/src/ExaMPM_TimeIntegrator.hpp @@ -16,7 +16,7 @@ #include #include -#include +#include #include @@ -59,8 +59,8 @@ void p2g( const ExecutionSpace& exec_space, const ProblemManagerType& pm ) double gamma = pm.gamma(); // Build the local mesh. - auto local_mesh = - Cajita::createLocalMesh( *( pm.mesh()->localGrid() ) ); + auto local_mesh = Cabana::Grid::createLocalMesh( + *( pm.mesh()->localGrid() ) ); // Loop over particles. Kokkos::parallel_for( @@ -71,16 +71,16 @@ void p2g( const ExecutionSpace& exec_space, const ProblemManagerType& pm ) double x[3] = { x_p( p, 0 ), x_p( p, 1 ), x_p( p, 2 ) }; // Setup interpolation to the nodes. - Cajita::SplineData sd; - Cajita::evaluateSpline( local_mesh, x, sd ); + Cabana::Grid::SplineData sd; + Cabana::Grid::evaluateSpline( local_mesh, x, sd ); // Compute the pressure on the particle with an equation of // state. double pressure = -bulk_mod * ( pow( j_p( p ), -gamma ) - 1.0 ); // Project the pressure gradient to the grid. - Cajita::P2G::gradient( -v_p( p ) * j_p( p ) * pressure, sd, - f_i_sv ); + Cabana::Grid::P2G::gradient( -v_p( p ) * j_p( p ) * pressure, sd, + f_i_sv ); // Extract the particle velocity double vel_p[3] = { u_p( p, 0 ), u_p( p, 1 ), u_p( p, 2 ) }; @@ -95,7 +95,7 @@ void p2g( const ExecutionSpace& exec_space, const ProblemManagerType& pm ) APIC::p2g( m_p( p ), vel_p, aff_p, sd, mu_i_sv ); // Project mass to the grid. - Cajita::P2G::value( m_p( p ), sd, m_i_sv ); + Cabana::Grid::P2G::value( m_p( p ), sd, m_i_sv ); } ); // Complete local scatter. @@ -127,12 +127,12 @@ void fieldSolve( const ExecutionSpace& exec_space, const ProblemManagerType& pm, double mass_epsilon = 1.0e-12; // Compute the velocity. - auto l2g = Cajita::IndexConversion::createL2G( *( pm.mesh()->localGrid() ), - Cajita::Node() ); + auto l2g = Cabana::Grid::IndexConversion::createL2G( + *( pm.mesh()->localGrid() ), Cabana::Grid::Node() ); auto local_nodes = pm.mesh()->localGrid()->indexSpace( - Cajita::Ghost(), Cajita::Node(), Cajita::Local() ); + Cabana::Grid::Ghost(), Cabana::Grid::Node(), Cabana::Grid::Local() ); Kokkos::parallel_for( - Cajita::createExecutionPolicy( local_nodes, exec_space ), + Cabana::Grid::createExecutionPolicy( local_nodes, exec_space ), KOKKOS_LAMBDA( const int li, const int lj, const int lk ) { int gi, gj, gk; l2g( li, lj, lk, gi, gj, gk ); @@ -188,8 +188,8 @@ void g2p( const ExecutionSpace& exec_space, const ProblemManagerType& pm, auto k_c_sv = Kokkos::Experimental::create_scatter_view( k_c ); // Build the local mesh. - auto local_mesh = - Cajita::createLocalMesh( *( pm.mesh()->localGrid() ) ); + auto local_mesh = Cabana::Grid::createLocalMesh( + *( pm.mesh()->localGrid() ) ); auto cell_size = pm.mesh()->localGrid()->globalGrid().globalMesh().cellSize( 0 ); auto cell_volume = cell_size * cell_size * cell_size; @@ -206,8 +206,8 @@ void g2p( const ExecutionSpace& exec_space, const ProblemManagerType& pm, double x[3] = { x_p( p, 0 ), x_p( p, 1 ), x_p( p, 2 ) }; // Setup interpolation from the nodes. - Cajita::SplineData sd_i; - Cajita::evaluateSpline( local_mesh, x, sd_i ); + Cabana::Grid::SplineData sd_i; + Cabana::Grid::evaluateSpline( local_mesh, x, sd_i ); // Update particle velocity. double vel_p[3]; @@ -222,7 +222,7 @@ void g2p( const ExecutionSpace& exec_space, const ProblemManagerType& pm, // Compute the velocity divergence (this is the trace of the // velocity gradient). double div_u; - Cajita::G2P::divergence( u_i, sd_i, div_u ); + Cabana::Grid::G2P::divergence( u_i, sd_i, div_u ); // Update the deformation gradient determinant. j_p( p ) *= exp( delta_t * div_u ); @@ -235,14 +235,14 @@ void g2p( const ExecutionSpace& exec_space, const ProblemManagerType& pm, } // Project density to cell. - Cajita::SplineData sd_c1; - Cajita::evaluateSpline( local_mesh, x, sd_c1 ); - Cajita::P2G::value( m_p( p ) / cell_volume, sd_c1, r_c_sv ); + Cabana::Grid::SplineData sd_c1; + Cabana::Grid::evaluateSpline( local_mesh, x, sd_c1 ); + Cabana::Grid::P2G::value( m_p( p ) / cell_volume, sd_c1, r_c_sv ); // Mark cells. Indicates whether or not cells have particles. - Cajita::SplineData sd_c0; - Cajita::evaluateSpline( local_mesh, x, sd_c0 ); - Cajita::P2G::value( 1.0, sd_c0, k_c_sv ); + Cabana::Grid::SplineData sd_c0; + Cabana::Grid::evaluateSpline( local_mesh, x, sd_c0 ); + Cabana::Grid::P2G::value( 1.0, sd_c0, k_c_sv ); } ); // Complete local scatter. @@ -280,24 +280,24 @@ void correctParticlePositions( const ExecutionSpace& exec_space, double density = pm.density(); // Build the local mesh. - auto local_mesh = - Cajita::createLocalMesh( *( pm.mesh()->localGrid() ) ); + auto local_mesh = Cabana::Grid::createLocalMesh( + *( pm.mesh()->localGrid() ) ); // Compute nodal correction. auto local_cells = pm.mesh()->localGrid()->indexSpace( - Cajita::Own(), Cajita::Cell(), Cajita::Local() ); + Cabana::Grid::Own(), Cabana::Grid::Cell(), Cabana::Grid::Local() ); Kokkos::parallel_for( "compute_position_correction", - Cajita::createExecutionPolicy( local_cells, exec_space ), + Cabana::Grid::createExecutionPolicy( local_cells, exec_space ), KOKKOS_LAMBDA( const int i, const int j, const int k ) { // Get the cell center. int idx[3] = { i, j, k }; double x[3]; - local_mesh.coordinates( Cajita::Cell(), idx, x ); + local_mesh.coordinates( Cabana::Grid::Cell(), idx, x ); // Setup interpolation from cell center to nodes. - Cajita::SplineData sd_i; - Cajita::evaluateSpline( local_mesh, x, sd_i ); + Cabana::Grid::SplineData sd_i; + Cabana::Grid::evaluateSpline( local_mesh, x, sd_i ); // Clamp the density outside the fluid. double rho = ( k_c( i, j, k, 0 ) > 0.0 ) @@ -307,7 +307,7 @@ void correctParticlePositions( const ExecutionSpace& exec_space, // Compute correction. double correction = -delta_t * delta_t * kappa * ( 1 - rho / density ) / density; - Cajita::P2G::gradient( correction, sd_i, x_i_sv ); + Cabana::Grid::P2G::gradient( correction, sd_i, x_i_sv ); } ); // Complete local scatter. @@ -321,12 +321,12 @@ void correctParticlePositions( const ExecutionSpace& exec_space, // Apply boundary condition to position correction. // Compute the velocity. - auto l2g = Cajita::IndexConversion::createL2G( *( pm.mesh()->localGrid() ), - Cajita::Node() ); + auto l2g = Cabana::Grid::IndexConversion::createL2G( + *( pm.mesh()->localGrid() ), Cabana::Grid::Node() ); auto local_nodes = pm.mesh()->localGrid()->indexSpace( - Cajita::Ghost(), Cajita::Node(), Cajita::Local() ); + Cabana::Grid::Ghost(), Cabana::Grid::Node(), Cabana::Grid::Local() ); Kokkos::parallel_for( - Cajita::createExecutionPolicy( local_nodes, exec_space ), + Cabana::Grid::createExecutionPolicy( local_nodes, exec_space ), KOKKOS_LAMBDA( const int li, const int lj, const int lk ) { int gi, gj, gk; l2g( li, lj, lk, gi, gj, gk ); @@ -343,12 +343,12 @@ void correctParticlePositions( const ExecutionSpace& exec_space, double x[3] = { x_p( p, 0 ), x_p( p, 1 ), x_p( p, 2 ) }; // Setup interpolation from the nodes. - Cajita::SplineData sd_i; - Cajita::evaluateSpline( local_mesh, x, sd_i ); + Cabana::Grid::SplineData sd_i; + Cabana::Grid::evaluateSpline( local_mesh, x, sd_i ); // Correct the particle position. double delta_x[3]; - Cajita::G2P::value( x_i, sd_i, delta_x ); + Cabana::Grid::G2P::value( x_i, sd_i, delta_x ); for ( int d = 0; d < 3; ++d ) x_p( p, d ) += delta_x[d]; } ); diff --git a/src/ExaMPM_TimeStepControl.hpp b/src/ExaMPM_TimeStepControl.hpp index 067a95f..04c13b2 100644 --- a/src/ExaMPM_TimeStepControl.hpp +++ b/src/ExaMPM_TimeStepControl.hpp @@ -14,7 +14,7 @@ #include -#include +#include #include diff --git a/src/ExaMPM_Types.hpp b/src/ExaMPM_Types.hpp index 59426d4..ac2cdc6 100644 --- a/src/ExaMPM_Types.hpp +++ b/src/ExaMPM_Types.hpp @@ -12,13 +12,13 @@ #ifndef EXAMPM_TYPES_HPP #define EXAMPM_TYPES_HPP -#include +#include namespace ExaMPM { //---------------------------------------------------------------------------// // Logical dimension index. -using Dim = Cajita::Dim; +using Dim = Cabana::Grid::Dim; //---------------------------------------------------------------------------// diff --git a/src/ExaMPM_VelocityInterpolation.hpp b/src/ExaMPM_VelocityInterpolation.hpp index d6050fb..1757db6 100644 --- a/src/ExaMPM_VelocityInterpolation.hpp +++ b/src/ExaMPM_VelocityInterpolation.hpp @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include @@ -55,11 +55,11 @@ p2g( const typename MomentumView::original_value_type m_p, const typename MomentumView::original_value_type B_p[3][3], const SplineDataType& sd, const MomentumView& node_momentum, typename std::enable_if< - ( Cajita::isNode::value && + ( Cabana::Grid::isNode::value && ( SplineDataType::order == 2 || SplineDataType::order == 3 ) ), void*>::type = 0 ) { - static_assert( Cajita::P2G::is_scatter_view::value, + static_assert( Cabana::Grid::P2G::is_scatter_view::value, "P2G requires a Kokkos::ScatterView" ); auto momentum_access = node_momentum.access(); @@ -106,11 +106,11 @@ p2g( const typename MomentumView::original_value_type m_p, const typename MomentumView::original_value_type B_p[3][3], const SplineDataType& sd, const MomentumView& node_momentum, typename std::enable_if< - ( Cajita::isNode::value && + ( Cabana::Grid::isNode::value && ( SplineDataType::order == 1 ) ), void*>::type = 0 ) { - static_assert( Cajita::P2G::is_scatter_view::value, + static_assert( Cabana::Grid::P2G::is_scatter_view::value, "P2G requires a Kokkos::ScatterView" ); auto momentum_access = node_momentum.access(); @@ -155,7 +155,7 @@ g2p( const VelocityView& node_velocity, const SplineDataType& sd, typename VelocityView::value_type u_p[3], typename VelocityView::value_type B_p[3][3], typename std::enable_if< - Cajita::isNode::value, + Cabana::Grid::isNode::value, void*>::type = 0 ) { using value_type = typename VelocityView::value_type; From 3417ee0fde323db50bb33fdef0cca1b466a34b08 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Sat, 21 Oct 2023 17:33:57 -0400 Subject: [PATCH 13/22] Update CI for Grid --- .github/workflows/CI.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1d0d7c2..238654f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -24,19 +24,16 @@ jobs: cxx: 'g++' backend: 'SERIAL' cmake_build_type: 'Debug' - kokkos_ver: '3.6.01' output: 'SILO' - distro: 'ubuntu:latest' cxx: 'g++' backend: 'SERIAL' cmake_build_type: 'Debug' - kokkos_ver: '3.6.01' output: 'NONE' - distro: 'ubuntu:latest' cxx: 'g++' backend: 'SERIAL' cmake_build_type: 'Debug' - kokkos_ver: '3.6.01' output: 'BOTH' runs-on: ubuntu-20.04 container: @@ -48,7 +45,7 @@ jobs: uses: actions/checkout@v2.2.0 with: repository: kokkos/kokkos - ref: 3.6.01 + ref: 3.7.02 path: kokkos - name: Build kokkos working-directory: kokkos @@ -60,8 +57,7 @@ jobs: uses: actions/checkout@v2.2.0 with: repository: ECP-copa/Cabana - # This version is post-release 0.5 - ref: a592efaadadeae2475c7e72efe3f62bf4f1fd3bd + ref: 0.6.0 path: Cabana - name: Build Cabana working-directory: Cabana @@ -78,6 +74,7 @@ jobs: cmake -B build \ -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \ -DCMAKE_PREFIX_PATH="$HOME/kokkos" \ + -DCMAKE_CXX_FLAGS="-Wall -Wextra -pedantic -Werror" \ -DCabana_REQUIRE_${{ matrix.backend }}=ON \ ${cabana_cmake_opts[@]} cmake --build build --parallel 2 From 390ff4e0115346bfef61ad780f67f5f4df64dd9a Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Mon, 23 Oct 2023 10:56:07 -0400 Subject: [PATCH 14/22] Remove Kokkos <3.7 math function guards --- src/ExaMPM_TimeStepControl.hpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/ExaMPM_TimeStepControl.hpp b/src/ExaMPM_TimeStepControl.hpp index 04c13b2..34a82a6 100644 --- a/src/ExaMPM_TimeStepControl.hpp +++ b/src/ExaMPM_TimeStepControl.hpp @@ -14,8 +14,6 @@ #include -#include - #include #include @@ -44,14 +42,8 @@ template double momentumCFL( MPI_Comm comm, ExecutionSpace, const ProblemManagerType& pm, const double current_dt, const double cfl ) { - -#if KOKKOS_VERSION >= 30700 using Kokkos::abs; using Kokkos::sqrt; -#else - using Kokkos::Experimental::abs; - using Kokkos::Experimental::sqrt; -#endif // Get the particle data we need. auto m_p = pm.get( Location::Particle(), Field::Mass() ); @@ -98,12 +90,7 @@ template double maxVelocity( MPI_Comm comm, ExecutionSpace, const ProblemManagerType& pm, const double current_dt, const double cfl ) { - -#if KOKKOS_VERSION >= 30700 using Kokkos::sqrt; -#else - using Kokkos::Experimental::sqrt; -#endif // Get the particle data we need. auto u_p = pm.get( Location::Particle(), Field::Velocity() ); From dd2ba4bba977dba7d3fe6f76b7222029ee4004e3 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Tue, 28 Nov 2023 09:34:51 -0500 Subject: [PATCH 15/22] Require Cabana 0.6.1 --- .github/workflows/CI.yml | 2 +- CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 238654f..5dc32d8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -57,7 +57,7 @@ jobs: uses: actions/checkout@v2.2.0 with: repository: ECP-copa/Cabana - ref: 0.6.0 + ref: 0.6.1 path: Cabana - name: Build Cabana working-directory: Cabana diff --git a/CMakeLists.txt b/CMakeLists.txt index 00a255e..810857c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ include(GNUInstallDirs) # find dependencies set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) -find_package(Cabana REQUIRED COMPONENTS Cabana::Grid Cabana::Core) +find_package(Cabana 0.6.1 REQUIRED COMPONENTS Cabana::Grid Cabana::Core) if( NOT Cabana_ENABLE_MPI ) message( FATAL_ERROR "Cabana must be compiled with MPI" ) endif() From 3c67cff65c70d208b64e234e7c7b40407ca251d8 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Thu, 29 Jun 2023 16:31:39 -0500 Subject: [PATCH 16/22] enable subfiling with h5fuse --- examples/dam_break.cpp | 15 ++++++++++ src/ExaMPM_Solver.hpp | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/examples/dam_break.cpp b/examples/dam_break.cpp index e394f10..9b6ee10 100644 --- a/examples/dam_break.cpp +++ b/examples/dam_break.cpp @@ -118,6 +118,21 @@ void damBreak( const double cell_size, const int ppc, const int halo_size, //---------------------------------------------------------------------------// int main( int argc, char* argv[] ) { + + const char* env_val = std::getenv("H5FD_SUBFILING"); + if(env_val != NULL) { + int mpi_thread_required = MPI_THREAD_MULTIPLE; + int mpi_thread_provided = 0; + + // HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE + + MPI_Init_thread(&argc, &argv, mpi_thread_required, &mpi_thread_provided); + if (mpi_thread_provided < mpi_thread_required) { + printf("MPI_THREAD_MULTIPLE not supported\n"); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + else MPI_Init( &argc, &argv ); Kokkos::initialize( argc, argv ); diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index ac2cfc6..379df1d 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -25,6 +25,10 @@ #include #include +#include +#include +#include +#include namespace ExaMPM { @@ -106,12 +110,72 @@ class Solver : public SolverBase // Prefer HDF5 output over Silo. Only output if one is enabled. #ifdef Cabana_ENABLE_HDF5 Cabana::Experimental::HDF5ParticleOutput::HDF5Config h5_config; + const char* env_val = std::getenv("H5FD_SUBFILING"); + if(env_val != NULL) + h5_config.subfiling = true; Cabana::Experimental::HDF5ParticleOutput::writeTimeStep( h5_config, "particles", _mesh->localGrid()->globalGrid().comm(), _step, _time, _pm->numParticle(), _pm->get( Location::Particle(), Field::Position() ), _pm->get( Location::Particle(), Field::Velocity() ), _pm->get( Location::Particle(), Field::J() ) ); + + if(h5_config.subfiling) { + + if ( _rank == 0) { + pid_t pid = 0; + pid_t tmppid; + int status; + + pid = fork(); + + if (pid == 0) { + + // stub HDF5 filename + std::stringstream filename_hdf5; + filename_hdf5 << "particles" << "_" << _step << ".h5"; + + // Directory containing the subfiling configuration file + std::stringstream config_dir; + if(const char* env_value = std::getenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX)) + config_dir << env_value; + else + config_dir << "."; + + // Find the name of the subfiling configuration file + struct stat file_info; + stat(filename_hdf5.str().c_str(), &file_info); + + char config_filename[PATH_MAX]; + snprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir.str().c_str(), + filename_hdf5.str().c_str(), (uint64_t)file_info.st_ino); + + // Call the h5fuse utility + char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; + + execvp(args[0], args); + + } + else { + tmppid = waitpid(pid, &status, 0); + + if (WIFEXITED(status)) { + int ret; + + if ((ret = WEXITSTATUS(status)) != 0) { + printf("h5fuse process exited with error code %d\n", ret); + fflush(stdout); + MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); + } + } + else { + printf("h5fuse process terminated abnormally\n"); + fflush(stdout); + MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); + } + } + } + } #else #ifdef Cabana_ENABLE_SILO Cabana::Grid::Experimental::SiloParticleOutput::writeTimeStep( From 5c87cc0208d7dbf58db22731021c56bbcea82978 Mon Sep 17 00:00:00 2001 From: Michael Breitenfeld Date: Wed, 2 Aug 2023 10:03:08 -0400 Subject: [PATCH 17/22] new fork schema --- src/ExaMPM_Solver.hpp | 163 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 152 insertions(+), 11 deletions(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index 379df1d..fae5048 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -32,6 +32,7 @@ namespace ExaMPM { +int nfork; //---------------------------------------------------------------------------// class SolverBase { @@ -84,7 +85,7 @@ class Solver : public SolverBase while ( _time < t_final ) { if ( 0 == _rank && 0 == _step % write_freq ) - printf( "Time %f / %f\n", _time, t_final ); + printf( "Time %12.5e / %12.5e\n", _time, t_final ); // Fixed timestep is guaranteed only when sufficently low dt // does not violate the CFL condition (otherwise user-set dt is @@ -103,6 +104,48 @@ class Solver : public SolverBase if ( 0 == ( _step ) % write_freq ) outputParticles(); } + Cabana::Experimental::HDF5ParticleOutput::HDF5Config h5_config; + const char* env_val = std::getenv("H5FUSE"); + if(env_val != NULL) { + + if(h5_config.subfiling) { + + MPI_Comm shmcomm; + MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, + MPI_INFO_NULL, &shmcomm); + + int shmrank; + MPI_Comm_rank(shmcomm, &shmrank); + int status; + if ( shmrank == 0) { + //pid_t pid; + for (int i = 0; i < nfork; i++) { + waitpid(-1, NULL, 0); + } +#if 0 + while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { + if (WIFEXITED(status)) { + int ret; + + if ((ret = WEXITSTATUS(status)) != 0) { + printf("h5fuse process exited with error code %d\n", ret); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + else { + printf("h5fuse process terminated abnormally\n"); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } +#endif + } + MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_free(&shmcomm); + } + } + } void outputParticles() @@ -113,24 +156,122 @@ class Solver : public SolverBase const char* env_val = std::getenv("H5FD_SUBFILING"); if(env_val != NULL) h5_config.subfiling = true; + + // Sets the HDF5 alignment equal to the SUBFILING STRIPE SIZE + env_val = std::getenv("SUBFILING STRIPE SIZE"); + if(env_val != NULL) { + h5_config.align = true; + h5_config.threshold = 0; + h5_config.alignment = std::atoi(env_val); + } + Cabana::Experimental::HDF5ParticleOutput::writeTimeStep( h5_config, "particles", _mesh->localGrid()->globalGrid().comm(), _step, _time, _pm->numParticle(), _pm->get( Location::Particle(), Field::Position() ), _pm->get( Location::Particle(), Field::Velocity() ), _pm->get( Location::Particle(), Field::J() ) ); - +#if 1 + env_val = std::getenv("H5FUSE"); + if(env_val != NULL) { + if(h5_config.subfiling) { - if ( _rank == 0) { - pid_t pid = 0; + MPI_Comm shmcomm; + MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, + MPI_INFO_NULL, &shmcomm); + + int shmrank; + MPI_Comm_rank(shmcomm, &shmrank); + + if ( shmrank == 0) { + pid_t pid = 0; + //pid_t tmppid; + int status; + //std::cout << "DOING OUTPUT " << _rank << std::endl; + //signal(SIGHUP, SIG_IGN); + //not interested in its childs, prevent zombies + //signal(SIGCHLD, SIG_IGN); + //signal(SIGQUIT, SIG_IGN); + + pid = fork(); + nfork++; + if (pid == 0) { + std::stringstream filename_hdf5; + filename_hdf5 << "particles" << "_" << _step << ".h5"; + + // Directory containing the subfiling configuration file + std::stringstream config_dir; + if(const char* env_value = std::getenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX)) + config_dir << env_value; + else + config_dir << "."; + // Find the name of the subfiling configuration file + struct stat file_info; + stat(filename_hdf5.str().c_str(), &file_info); + + char config_filename[PATH_MAX]; + snprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir.str().c_str(), + filename_hdf5.str().c_str(), (uint64_t)file_info.st_ino); + + // Call the h5fuse utility + char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-f"), config_filename, NULL}; + //char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; + + execvp(args[0], args); + //exit(0); + +#if 0 + // char *tmp_filename; + char *args[7]; + + //tmp_filename = (char *)malloc(PATH_MAX); + + /* Generate name for configuration file */ + //HDsnprintf(tmp_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir, + // SUBF_FILENAME, file_inode); + args[0] = strdup("ls"); + args[1] = strdup("-l"); + args[2] = NULL; + execvp("ls", args); +#endif + } +#if 0 + else { + //tmppid = waitpid(pid, &status, WNOHANG); + waitpid(pid, &status, 0); + if (WIFEXITED(status)) { + int ret; + + if ((ret = WEXITSTATUS(status)) != 0) { + printf("h5fuse process exited with error code %d\n", ret); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + else { + printf("h5fuse process terminated abnormally\n"); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } +#endif + } +} +} +#endif +#if 0 pid_t tmppid; int status; - pid = fork(); - + pid_t pid = fork(); + if (pid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } +#endif +#if 0 if (pid == 0) { - // stub HDF5 filename std::stringstream filename_hdf5; filename_hdf5 << "particles" << "_" << _step << ".h5"; @@ -141,7 +282,6 @@ class Solver : public SolverBase config_dir << env_value; else config_dir << "."; - // Find the name of the subfiling configuration file struct stat file_info; stat(filename_hdf5.str().c_str(), &file_info); @@ -154,7 +294,6 @@ class Solver : public SolverBase char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; execvp(args[0], args); - } else { tmppid = waitpid(pid, &status, 0); @@ -174,8 +313,10 @@ class Solver : public SolverBase MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); } } - } - } +#endif +// } +// } +//#endif #else #ifdef Cabana_ENABLE_SILO Cabana::Grid::Experimental::SiloParticleOutput::writeTimeStep( From 17d5760c99cb6f47d04d5c54a62f2183f4f55958 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Tue, 15 Aug 2023 08:36:19 -0500 Subject: [PATCH 18/22] fixed env. var. --- src/ExaMPM_Solver.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index fae5048..1eee87f 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -141,7 +141,6 @@ class Solver : public SolverBase } #endif } - MPI_Barrier(MPI_COMM_WORLD); MPI_Comm_free(&shmcomm); } } @@ -157,8 +156,8 @@ class Solver : public SolverBase if(env_val != NULL) h5_config.subfiling = true; - // Sets the HDF5 alignment equal to the SUBFILING STRIPE SIZE - env_val = std::getenv("SUBFILING STRIPE SIZE"); + // Sets the HDF5 alignment equal subfiling's stripe size + env_val = std::getenv("H5FD_SUBFILING_STRIPE_SIZE"); if(env_val != NULL) { h5_config.align = true; h5_config.threshold = 0; From 047dd8d252472759d152ce414553bafa4595003a Mon Sep 17 00:00:00 2001 From: Michael Breitenfeld Date: Tue, 15 Aug 2023 16:11:14 -0400 Subject: [PATCH 19/22] subfiling h5fuse fix --- src/ExaMPM_Solver.hpp | 82 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index 1eee87f..09ea349 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -33,6 +33,52 @@ namespace ExaMPM { int nfork; + +struct timer_statsinfo { + double min; + double max; + double mean; + double std; +}; + + timer_statsinfo io_stats; + + // Collect statistics of timers on all ranks + // timer - elapsed time for rank + // comm - communicator for collecting stats + // destrank - the rank to which to collect stats + // stats - pointer to timer stats + // + + void timer_stats(double timer, MPI_Comm comm, int destrank, timer_statsinfo *stats) + { + int rank, nprocs, i; + double *rtimers=NULL; /* All timers from ranks */ + + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &nprocs); + if(rank == destrank) { + rtimers = (double *) malloc(nprocs*sizeof(double)); + stats->mean = 0.; + stats->min = timer; + stats->max = timer; + stats->std = 0.f; + } + MPI_Gather(&timer, 1, MPI_DOUBLE, rtimers, 1, MPI_DOUBLE, destrank, comm); + if(rank == destrank) { + for(i = 0; i < nprocs; i++) { + if(rtimers[i] > stats->max) stats->max = rtimers[i]; + if(rtimers[i] < stats->min) stats->min = rtimers[i]; + stats->mean += rtimers[i]; + } + stats->mean /= nprocs; + for(i = 0; i < nprocs; i++) + stats->std += (rtimers[i]-stats->mean)*(rtimers[i]-stats->mean); + stats->std = sqrt(stats->std / nprocs); + free(rtimers); + } + } + //---------------------------------------------------------------------------// class SolverBase { @@ -81,11 +127,12 @@ class Solver : public SolverBase { // Output initial state. outputParticles(); + while ( _time < t_final ) { if ( 0 == _rank && 0 == _step % write_freq ) - printf( "Time %12.5e / %12.5e\n", _time, t_final ); + printf( "Time %12.5e / %12.5e [iostats (s), mean min max: %12.5e %12.5e %12.5e] \n", _time, t_final, io_stats.mean, io_stats.min, io_stats.max ); // Fixed timestep is guaranteed only when sufficently low dt // does not violate the CFL condition (otherwise user-set dt is @@ -104,12 +151,10 @@ class Solver : public SolverBase if ( 0 == ( _step ) % write_freq ) outputParticles(); } - Cabana::Experimental::HDF5ParticleOutput::HDF5Config h5_config; + const char* env_val = std::getenv("H5FUSE"); if(env_val != NULL) { - - if(h5_config.subfiling) { - + MPI_Comm shmcomm; MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shmcomm); @@ -118,9 +163,24 @@ class Solver : public SolverBase MPI_Comm_rank(shmcomm, &shmrank); int status; if ( shmrank == 0) { - //pid_t pid; + for (int i = 0; i < nfork; i++) { - waitpid(-1, NULL, 0); + waitpid(-1, &status, 0); + if (WIFEXITED(status)) { + int ret; + + if ((ret = WEXITSTATUS(status)) != 0) { + printf("h5fuse process exited with error code %d\n", ret); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + else { + printf("h5fuse process terminated abnormally\n"); + fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } #if 0 while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { @@ -142,7 +202,6 @@ class Solver : public SolverBase #endif } MPI_Comm_free(&shmcomm); - } } } @@ -163,13 +222,17 @@ class Solver : public SolverBase h5_config.threshold = 0; h5_config.alignment = std::atoi(env_val); } - + + double t1, t2; + t1 = MPI_Wtime(); Cabana::Experimental::HDF5ParticleOutput::writeTimeStep( h5_config, "particles", _mesh->localGrid()->globalGrid().comm(), _step, _time, _pm->numParticle(), _pm->get( Location::Particle(), Field::Position() ), _pm->get( Location::Particle(), Field::Velocity() ), _pm->get( Location::Particle(), Field::J() ) ); + t2 = MPI_Wtime(); + timer_stats(t2-t1, MPI_COMM_WORLD, 0, &io_stats); #if 1 env_val = std::getenv("H5FUSE"); if(env_val != NULL) { @@ -342,6 +405,7 @@ class Solver : public SolverBase std::shared_ptr> _mesh; std::shared_ptr> _pm; int _rank; + }; //---------------------------------------------------------------------------// From f650a3897c9d07836fd1c0567176e92c4f49b7a3 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Tue, 15 Aug 2023 16:13:25 -0500 Subject: [PATCH 20/22] code clean-up --- src/ExaMPM_Solver.hpp | 404 ++++++++++++++++-------------------------- 1 file changed, 151 insertions(+), 253 deletions(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index 09ea349..c97e8a0 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -25,60 +25,15 @@ #include #include -#include -#include -#include #include +#include +#include +#include namespace ExaMPM { int nfork; -struct timer_statsinfo { - double min; - double max; - double mean; - double std; -}; - - timer_statsinfo io_stats; - - // Collect statistics of timers on all ranks - // timer - elapsed time for rank - // comm - communicator for collecting stats - // destrank - the rank to which to collect stats - // stats - pointer to timer stats - // - - void timer_stats(double timer, MPI_Comm comm, int destrank, timer_statsinfo *stats) - { - int rank, nprocs, i; - double *rtimers=NULL; /* All timers from ranks */ - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &nprocs); - if(rank == destrank) { - rtimers = (double *) malloc(nprocs*sizeof(double)); - stats->mean = 0.; - stats->min = timer; - stats->max = timer; - stats->std = 0.f; - } - MPI_Gather(&timer, 1, MPI_DOUBLE, rtimers, 1, MPI_DOUBLE, destrank, comm); - if(rank == destrank) { - for(i = 0; i < nprocs; i++) { - if(rtimers[i] > stats->max) stats->max = rtimers[i]; - if(rtimers[i] < stats->min) stats->min = rtimers[i]; - stats->mean += rtimers[i]; - } - stats->mean /= nprocs; - for(i = 0; i < nprocs; i++) - stats->std += (rtimers[i]-stats->mean)*(rtimers[i]-stats->mean); - stats->std = sqrt(stats->std / nprocs); - free(rtimers); - } - } - //---------------------------------------------------------------------------// class SolverBase { @@ -127,12 +82,14 @@ class Solver : public SolverBase { // Output initial state. outputParticles(); - while ( _time < t_final ) { if ( 0 == _rank && 0 == _step % write_freq ) - printf( "Time %12.5e / %12.5e [iostats (s), mean min max: %12.5e %12.5e %12.5e] \n", _time, t_final, io_stats.mean, io_stats.min, io_stats.max ); + printf( "Time %12.5e / %12.5e [iostats, mean min max (s): " + "%12.5e %12.5e %12.5e] \n", + _time, t_final, io_stats.mean, io_stats.min, + io_stats.max ); // Fixed timestep is guaranteed only when sufficently low dt // does not violate the CFL condition (otherwise user-set dt is @@ -151,59 +108,33 @@ class Solver : public SolverBase if ( 0 == ( _step ) % write_freq ) outputParticles(); } - - const char* env_val = std::getenv("H5FUSE"); - if(env_val != NULL) { - - MPI_Comm shmcomm; - MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, - MPI_INFO_NULL, &shmcomm); - - int shmrank; - MPI_Comm_rank(shmcomm, &shmrank); - int status; - if ( shmrank == 0) { - for (int i = 0; i < nfork; i++) { - waitpid(-1, &status, 0); - if (WIFEXITED(status)) { + // Wait for all the h5fuse processes to complete + if ( shmrank == 0 ) + { + int status; + for ( int i = 0; i < nfork; i++ ) + { + waitpid( -1, &status, 0 ); + if ( WIFEXITED( status ) ) + { int ret; - - if ((ret = WEXITSTATUS(status)) != 0) { - printf("h5fuse process exited with error code %d\n", ret); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); + if ( ( ret = WEXITSTATUS( status ) ) != 0 ) + { + printf( "h5fuse process exited with error code %d\n", + ret ); + fflush( stdout ); + MPI_Abort( MPI_COMM_WORLD, -1 ); } } - else { - printf("h5fuse process terminated abnormally\n"); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - } -#if 0 - while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { - if (WIFEXITED(status)) { - int ret; - - if ((ret = WEXITSTATUS(status)) != 0) { - printf("h5fuse process exited with error code %d\n", ret); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } + else + { + printf( "h5fuse process terminated abnormally\n" ); + fflush( stdout ); + MPI_Abort( MPI_COMM_WORLD, -1 ); } - else { - printf("h5fuse process terminated abnormally\n"); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } - } -#endif } - MPI_Comm_free(&shmcomm); - } - + } } void outputParticles() @@ -211,20 +142,21 @@ class Solver : public SolverBase // Prefer HDF5 output over Silo. Only output if one is enabled. #ifdef Cabana_ENABLE_HDF5 Cabana::Experimental::HDF5ParticleOutput::HDF5Config h5_config; - const char* env_val = std::getenv("H5FD_SUBFILING"); - if(env_val != NULL) - h5_config.subfiling = true; + const char* env_val = std::getenv( "H5FD_SUBFILING" ); + if ( env_val != NULL ) + h5_config.subfiling = true; // Sets the HDF5 alignment equal subfiling's stripe size - env_val = std::getenv("H5FD_SUBFILING_STRIPE_SIZE"); - if(env_val != NULL) { - h5_config.align = true; - h5_config.threshold = 0; - h5_config.alignment = std::atoi(env_val); - } - - double t1, t2; - t1 = MPI_Wtime(); + env_val = std::getenv( "H5FD_SUBFILING_STRIPE_SIZE" ); + if ( env_val != NULL ) + { + h5_config.align = true; + h5_config.threshold = 0; + h5_config.alignment = std::atoi( env_val ); + } + + double t1, t2; + t1 = MPI_Wtime(); Cabana::Experimental::HDF5ParticleOutput::writeTimeStep( h5_config, "particles", _mesh->localGrid()->globalGrid().comm(), _step, _time, _pm->numParticle(), @@ -232,153 +164,65 @@ class Solver : public SolverBase _pm->get( Location::Particle(), Field::Velocity() ), _pm->get( Location::Particle(), Field::J() ) ); t2 = MPI_Wtime(); - timer_stats(t2-t1, MPI_COMM_WORLD, 0, &io_stats); -#if 1 - env_val = std::getenv("H5FUSE"); - if(env_val != NULL) { - - if(h5_config.subfiling) { - - MPI_Comm shmcomm; - MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, - MPI_INFO_NULL, &shmcomm); - - int shmrank; - MPI_Comm_rank(shmcomm, &shmrank); - - if ( shmrank == 0) { - pid_t pid = 0; - //pid_t tmppid; - int status; - //std::cout << "DOING OUTPUT " << _rank << std::endl; - //signal(SIGHUP, SIG_IGN); - //not interested in its childs, prevent zombies - //signal(SIGCHLD, SIG_IGN); - //signal(SIGQUIT, SIG_IGN); - - pid = fork(); - nfork++; - if (pid == 0) { - std::stringstream filename_hdf5; - filename_hdf5 << "particles" << "_" << _step << ".h5"; - - // Directory containing the subfiling configuration file - std::stringstream config_dir; - if(const char* env_value = std::getenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX)) - config_dir << env_value; - else - config_dir << "."; - // Find the name of the subfiling configuration file - struct stat file_info; - stat(filename_hdf5.str().c_str(), &file_info); - - char config_filename[PATH_MAX]; - snprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir.str().c_str(), - filename_hdf5.str().c_str(), (uint64_t)file_info.st_ino); - - // Call the h5fuse utility - char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-f"), config_filename, NULL}; - //char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; - - execvp(args[0], args); - //exit(0); - -#if 0 - // char *tmp_filename; - char *args[7]; - - //tmp_filename = (char *)malloc(PATH_MAX); - - /* Generate name for configuration file */ - //HDsnprintf(tmp_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir, - // SUBF_FILENAME, file_inode); - args[0] = strdup("ls"); - args[1] = strdup("-l"); - args[2] = NULL; - execvp("ls", args); -#endif - } -#if 0 - else { - //tmppid = waitpid(pid, &status, WNOHANG); - waitpid(pid, &status, 0); - if (WIFEXITED(status)) { - int ret; - - if ((ret = WEXITSTATUS(status)) != 0) { - printf("h5fuse process exited with error code %d\n", ret); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } - } - else { - printf("h5fuse process terminated abnormally\n"); - fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, -1); - } - } -#endif - } -} -} -#endif -#if 0 - pid_t tmppid; - int status; - - pid_t pid = fork(); - if (pid == -1) { - perror("fork"); - exit(EXIT_FAILURE); - } -#endif -#if 0 - if (pid == 0) { - // stub HDF5 filename - std::stringstream filename_hdf5; - filename_hdf5 << "particles" << "_" << _step << ".h5"; - - // Directory containing the subfiling configuration file - std::stringstream config_dir; - if(const char* env_value = std::getenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX)) - config_dir << env_value; - else - config_dir << "."; - // Find the name of the subfiling configuration file - struct stat file_info; - stat(filename_hdf5.str().c_str(), &file_info); - - char config_filename[PATH_MAX]; - snprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir.str().c_str(), - filename_hdf5.str().c_str(), (uint64_t)file_info.st_ino); - - // Call the h5fuse utility - char* args[] = { strdup("./h5fuse.sh"), strdup("-r"), strdup("-v"), strdup("-f"), config_filename, NULL}; - - execvp(args[0], args); - } - else { - tmppid = waitpid(pid, &status, 0); - - if (WIFEXITED(status)) { - int ret; + timer_stats( t2 - t1, MPI_COMM_WORLD, 0, &io_stats ); - if ((ret = WEXITSTATUS(status)) != 0) { - printf("h5fuse process exited with error code %d\n", ret); - fflush(stdout); - MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); + env_val = std::getenv( "H5FUSE" ); + if ( env_val != NULL ) + { + if ( h5_config.subfiling ) + { + + MPI_Comm shmcomm; + MPI_Comm_split_type( MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, + MPI_INFO_NULL, &shmcomm ); + + MPI_Comm_rank( shmcomm, &shmrank ); + + // One rank from each node executes h5fuse.sh + + if ( shmrank == 0 ) + { + pid_t pid = 0; + int status; + + pid = fork(); + nfork++; + if ( pid == 0 ) + { + std::stringstream filename_hdf5; + filename_hdf5 << "particles" + << "_" << _step << ".h5"; + + // Directory containing the subfiling configuration file + std::stringstream config_dir; + if ( const char* env_value = std::getenv( + H5FD_SUBFILING_CONFIG_FILE_PREFIX ) ) + config_dir << env_value; + else + config_dir << "."; + // Find the name of the subfiling configuration file + struct stat file_info; + stat( filename_hdf5.str().c_str(), &file_info ); + + char config_filename[PATH_MAX]; + snprintf( config_filename, PATH_MAX, + "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, + config_dir.str().c_str(), + filename_hdf5.str().c_str(), + (uint64_t)file_info.st_ino ); + + // Call the h5fuse utility + // Removes the subfiles in the process + char* args[] = { strdup( "./h5fuse.sh" ), + strdup( "-r" ), strdup( "-f" ), + config_filename, NULL }; + + execvp( args[0], args ); + } } - } - else { - printf("h5fuse process terminated abnormally\n"); - fflush(stdout); - MPI_Abort( _mesh->localGrid()->globalGrid().comm(), -1); - } + MPI_Comm_free( &shmcomm ); } -#endif -// } -// } -//#endif + } #else #ifdef Cabana_ENABLE_SILO Cabana::Grid::Experimental::SiloParticleOutput::writeTimeStep( @@ -405,7 +249,61 @@ class Solver : public SolverBase std::shared_ptr> _mesh; std::shared_ptr> _pm; int _rank; + int shmrank; + + struct timer_statsinfo + { + double min; + double max; + double mean; + double std; + }; + timer_statsinfo io_stats; + + // Collect statistics of timers on all ranks + // timer - elapsed time for rank + // comm - communicator for collecting stats + // destrank - the rank to which to collect stats + // stats - pointer to timer stats + // + + void timer_stats( double timer, MPI_Comm comm, int destrank, + timer_statsinfo* stats ) + { + int rank, nprocs, i; + double* rtimers = NULL; /* All timers from ranks */ + + MPI_Comm_rank( comm, &rank ); + MPI_Comm_size( comm, &nprocs ); + if ( rank == destrank ) + { + rtimers = (double*)malloc( nprocs * sizeof( double ) ); + stats->mean = 0.; + stats->min = timer; + stats->max = timer; + stats->std = 0.f; + } + MPI_Gather( &timer, 1, MPI_DOUBLE, rtimers, 1, MPI_DOUBLE, destrank, + comm ); + if ( rank == destrank ) + { + for ( i = 0; i < nprocs; i++ ) + { + if ( rtimers[i] > stats->max ) + stats->max = rtimers[i]; + if ( rtimers[i] < stats->min ) + stats->min = rtimers[i]; + stats->mean += rtimers[i]; + } + stats->mean /= nprocs; + for ( i = 0; i < nprocs; i++ ) + stats->std += + ( rtimers[i] - stats->mean ) * ( rtimers[i] - stats->mean ); + stats->std = sqrt( stats->std / nprocs ); + free( rtimers ); + } + } }; //---------------------------------------------------------------------------// From 35ba4b905ec4dc9c2db754820f877bdce922f999 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Tue, 15 Aug 2023 16:41:56 -0500 Subject: [PATCH 21/22] clean-up --- examples/dam_break.cpp | 28 ++++++++++++++++------------ src/ExaMPM_Solver.hpp | 3 +++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/examples/dam_break.cpp b/examples/dam_break.cpp index 9b6ee10..b38e240 100644 --- a/examples/dam_break.cpp +++ b/examples/dam_break.cpp @@ -119,21 +119,25 @@ void damBreak( const double cell_size, const int ppc, const int halo_size, int main( int argc, char* argv[] ) { - const char* env_val = std::getenv("H5FD_SUBFILING"); - if(env_val != NULL) { - int mpi_thread_required = MPI_THREAD_MULTIPLE; - int mpi_thread_provided = 0; + // enable the use of subfiling by setting enviroment H5FD_SUBFILING + const char* env_val = std::getenv( "H5FD_SUBFILING" ); + if ( env_val != NULL ) + { + int mpi_thread_required = MPI_THREAD_MULTIPLE; + int mpi_thread_provided = 0; - // HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE + // HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE - MPI_Init_thread(&argc, &argv, mpi_thread_required, &mpi_thread_provided); - if (mpi_thread_provided < mpi_thread_required) { - printf("MPI_THREAD_MULTIPLE not supported\n"); - MPI_Abort(MPI_COMM_WORLD, -1); + MPI_Init_thread( &argc, &argv, mpi_thread_required, + &mpi_thread_provided ); + if ( mpi_thread_provided < mpi_thread_required ) + { + printf( "MPI_THREAD_MULTIPLE not supported\n" ); + MPI_Abort( MPI_COMM_WORLD, -1 ); + } } - } - else - MPI_Init( &argc, &argv ); + else + MPI_Init( &argc, &argv ); Kokkos::initialize( argc, argv ); diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index c97e8a0..6661745 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -166,6 +166,9 @@ class Solver : public SolverBase t2 = MPI_Wtime(); timer_stats( t2 - t1, MPI_COMM_WORLD, 0, &io_stats ); + // Setting enviroment H5FUSE enables fusing the subfiles into + // an HDF5 file. Assumes h5fuse.sh is in the same directory + // as the executable. env_val = std::getenv( "H5FUSE" ); if ( env_val != NULL ) { From 6a9c2c4423e8a56e88b7dcaef93868ba975ad09c Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Mon, 5 Feb 2024 23:47:23 -0600 Subject: [PATCH 22/22] added PFS processing --- src/ExaMPM_Solver.hpp | 73 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/src/ExaMPM_Solver.hpp b/src/ExaMPM_Solver.hpp index 6661745..f918611 100644 --- a/src/ExaMPM_Solver.hpp +++ b/src/ExaMPM_Solver.hpp @@ -155,6 +155,14 @@ class Solver : public SolverBase h5_config.alignment = std::atoi( env_val ); } + env_val = std::getenv( "H5FUSE" ); + if ( env_val != NULL ) { + h5_config.h5fuse_info = true; + env_val = std::getenv( "LOC" ); + if ( env_val != NULL ) + h5_config.h5fuse_local = true; + } + double t1, t2; t1 = MPI_Wtime(); Cabana::Experimental::HDF5ParticleOutput::writeTimeStep( @@ -167,7 +175,7 @@ class Solver : public SolverBase timer_stats( t2 - t1, MPI_COMM_WORLD, 0, &io_stats ); // Setting enviroment H5FUSE enables fusing the subfiles into - // an HDF5 file. Assumes h5fuse.sh is in the same directory + // an HDF5 file. Assumes h5fuse is in the same directory // as the executable. env_val = std::getenv( "H5FUSE" ); if ( env_val != NULL ) @@ -175,13 +183,67 @@ class Solver : public SolverBase if ( h5_config.subfiling ) { + // if (h5_config.h5fuse_info) + // std::cout << "LEN " << h5_config.subfilenames_len << std::endl; + + //if(!h5_config.subfilenames.empty()) { + // int l_mpi_rank; + // MPI_Comm_rank(MPI_COMM_WORLD, &l_mpi_rank); + // std::cout << "ExaMPM " << h5_config.subfilenames << std::endl; + //} + + if (!h5_config.h5fuse_local) { + + if(!h5_config.subfilenames.empty()) { + { + pid_t pid = 0; + int status; + + pid = fork(); + nfork++; + if ( pid == 0 ) + { + std::stringstream filename_hdf5; + filename_hdf5 << "particles" + << "_" << _step << ".h5"; + + // Directory containing the subfiling configuration file + std::stringstream config_dir; + if ( const char* env_value = std::getenv( + H5FD_SUBFILING_CONFIG_FILE_PREFIX ) ) + config_dir << env_value; + else + config_dir << "."; + // Find the name of the subfiling configuration file + struct stat file_info; + stat( filename_hdf5.str().c_str(), &file_info ); + + char config_filename[PATH_MAX]; + snprintf( config_filename, PATH_MAX, + "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, + config_dir.str().c_str(), + filename_hdf5.str().c_str(), + (uint64_t)file_info.st_ino ); + + // Call the h5fuse utility + // Removes the subfiles in the process + char* args[] = { strdup( "./h5fuse" ), + strdup( "-l" ), strdup( h5_config.subfilenames.c_str() ), + //strdup( "-v" ), + strdup( "-f" ), config_filename, NULL }; + execvp( args[0], args ); + } + } + } + } else { + MPI_Comm shmcomm; MPI_Comm_split_type( MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shmcomm ); MPI_Comm_rank( shmcomm, &shmrank ); - // One rank from each node executes h5fuse.sh + // One rank from each node executes h5fuse if ( shmrank == 0 ) { @@ -216,14 +278,15 @@ class Solver : public SolverBase // Call the h5fuse utility // Removes the subfiles in the process - char* args[] = { strdup( "./h5fuse.sh" ), - strdup( "-r" ), strdup( "-f" ), - config_filename, NULL }; + char* args[] = { strdup( "./h5fuse" ), + strdup( "-r" ), + strdup( "-f" ), config_filename, NULL }; execvp( args[0], args ); } } MPI_Comm_free( &shmcomm ); + } } } #else