diff --git a/.github/workflows/dependencies/install_spack b/.github/workflows/dependencies/install_spack index a2e281dece..9c54127010 100755 --- a/.github/workflows/dependencies/install_spack +++ b/.github/workflows/dependencies/install_spack @@ -3,15 +3,20 @@ set -eu -o pipefail -spack_ver="0.16.0" +spack_ver="2b6f896ca744081a38579573a52824bf334fb54b" cd /opt +if [[ -d spack && ! -f spack_${spack_ver} ]] +then + rm -rf spack /usr/bin/spack $HOME/.spack/ +fi if [ ! -d spack ] then # download - curl -sOL https://github.com/spack/spack/archive/v${spack_ver}.tar.gz - tar -xf v${spack_ver}.tar.gz && rm v${spack_ver}.tar.gz + curl -sOL https://github.com/spack/spack/archive/${spack_ver}.tar.gz + tar -xf ${spack_ver}.tar.gz && rm ${spack_ver}.tar.gz mv spack-${spack_ver} spack + touch spack_${spack_ver} fi # install diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 63fd020ba6..766515faaf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -23,6 +23,8 @@ Bug Fixes Other """"" +- ADIOS2: require version 2.7.0+ #927 + 0.13.2 ------ diff --git a/CMakeLists.txt b/CMakeLists.txt index ab38b77121..bf5b90e60f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -273,14 +273,14 @@ endif() # external library: ADIOS2 (optional) if(openPMD_USE_ADIOS2 STREQUAL AUTO) - find_package(ADIOS2 2.6.0 CONFIG) + find_package(ADIOS2 2.7.0 CONFIG) if(ADIOS2_FOUND) set(openPMD_HAVE_ADIOS2 TRUE) else() set(openPMD_HAVE_ADIOS2 FALSE) endif() elseif(openPMD_USE_ADIOS2) - find_package(ADIOS2 2.6.0 REQUIRED CONFIG) + find_package(ADIOS2 2.7.0 REQUIRED CONFIG) set(openPMD_HAVE_ADIOS2 TRUE) else() set(openPMD_HAVE_ADIOS2 FALSE) diff --git a/Dockerfile b/Dockerfile index 77ac81b129..0986b1a1cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,7 +65,7 @@ RUN curl -sLo adios-1.13.1.tar.gz http://users.nccs.gov/~pnorbert/adios-1 && make \ && make install -RUN curl -sLo adios2-2.6.0.tar.gz https://github.com/ornladios/ADIOS2/archive/v2.6.0.tar.gz \ +RUN curl -sLo adios2-2.7.1.tar.gz https://github.com/ornladios/ADIOS2/archive/v2.7.1.tar.gz \ && file adios2*.tar.gz \ && tar -xzf adios2*.tar.gz \ && rm adios2*.tar.gz \ diff --git a/NEWS.rst b/NEWS.rst index e776de9780..327ebea428 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,6 +3,12 @@ Upgrade Guide ============= +0.14.0 +------ + +ADIOS 2.7.0 is now the minimally supported version for ADIOS2 support. + + 0.13.0 ------ diff --git a/README.md b/README.md index ad543bebae..8e0ddac719 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ I/O backends: * [JSON](https://en.wikipedia.org/wiki/JSON) * [HDF5](https://support.hdfgroup.org/HDF5) 1.8.13+ (optional) * [ADIOS1](https://www.olcf.ornl.gov/center-projects/adios) 1.13.1+ (optional) -* [ADIOS2](https://github.com/ornladios/ADIOS2) 2.6.0+ (optional) +* [ADIOS2](https://github.com/ornladios/ADIOS2) 2.7.0+ (optional) while those can be built either with or without: * MPI 2.1+, e.g. OpenMPI 1.6.5+ or MPICH2 diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index 7fed72116b..dd2e40631a 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -22,8 +22,11 @@ Steps ADIOS2 is optimized towards organizing the process of reading/writing data into IO steps. In order to activate steps, it is imperative to use the :ref:`Streaming API ` (which can be used for either file-based or streaming-based workflows). -With ADIOS2 release 2.6.0 containing a bug (fixed in development versions, see `PR #2348 `_) that disallows random-accessing steps in file-based engines, step-based processing must currently be opted in to via use of the :ref:`JSON parameter` ``adios2.engine.usesteps = true`` when using a file-based engine such as BP3 or BP4. -With these ADIOS2 releases, files written in such a way may only be read using the streaming API. + +ADIOS2 release 2.6.0 contained a bug (fixed in ADIOS 2.7.0, see `PR #2348 `_) that disallows random-accessing steps in file-based engines. +With this ADIOS2 release, files written with steps may only be read using the streaming API. +In order to keep compatibility with older codes reading ADIOS2 files, step-based processing must currently be opted in to via use of the :ref:`JSON parameter` ``adios2.engine.usesteps = true`` when using a file-based engine such as BP3 or BP4. + Upon reading a file, the ADIOS2 backend will automatically recognize whether it has been written with or without steps, ignoring the JSON option mentioned above. Steps are mandatory for streaming-based engines and trying to switch them off will result in a runtime error. diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index 183a5335ab..2b4278652b 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -28,7 +28,7 @@ Optional: I/O backends * `JSON `_ * `HDF5 `_ 1.8.13+ * `ADIOS1 `_ 1.13.1+ -* `ADIOS2 `_ 2.6.0+ +* `ADIOS2 `_ 2.7.0+ while those can be build either with or without: diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 60b933a84d..ac3754e430 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -1127,10 +1127,13 @@ namespace detail * step are shown which hinders parsing. So, until a step is * explicitly opened via ADIOS2IOHandlerImpl::advance, do not open * one. - * (This is a workaround for the fact that attributes - * are not associated with steps in ADIOS -- seeing all attributes - * from all steps in file-based engines, but only the current - * variables breaks parsing otherwise.) + * This is to enable use of ADIOS files without the Streaming API + * (i.e. all iterations should be visible to the user upon opening + * the Series.) + * @todo Add a workflow without up-front parsing of all iterations + * for file-based engines. + * (This would merely be an optimization since the streaming + * API still works with files as intended.) * */ enum class StreamStatus @@ -1155,23 +1158,40 @@ namespace detail * API. This is due to the fact that ADIOS2.6.0 requires using * steps to read BP4 files written with steps, so using steps * is opt-in for now. + * Notice that while the openPMD API requires ADIOS >= 2.7.0, + * the resulting files need to be readable from ADIOS 2.6.0 as + * well. This workaround is hence staying until switching to + * a new ADIOS schema. * 2) Reading with the Streaming API any file that has been written - * without steps. + * without steps. This is not a workaround since not using steps, + * while inefficient in ADIOS2, is something that we support. */ NoStream, /** - * Necessary workaround under the following circumstances: - * 1) Using ADIOS2.6.0 - * 2) Using attribute-based layout - * 3) Reading from a file-based engine a Series written with steps - * Up until ADIOS2.6.0, attributes are not associated with ADIOS - * steps in file-based engines. As a consequence, parsing one - * ADIOS step will show only the variables of that step, but the - * attributes of all steps which breaks our parsing logic. - * Workaround: If parsing before opening any step, all variables - * and attributes in the file will be shown. - * Hence, streamStatus == Parsing means that the first step has yet - * to be opened. + * Rationale behind this state: + * When user code opens a Series, series.iterations should contain + * all available iterations. + * If accessing a file without opening a step, ADIOS2 will grant + * access to variables and attributes from all steps, allowing us + * to parse the complete dump. + * This state indicates that no step should be opened for parsing + * purposes (which is necessary in streaming engines, hence they + * are initialized with the OutsideOfStep state). + * A step should only be opened if an explicit ADVANCE task arrives + * at the backend. + * + * @todo If the streaming API is used on files, parsing the whole + * Series up front is unnecessary work. + * Our frontend does not yet allow to distinguish whether + * parsing the whole series will be necessary since parsing + * happens upon construction time of Series, + * but the classical and the streaming API are both activated + * afterwards from the created Series object. + * Hence, improving this requires refactoring in our + * user-facing API. Ideas: + * (1) Delayed lazy parsing of iterations upon accessing + * (would bring other benefits also). + * (2) Introduce a restricted class StreamingSeries. */ Parsing, /** diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index d41b122354..7b277b72b1 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -33,6 +33,7 @@ namespace openPMD ADIOS1, ADIOS2, ADIOS2_SST, + ADIOS2_SSC, JSON, DUMMY }; diff --git a/src/Format.cpp b/src/Format.cpp index 8914257f35..98defa14b3 100644 --- a/src/Format.cpp +++ b/src/Format.cpp @@ -55,6 +55,8 @@ namespace openPMD { } if (auxiliary::ends_with(filename, ".sst")) return Format::ADIOS2_SST; + if (auxiliary::ends_with(filename, ".ssc")) + return Format::ADIOS2_SSC; if (auxiliary::ends_with(filename, ".json")) return Format::JSON; if (std::string::npos != filename.find('.') /* extension is provided */ ) @@ -73,6 +75,8 @@ namespace openPMD { return ".bp"; case Format::ADIOS2_SST: return ".sst"; + case Format::ADIOS2_SSC: + return ".ssc"; case Format::JSON: return ".json"; default: diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index cf73539272..e12622493c 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -96,7 +96,7 @@ namespace detail { "long double", Datatype::LONG_DOUBLE }, { "float complex", Datatype::CFLOAT }, { "double complex", Datatype::CDOUBLE }, - { "long double complex", Datatype::CLONG_DOUBLE }, // does not exist as of 2.6.0 but might come later + { "long double complex", Datatype::CLONG_DOUBLE }, // does not exist as of 2.7.0 but might come later { "uint8_t", Datatype::UCHAR }, { "int8_t", Datatype::CHAR }, { "uint16_t", determineDatatype< uint16_t >() }, diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp index ac8dadcc5b..ca32095f68 100644 --- a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp +++ b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp @@ -28,6 +28,7 @@ #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include #include #include #include @@ -52,7 +53,7 @@ namespace detail constexpr size_t operator()( Args &&... ) const { - return 0; + return alignof(std::max_align_t); } }; @@ -102,7 +103,15 @@ namespace detail { numItems *= extent; } - new( dest ) T[ numItems ]{}; + /* + * MSVC does not like placement new of arrays, so we do it + * in a loop instead. + * https://developercommunity.visualstudio.com/t/c-placement-new-is-incorrectly-compiled/206439 + */ + for( size_t i = 0; i < numItems; ++i ) + { + new( dest + i ) T(); + } location.destroy = buffer; engine.Get( var, dest, adios2::Mode::Deferred ); } diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 0a5b443a54..5a37ec6fe7 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -57,6 +57,9 @@ namespace openPMD case Format::ADIOS2_SST: return std::make_shared< ADIOS2IOHandler >( path, access, comm, std::move( optionsJson ), "sst" ); + case Format::ADIOS2_SSC: + return std::make_shared< ADIOS2IOHandler >( + path, access, comm, std::move( optionsJson ), "ssc" ); default: throw std::runtime_error( "Unknown file format! Did you specify a file ending?" ); @@ -89,6 +92,9 @@ namespace openPMD case Format::ADIOS2_SST: return std::make_shared< ADIOS2IOHandler >( path, access, std::move( optionsJson ), "sst" ); + case Format::ADIOS2_SSC: + return std::make_shared< ADIOS2IOHandler >( + path, access, std::move( optionsJson ), "ssc" ); #endif // openPMD_HAVE_ADIOS2 case Format::JSON: return std::make_shared< JSONIOHandler >( path, access ); diff --git a/src/Series.cpp b/src/Series.cpp index 92c6e422f3..146981d008 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1245,6 +1245,7 @@ namespace case Format::ADIOS1: case Format::ADIOS2: case Format::ADIOS2_SST: + case Format::ADIOS2_SSC: case Format::JSON: return auxiliary::replace_last(filename, suffix(f), ""); default: @@ -1296,6 +1297,16 @@ namespace nameReg += + ")" + postfix + ".sst$"; return buildMatcher(nameReg); } + case Format::ADIOS2_SSC: + { + std::string nameReg = "^" + prefix + "([[:digit:]]"; + if( padding != 0 ) + nameReg += "{" + std::to_string(padding) + "}"; + else + nameReg += "+"; + nameReg += + ")" + postfix + ".ssc$"; + return buildMatcher(nameReg); + } case Format::JSON: { std::string nameReg = "^" + prefix + "([[:digit:]]"; if (padding != 0) diff --git a/src/config.cpp b/src/config.cpp index b50b040765..06b9c58640 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -52,6 +52,9 @@ openPMD::getFileExtensions() #ifdef ADIOS2_HAVE_SST fext.emplace_back("sst"); #endif +#ifdef ADIOS2_HAVE_SSC + fext.emplace_back("ssc"); +#endif #if openPMD_HAVE_HDF5 fext.emplace_back("h5"); #endif diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 73d71e6b69..9aa2334b7e 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1022,6 +1022,13 @@ TEST_CASE( "parallel_adios2_json_config", "[parallel][adios2]" ) void adios2_ssc() { + auto const extensions = openPMD::getFileExtensions(); + if( std::find( extensions.begin(), extensions.end(), "ssc" ) == + extensions.end() ) + { + // SSC engine not available in ADIOS2 + return; + } int global_size{ -1 }; int global_rank{ -1 }; MPI_Comm_size( MPI_COMM_WORLD, &global_size ); @@ -1047,23 +1054,13 @@ adios2_ssc() constexpr size_t extent = 10; - std::string options = R"( - { - "adios2": { - "engine": { - "type": "ssc" - } - } - })"; - if( color == 0 ) { // write Series writeSeries( - "../samples/adios2_stream.bp", + "../samples/adios2_stream.ssc", Access::CREATE, - local_comm, - options ); + local_comm ); auto iterations = writeSeries.writeIterations(); for( size_t i = 0; i < 10; ++i ) { @@ -1082,10 +1079,9 @@ adios2_ssc() { // read Series readSeries( - "../samples/adios2_stream.bp", + "../samples/adios2_stream.ssc", Access::READ_ONLY, - local_comm, - options ); + local_comm ); size_t last_iteration_index = 0; for( auto iteration : readSeries.readIterations() ) @@ -1110,15 +1106,6 @@ adios2_ssc() TEST_CASE( "adios2_ssc", "[parallel][adios2]" ) { - /* - * @todo Activate this test as soon as we rely upon an ADIOS2 version - * including this fix https://github.com/ornladios/ADIOS2/pull/2568 - * (e.g. ADIOS 2.7.0). - */ - constexpr bool testAdiosSSC = false; - if( testAdiosSSC ) - { - adios2_ssc(); - } + adios2_ssc(); } #endif diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 1425d21689..41dd74bf27 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -30,8 +30,11 @@ using namespace openPMD; std::vector< std::string > testedFileExtensions() { auto allExtensions = getFileExtensions(); - auto newEnd = - std::remove( allExtensions.begin(), allExtensions.end(), "sst" ); + auto newEnd = std::remove_if( + allExtensions.begin(), + allExtensions.end(), + []( std::string const & ext ) + { return ext == "sst" || ext == "ssc"; } ); return { allExtensions.begin(), newEnd }; } @@ -1170,7 +1173,7 @@ void test_complex(const std::string & backend) { TEST_CASE( "test_complex", "[serial]" ) { // Notes: - // - ADIOS1 and ADIOS 2.6.0 have no complex long double + // - ADIOS1 and ADIOS 2.7.0 have no complex long double // - JSON read-back not distinguishable yet from N+1 shaped data set for (auto const & t : testedFileExtensions()) { @@ -3232,12 +3235,6 @@ TEST_CASE( "bp4_steps", "[serial][adios2]" ) } } )"; - /* - * @todo Activate these tests for Windows as soon as we bump the required - * ADIOS2 version to 2.7.0. Read here: - * https://github.com/openPMD/openPMD-api/pull/813#issuecomment-762235260 - */ -#ifndef _WIN32 // sing the yes no song bp4_steps( "../samples/newlayout_bp4steps_yes_yes.bp", useSteps, useSteps ); bp4_steps( @@ -3246,7 +3243,6 @@ TEST_CASE( "bp4_steps", "[serial][adios2]" ) "../samples/newlayout_bp4steps_yes_no.bp", useSteps, dontUseSteps ); bp4_steps( "../samples/newlayout_bp4steps_no_no.bp", dontUseSteps, dontUseSteps ); -#endif } #endif diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 65665b519f..390462d216 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -23,7 +23,9 @@ from TestUtilities.TestUtilities import generateTestFilePath -tested_file_extensions = [ext for ext in io.file_extensions if ext != 'sst'] +tested_file_extensions = [ + ext for ext in io.file_extensions if ext != 'sst' and ext != 'ssc' +] class APITest(unittest.TestCase): @@ -339,7 +341,7 @@ def attributeRoundTrip(self, file_ending): np.testing.assert_almost_equal( series.get_attribute("nparr_cdouble"), [4.5 + 1.1j, 6.7 - 2.2j]) - if file_ending != "bp": # not in ADIOS 1.13.1 nor ADIOS 2.6.0 + if file_ending != "bp": # not in ADIOS 1.13.1 nor ADIOS 2.7.0 np.testing.assert_almost_equal( series.get_attribute("nparr_clongdouble"), [8.9 + 7.8j, 7.6 + 9.2j])