diff --git a/.github/workflows/unix.yml b/.github/workflows/unix.yml index e65060b4eb..ce9f20aa3c 100644 --- a/.github/workflows/unix.yml +++ b/.github/workflows/unix.yml @@ -103,7 +103,7 @@ jobs: ../share/openPMD/download_samples.sh && chmod u-w samples/git-sample/*.h5 cmake -S .. -B . -DopenPMD_USE_PYTHON=OFF -DopenPMD_USE_MPI=ON -DopenPMD_USE_HDF5=ON -DopenPMD_USE_ADIOS1=OFF -DopenPMD_USE_ADIOS2=ON -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build . --parallel 2 - export OPENPMD_NEW_ATTRIBUTE_LAYOUT=1 + export OPENPMD2_ADIOS2_SCHEMA=20210209 ctest --output-on-failure # TODO diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index dd2e40631a..8a57fb4005 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -51,7 +51,7 @@ environment variable default description ``OPENPMD_ADIOS2_HAVE_METADATA_FILE`` ``1`` Online creation of the adios journal file (``1``: yes, ``0``: no). ``OPENPMD_ADIOS2_NUM_SUBSTREAMS`` ``0`` Number of files to be created, 0 indicates maximum number possible. ``OPENPMD_ADIOS2_ENGINE`` ``File`` `ADIOS2 engine `_ -``OPENPMD_NEW_ATTRIBUTE_LAYOUT`` ``0`` Experimental: new attribute layout (see below) +``OPENPMD2_ADIOS2_SCHEMA`` ``0`` ADIOS2 schema version (see below) ``OPENPMD_BP_BACKEND`` ``ADIOS2`` Chose preferred ``.bp`` file backend if ``ADIOS1`` and ``ADIOS2`` are available. ===================================== ========== ================================================================================ @@ -79,8 +79,8 @@ A good number for substreams is usually the number of contributing nodes divided For fine-tuning at extreme scale or for exotic systems, please refer to the ADIOS2 manual and talk to your filesystem admins and the ADIOS2 authors. Be aware that extreme-scale I/O is a research topic after all. -Experimental new attribute layout ---------------------------------- +Experimental new ADIOS2 schema +------------------------------ We are experimenting with a breaking change to our layout of openPMD datasets in ADIOS2. It is likely that we will in future use ADIOS attributes only for a handful of internal flags. @@ -96,11 +96,10 @@ We hope that this will bring several advantages: The new layout may be activated **for experimental purposes** in two ways: -* Via the JSON parameter ``adios2.new_attribute_layout = true``. -* Via the environment variable ``export OPENPMD_NEW_ATTRIBUTE_LAYOUT=1``. +* Via the JSON parameter ``adios2.schema = 20210209``. +* Via the environment variable ``export OPENPMD2_ADIOS2_SCHEMA=20210209``. -The classical and the new layout are absolutely incompatible with one another. -The ADIOS2 backend will **not** (yet) automatically recognize the layout that has been used by a writer when reading a dataset. +The ADIOS2 backend will automatically recognize the layout that has been used by a writer when reading a dataset. Selected References ------------------- diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index a91372797c..3cfe22272c 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -80,6 +80,26 @@ namespace detail } // namespace detail +namespace ADIOS2Schema +{ + using schema_t = uint64_t; + /* + * Original ADIOS schema. + */ + constexpr schema_t schema_0000_00_00 = 00000000; + /* + * This introduces attribute layout via scalar ADIOS variables. + */ + constexpr schema_t schema_2021_02_09 = 20210209; + + enum class SupportedSchema : char + { + s_0000_00_00, + s_2021_02_09 + }; +} +using SupportedSchema = ADIOS2Schema::SupportedSchema; + class ADIOS2IOHandlerImpl : public AbstractIOHandlerImplCommon< ADIOS2FilePosition > { @@ -203,6 +223,7 @@ class ADIOS2IOHandlerImpl * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine */ std::string m_engineType; + ADIOS2Schema::schema_t m_schema = ADIOS2Schema::schema_0000_00_00; enum class AttributeLayout : char { @@ -210,7 +231,32 @@ class ADIOS2IOHandlerImpl ByAdiosVariables }; - AttributeLayout m_attributeLayout = AttributeLayout::ByAdiosAttributes; + inline SupportedSchema schema() const + { + switch( m_schema ) + { + case ADIOS2Schema::schema_0000_00_00: + return SupportedSchema::s_0000_00_00; + case ADIOS2Schema::schema_2021_02_09: + return SupportedSchema::s_2021_02_09; + default: + throw std::runtime_error( + "[ADIOS2] Encountered unsupported schema version: " + + std::to_string( m_schema ) ); + } + } + + inline AttributeLayout attributeLayout() const + { + switch( schema() ) + { + case SupportedSchema::s_0000_00_00: + return AttributeLayout::ByAdiosAttributes; + case SupportedSchema::s_2021_02_09: + return AttributeLayout::ByAdiosVariables; + } + throw std::runtime_error( "Unreachable!" ); + } struct ParameterizedOperator { @@ -357,6 +403,11 @@ namespace ADIOS2Defaults constexpr const_str str_params = "parameters"; constexpr const_str str_usesteps = "usesteps"; constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps"; + constexpr const_str str_adios2Schema = + "__openPMD_internal/openPMD2_adios2_schema"; + constexpr const_str str_isBooleanOldLayout = "__is_boolean__"; + constexpr const_str str_isBooleanNewLayout = + "__openPMD_internal/is_boolean"; } // namespace ADIOS2Defaults namespace detail @@ -995,8 +1046,6 @@ namespace detail detail::DatasetReader const m_readDataset; detail::AttributeReader const m_attributeReader; PreloadAdiosAttributes preloadAttributes; - using AttributeLayout = ADIOS2IOHandlerImpl::AttributeLayout; - AttributeLayout m_attributeLayout = AttributeLayout::ByAdiosAttributes; /* * We call an attribute committed if the step during which it was @@ -1106,6 +1155,7 @@ namespace detail invalidateVariablesMap(); private: + ADIOS2IOHandlerImpl * m_impl; auxiliary::Option< adios2::Engine > m_engine; //! ADIOS engine /** * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine @@ -1230,10 +1280,20 @@ namespace detail */ bool finalized = false; + inline SupportedSchema schema() const + { + return m_impl->schema(); + } + void configure_IO( ADIOS2IOHandlerImpl & impl ); - }; + using AttributeLayout = ADIOS2IOHandlerImpl::AttributeLayout; + inline AttributeLayout attributeLayout() const + { + return m_impl->attributeLayout(); + } + }; } // namespace detail #endif // openPMD_HAVE_ADIOS2 diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index df46ab5ee5..49773bbae4 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -126,12 +126,10 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) { m_config = std::move( cfg[ "adios2" ] ); - if( m_config.json().contains( "new_attribute_layout" ) ) + if( m_config.json().contains( "schema" ) ) { - m_attributeLayout = - static_cast< bool >( m_config[ "new_attribute_layout" ].json() ) - ? AttributeLayout::ByAdiosVariables - : AttributeLayout::ByAdiosAttributes; + m_schema = + m_config[ "schema" ].json().get< ADIOS2Schema::schema_t >(); } auto engineConfig = config( ADIOS2Defaults::str_engine ); @@ -157,11 +155,7 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) } } // environment-variable based configuration - int useNewLayout = auxiliary::getEnvNum( - "OPENPMD_NEW_ATTRIBUTE_LAYOUT", - m_attributeLayout == AttributeLayout::ByAdiosVariables ); - m_attributeLayout = useNewLayout == 0 ? AttributeLayout::ByAdiosAttributes - : AttributeLayout::ByAdiosVariables; + m_schema = auxiliary::getEnvNum( "OPENPMD2_ADIOS2_SCHEMA", m_schema ); } auxiliary::Option< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > > @@ -595,7 +589,7 @@ void ADIOS2IOHandlerImpl::writeDataset( void ADIOS2IOHandlerImpl::writeAttribute( Writable * writable, const Parameter< Operation::WRITE_ATT > & parameters ) { - switch( m_attributeLayout ) + switch( attributeLayout() ) { case AttributeLayout::ByAdiosAttributes: switchType( @@ -650,7 +644,7 @@ void ADIOS2IOHandlerImpl::readAttribute( auto file = refreshFileFromParent( writable ); auto pos = setAndGetFilePosition( writable ); detail::BufferedActions & ba = getFileData( file ); - switch( m_attributeLayout ) + switch( attributeLayout() ) { using AL = AttributeLayout; case AL::ByAdiosAttributes: @@ -711,7 +705,7 @@ void ADIOS2IOHandlerImpl::listPaths( */ std::vector< std::string > delete_me; - switch( m_attributeLayout ) + switch( attributeLayout() ) { using AL = AttributeLayout; case AL::ByAdiosVariables: @@ -818,7 +812,7 @@ void ADIOS2IOHandlerImpl::listDatasets( std::unordered_set< std::string > subdirs; for( auto var : fileData.availableVariablesPrefixed( myName ) ) { - if( m_attributeLayout == AttributeLayout::ByAdiosVariables ) + if( attributeLayout() == AttributeLayout::ByAdiosVariables ) { // since current Writable is a group and no dataset, // var == "__data__" is not possible @@ -862,7 +856,7 @@ void ADIOS2IOHandlerImpl::listAttributes( ba.requireActiveStep(); // make sure that the attributes are present std::vector< std::string > attrs; - switch( m_attributeLayout ) + switch( attributeLayout() ) { using AL = AttributeLayout; case AL::ByAdiosAttributes: @@ -874,7 +868,7 @@ void ADIOS2IOHandlerImpl::listAttributes( } for( auto & rawAttr : attrs ) { - if( m_attributeLayout == AttributeLayout::ByAdiosVariables && + if( attributeLayout() == AttributeLayout::ByAdiosVariables && ( auxiliary::ends_with( rawAttr, "/__data__" ) || rawAttr == "__data__" ) ) { @@ -1036,7 +1030,7 @@ ADIOS2IOHandlerImpl::nameOfVariable( Writable * writable ) { auto filepos = setAndGetFilePosition( writable ); auto res = filePositionToString( filepos ); - if( m_attributeLayout == AttributeLayout::ByAdiosAttributes ) + if( attributeLayout() == AttributeLayout::ByAdiosAttributes ) { return res; } @@ -1191,7 +1185,8 @@ namespace detail # endif ( std::is_same< T, rep >::value ) { - std::string metaAttr = "__is_boolean__" + name; + std::string metaAttr = + ADIOS2Defaults::str_isBooleanOldLayout + name; /* * In verbose mode, attributeInfo will yield a warning if not * finding the requested attribute. Since we expect the attribute @@ -1199,7 +1194,9 @@ namespace detail * a boolean), let's tell attributeInfo to be quiet. */ auto type = attributeInfo( - IO, "__is_boolean__" + name, /* verbose = */ false ); + IO, + ADIOS2Defaults::str_isBooleanOldLayout + name, + /* verbose = */ false ); if( type == determineDatatype< rep >() ) { auto attr = IO.InquireAttribute< rep >( metaAttr ); @@ -1244,7 +1241,8 @@ namespace detail #endif ( std::is_same< T, rep >::value ) { - std::string metaAttr = "__is_boolean__" + name; + std::string metaAttr = + ADIOS2Defaults::str_isBooleanNewLayout + name; /* * In verbose mode, attributeInfo will yield a warning if not * finding the requested attribute. Since we expect the attribute @@ -1252,7 +1250,9 @@ namespace detail * a boolean), let's tell attributeInfo to be quiet. */ auto type = attributeInfo( - IO, "__is_boolean__" + name, /* verbose = */ false ); + IO, + ADIOS2Defaults::str_isBooleanNewLayout + name, + /* verbose = */ false ); if( type == determineDatatype< rep >() ) { auto attr = IO.InquireAttribute< rep >( metaAttr ); @@ -1905,7 +1905,8 @@ namespace detail std::string name, const bool value ) { - IO.DefineAttribute< bool_representation >( "__is_boolean__" + name, 1 ); + IO.DefineAttribute< bool_representation >( + ADIOS2Defaults::str_isBooleanOldLayout + name, 1 ); AttributeTypes< bool_representation >::oldCreateAttribute( IO, name, toRep( value ) ); } @@ -1935,7 +1936,7 @@ namespace detail const bool value ) { IO.DefineAttribute< bool_representation >( - "__is_boolean__" + params.name, 1 ); + ADIOS2Defaults::str_isBooleanNewLayout + params.name, 1 ); AttributeTypes< bool_representation >::createAttribute( IO, engine, params, toRep( value ) ); } @@ -2027,8 +2028,7 @@ namespace detail } BufferedActions::BufferedActions( - ADIOS2IOHandlerImpl & impl, - InvalidatableFile file ) + ADIOS2IOHandlerImpl & impl, InvalidatableFile file ) : m_file( impl.fullPath( std::move( file ) ) ) , m_IOName( std::to_string( impl.nameCounter++ ) ) , m_ADIOS( impl.m_ADIOS ) @@ -2037,7 +2037,7 @@ namespace detail , m_writeDataset( &impl ) , m_readDataset( &impl ) , m_attributeReader() - , m_attributeLayout( impl.m_attributeLayout ) + , m_impl( &impl ) , m_engineType( impl.m_engineType ) { if( !m_IO ) @@ -2110,6 +2110,7 @@ namespace detail }; // set engine type + bool isStreaming = false; { // allow overriding through environment variable m_engineType = auxiliary::getEnvString( @@ -2124,8 +2125,9 @@ namespace detail auto it = streamingEngines.find( m_engineType ); if( it != streamingEngines.end() ) { + isStreaming = true; optimizeAttributesStreaming = - m_attributeLayout == AttributeLayout::ByAdiosAttributes; + schema() == SupportedSchema::s_0000_00_00; streamStatus = StreamStatus::OutsideOfStep; } else @@ -2135,16 +2137,34 @@ namespace detail { switch( m_mode ) { - case adios2::Mode::Read: - streamStatus = StreamStatus::Undecided; - delayOpeningTheFirstStep = m_attributeLayout == - AttributeLayout::ByAdiosAttributes; - break; - case adios2::Mode::Write: + case adios2::Mode::Read: + /* + * File engines, read mode: + * Use of steps is dictated by what is detected in the + * file being read. + */ + streamStatus = StreamStatus::Undecided; + // @todo no?? should be default in both modes + delayOpeningTheFirstStep = true; + break; + case adios2::Mode::Write: + /* + * File engines, write mode: + * Default for old layout is no steps. + * Default for new layout is to use steps. + */ + switch( schema() ) + { + case SupportedSchema::s_0000_00_00: streamStatus = StreamStatus::NoStream; break; - default: - throw std::runtime_error( "Unreachable!" ); + case SupportedSchema::s_2021_02_09: + streamStatus = StreamStatus::OutsideOfStep; + break; + } + break; + default: + throw std::runtime_error( "Unreachable!" ); } optimizeAttributesStreaming = false; } @@ -2182,8 +2202,7 @@ namespace detail m_mode != adios2::Mode::Read ) { bool tmp = _useAdiosSteps.json(); - if( streamStatus == StreamStatus::OutsideOfStep && - !bool( tmp ) ) + if( isStreaming && !bool( tmp ) ) { throw std::runtime_error( "Cannot switch off steps for streaming engines." ); @@ -2255,72 +2274,89 @@ namespace detail */ m_IO.SetParameter( "StatsLevel", "0" ); } + + // We need to open the engine now already to inquire configuration + // options stored in there + getEngine(); } - adios2::Engine & - BufferedActions::getEngine() + adios2::Engine & BufferedActions::getEngine() { if( !m_engine ) { switch( m_mode ) { - case adios2::Mode::Write: - { - bool_representation usesSteps = - streamStatus == StreamStatus::NoStream ? 0 : 1; - m_IO.DefineAttribute< bool_representation >( - ADIOS2Defaults::str_usesstepsAttribute, usesSteps ); - m_engine = auxiliary::makeOption( - adios2::Engine( m_IO.Open( m_file, m_mode ) ) ); - break; - } - case adios2::Mode::Read: + case adios2::Mode::Write: { + // usesSteps attribute only written upon ::advance() + // this makes sure that the attribute is only put in case + // the streaming API was used. + m_IO.DefineAttribute< ADIOS2Schema::schema_t >( + ADIOS2Defaults::str_adios2Schema, m_impl->m_schema ); + m_engine = auxiliary::makeOption( + adios2::Engine( m_IO.Open( m_file, m_mode ) ) ); + break; + } + case adios2::Mode::Read: { + m_engine = auxiliary::makeOption( + adios2::Engine( m_IO.Open( m_file, m_mode ) ) ); + // decide attribute layout + // in streaming mode, this needs to be done after opening + // a step + // in file-based mode, we do it before + auto layoutVersion = [ IO{ m_IO } ]() mutable { + auto attr = IO.InquireAttribute< ADIOS2Schema::schema_t >( + ADIOS2Defaults::str_adios2Schema ); + if( !attr ) + { + return ADIOS2Schema::schema_0000_00_00; + } + else + { + return attr.Data()[ 0 ]; + } + }; + // decide streaming mode + switch( streamStatus ) { - m_engine = auxiliary::makeOption( - adios2::Engine( m_IO.Open( m_file, m_mode ) ) ); - switch( streamStatus ) + case StreamStatus::Undecided: { + m_impl->m_schema = layoutVersion(); + auto attr = m_IO.InquireAttribute< bool_representation >( + ADIOS2Defaults::str_usesstepsAttribute ); + if( attr && attr.Data()[ 0 ] == 1 ) { - case StreamStatus::Undecided: + if( delayOpeningTheFirstStep ) { - auto attr = - m_IO.InquireAttribute< bool_representation >( - ADIOS2Defaults::str_usesstepsAttribute ); - if( attr && attr.Data()[ 0 ] == 1 ) - { - if( delayOpeningTheFirstStep ) - { - streamStatus = StreamStatus::Parsing; - } - else - { - m_engine.get().BeginStep(); - streamStatus = StreamStatus::DuringStep; - } - } - else - { - streamStatus = StreamStatus::NoStream; - } - break; + streamStatus = StreamStatus::Parsing; } - case StreamStatus::OutsideOfStep: + else + { m_engine.get().BeginStep(); streamStatus = StreamStatus::DuringStep; - break; - default: - throw std::runtime_error( - "[ADIOS2] Control flow error!" ); + } } - if( m_attributeLayout == AttributeLayout::ByAdiosVariables ) + else { - preloadAttributes.preloadAttributes( - m_IO, m_engine.get() ); + streamStatus = StreamStatus::NoStream; } break; } + case StreamStatus::OutsideOfStep: + m_engine.get().BeginStep(); + m_impl->m_schema = layoutVersion(); + streamStatus = StreamStatus::DuringStep; + break; default: - throw std::runtime_error( - "[ADIOS2] Invalid ADIOS access mode" ); + throw std::runtime_error( "[ADIOS2] Control flow error!" ); + } + if( attributeLayout() == AttributeLayout::ByAdiosVariables ) + { + preloadAttributes.preloadAttributes( m_IO, m_engine.get() ); + } + break; + } + default: + throw std::runtime_error( + "[ADIOS2] Invalid ADIOS access mode" ); } if( !m_engine ) @@ -2338,7 +2374,7 @@ namespace detail { m_lastStepStatus = eng.BeginStep(); if( m_mode == adios2::Mode::Read && - m_attributeLayout == AttributeLayout::ByAdiosVariables ) + attributeLayout() == AttributeLayout::ByAdiosVariables ) { preloadAttributes.preloadAttributes( m_IO, m_engine.get() ); } @@ -2462,9 +2498,14 @@ namespace detail // sic! no else if( streamStatus == StreamStatus::NoStream ) { + m_IO.DefineAttribute< bool_representation >( + ADIOS2Defaults::str_usesstepsAttribute, 0 ); flush( /* writeAttributes = */ false ); return AdvanceStatus::OK; } + + m_IO.DefineAttribute< bool_representation >( + ADIOS2Defaults::str_usesstepsAttribute, 1 ); switch( mode ) { case AdvanceMode::ENDSTEP: @@ -2511,7 +2552,7 @@ namespace detail /* flushUnconditionally = */ true ); if( adiosStatus == adios2::StepStatus::OK && m_mode == adios2::Mode::Read && - m_attributeLayout == AttributeLayout::ByAdiosVariables ) + attributeLayout() == AttributeLayout::ByAdiosVariables ) { preloadAttributes.preloadAttributes( m_IO, m_engine.get() ); diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index adbc894f1c..fe6f63f6f1 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1017,13 +1017,11 @@ TEST_CASE( "parallel_adios2_json_config", "[parallel][adios2]" ) write( "../samples/jsonConfiguredBP4Parallel.bp", writeConfigBP4 ); write( "../samples/jsonConfiguredBP3Parallel.bp", writeConfigBP3 ); - MPI_Barrier( MPI_COMM_WORLD ); - // BP3 engine writes files, BP4 writes directories - REQUIRE( openPMD::auxiliary::file_exists( - "../samples/jsonConfiguredBP3Parallel.bp" ) ); + REQUIRE( + openPMD::auxiliary::file_exists( "../samples/jsonConfiguredBP3.bp" ) ); REQUIRE( openPMD::auxiliary::directory_exists( - "../samples/jsonConfiguredBP4Parallel.bp" ) ); + "../samples/jsonConfiguredBP4.bp" ) ); std::string readConfigBP3 = R"END( { diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index f0588ac2a0..d3682ad3c6 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -3374,7 +3374,7 @@ TEST_CASE( "bp4_steps", "[serial][adios2]" ) useSteps = R"( { "adios2": { - "new_attribute_layout": true, + "schema": 20210209, "engine": { "type": "bp4", "usesteps": true @@ -3385,7 +3385,7 @@ TEST_CASE( "bp4_steps", "[serial][adios2]" ) dontUseSteps = R"( { "adios2": { - "new_attribute_layout": true, + "schema": 20210209, "engine": { "type": "bp4", "usesteps": false @@ -3395,10 +3395,10 @@ TEST_CASE( "bp4_steps", "[serial][adios2]" ) )"; // sing the yes no song bp4_steps( "../samples/newlayout_bp4steps_yes_yes.bp", useSteps, useSteps ); - bp4_steps( - "../samples/newlayout_bp4steps_no_yes.bp", dontUseSteps, useSteps ); bp4_steps( "../samples/newlayout_bp4steps_yes_no.bp", useSteps, dontUseSteps ); + bp4_steps( + "../samples/newlayout_bp4steps_no_yes.bp", dontUseSteps, useSteps ); bp4_steps( "../samples/newlayout_bp4steps_no_no.bp", dontUseSteps, dontUseSteps ); }