From 36de47774cda8e7e27393288f9a2aaa92b5c62e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 11 Feb 2020 11:59:45 +0100 Subject: [PATCH 01/24] Add TracingJSON class This will allow warning users of unused entries in the JSON config. Invert TracingJSON shadow logic Instead of removing all accessed entries one after another, add them to the "shadow" value one after another. This is a bit more tricky since the result has to be inverted again in the end, but it is less prone to creating dangling pointer situations. --- CMakeLists.txt | 1 + include/openPMD/auxiliary/JSON.hpp | 96 ++++++++++++++++++++++++++++++ src/auxiliary/JSON.cpp | 91 ++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 include/openPMD/auxiliary/JSON.hpp create mode 100644 src/auxiliary/JSON.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ab4d090919..fdd27320c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,6 +307,7 @@ set(CORE_SOURCE src/version.cpp src/auxiliary/Date.cpp src/auxiliary/Filesystem.cpp + src/auxiliary/JSON.cpp src/backend/Attributable.cpp src/backend/BaseRecordComponent.cpp src/backend/MeshRecordComponent.cpp diff --git a/include/openPMD/auxiliary/JSON.hpp b/include/openPMD/auxiliary/JSON.hpp new file mode 100644 index 0000000000..2b42544901 --- /dev/null +++ b/include/openPMD/auxiliary/JSON.hpp @@ -0,0 +1,96 @@ +#pragma once + +#include "openPMD/config.hpp" + +#if openPMD_HAVE_JSON + +# include // std::shared_ptr +# include // std::forward + +# include + +namespace openPMD +{ +namespace auxiliary +{ + /** + * @brief Extend nlohmann::json with tracing of which keys have been + * accessed by operator[](). + * An access is only registered if the current JSON value is a JSON object + * (not an array) and if the accessed JSON value is a leaf, i.e. anything + * but an object. This means that objects contained in arrays will not be + * traced. + * + * Warning: The current implementation is inefficient because it will copy + * the accessed JSON value upon access. Currently not to be used for deeply + * nested / large JSON structures. + * + */ + class TracingJSON : public nlohmann::json + { + public: + TracingJSON(); + TracingJSON( nlohmann::json ); + + template< typename Key > + TracingJSON operator[]( Key && key ); + + /** + * @brief Get the "shadow", i.e. a copy of the original JSON value + * containing all accessed object keys. + * + * @return nlohmann::json const& + */ + nlohmann::json const & + getShadow(); + + /** + * @brief Invert the "shadow", i.e. a copy of the original JSON value + * that contains exactly those values that have not been accessed yet. + * + * @return nlohmann::json + */ + nlohmann::json + invertShadow(); + + /** + * @brief Declare all keys of the current object read. + * + */ + void + declareFullyRead(); + + private: + std::shared_ptr< nlohmann::json > m_shadow; + nlohmann::json * m_position; + bool m_trace = true; + + void + invertShadow( nlohmann::json & result, nlohmann::json & shadow ); + + TracingJSON( + nlohmann::json, + std::shared_ptr< nlohmann::json > shadow, + nlohmann::json * position, + bool trace ); + }; + + template< typename Key > + TracingJSON TracingJSON::operator[]( Key && key ) + { + nlohmann::json::const_reference res = nlohmann::json::operator[]( key ); + // If accessing a leaf in the JSON tree from an object (not an array!) + // erase the corresponding key + static nlohmann::json nullvalue; + nlohmann::json * emplaced = &nullvalue; + if( m_trace && this->is_object() ) + { + emplaced = &m_position->operator[]( key ); + } + bool traceFurther = res.is_object(); + return TracingJSON( res, m_shadow, emplaced, traceFurther ); + } +} // namespace auxiliary +} // namespace openPMD + +#endif // openPMD_HAVE_JSON \ No newline at end of file diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp new file mode 100644 index 0000000000..4b6157fc6b --- /dev/null +++ b/src/auxiliary/JSON.cpp @@ -0,0 +1,91 @@ +#include "openPMD/config.hpp" +#if openPMD_HAVE_JSON +# include + +# include "openPMD/auxiliary/JSON.hpp" + +namespace openPMD +{ +namespace auxiliary +{ + TracingJSON::TracingJSON() : TracingJSON( nlohmann::json() ) + { + } + + TracingJSON::TracingJSON( nlohmann::json json ) + : nlohmann::json( std::move( json ) ) + , m_shadow( std::make_shared< nlohmann::json >() ) + , m_position( &*m_shadow ) + { + } + + nlohmann::json const & + TracingJSON::getShadow() + { + return *m_position; + } + + nlohmann::json + TracingJSON::invertShadow() + { + nlohmann::json inverted = *this; + invertShadow( inverted, *m_position ); + return inverted; + } + + void + TracingJSON::invertShadow( + nlohmann::json & result, + nlohmann::json & shadow ) + { + if( !shadow.is_object() ) + { + return; + } + std::vector< std::string > toRemove; + for( auto it = shadow.begin(); it != shadow.end(); ++it ) + { + nlohmann::json & partialResult = result[ it.key() ]; + if( partialResult.is_object() ) + { + invertShadow( partialResult, it.value() ); + if( partialResult.size() == 0 ) + { + toRemove.emplace_back( it.key() ); + } + } + else + { + toRemove.emplace_back( it.key() ); + } + } + for( auto const & key : toRemove ) + { + result.erase( key ); + } + } + + void + TracingJSON::declareFullyRead() + { + if( m_trace ) + { + *m_position = *this; + } + } + + TracingJSON::TracingJSON( + nlohmann::json json, + std::shared_ptr< nlohmann::json > shadow, + nlohmann::json * position, + bool trace ) + : nlohmann::json( json ) + , m_shadow( std::move( shadow ) ) + , m_position( position ) + , m_trace( trace ) + { + } +} // namespace auxiliary +} // namespace openPMD + +#endif \ No newline at end of file From 7ce3ed33eb4de824da2abae835c12e4a5cae49e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Jan 2020 13:55:18 +0100 Subject: [PATCH 02/24] Add JSON-formatted string parameter to frontend for backend configuration --- .../openPMD/IO/AbstractIOHandlerHelper.hpp | 44 ++++++++++--------- include/openPMD/Series.hpp | 15 ++++--- src/IO/AbstractIOHandlerHelper.cpp | 11 +++-- src/Series.cpp | 35 ++++++++------- 4 files changed, 60 insertions(+), 45 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandlerHelper.hpp b/include/openPMD/IO/AbstractIOHandlerHelper.hpp index deb52de92e..ca6ae930a9 100644 --- a/include/openPMD/IO/AbstractIOHandlerHelper.hpp +++ b/include/openPMD/IO/AbstractIOHandlerHelper.hpp @@ -36,26 +36,28 @@ namespace openPMD * @param comm MPI communicator used for IO. * @return Smart pointer to created IOHandler. */ - std::shared_ptr< AbstractIOHandler > - createIOHandler( - std::string path, - AccessType accessType, - Format format, - MPI_Comm comm - ); +std::shared_ptr< AbstractIOHandler > +createIOHandler( + std::string path, + AccessType accessType, + Format format, + MPI_Comm comm, + std::string const & options = "{}" ); #endif - /** Construct an appropriate specific IOHandler for the desired IO mode. - * - * @param path Path to root folder for all operations associated with the desired handler. - * @param accessType AccessType describing desired operations and permissions of the desired handler. - * @param format Format describing the IO backend of the desired handler. - * @return Smart pointer to created IOHandler. - */ - std::shared_ptr< AbstractIOHandler > - createIOHandler( - std::string path, - AccessType accessType, - Format format - ); -} // openPMD +/** Construct an appropriate specific IOHandler for the desired IO mode. + * + * @param path Path to root folder for all operations associated with + * the desired handler. + * @param accessType AccessType describing desired operations and permissions + * of the desired handler. + * @param format Format describing the IO backend of the desired handler. + * @return Smart pointer to created IOHandler. + */ +std::shared_ptr< AbstractIOHandler > +createIOHandler( + std::string path, + AccessType accessType, + Format format, + std::string const & options = "{}" ); +} // namespace openPMD diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 358b412db3..340898974f 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -58,12 +58,17 @@ class Series : public Attributable public: #if openPMD_HAVE_MPI - Series(std::string const& filepath, - AccessType at, - MPI_Comm comm); + Series( + std::string const & filepath, + AccessType at, + MPI_Comm comm, + std::string const & options = "{}" ); #endif - Series(std::string const& filepath, - AccessType at); + + Series( + std::string const & filepath, + AccessType at, + std::string const & options = "{}" ); ~Series(); /** diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 331f9bb0fc..aba0cecb96 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -27,6 +27,7 @@ #include "openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp" #include "openPMD/IO/JSON/JSONIOHandler.hpp" +#include namespace openPMD { @@ -36,9 +37,10 @@ namespace openPMD std::string path, AccessType accessTypeBackend, Format format, - MPI_Comm comm - ) + MPI_Comm comm, + std::string const & options ) { + nlohmann::json optionsJson = nlohmann::json::parse( options ); switch( format ) { case Format::HDF5: @@ -60,9 +62,10 @@ namespace openPMD createIOHandler( std::string path, AccessType accessType, - Format format - ) + Format format, + std::string const & options ) { + nlohmann::json optionsJson = nlohmann::json::parse( options ); switch( format ) { case Format::HDF5: diff --git a/src/Series.cpp b/src/Series.cpp index 4aea6aa7f9..ca91c05a6d 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -81,25 +81,30 @@ struct Series::ParsedInput }; //ParsedInput #if openPMD_HAVE_MPI -Series::Series(std::string const& filepath, - AccessType at, - MPI_Comm comm) - : iterations{Container< Iteration, uint64_t >()}, - m_iterationEncoding{std::make_shared< IterationEncoding >()} -{ - auto input = parseInput(filepath); - auto handler = createIOHandler(input->path, at, input->format, comm); - init(handler, std::move(input)); +Series::Series( + std::string const & filepath, + AccessType at, + MPI_Comm comm, + std::string const & options ) + : iterations{ Container< Iteration, uint64_t >() } + , m_iterationEncoding{ std::make_shared< IterationEncoding >() } +{ + auto input = parseInput( filepath ); + auto handler = + createIOHandler( input->path, at, input->format, comm, options ); + init( handler, std::move( input ) ); } #endif -Series::Series(std::string const& filepath, - AccessType at) - : iterations{Container< Iteration, uint64_t >()}, - m_iterationEncoding{std::make_shared< IterationEncoding >()} +Series::Series( + std::string const & filepath, + AccessType at, + std::string const & options ) + : iterations{ Container< Iteration, uint64_t >() } + , m_iterationEncoding{ std::make_shared< IterationEncoding >() } { - auto input = parseInput(filepath); - auto handler = createIOHandler(input->path, at, input->format); + auto input = parseInput( filepath ); + auto handler = createIOHandler( input->path, at, input->format, options ); init(handler, std::move(input)); } From 7ecd54bb62d564ceab503be1bd968f507e68899f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Jan 2020 14:02:33 +0100 Subject: [PATCH 03/24] Add a simple implementation of JSON parsing to ADIOS2 backend --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 57 +++++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 175 +++++++++++++------ src/IO/AbstractIOHandlerHelper.cpp | 7 +- 3 files changed, 177 insertions(+), 62 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index d6706df1f8..d5ea913d43 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -20,8 +20,6 @@ */ #pragma once -#include "openPMD/config.hpp" - #include "ADIOS2FilePosition.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" @@ -40,6 +38,9 @@ #include // pair #include +#include + +#include "openPMD/config.hpp" #if openPMD_HAVE_ADIOS2 # include @@ -95,13 +96,13 @@ class ADIOS2IOHandlerImpl #if openPMD_HAVE_MPI - ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm ); + ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm, nlohmann::json config ); MPI_Comm m_comm; #endif // openPMD_HAVE_MPI - explicit ADIOS2IOHandlerImpl( AbstractIOHandler * ); + explicit ADIOS2IOHandlerImpl( AbstractIOHandler *, nlohmann::json config ); ~ADIOS2IOHandlerImpl( ) override = default; @@ -165,8 +166,6 @@ class ADIOS2IOHandlerImpl listAttributes( Writable *, Parameter< Operation::LIST_ATTS > & parameters ) override; - - /** * @brief The ADIOS2 access type to chose for Engines opened * within this instance. @@ -176,6 +175,32 @@ class ADIOS2IOHandlerImpl private: adios2::ADIOS m_ADIOS; + nlohmann::json m_config; + static nlohmann::json nullvalue; + + void + init( nlohmann::json config ); + + template< typename Key > + nlohmann::json & + config( Key && key, nlohmann::json & cfg ) + { + if( cfg.is_object() && cfg.contains( key ) ) + { + return cfg[ key ]; + } + else + { + return nullvalue; + } + } + + template< typename Key > + nlohmann::json & + config( Key && key ) + { + return config< Key >( std::forward< Key >( key ), m_config ); + } /* * We need to give names to IO objects. These names are irrelevant @@ -260,6 +285,14 @@ class ADIOS2IOHandlerImpl namespace detail { + /* + * The following strings are used during parsing of the JSON configuration + * string for the ADIOS2 backend. + */ + constexpr char const * const str_engine = "engine"; + constexpr char const * str_type = "type"; + constexpr char const * str_params = "parameters"; + // Helper structs for calls to the switchType function struct DatasetReader @@ -570,6 +603,9 @@ namespace detail ~BufferedActions( ); + void + configure_IO( ADIOS2IOHandlerImpl & impl ); + adios2::Engine & getEngine( ); template < typename BA > void enqueue( BA && ba ); @@ -666,11 +702,16 @@ friend class ADIOS2IOHandlerImpl; #if openPMD_HAVE_MPI - ADIOS2IOHandler( std::string path, AccessType, MPI_Comm ); + ADIOS2IOHandler( + std::string path, + AccessType, + MPI_Comm, + nlohmann::json options ); #endif - ADIOS2IOHandler( std::string path, AccessType ); + ADIOS2IOHandler( std::string path, AccessType, + nlohmann::json options); std::string backendName() const override { return "ADIOS2"; } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 99427bfee3..cda6775bef 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -20,15 +20,17 @@ */ #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" + +#include +#include +#include + #include "openPMD/Datatype.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" - -#include -#include #include #include #include @@ -58,22 +60,37 @@ namespace openPMD #if openPMD_HAVE_ADIOS2 -#if openPMD_HAVE_MPI +# if openPMD_HAVE_MPI ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler * handler, - MPI_Comm communicator ) + MPI_Comm communicator, + nlohmann::json cfg ) : AbstractIOHandlerImplCommon( handler ) , m_comm{ communicator } , m_ADIOS{ communicator, ADIOS2_DEBUG_MODE } { + init( std::move( cfg ) ); } -#endif // openPMD_HAVE_MPI +# endif // openPMD_HAVE_MPI -ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler * handler ) +ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( + AbstractIOHandler * handler, + nlohmann::json cfg ) : AbstractIOHandlerImplCommon( handler ), m_ADIOS{ ADIOS2_DEBUG_MODE } { + init( std::move( cfg ) ); +} + + +void +ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) +{ + if( cfg.contains( "adios2" ) ) + { + m_config = std::move( cfg[ "adios2" ] ); + } } std::future< void > ADIOS2IOHandlerImpl::flush( ) @@ -496,8 +513,7 @@ void ADIOS2IOHandlerImpl::listAttributes( } } -adios2::Mode -ADIOS2IOHandlerImpl::adios2Accesstype() +adios2::Mode ADIOS2IOHandlerImpl::adios2Accesstype( ) { switch ( m_handler->accessTypeBackend ) { @@ -515,6 +531,8 @@ ADIOS2IOHandlerImpl::adios2Accesstype() } } +nlohmann::json ADIOS2IOHandlerImpl::nullvalue = nlohmann::json(); + std::string ADIOS2IOHandlerImpl::filePositionToString( std::shared_ptr< ADIOS2FilePosition > filepos ) { @@ -1101,6 +1119,7 @@ namespace detail *param.dtype = ret; } + BufferedActions::BufferedActions( ADIOS2IOHandlerImpl & impl, InvalidatableFile file ) : m_file( impl.fullPath( std::move( file ) ) ), @@ -1116,13 +1135,61 @@ namespace detail } else { - // read parameters from environment - auto const engine = auxiliary::getEnvString( "OPENPMD_ADIOS2_ENGINE", "File" ); + configure_IO(impl); + } + } + + BufferedActions::~BufferedActions() + { + // if write accessing, ensure that the engine is opened + if( !m_engine && m_mode != adios2::Mode::Read ) + { + getEngine(); + } + if( m_engine ) + { + m_engine->Close(); + } + } + + void + BufferedActions::configure_IO( ADIOS2IOHandlerImpl & impl ) + { + std::set< std::string > alreadyConfigured; + auto & engineConfig = impl.config( detail::str_engine ); + if( !engineConfig.is_null() ) + { + m_IO.SetEngine( impl.config( detail::str_type, engineConfig ) ); + auto & params = impl.config( detail::str_params, engineConfig ); + if( params.is_object() ) + { + for( auto it = params.begin(); it != params.end(); it++ ) + { + m_IO.SetParameter( it.key(), it.value() ); + alreadyConfigured.emplace( it.key() ); + } + } + alreadyConfigured.emplace( "Engine" ); + } + + auto notYetConfigured = + [&alreadyConfigured]( std::string const & param ) { + auto it = alreadyConfigured.find( param ); + return it == alreadyConfigured.end(); + }; + + // read parameters from environment + if( notYetConfigured( "Engine" ) ) + { + auto const engine = + auxiliary::getEnvString( "OPENPMD_ADIOS2_ENGINE", "File" ); m_IO.SetEngine( engine ); + } - if ( 1 == - auxiliary::getEnvNum( - "OPENPMD_ADIOS2_HAVE_METADATA_FILE", 1 ) ) + if( notYetConfigured( "CollectiveMetadata" ) ) + { + if( 1 == + auxiliary::getEnvNum( "OPENPMD_ADIOS2_HAVE_METADATA_FILE", 1 ) ) { m_IO.SetParameter( "CollectiveMetadata", "On" ); } @@ -1130,9 +1197,13 @@ namespace detail { m_IO.SetParameter( "CollectiveMetadata", "Off" ); } - - if ( 1 == - auxiliary::getEnvNum( "OPENPMD_ADIOS2_HAVE_PROFILING", 1 ) ) + } + if( notYetConfigured( "Profile" ) ) + { + if( 1 == + auxiliary::getEnvNum( + "OPENPMD_ADIOS2_HAVE_PROFILING", 1 ) && + notYetConfigured( "Profile" ) ) { m_IO.SetParameter( "Profile", "On" ); } @@ -1140,34 +1211,22 @@ namespace detail { m_IO.SetParameter( "Profile", "Off" ); } + } #if openPMD_HAVE_MPI + { + auto num_substreams = + auxiliary::getEnvNum( "OPENPMD_ADIOS2_NUM_SUBSTREAMS", 0 ); + if( notYetConfigured( "SubStreams" ) && 0 != num_substreams ) { - auto num_substreams = - auxiliary::getEnvNum( "OPENPMD_ADIOS2_NUM_SUBSTREAMS", 0 ); - if ( 0 != num_substreams ) - { - m_IO.SetParameter( "SubStreams", - std::to_string( num_substreams ) ); - } + m_IO.SetParameter( + "SubStreams", std::to_string( num_substreams ) ); } -#endif - } - } - - BufferedActions::~BufferedActions( ) - { - // if write accessing, ensure that the engine is opened - if ( !m_engine && m_mode != adios2::Mode::Read ) - { - getEngine( ); - } - if ( m_engine ) - { - m_engine->Close( ); } +#endif } - adios2::Engine & BufferedActions::getEngine( ) + adios2::Engine & + BufferedActions::getEngine() { if ( !m_engine ) { @@ -1317,18 +1376,26 @@ namespace detail #if openPMD_HAVE_MPI -ADIOS2IOHandler::ADIOS2IOHandler( std::string path, openPMD::AccessType at, - MPI_Comm comm ) -: AbstractIOHandler( std::move( path ), at, comm ), m_impl{this, comm +ADIOS2IOHandler::ADIOS2IOHandler( + std::string path, + openPMD::AccessType at, + MPI_Comm comm, + nlohmann::json options ) + : AbstractIOHandler( std::move( path ), at, comm ) + , m_impl{ this, comm, std::move( options ) - } + } { } #endif -ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at ) -: AbstractIOHandler( std::move( path ), at ), m_impl{this} +ADIOS2IOHandler::ADIOS2IOHandler( + std::string path, + AccessType at, + nlohmann::json options ) + : AbstractIOHandler( std::move( path ), at ) + , m_impl{ this, std::move( options ) } { } @@ -1340,16 +1407,22 @@ std::future< void > ADIOS2IOHandler::flush( ) #else // openPMD_HAVE_ADIOS2 #if openPMD_HAVE_MPI -ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at, - MPI_Comm comm ) -: AbstractIOHandler( std::move( path ), at, comm ) +ADIOS2IOHandler::ADIOS2IOHandler( + std::string path, + AccessType at, + MPI_Comm comm, + nlohmann::json ) + : AbstractIOHandler( std::move( path ), at, comm ) { } -#endif +# endif -ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at ) -: AbstractIOHandler( std::move( path ), at ) +ADIOS2IOHandler::ADIOS2IOHandler( + std::string path, + AccessType at, + nlohmann::json ) + : AbstractIOHandler( std::move( path ), at ) { } diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index aba0cecb96..6309be58c4 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -52,7 +52,7 @@ namespace openPMD throw std::runtime_error("openPMD-api built without ADIOS1 support"); # endif case Format::ADIOS2: - return std::make_shared(path, accessTypeBackend, comm); + return std::make_shared(path, accessTypeBackend, comm, std::move(optionsJson)); default: throw std::runtime_error("Unknown file format! Did you specify a file ending?" ); } @@ -77,8 +77,9 @@ namespace openPMD throw std::runtime_error("openPMD-api built without ADIOS1 support"); #endif #if openPMD_HAVE_ADIOS2 - case Format::ADIOS2: - return std::make_shared(path, accessType); + case Format::ADIOS2: + return std::make_shared< ADIOS2IOHandler >( + path, accessType, std::move( optionsJson ) ); #endif case Format::JSON: return std::make_shared< JSONIOHandler >(path, accessType); From f44967c0b3b88d4ddc270c7ec2610f0f19b2f753 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 17 Oct 2019 16:22:09 -0700 Subject: [PATCH 04/24] Gracefully handle missing JSON JSON is still optional, so we should gracefully throw if a user tries to pass options that we cannot parse without it. https://travis-ci.org/openPMD/openPMD-api/builds/599388433 --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 35 ++++++++-- src/IO/ADIOS/ADIOS2IOHandler.cpp | 72 +++++++++++++++----- src/IO/AbstractIOHandlerHelper.cpp | 32 +++++++-- 3 files changed, 110 insertions(+), 29 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index d5ea913d43..60470f4f8a 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -96,13 +96,24 @@ class ADIOS2IOHandlerImpl #if openPMD_HAVE_MPI - ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm, nlohmann::json config ); + ADIOS2IOHandlerImpl( + AbstractIOHandler *, + MPI_Comm +# if openPMD_HAVE_JSON + , nlohmann::json config +# endif + ); MPI_Comm m_comm; #endif // openPMD_HAVE_MPI - explicit ADIOS2IOHandlerImpl( AbstractIOHandler *, nlohmann::json config ); + explicit ADIOS2IOHandlerImpl( + AbstractIOHandler * +# if openPMD_HAVE_JSON + , nlohmann::json config +# endif + ); ~ADIOS2IOHandlerImpl( ) override = default; @@ -175,6 +186,8 @@ class ADIOS2IOHandlerImpl private: adios2::ADIOS m_ADIOS; + +# if openPMD_HAVE_JSON nlohmann::json m_config; static nlohmann::json nullvalue; @@ -201,7 +214,7 @@ class ADIOS2IOHandlerImpl { return config< Key >( std::forward< Key >( key ), m_config ); } - +# endif // openPMD_HAVE_JSON /* * We need to give names to IO objects. These names are irrelevant * within this application, since: @@ -705,13 +718,21 @@ friend class ADIOS2IOHandlerImpl; ADIOS2IOHandler( std::string path, AccessType, - MPI_Comm, - nlohmann::json options ); + MPI_Comm +# if openPMD_HAVE_JSON + , nlohmann::json options +# endif + ); #endif - ADIOS2IOHandler( std::string path, AccessType, - nlohmann::json options); + ADIOS2IOHandler( + std::string path, + AccessType +#if openPMD_HAVE_JSON + , nlohmann::json options +#endif + ); std::string backendName() const override { return "ADIOS2"; } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index cda6775bef..620600e7a2 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -64,26 +64,36 @@ namespace openPMD ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler * handler, - MPI_Comm communicator, - nlohmann::json cfg ) + MPI_Comm communicator +# if openPMD_HAVE_JSON + , nlohmann::json cfg +# endif // openPMD_HAVE_JSON +) : AbstractIOHandlerImplCommon( handler ) , m_comm{ communicator } , m_ADIOS{ communicator, ADIOS2_DEBUG_MODE } { +# if openPMD_HAVE_JSON init( std::move( cfg ) ); +# endif // openPMD_HAVE_JSON } # endif // openPMD_HAVE_MPI ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( - AbstractIOHandler * handler, - nlohmann::json cfg ) + AbstractIOHandler * handler +# if openPMD_HAVE_JSON + , nlohmann::json cfg +# endif // openPMD_HAVE_JSON +) : AbstractIOHandlerImplCommon( handler ), m_ADIOS{ ADIOS2_DEBUG_MODE } { +# if openPMD_HAVE_JSON init( std::move( cfg ) ); +# endif // openPMD_HAVE_JSON } - +# if openPMD_HAVE_JSON void ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) { @@ -92,6 +102,7 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) m_config = std::move( cfg[ "adios2" ] ); } } +# endif // openPMD_HAVE_JSON std::future< void > ADIOS2IOHandlerImpl::flush( ) { @@ -531,7 +542,9 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2Accesstype( ) } } +# if openPMD_HAVE_JSON nlohmann::json ADIOS2IOHandlerImpl::nullvalue = nlohmann::json(); +# endif // openPMD_HAVE_JSON std::string ADIOS2IOHandlerImpl::filePositionToString( std::shared_ptr< ADIOS2FilePosition > filepos ) @@ -1155,7 +1168,9 @@ namespace detail void BufferedActions::configure_IO( ADIOS2IOHandlerImpl & impl ) { + (void)impl; std::set< std::string > alreadyConfigured; +#if openPMD_HAVE_JSON auto & engineConfig = impl.config( detail::str_engine ); if( !engineConfig.is_null() ) { @@ -1171,7 +1186,7 @@ namespace detail } alreadyConfigured.emplace( "Engine" ); } - +#endif // openPMD_HAVE_JSON auto notYetConfigured = [&alreadyConfigured]( std::string const & param ) { auto it = alreadyConfigured.find( param ); @@ -1379,11 +1394,18 @@ namespace detail ADIOS2IOHandler::ADIOS2IOHandler( std::string path, openPMD::AccessType at, - MPI_Comm comm, - nlohmann::json options ) + MPI_Comm comm +# if openPMD_HAVE_JSON + , nlohmann::json options +# endif +) : AbstractIOHandler( std::move( path ), at, comm ) - , m_impl{ this, comm, std::move( options ) - + , m_impl{ + this, + comm +# if openPMD_HAVE_JSON + , std::move( options ) +# endif } { } @@ -1392,10 +1414,18 @@ ADIOS2IOHandler::ADIOS2IOHandler( ADIOS2IOHandler::ADIOS2IOHandler( std::string path, - AccessType at, - nlohmann::json options ) + AccessType at +# if openPMD_HAVE_JSON + , nlohmann::json options +# endif +) : AbstractIOHandler( std::move( path ), at ) - , m_impl{ this, std::move( options ) } + , m_impl{ + this +# if openPMD_HAVE_JSON + , std::move( options ) +# endif + } { } @@ -1410,18 +1440,24 @@ std::future< void > ADIOS2IOHandler::flush( ) ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at, - MPI_Comm comm, - nlohmann::json ) + MPI_Comm comm +# if openPMD_HAVE_JSON + , nlohmann::json +# endif +) : AbstractIOHandler( std::move( path ), at, comm ) { } -# endif +#endif // openPMD_HAVE_MPI ADIOS2IOHandler::ADIOS2IOHandler( std::string path, - AccessType at, - nlohmann::json ) + AccessType at +#if openPMD_HAVE_JSON + , nlohmann::json +#endif +) : AbstractIOHandler( std::move( path ), at ) { } diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 6309be58c4..0c421b54e1 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -27,7 +27,9 @@ #include "openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp" #include "openPMD/IO/JSON/JSONIOHandler.hpp" -#include +#if openPMD_HAVE_JSON +# include +#endif namespace openPMD { @@ -40,7 +42,12 @@ namespace openPMD MPI_Comm comm, std::string const & options ) { +# if openPMD_HAVE_JSON nlohmann::json optionsJson = nlohmann::json::parse( options ); +# else + if( options.size() > 0u && options != "{}" ) + throw std::runtime_error("openPMD-api built without JSON support which is required for runtime options!"); +# endif switch( format ) { case Format::HDF5: @@ -52,7 +59,14 @@ namespace openPMD throw std::runtime_error("openPMD-api built without ADIOS1 support"); # endif case Format::ADIOS2: - return std::make_shared(path, accessTypeBackend, comm, std::move(optionsJson)); + return std::make_shared( + path, + accessTypeBackend, + comm +# if openPMD_HAVE_JSON + , std::move(optionsJson) +# endif + ); default: throw std::runtime_error("Unknown file format! Did you specify a file ending?" ); } @@ -65,7 +79,12 @@ namespace openPMD Format format, std::string const & options ) { +#if openPMD_HAVE_JSON nlohmann::json optionsJson = nlohmann::json::parse( options ); +#else + if( options.size() > 0u && options != "{}" ) + throw std::runtime_error("openPMD-api built without JSON support which is required for runtime options!"); +#endif switch( format ) { case Format::HDF5: @@ -79,8 +98,13 @@ namespace openPMD #if openPMD_HAVE_ADIOS2 case Format::ADIOS2: return std::make_shared< ADIOS2IOHandler >( - path, accessType, std::move( optionsJson ) ); -#endif + path, + accessType +# if openPMD_HAVE_JSON + , std::move( optionsJson ) +# endif // openPMD_HAVE_JSON + ); +#endif // openPMD_HAVE_ADIOS2 case Format::JSON: return std::make_shared< JSONIOHandler >(path, accessType); default: From 176b107a16a8bd92944fe6788cd21f03eb96854b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 11 Feb 2020 14:45:27 +0100 Subject: [PATCH 05/24] Move ADIOS2 backend to TracingJSON Warn in ADIOS2 when not using all of the JSON configuration --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 12 ++--- src/IO/ADIOS/ADIOS2IOHandler.cpp | 57 +++++++++++++------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 60470f4f8a..215f5d4af9 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -187,16 +187,16 @@ class ADIOS2IOHandlerImpl private: adios2::ADIOS m_ADIOS; -# if openPMD_HAVE_JSON - nlohmann::json m_config; - static nlohmann::json nullvalue; +# if openPMD_HAVE_JSON + auxiliary::TracingJSON m_config; + static auxiliary::TracingJSON nullvalue; void init( nlohmann::json config ); template< typename Key > - nlohmann::json & - config( Key && key, nlohmann::json & cfg ) + auxiliary::TracingJSON + config( Key && key, auxiliary::TracingJSON & cfg ) { if( cfg.is_object() && cfg.contains( key ) ) { @@ -209,7 +209,7 @@ class ADIOS2IOHandlerImpl } template< typename Key > - nlohmann::json & + auxiliary::TracingJSON config( Key && key ) { return config< Key >( std::forward< Key >( key ), m_config ); diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 620600e7a2..9e3af7bf88 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -21,7 +21,9 @@ #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" +#include #include +#include #include #include @@ -30,10 +32,9 @@ #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +// #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include -#include -#include namespace openPMD @@ -82,18 +83,19 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler * handler -# if openPMD_HAVE_JSON - , nlohmann::json cfg -# endif // openPMD_HAVE_JSON -) +# if openPMD_HAVE_JSON + , + nlohmann::json cfg +# endif // openPMD_HAVE_JSON + ) : AbstractIOHandlerImplCommon( handler ), m_ADIOS{ ADIOS2_DEBUG_MODE } { -# if openPMD_HAVE_JSON +# if openPMD_HAVE_JSON init( std::move( cfg ) ); -# endif // openPMD_HAVE_JSON +# endif // openPMD_HAVE_JSON } -# if openPMD_HAVE_JSON +# if openPMD_HAVE_JSON void ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) { @@ -101,10 +103,17 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) { m_config = std::move( cfg[ "adios2" ] ); } + else + { + std::cerr << "Warning: ADIOS2 is not configured in the JSON " + "configuration. Running with default settings." + << std::endl; + } } -# endif // openPMD_HAVE_JSON +# endif // openPMD_HAVE_JSON -std::future< void > ADIOS2IOHandlerImpl::flush( ) +std::future< void > +ADIOS2IOHandlerImpl::flush() { auto res = AbstractIOHandlerImpl::flush( ); for ( auto & p : m_fileData ) @@ -542,9 +551,9 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2Accesstype( ) } } -# if openPMD_HAVE_JSON -nlohmann::json ADIOS2IOHandlerImpl::nullvalue = nlohmann::json(); -# endif // openPMD_HAVE_JSON +# if openPMD_HAVE_JSON +auxiliary::TracingJSON ADIOS2IOHandlerImpl::nullvalue = nlohmann::json(); +# endif // openPMD_HAVE_JSON std::string ADIOS2IOHandlerImpl::filePositionToString( std::shared_ptr< ADIOS2FilePosition > filepos ) @@ -1170,12 +1179,17 @@ namespace detail { (void)impl; std::set< std::string > alreadyConfigured; -#if openPMD_HAVE_JSON - auto & engineConfig = impl.config( detail::str_engine ); +# if openPMD_HAVE_JSON + // TODO if changing TracingJSON to return references upon operator[](), + // change this to a reference + auto engineConfig = impl.config( detail::str_engine ); if( !engineConfig.is_null() ) { m_IO.SetEngine( impl.config( detail::str_type, engineConfig ) ); - auto & params = impl.config( detail::str_params, engineConfig ); + // TODO if changing TracingJSON to return references upon + // operator[](), change this to a reference + auto params = impl.config( detail::str_params, engineConfig ); + params.declareFullyRead(); if( params.is_object() ) { for( auto it = params.begin(); it != params.end(); it++ ) @@ -1186,7 +1200,14 @@ namespace detail } alreadyConfigured.emplace( "Engine" ); } -#endif // openPMD_HAVE_JSON + auto shadow = impl.m_config.invertShadow(); + if( shadow.size() > 0 ) + { + std::cerr << "Warning: parts of the JSON configuration for ADIOS2 " + "remain unused:\n" + << shadow << std::endl; + } +# endif // openPMD_HAVE_JSON auto notYetConfigured = [&alreadyConfigured]( std::string const & param ) { auto it = alreadyConfigured.find( param ); From c4387925a6b426016defb290c0b516c19b05b9b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 18 Feb 2020 16:27:58 +0100 Subject: [PATCH 06/24] Expose Python bindings for JSON configuration --- src/binding/python/Series.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index af3ca74cec..03c57b7f75 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -57,10 +57,14 @@ using namespace openPMD; void init_Series(py::module &m) { py::class_(m, "Series") - .def(py::init(), - py::arg("filepath"), py::arg("access_type")) + .def(py::init(), + py::arg("filepath"), py::arg("access_type"), py::arg("options") = "{}") #if openPMD_HAVE_MPI - .def(py::init([](std::string const& filepath, AccessType at, py::object &comm){ + .def(py::init([]( + std::string const& filepath, + AccessType at, + py::object &comm, + std::string const& options){ //! @todo perform mpi4py import test and check min-version //! careful: double MPI_Init risk? only import mpi4py.MPI? //! required C-API init? probably just checks: @@ -106,9 +110,12 @@ void init_Series(py::module &m) { "(Mismatched MPI at compile vs. runtime?)"); } - return new Series(filepath, at, *mpiCommPtr); + return new Series(filepath, at, *mpiCommPtr, options); }), - py::arg("filepath"), py::arg("access_type"), py::arg("mpi_communicator") + py::arg("filepath"), + py::arg("access_type"), + py::arg("mpi_communicator"), + py::arg("options") = "{}" ) #endif From cf44a7c8c9f1bf374a7bfb038e1b66bc94b09fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 19 Feb 2020 14:50:46 +0100 Subject: [PATCH 07/24] Make TracingJSON more efficient operator[] will no longer copy the whole JSON structure Make TracingJSON more efficient operator[] will no longer copy the whole JSON structure --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 2 +- include/openPMD/auxiliary/JSON.hpp | 46 ++++++++++++++------ src/IO/ADIOS/ADIOS2IOHandler.cpp | 10 +++-- src/auxiliary/JSON.cpp | 25 ++++++----- 4 files changed, 55 insertions(+), 28 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 215f5d4af9..92349a5236 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -198,7 +198,7 @@ class ADIOS2IOHandlerImpl auxiliary::TracingJSON config( Key && key, auxiliary::TracingJSON & cfg ) { - if( cfg.is_object() && cfg.contains( key ) ) + if( cfg.json().is_object() && cfg.json().contains( key ) ) { return cfg[ key ]; } diff --git a/include/openPMD/auxiliary/JSON.hpp b/include/openPMD/auxiliary/JSON.hpp index 2b42544901..7e501e37e8 100644 --- a/include/openPMD/auxiliary/JSON.hpp +++ b/include/openPMD/auxiliary/JSON.hpp @@ -21,17 +21,28 @@ namespace auxiliary * but an object. This means that objects contained in arrays will not be * traced. * - * Warning: The current implementation is inefficient because it will copy - * the accessed JSON value upon access. Currently not to be used for deeply - * nested / large JSON structures. + * If working directly with the underlying JSON value (necessary since this + * class only redefines operator[]), declareFullyRead() may be used to + * declare keys read manually. * */ - class TracingJSON : public nlohmann::json + class TracingJSON { public: TracingJSON(); TracingJSON( nlohmann::json ); + /** + * @brief Access the underlying JSON value + * + * @return nlohmann::json& + */ + inline nlohmann::json & + json() + { + return *m_positionInOriginal; + } + template< typename Key > TracingJSON operator[]( Key && key ); @@ -61,34 +72,43 @@ namespace auxiliary declareFullyRead(); private: + std::shared_ptr< nlohmann::json > m_originalJSON; std::shared_ptr< nlohmann::json > m_shadow; - nlohmann::json * m_position; + nlohmann::json * m_positionInOriginal; + nlohmann::json * m_positionInShadow; bool m_trace = true; void invertShadow( nlohmann::json & result, nlohmann::json & shadow ); TracingJSON( - nlohmann::json, + std::shared_ptr< nlohmann::json > originalJSON, std::shared_ptr< nlohmann::json > shadow, - nlohmann::json * position, + nlohmann::json * positionInOriginal, + nlohmann::json * positionInShadow, bool trace ); }; template< typename Key > TracingJSON TracingJSON::operator[]( Key && key ) { - nlohmann::json::const_reference res = nlohmann::json::operator[]( key ); + nlohmann::json * newPositionInOriginal = + &m_positionInOriginal->operator[]( key ); // If accessing a leaf in the JSON tree from an object (not an array!) // erase the corresponding key static nlohmann::json nullvalue; - nlohmann::json * emplaced = &nullvalue; - if( m_trace && this->is_object() ) + nlohmann::json * newPositionInShadow = &nullvalue; + if( m_trace && m_positionInOriginal->is_object() ) { - emplaced = &m_position->operator[]( key ); + newPositionInShadow = &m_positionInShadow->operator[]( key ); } - bool traceFurther = res.is_object(); - return TracingJSON( res, m_shadow, emplaced, traceFurther ); + bool traceFurther = newPositionInOriginal->is_object(); + return TracingJSON( + m_originalJSON, + m_shadow, + newPositionInOriginal, + newPositionInShadow, + traceFurther ); } } // namespace auxiliary } // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 9e3af7bf88..ac0cb92f44 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1183,16 +1183,18 @@ namespace detail // TODO if changing TracingJSON to return references upon operator[](), // change this to a reference auto engineConfig = impl.config( detail::str_engine ); - if( !engineConfig.is_null() ) + if( !engineConfig.json().is_null() ) { - m_IO.SetEngine( impl.config( detail::str_type, engineConfig ) ); + m_IO.SetEngine( + impl.config( detail::str_type, engineConfig ).json() ); // TODO if changing TracingJSON to return references upon // operator[](), change this to a reference auto params = impl.config( detail::str_params, engineConfig ); params.declareFullyRead(); - if( params.is_object() ) + if( params.json().is_object() ) { - for( auto it = params.begin(); it != params.end(); it++ ) + for( auto it = params.json().begin(); it != params.json().end(); + it++ ) { m_IO.SetParameter( it.key(), it.value() ); alreadyConfigured.emplace( it.key() ); diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index 4b6157fc6b..b2f70d14d3 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -13,23 +13,25 @@ namespace auxiliary } TracingJSON::TracingJSON( nlohmann::json json ) - : nlohmann::json( std::move( json ) ) + : m_originalJSON( + std::make_shared< nlohmann::json >( std::move( json ) ) ) , m_shadow( std::make_shared< nlohmann::json >() ) - , m_position( &*m_shadow ) + , m_positionInOriginal( &*m_originalJSON ) + , m_positionInShadow( &*m_shadow ) { } nlohmann::json const & TracingJSON::getShadow() { - return *m_position; + return *m_positionInShadow; } nlohmann::json TracingJSON::invertShadow() { - nlohmann::json inverted = *this; - invertShadow( inverted, *m_position ); + nlohmann::json inverted = *m_positionInOriginal; + invertShadow( inverted, *m_positionInShadow ); return inverted; } @@ -70,18 +72,21 @@ namespace auxiliary { if( m_trace ) { - *m_position = *this; + // copy over + *m_positionInShadow = *m_positionInOriginal; } } TracingJSON::TracingJSON( - nlohmann::json json, + std::shared_ptr< nlohmann::json > originalJSON, std::shared_ptr< nlohmann::json > shadow, - nlohmann::json * position, + nlohmann::json * positionInOriginal, + nlohmann::json * positionInShadow, bool trace ) - : nlohmann::json( json ) + : m_originalJSON( std::move( originalJSON ) ) , m_shadow( std::move( shadow ) ) - , m_position( position ) + , m_positionInOriginal( positionInOriginal ) + , m_positionInShadow( positionInShadow ) , m_trace( trace ) { } From b4c3f10716fdc0c41a594456787be0b76d8d24b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 19 Feb 2020 18:17:58 +0100 Subject: [PATCH 08/24] Add user documentation for backend-specific configuration Update ADIOS2 configuration documentation --- docs/source/details/adios2.json | 19 +++++++++++ docs/source/details/backendconfig.rst | 47 ++++++++++++++++++++++++++ docs/source/details/config_layout.json | 5 +++ docs/source/index.rst | 1 + 4 files changed, 72 insertions(+) create mode 100644 docs/source/details/adios2.json create mode 100644 docs/source/details/backendconfig.rst create mode 100644 docs/source/details/config_layout.json diff --git a/docs/source/details/adios2.json b/docs/source/details/adios2.json new file mode 100644 index 0000000000..9c5ccb59ba --- /dev/null +++ b/docs/source/details/adios2.json @@ -0,0 +1,19 @@ +{ + "adios2": { + "engine": { + "type": "sst", + "parameters": { + "BufferGrowthFactor": "2.0", + "QueueLimit": "2" + } + }, + "dataset": { + "operators": [ + { + "type": "bzip2", + "parameters": {} + } + ] + } + } +} diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst new file mode 100644 index 0000000000..8529800525 --- /dev/null +++ b/docs/source/details/backendconfig.rst @@ -0,0 +1,47 @@ +.. _backendconfig + +Backend-specific configuration +============================== + +While the openPMD API intends to be a backend-*independent* implementation of the openPMD standard, it is often necessary to pass configuration parameters to the concrete backend in use. Some backends recognize a number of configuration options via environment variables (see the backends' documentations), the preferred way to do so, however, *– as far as it is implemented –* is via a JSON-formatted string. + +The fundamental structure of this JSON configuration string is given as follows: + +.. literalinclude:: config_layout.json + +This structure allows keeping one configuration string for several backends at once, with the concrete backend configuration being chosen upon choosing the backend itself. + +The configuration is currently read in a case-sensitive manner. Since this may change in the future, all parts of the configuration are given in *lower case*. Parameters that are directly passed through to an external library and not interpreted within the openPMD API (e.g. ``adios2.engine.parameters``) are unaffected by this and follow the respective library's conventions. + +The configuration string may refer to the complete ``openPMD::Series`` and additionally be specified per ``openPMD::Dataset``, passed in the respective constructors. *Configuration per dataset is currently not yet implemented.* This reflects the fact that certain backend-specific parameters may refer to the whole Series (such as storage engines and their parameters) and others refer to actual datasets (such as compression). + +For a consistent user interface, backends shall follow the following rules: + +* The configuration structures for the Series and for each dataset should be defined equivalently. +* Any setting referring to single datasets should also be applicable globally, affecting all datasets. +* If a setting is defined globally, but also for a concrete dataset, the dataset-specific setting should override the global one. +* If a setting is passed to a dataset that only makes sense globally (such as the storage engine), the setting should be ignored except for printing a warning. Backends should define clearly which keys are applicable to datasets and which are not. + +Configuration structure per backend +----------------------------------- + +ADIOS2 +^^^^^^ + +A full configuration of the ADIOS2 backend: + +.. literalinclude:: adios2.json + +All keys found under ``adios2.dataset`` are applicable globally as well as per dataset, keys found under ``adios2.engine`` only globally. Explanation of the single keys: + +* ``adios2.engine.type``: A string that is passed directly to ``adios2::IO:::SetEngine`` for choosing the ADIOS2 engine to be used. Please refer to the `official ADIOS2 documentation `_ for a list of available engines. +* ``adios2.engine.type``: An associative array of string-formatted engine parameters, passed directly through to ``adios2::IO::SetParameters``. Please refer to the official ADIOS2 documentation for the allowable engine parameters. +* ``adios2.dataset.operators``: (*currently unimplemented* – please use the ``openPMD::Dataset::compression`` for the meantime) This key contains a list of ADIOS2 `operators `_, used to enable compression or dataset transformations. Each object in the list has three keys: + + * ``type`` supported ADIOS operator type, e.g. zfp, sz + * ``parameters`` is an associative map of string parameters for the operator (e.g. compression levels) + +Other backends +^^^^^^^^^^^^^^ + +Do currently not read the configuration string. Please refer to the respective backends' documentations for further information on their configuration. \ No newline at end of file diff --git a/docs/source/details/config_layout.json b/docs/source/details/config_layout.json new file mode 100644 index 0000000000..691a6ba1ae --- /dev/null +++ b/docs/source/details/config_layout.json @@ -0,0 +1,5 @@ +{ + "adios": "put ADIOS config here", + "adios2": "put ADIOS2 config here", + "hdf5": "put HDF5 config here" +} diff --git a/docs/source/index.rst b/docs/source/index.rst index 6a91b65326..d8f187d31e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -85,6 +85,7 @@ API Details details/doxygen.rst details/python.rst details/mpi.rst + details/backendconfig.rst Utilities --------- From 7df918413e37b8f40fad06e356d9fa99834cac81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 20 Feb 2020 13:22:19 +0100 Subject: [PATCH 09/24] Fix CI errors name shadowing and documentation --- include/openPMD/IO/AbstractIOHandlerHelper.hpp | 4 ++++ src/auxiliary/JSON.cpp | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandlerHelper.hpp b/include/openPMD/IO/AbstractIOHandlerHelper.hpp index ca6ae930a9..92e0b05380 100644 --- a/include/openPMD/IO/AbstractIOHandlerHelper.hpp +++ b/include/openPMD/IO/AbstractIOHandlerHelper.hpp @@ -34,6 +34,8 @@ namespace openPMD * @param accessType AccessType describing desired operations and permissions of the desired handler. * @param format Format describing the IO backend of the desired handler. * @param comm MPI communicator used for IO. + * @param options JSON-formatted option string, to be interpreted by + * the backend. * @return Smart pointer to created IOHandler. */ std::shared_ptr< AbstractIOHandler > @@ -52,6 +54,8 @@ createIOHandler( * @param accessType AccessType describing desired operations and permissions * of the desired handler. * @param format Format describing the IO backend of the desired handler. + * @param options JSON-formatted option string, to be interpreted by + * the backend. * @return Smart pointer to created IOHandler. */ std::shared_ptr< AbstractIOHandler > diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index b2f70d14d3..5baf6e920d 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -12,9 +12,9 @@ namespace auxiliary { } - TracingJSON::TracingJSON( nlohmann::json json ) + TracingJSON::TracingJSON( nlohmann::json originalJSON ) : m_originalJSON( - std::make_shared< nlohmann::json >( std::move( json ) ) ) + std::make_shared< nlohmann::json >( std::move( originalJSON ) ) ) , m_shadow( std::make_shared< nlohmann::json >() ) , m_positionInOriginal( &*m_originalJSON ) , m_positionInShadow( &*m_shadow ) From d3dee1daabab102c87ad28022b74b9f1c2f65865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 3 Mar 2020 13:49:54 +0100 Subject: [PATCH 10/24] Add reviewer's comments concerning documentation --- docs/source/details/backendconfig.rst | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index 8529800525..098c0dd495 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -1,9 +1,11 @@ .. _backendconfig -Backend-specific configuration +Backend-Specific configuration ============================== -While the openPMD API intends to be a backend-*independent* implementation of the openPMD standard, it is often necessary to pass configuration parameters to the concrete backend in use. Some backends recognize a number of configuration options via environment variables (see the backends' documentations), the preferred way to do so, however, *– as far as it is implemented –* is via a JSON-formatted string. +While the openPMD API intends to be a backend-*independent* implementation of the openPMD standard, it is sometimes useful to pass configuration parameters to the specific backend in use. +:ref:`For each backend `, configuration options can be passed via a JSON-formatted string or via environment variables. +A JSON option always takes precedence over an environment variable. The fundamental structure of this JSON configuration string is given as follows: @@ -11,18 +13,24 @@ The fundamental structure of this JSON configuration string is given as follows: This structure allows keeping one configuration string for several backends at once, with the concrete backend configuration being chosen upon choosing the backend itself. -The configuration is currently read in a case-sensitive manner. Since this may change in the future, all parts of the configuration are given in *lower case*. Parameters that are directly passed through to an external library and not interpreted within the openPMD API (e.g. ``adios2.engine.parameters``) are unaffected by this and follow the respective library's conventions. +The configuration is read in a case-sensitive manner. +Generally, keys of the configuration are *lower case*. +Parameters that are directly passed through to an external library and not interpreted within openPMD API (e.g. ``adios2.engine.parameters``) are unaffected by this and follow the respective library's conventions. -The configuration string may refer to the complete ``openPMD::Series`` and additionally be specified per ``openPMD::Dataset``, passed in the respective constructors. *Configuration per dataset is currently not yet implemented.* This reflects the fact that certain backend-specific parameters may refer to the whole Series (such as storage engines and their parameters) and others refer to actual datasets (such as compression). +The configuration string may refer to the complete ``openPMD::Series`` or may additionally be specified per ``openPMD::Dataset``, passed in the respective constructors. +*A configuration per dataset is currently not yet implemented.* +This reflects the fact that certain backend-specific parameters may refer to the whole Series (such as storage engines and their parameters) and others refer to actual datasets (such as compression). For a consistent user interface, backends shall follow the following rules: * The configuration structures for the Series and for each dataset should be defined equivalently. * Any setting referring to single datasets should also be applicable globally, affecting all datasets. * If a setting is defined globally, but also for a concrete dataset, the dataset-specific setting should override the global one. -* If a setting is passed to a dataset that only makes sense globally (such as the storage engine), the setting should be ignored except for printing a warning. Backends should define clearly which keys are applicable to datasets and which are not. +* If a setting is passed to a dataset that only makes sense globally (such as the storage engine), the setting should be ignored except for printing a warning. + Backends should define clearly which keys are applicable to datasets and which are not. -Configuration structure per backend + +Configuration Structure per Backend ----------------------------------- ADIOS2 From 2339f591ce67388cbbf06779c5b5a90e8c6bf493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 3 Mar 2020 14:04:58 +0100 Subject: [PATCH 11/24] Remove openPMD_HAVE_JSON macros JSON is now a mandatory dependency --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 27 ++----- include/openPMD/auxiliary/JSON.hpp | 13 +--- src/IO/ADIOS/ADIOS2IOHandler.cpp | 78 +++++--------------- src/auxiliary/JSON.cpp | 10 +-- 4 files changed, 33 insertions(+), 95 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 92349a5236..3ba63b63b7 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -96,13 +96,7 @@ class ADIOS2IOHandlerImpl #if openPMD_HAVE_MPI - ADIOS2IOHandlerImpl( - AbstractIOHandler *, - MPI_Comm -# if openPMD_HAVE_JSON - , nlohmann::json config -# endif - ); + ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm, nlohmann::json config ); MPI_Comm m_comm; @@ -110,15 +104,14 @@ class ADIOS2IOHandlerImpl explicit ADIOS2IOHandlerImpl( AbstractIOHandler * -# if openPMD_HAVE_JSON , nlohmann::json config -# endif ); - ~ADIOS2IOHandlerImpl( ) override = default; + ~ADIOS2IOHandlerImpl() override = default; - std::future< void > flush( ) override; + std::future< void > + flush() override; void createFile( Writable *, Parameter< Operation::CREATE_FILE > const & ) override; @@ -187,7 +180,6 @@ class ADIOS2IOHandlerImpl private: adios2::ADIOS m_ADIOS; -# if openPMD_HAVE_JSON auxiliary::TracingJSON m_config; static auxiliary::TracingJSON nullvalue; @@ -214,7 +206,6 @@ class ADIOS2IOHandlerImpl { return config< Key >( std::forward< Key >( key ), m_config ); } -# endif // openPMD_HAVE_JSON /* * We need to give names to IO objects. These names are irrelevant * within this application, since: @@ -719,20 +710,12 @@ friend class ADIOS2IOHandlerImpl; std::string path, AccessType, MPI_Comm -# if openPMD_HAVE_JSON , nlohmann::json options -# endif ); #endif - ADIOS2IOHandler( - std::string path, - AccessType -#if openPMD_HAVE_JSON - , nlohmann::json options -#endif - ); + ADIOS2IOHandler( std::string path, AccessType, nlohmann::json options ); std::string backendName() const override { return "ADIOS2"; } diff --git a/include/openPMD/auxiliary/JSON.hpp b/include/openPMD/auxiliary/JSON.hpp index 7e501e37e8..6979355960 100644 --- a/include/openPMD/auxiliary/JSON.hpp +++ b/include/openPMD/auxiliary/JSON.hpp @@ -1,13 +1,10 @@ #pragma once -#include "openPMD/config.hpp" - -#if openPMD_HAVE_JSON - -# include // std::shared_ptr -# include // std::forward +#include // std::shared_ptr +#include // std::forward -# include +#include "openPMD/config.hpp" +#include namespace openPMD { @@ -112,5 +109,3 @@ namespace auxiliary } } // namespace auxiliary } // namespace openPMD - -#endif // openPMD_HAVE_JSON \ No newline at end of file diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index ac0cb92f44..3068fb9e74 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -32,7 +32,6 @@ #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" -// #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include @@ -65,37 +64,25 @@ namespace openPMD ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler * handler, - MPI_Comm communicator -# if openPMD_HAVE_JSON - , nlohmann::json cfg -# endif // openPMD_HAVE_JSON -) + MPI_Comm communicator, + nlohmann::json cfg ) : AbstractIOHandlerImplCommon( handler ) , m_comm{ communicator } , m_ADIOS{ communicator, ADIOS2_DEBUG_MODE } { -# if openPMD_HAVE_JSON init( std::move( cfg ) ); -# endif // openPMD_HAVE_JSON } # endif // openPMD_HAVE_MPI ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( - AbstractIOHandler * handler -# if openPMD_HAVE_JSON - , - nlohmann::json cfg -# endif // openPMD_HAVE_JSON - ) + AbstractIOHandler * handler, + nlohmann::json cfg ) : AbstractIOHandlerImplCommon( handler ), m_ADIOS{ ADIOS2_DEBUG_MODE } { -# if openPMD_HAVE_JSON init( std::move( cfg ) ); -# endif // openPMD_HAVE_JSON } -# if openPMD_HAVE_JSON void ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) { @@ -110,12 +97,11 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) << std::endl; } } -# endif // openPMD_HAVE_JSON std::future< void > ADIOS2IOHandlerImpl::flush() { - auto res = AbstractIOHandlerImpl::flush( ); + auto res = AbstractIOHandlerImpl::flush(); for ( auto & p : m_fileData ) { if ( m_dirty.find( p.first ) != m_dirty.end( ) ) @@ -551,11 +537,10 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2Accesstype( ) } } -# if openPMD_HAVE_JSON auxiliary::TracingJSON ADIOS2IOHandlerImpl::nullvalue = nlohmann::json(); -# endif // openPMD_HAVE_JSON -std::string ADIOS2IOHandlerImpl::filePositionToString( +std::string +ADIOS2IOHandlerImpl::filePositionToString( std::shared_ptr< ADIOS2FilePosition > filepos ) { return filepos->location; @@ -1179,7 +1164,6 @@ namespace detail { (void)impl; std::set< std::string > alreadyConfigured; -# if openPMD_HAVE_JSON // TODO if changing TracingJSON to return references upon operator[](), // change this to a reference auto engineConfig = impl.config( detail::str_engine ); @@ -1209,7 +1193,6 @@ namespace detail "remain unused:\n" << shadow << std::endl; } -# endif // openPMD_HAVE_JSON auto notYetConfigured = [&alreadyConfigured]( std::string const & param ) { auto it = alreadyConfigured.find( param ); @@ -1417,19 +1400,10 @@ namespace detail ADIOS2IOHandler::ADIOS2IOHandler( std::string path, openPMD::AccessType at, - MPI_Comm comm -# if openPMD_HAVE_JSON - , nlohmann::json options -# endif -) + MPI_Comm comm, + nlohmann::json options ) : AbstractIOHandler( std::move( path ), at, comm ) - , m_impl{ - this, - comm -# if openPMD_HAVE_JSON - , std::move( options ) -# endif - } + , m_impl{ this, comm, std::move( options ) } { } @@ -1437,50 +1411,38 @@ ADIOS2IOHandler::ADIOS2IOHandler( ADIOS2IOHandler::ADIOS2IOHandler( std::string path, - AccessType at -# if openPMD_HAVE_JSON - , nlohmann::json options -# endif -) + AccessType at, + nlohmann::json options ) : AbstractIOHandler( std::move( path ), at ) - , m_impl{ - this -# if openPMD_HAVE_JSON - , std::move( options ) -# endif - } + , m_impl{ this, std::move( options ) } { } -std::future< void > ADIOS2IOHandler::flush( ) +std::future< void > +ADIOS2IOHandler::flush() { - return m_impl.flush( ); + return m_impl.flush(); } #else // openPMD_HAVE_ADIOS2 -#if openPMD_HAVE_MPI +# if openPMD_HAVE_MPI ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at, MPI_Comm comm -# if openPMD_HAVE_JSON , nlohmann::json -# endif ) : AbstractIOHandler( std::move( path ), at, comm ) { } -#endif // openPMD_HAVE_MPI +# endif // openPMD_HAVE_MPI ADIOS2IOHandler::ADIOS2IOHandler( std::string path, - AccessType at -#if openPMD_HAVE_JSON - , nlohmann::json -#endif -) + AccessType at, + nlohmann::json ) : AbstractIOHandler( std::move( path ), at ) { } diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index 5baf6e920d..d0b10b152b 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -1,8 +1,8 @@ -#include "openPMD/config.hpp" -#if openPMD_HAVE_JSON -# include +#include "openPMD/auxiliary/JSON.hpp" + +#include -# include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/config.hpp" namespace openPMD { @@ -92,5 +92,3 @@ namespace auxiliary } } // namespace auxiliary } // namespace openPMD - -#endif \ No newline at end of file From e84ced8327f487a9a501938805ec930151f2f3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 3 Mar 2020 14:18:56 +0100 Subject: [PATCH 12/24] Move ADIOS2 defaults to struct of their own --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 22 +++++++++++++------- src/IO/ADIOS/ADIOS2IOHandler.cpp | 22 ++++++++++++++++---- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 3ba63b63b7..4843d36ce9 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -287,16 +287,22 @@ class ADIOS2IOHandlerImpl std::string const & var ); }; // ADIOS2IOHandlerImpl -namespace detail +/* + * The following strings are used during parsing of the JSON configuration + * string for the ADIOS2 backend. + * Note that this struct's members additionally need to be defined at namespace + * scope up until C++17, see ADIOS2IOHandlerImpl.cpp + */ +struct ADIOS2Defaults { - /* - * The following strings are used during parsing of the JSON configuration - * string for the ADIOS2 backend. - */ - constexpr char const * const str_engine = "engine"; - constexpr char const * str_type = "type"; - constexpr char const * str_params = "parameters"; + using const_str = char const * const; + constexpr static const_str str_engine = "engine"; + constexpr static const_str str_type = "type"; + constexpr static const_str str_params = "parameters"; +}; +namespace detail +{ // Helper structs for calls to the switchType function struct DatasetReader diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 3068fb9e74..fc0ee2e780 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -95,6 +95,7 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) std::cerr << "Warning: ADIOS2 is not configured in the JSON " "configuration. Running with default settings." << std::endl; + return; } } @@ -684,6 +685,18 @@ ADIOS2IOHandlerImpl::verifyDataset( Offset const & offset, return var; } +# if __cplusplus < 201703L +// https://en.cppreference.com/w/cpp/language/static: +// If a static data member is declared constexpr, it is implicitly inline and +// does not need to be redeclared at namespace scope. This redeclaration +// without an initializer (formerly required as shown above) +// is still permitted, but is deprecated. (since C++17) + +constexpr ADIOS2Defaults::const_str ADIOS2Defaults::str_engine; +constexpr ADIOS2Defaults::const_str ADIOS2Defaults::str_type; +constexpr ADIOS2Defaults::const_str ADIOS2Defaults::str_params; +# endif + namespace detail { DatasetReader::DatasetReader( openPMD::ADIOS2IOHandlerImpl * impl ) @@ -1166,14 +1179,15 @@ namespace detail std::set< std::string > alreadyConfigured; // TODO if changing TracingJSON to return references upon operator[](), // change this to a reference - auto engineConfig = impl.config( detail::str_engine ); + auto engineConfig = impl.config( ADIOS2Defaults::str_engine ); if( !engineConfig.json().is_null() ) { m_IO.SetEngine( - impl.config( detail::str_type, engineConfig ).json() ); + impl.config( ADIOS2Defaults::str_type, engineConfig ).json() ); // TODO if changing TracingJSON to return references upon // operator[](), change this to a reference - auto params = impl.config( detail::str_params, engineConfig ); + auto params = + impl.config( ADIOS2Defaults::str_params, engineConfig ); params.declareFullyRead(); if( params.json().is_object() ) { @@ -1395,7 +1409,7 @@ namespace detail } // namespace detail -#if openPMD_HAVE_MPI +# if openPMD_HAVE_MPI ADIOS2IOHandler::ADIOS2IOHandler( std::string path, From 3604b8b97ecca2f22c1d284ae96383ea6c7a64d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 20 Feb 2020 15:39:28 +0100 Subject: [PATCH 13/24] Some cleanup in ADIOS2IOHandlerImpl --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 28 ++++++++++---------- src/IO/ADIOS/ADIOS2IOHandler.cpp | 19 +++++++------ 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 4843d36ce9..e445e267ac 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -375,17 +375,13 @@ namespace detail struct VariableDefiner { + // Parameters such as DatasetHelper< T >::defineVariable + template < typename T, typename... Params > + void operator( )( Params &&... params ); - template < typename T > - void operator( )( adios2::IO & IO, const std::string & name, - std::unique_ptr< adios2::Operator > compression, - const adios2::Dims & shape = adios2::Dims( ), - const adios2::Dims & start = adios2::Dims( ), - const adios2::Dims & count = adios2::Dims( ), - const bool constantDims = false ); - - template < int n, typename... Params > - void operator( )( adios2::IO & IO, Params &&... ); + template< int n, typename... Params > + void + operator()( Params &&... ); }; @@ -517,10 +513,14 @@ namespace detail std::string const & fileName ); static void - defineVariable( adios2::IO & IO, const std::string & name, - std::unique_ptr< adios2::Operator > compression, - const adios2::Dims & shape, const adios2::Dims & start, - const adios2::Dims & count, bool constantDims ); + defineVariable( + adios2::IO & IO, + std::string const & name, + std::unique_ptr< adios2::Operator > compression, + adios2::Dims const & shape = adios2::Dims(), + adios2::Dims const & start = adios2::Dims(), + adios2::Dims const & count = adios2::Dims(), + bool const constantDims = false ); void writeDataset( BufferedPut &, adios2::IO &, adios2::Engine & ); }; diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index fc0ee2e780..805dc7eba4 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -834,21 +834,20 @@ namespace detail throw std::runtime_error( "[ADIOS2] WRITE_DATASET: Invalid datatype." ); } - template < typename T > + template < typename T, typename... Params > void VariableDefiner:: - operator( )( adios2::IO & IO, const std::string & name, - std::unique_ptr< adios2::Operator > compression, - const adios2::Dims & shape, const adios2::Dims & start, - const adios2::Dims & count, const bool constantDims ) + operator( )( Params &&... params ) { - DatasetHelper< T >::defineVariable( IO, name, std::move( compression ), - shape, start, count, constantDims ); + DatasetHelper< T >::defineVariable( + std::forward< Params >( params )... ); } - template < int n, typename... Params > - void VariableDefiner::operator( )( adios2::IO &, Params &&... ) + template< int n, typename... Params > + void + VariableDefiner::operator()( Params &&... ) { - throw std::runtime_error( "[ADIOS2] Defining a variable with undefined type." ); + throw std::runtime_error( + "[ADIOS2] Defining a variable with undefined type." ); } From 97da62a3caffe3319998515ac0f61afde24f1bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 20 Feb 2020 16:55:46 +0100 Subject: [PATCH 14/24] Enable ADIOS2 operator reading from JSON config Also fix a bug where a null pointer is dereferenced if an invalid compression is requested. --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 26 ++++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 115 ++++++++++++++++--- 2 files changed, 121 insertions(+), 20 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index e445e267ac..721f066c43 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -180,6 +180,14 @@ class ADIOS2IOHandlerImpl private: adios2::ADIOS m_ADIOS; + struct ParameterizedOperator + { + adios2::Operator const op; + adios2::Params const params; + }; + + std::vector< ParameterizedOperator > defaultOperators; + auxiliary::TracingJSON m_config; static auxiliary::TracingJSON nullvalue; @@ -206,6 +214,21 @@ class ADIOS2IOHandlerImpl { return config< Key >( std::forward< Key >( key ), m_config ); } + + /** + * + * @param config The top-level of the ADIOS2 configuration JSON object + * with operators to be found under dataset.operators + * @return first parameter: the operators, second parameters: whether + * operators have been configured + */ + std::pair< std::vector< ParameterizedOperator >, bool > + getOperators( auxiliary::TracingJSON config ); + + // use m_config + std::pair< std::vector< ParameterizedOperator >, bool > + getOperators(); + /* * We need to give names to IO objects. These names are irrelevant * within this application, since: @@ -516,7 +539,8 @@ namespace detail defineVariable( adios2::IO & IO, std::string const & name, - std::unique_ptr< adios2::Operator > compression, + std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > + compressions, adios2::Dims const & shape = adios2::Dims(), adios2::Dims const & start = adios2::Dims(), adios2::Dims const & count = adios2::Dims(), diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 805dc7eba4..71fbf3ddcd 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -86,17 +86,66 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( void ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) { - if( cfg.contains( "adios2" ) ) - { - m_config = std::move( cfg[ "adios2" ] ); - } - else + if( !cfg.contains( "adios2" ) ) { std::cerr << "Warning: ADIOS2 is not configured in the JSON " "configuration. Running with default settings." << std::endl; return; } + m_config = std::move( cfg[ "adios2" ] ); + defaultOperators = getOperators().first; +} + +std::pair< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator >, bool > +ADIOS2IOHandlerImpl::getOperators( auxiliary::TracingJSON cfg ) +{ + std::vector< ParameterizedOperator > res; + if( !cfg.json().contains( "dataset" ) ) + { + return std::make_pair( res, false ); + } + auto datasetConfig = cfg[ "dataset" ]; + if( !datasetConfig.json().contains( "operators" ) ) + { + return std::make_pair( res, false ); + } + auto _operators = datasetConfig[ "operators" ]; + nlohmann::json const & operators = _operators.json(); + for( auto operatorIterator = operators.begin(); + operatorIterator != operators.end(); + ++operatorIterator ) + { + nlohmann::json const & op = operatorIterator.value(); + std::string const & type = op[ "type" ]; + adios2::Params adiosParams; + if( op.contains( "parameters" ) ) + { + nlohmann::json const & params = op[ "parameters" ]; + for( auto paramIterator = params.begin(); + paramIterator != params.end(); + ++paramIterator ) + { + adiosParams[ paramIterator.key() ] = + std::string( paramIterator.value() ); + } + } + std::unique_ptr< adios2::Operator > adiosOperator = + getCompressionOperator( type ); + if( adiosOperator ) + { + res.emplace_back( ParameterizedOperator{ + *adiosOperator, std::move( adiosParams ) } ); + } + } + _operators.declareFullyRead(); + return std::make_pair( res, true ); +} + +std::pair< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator >, bool > +ADIOS2IOHandlerImpl::getOperators() +{ + return getOperators( m_config ); } std::future< void > @@ -216,17 +265,33 @@ void ADIOS2IOHandlerImpl::createDataset( * the C++11 standard * @todo replace with std::optional upon switching to C++17 */ + + auto operators = defaultOperators; + std::unique_ptr< adios2::Operator > compression; - if ( !parameters.compression.empty( ) ) - compression = getCompressionOperator( parameters.compression ); + if( !parameters.compression.empty() ) + { + std::unique_ptr< adios2::Operator > adiosOperator = + getCompressionOperator( parameters.compression ); + if( adiosOperator ) + { + operators.push_back( ParameterizedOperator{ + *adiosOperator, + adios2::Params() } ); + } + } // cast from openPMD::Extent to adios2::Dims adios2::Dims const shape( parameters.extent.begin(), parameters.extent.end() ); auto & fileData = getFileData( file ); - switchType( parameters.dtype, detail::VariableDefiner( ), - fileData.m_IO, varName, - std::move( compression ), shape ); + switchType( + parameters.dtype, + detail::VariableDefiner(), + fileData.m_IO, + varName, + operators, + shape ); fileData.invalidateVariablesMap(); writable->written = true; m_dirty.emplace( file ); @@ -1012,13 +1077,20 @@ namespace detail engine.Get( var, ptr ); } - template < typename T > - void DatasetHelper< - T, typename std::enable_if< DatasetTypes< T >::validType >::type >:: - defineVariable( adios2::IO & IO, const std::string & name, - std::unique_ptr< adios2::Operator > compression, - const adios2::Dims & shape, const adios2::Dims & start, - const adios2::Dims & count, const bool constantDims ) + template< typename T > + void + DatasetHelper< + T, + typename std::enable_if< DatasetTypes< T >::validType >::type >:: + defineVariable( + adios2::IO & IO, + const std::string & name, + std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > + compressions, + const adios2::Dims & shape, + const adios2::Dims & start, + const adios2::Dims & count, + const bool constantDims ) { adios2::Variable< T > var = IO.DefineVariable< T >( name, shape, start, count, constantDims ); @@ -1029,9 +1101,14 @@ namespace detail } // check whether the unique_ptr has an element // and whether the held operator is valid - if ( compression && *compression ) + for( auto const & compression : compressions ) { - var.AddOperation( *compression ); + if( compression.op ) + { + var.AddOperation( + std::move( compression.op ), + std::move( compression.params ) ); + } } } From 784c0c67b1a927ae5f47b1081060ceecebfee93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 3 Mar 2020 17:22:33 +0100 Subject: [PATCH 15/24] Add JSON config test --- test/SerialIOTest.cpp | 91 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 5220cb061a..1b69d7fe66 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2320,3 +2320,94 @@ TEST_CASE( "no_serial_adios1", "[serial][adios]") } #endif +#if openPMD_HAVE_ADIOS2 +TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) +{ + std::string writeConfig = R"END( +{ + "adios2": { + "engine": { + "type": "bp4", + "unused": "parameter", + "parameters": { + "BufferGrowthFactor": "2.0", + "Profile": "On" + } + }, + "unused": "as well", + "dataset": { + "operators": [ + { "type": "BZip2" } + ] + } + } +} +)END"; + { + openPMD::Series series( + "../samples/jsonConfigured.bp", + openPMD::AccessType::CREATE, + writeConfig ); + auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ]; + E_x.resetDataset( openPMD::Dataset( openPMD::Datatype::INT, { 1000 } ) ); + std::vector< int > data( 1000, 0 ); + E_x.storeChunk( data, { 0 }, { 1000 } ); + series.flush(); + } + std::string wrongReadConfig = R"END( +{ + "adios2": { + "engine": { + "type": "bp3", + "unused": "parameter" + } + } +} +)END"; + // catching the exception thrown in ADIOS2 does not work + // maybe rel.:https://stackoverflow.com/questions/38878999/try-catch-doesnt-work-in-shared-library +# if 0 + { + // wrong engine + // Catching the exception from the constructor will not work with + // REQUIRE_THROWS, so we use a plain old try. + try { + openPMD::Series( + "../samples/jsonConfigured.bp", + openPMD::AccessType::READ_ONLY, + wrongReadConfig ); + FAIL_CHECK( "Reading with a wrong engine should fail." ); + } catch (...) + { + REQUIRE( true ); + } + } +# endif + + std::string correctReadConfig = R"END( +{ + "adios2": { + "engine": { + "type": "bp4", + "unused": "parameter" + } + } +} +)END"; + { + openPMD::Series series( + "../samples/jsonConfigured.bp", + openPMD::AccessType::READ_ONLY, + correctReadConfig ); + auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ]; + REQUIRE( E_x.getDimensionality() == 1 ); + REQUIRE( E_x.getExtent()[ 0 ] == 1000 ); + auto chunk = E_x.loadChunk< int >( { 0 }, { 1000 } ); + series.flush(); + for( size_t i = 0; i < 1000; ++i ) + { + REQUIRE( chunk.get()[ i ] == 0 ); + } + } +} +#endif From 56c3451c2950fdc43c3e37e9fd7579e7a83a4f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 4 Mar 2020 13:57:18 +0100 Subject: [PATCH 16/24] Cleanup operator handling --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 2 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 721f066c43..c9f3bd9561 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -539,7 +539,7 @@ namespace detail defineVariable( adios2::IO & IO, std::string const & name, - std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > + std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > const & compressions, adios2::Dims const & shape = adios2::Dims(), adios2::Dims const & start = adios2::Dims(), diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 71fbf3ddcd..abcaaa6767 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -268,7 +268,6 @@ void ADIOS2IOHandlerImpl::createDataset( auto operators = defaultOperators; - std::unique_ptr< adios2::Operator > compression; if( !parameters.compression.empty() ) { std::unique_ptr< adios2::Operator > adiosOperator = @@ -1085,7 +1084,7 @@ namespace detail defineVariable( adios2::IO & IO, const std::string & name, - std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > + std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > const & compressions, const adios2::Dims & shape, const adios2::Dims & start, @@ -1099,15 +1098,11 @@ namespace detail throw std::runtime_error( "[ADIOS2] Internal error: Could not create Variable '" + name + "'." ); } - // check whether the unique_ptr has an element - // and whether the held operator is valid for( auto const & compression : compressions ) { if( compression.op ) { - var.AddOperation( - std::move( compression.op ), - std::move( compression.params ) ); + var.AddOperation( compression.op, compression.params ); } } } From 74e9d3802655024548f156c9a4dec94233e42778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 4 Mar 2020 14:41:08 +0100 Subject: [PATCH 17/24] Check whether files have been written with correct ADIOS2 engine JSON configuration test will verify that the configuration of the ADIOS2 engine has actually had an effect. --- test/SerialIOTest.cpp | 78 ++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 1b69d7fe66..d126946b03 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2323,7 +2323,27 @@ TEST_CASE( "no_serial_adios1", "[serial][adios]") #if openPMD_HAVE_ADIOS2 TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) { - std::string writeConfig = R"END( + std::string writeConfigBP3 = R"END( +{ + "adios2": { + "engine": { + "type": "bp3", + "unused": "parameter", + "parameters": { + "BufferGrowthFactor": "2.0", + "Profile": "On" + } + }, + "unused": "as well", + "dataset": { + "operators": [ + { "type": "BZip2" } + ] + } + } +} +)END"; + std::string writeConfigBP4 = R"END( { "adios2": { "engine": { @@ -2343,18 +2363,26 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) } } )END"; - { - openPMD::Series series( - "../samples/jsonConfigured.bp", - openPMD::AccessType::CREATE, - writeConfig ); + auto write = []( std::string const & filename, + std::string const & config ) { + openPMD::Series series( filename, openPMD::AccessType::CREATE, config ); auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ]; - E_x.resetDataset( openPMD::Dataset( openPMD::Datatype::INT, { 1000 } ) ); + E_x.resetDataset( + openPMD::Dataset( openPMD::Datatype::INT, { 1000 } ) ); std::vector< int > data( 1000, 0 ); E_x.storeChunk( data, { 0 }, { 1000 } ); series.flush(); - } - std::string wrongReadConfig = R"END( + }; + write( "../samples/jsonConfiguredBP4.bp", writeConfigBP4 ); + write( "../samples/jsonConfiguredBP3.bp", writeConfigBP3 ); + + // BP3 engine writes files, BP4 writes directories + REQUIRE( + openPMD::auxiliary::file_exists( "../samples/jsonConfiguredBP3.bp" ) ); + REQUIRE( openPMD::auxiliary::directory_exists( + "../samples/jsonConfiguredBP4.bp" ) ); + + std::string readConfigBP3 = R"END( { "adios2": { "engine": { @@ -2364,27 +2392,7 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) } } )END"; - // catching the exception thrown in ADIOS2 does not work - // maybe rel.:https://stackoverflow.com/questions/38878999/try-catch-doesnt-work-in-shared-library -# if 0 - { - // wrong engine - // Catching the exception from the constructor will not work with - // REQUIRE_THROWS, so we use a plain old try. - try { - openPMD::Series( - "../samples/jsonConfigured.bp", - openPMD::AccessType::READ_ONLY, - wrongReadConfig ); - FAIL_CHECK( "Reading with a wrong engine should fail." ); - } catch (...) - { - REQUIRE( true ); - } - } -# endif - - std::string correctReadConfig = R"END( + std::string readConfigBP4 = R"END( { "adios2": { "engine": { @@ -2394,11 +2402,9 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) } } )END"; - { + auto read = []( std::string const & filename, std::string const & config ) { openPMD::Series series( - "../samples/jsonConfigured.bp", - openPMD::AccessType::READ_ONLY, - correctReadConfig ); + filename, openPMD::AccessType::READ_ONLY, config ); auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ]; REQUIRE( E_x.getDimensionality() == 1 ); REQUIRE( E_x.getExtent()[ 0 ] == 1000 ); @@ -2408,6 +2414,8 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) { REQUIRE( chunk.get()[ i ] == 0 ); } - } + }; + read( "../samples/jsonConfiguredBP3.bp", readConfigBP3 ); + read( "../samples/jsonConfiguredBP4.bp", readConfigBP4 ); } #endif From 9a6569ef77490f1e3528b3e78a5772d0a736cc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 4 Mar 2020 14:46:47 +0100 Subject: [PATCH 18/24] Add EOF newline to JSON.hpp --- include/openPMD/auxiliary/JSON.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/openPMD/auxiliary/JSON.hpp b/include/openPMD/auxiliary/JSON.hpp index 6979355960..81e586283b 100644 --- a/include/openPMD/auxiliary/JSON.hpp +++ b/include/openPMD/auxiliary/JSON.hpp @@ -109,3 +109,4 @@ namespace auxiliary } } // namespace auxiliary } // namespace openPMD + From f683b25e227a7ecb5742a4d352b1fed2a12f507c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2020 17:27:07 +0100 Subject: [PATCH 19/24] Extend tests to include adios operator parameters --- test/SerialIOTest.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index d126946b03..36fd24a664 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2337,7 +2337,13 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) "unused": "as well", "dataset": { "operators": [ - { "type": "BZip2" } + { + "type": "blosc", + "parameters": { + "clevel": "1", + "doshuffle": "BLOSC_BITSHUFFLE" + } + } ] } } @@ -2357,7 +2363,13 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) "unused": "as well", "dataset": { "operators": [ - { "type": "BZip2" } + { + "type": "blosc", + "parameters": { + "clevel": "1", + "doshuffle": "BLOSC_BITSHUFFLE" + } + } ] } } From f824c4e38e2a735e82ce347fa570bc39ba135d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2020 17:33:42 +0100 Subject: [PATCH 20/24] Make ADIOS2Defaults a namespace (was previously a struct) --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 10 +++++----- src/IO/ADIOS/ADIOS2IOHandler.cpp | 12 ------------ 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index c9f3bd9561..7adfa83a03 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -316,13 +316,13 @@ class ADIOS2IOHandlerImpl * Note that this struct's members additionally need to be defined at namespace * scope up until C++17, see ADIOS2IOHandlerImpl.cpp */ -struct ADIOS2Defaults +namespace ADIOS2Defaults { using const_str = char const * const; - constexpr static const_str str_engine = "engine"; - constexpr static const_str str_type = "type"; - constexpr static const_str str_params = "parameters"; -}; + constexpr const_str str_engine = "engine"; + constexpr const_str str_type = "type"; + constexpr const_str str_params = "parameters"; +} namespace detail { diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index abcaaa6767..845b0496a1 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -749,18 +749,6 @@ ADIOS2IOHandlerImpl::verifyDataset( Offset const & offset, return var; } -# if __cplusplus < 201703L -// https://en.cppreference.com/w/cpp/language/static: -// If a static data member is declared constexpr, it is implicitly inline and -// does not need to be redeclared at namespace scope. This redeclaration -// without an initializer (formerly required as shown above) -// is still permitted, but is deprecated. (since C++17) - -constexpr ADIOS2Defaults::const_str ADIOS2Defaults::str_engine; -constexpr ADIOS2Defaults::const_str ADIOS2Defaults::str_type; -constexpr ADIOS2Defaults::const_str ADIOS2Defaults::str_params; -# endif - namespace detail { DatasetReader::DatasetReader( openPMD::ADIOS2IOHandlerImpl * impl ) From 4c412cd39d39b970dbe23e25ee70ef18eaaa62d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2020 17:39:49 +0100 Subject: [PATCH 21/24] Implement reviewer's comments Remove traces of openPMD_HAVE_JSON Improve JSON highlighting in documentation Formatting improvements in documentation Includes reordering Documentation --- docs/source/backends/json.rst | 1 + docs/source/details/backendconfig.rst | 19 +++++--- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 18 +++++++ src/IO/AbstractIOHandlerHelper.cpp | 51 ++++++-------------- 4 files changed, 47 insertions(+), 42 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index c247afafa2..a02eeee831 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -85,4 +85,5 @@ The example code in the :ref:`usage section ` will produce the fol when picking the JSON backend: .. literalinclude:: json_example.json + :language: json diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index 098c0dd495..103197950d 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -1,6 +1,6 @@ .. _backendconfig -Backend-Specific configuration +Backend-Specific Configuration ============================== While the openPMD API intends to be a backend-*independent* implementation of the openPMD standard, it is sometimes useful to pass configuration parameters to the specific backend in use. @@ -10,6 +10,7 @@ A JSON option always takes precedence over an environment variable. The fundamental structure of this JSON configuration string is given as follows: .. literalinclude:: config_layout.json + :language: json This structure allows keeping one configuration string for several backends at once, with the concrete backend configuration being chosen upon choosing the backend itself. @@ -39,12 +40,17 @@ ADIOS2 A full configuration of the ADIOS2 backend: .. literalinclude:: adios2.json + :language: json -All keys found under ``adios2.dataset`` are applicable globally as well as per dataset, keys found under ``adios2.engine`` only globally. Explanation of the single keys: +All keys found under ``adios2.dataset`` are applicable globally as well as per dataset, keys found under ``adios2.engine`` only globally. +Explanation of the single keys: -* ``adios2.engine.type``: A string that is passed directly to ``adios2::IO:::SetEngine`` for choosing the ADIOS2 engine to be used. Please refer to the `official ADIOS2 documentation `_ for a list of available engines. -* ``adios2.engine.type``: An associative array of string-formatted engine parameters, passed directly through to ``adios2::IO::SetParameters``. Please refer to the official ADIOS2 documentation for the allowable engine parameters. -* ``adios2.dataset.operators``: (*currently unimplemented* – please use the ``openPMD::Dataset::compression`` for the meantime) This key contains a list of ADIOS2 `operators `_, used to enable compression or dataset transformations. Each object in the list has three keys: +* ``adios2.engine.type``: A string that is passed directly to ``adios2::IO:::SetEngine`` for choosing the ADIOS2 engine to be used. + Please refer to the `official ADIOS2 documentation `_ for a list of available engines. +* ``adios2.engine.type``: An associative array of string-formatted engine parameters, passed directly through to ``adios2::IO::SetParameters``. + Please refer to the official ADIOS2 documentation for the allowable engine parameters. +* ``adios2.dataset.operators``: (*currently unimplemented* – please use the ``openPMD::Dataset::compression`` for the meantime) This key contains a list of ADIOS2 `operators `_, used to enable compression or dataset transformations. + Each object in the list has three keys: * ``type`` supported ADIOS operator type, e.g. zfp, sz * ``parameters`` is an associative map of string parameters for the operator (e.g. compression levels) @@ -52,4 +58,5 @@ All keys found under ``adios2.dataset`` are applicable globally as well as per d Other backends ^^^^^^^^^^^^^^ -Do currently not read the configuration string. Please refer to the respective backends' documentations for further information on their configuration. \ No newline at end of file +Do currently not read the configuration string. +Please refer to the respective backends' documentations for further information on their configuration. diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 7adfa83a03..b9233a368a 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IO/InvalidatableFile.hpp" +#include "openPMD/auxiliary/JSON.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -42,6 +43,8 @@ #include "openPMD/config.hpp" +#include + #if openPMD_HAVE_ADIOS2 # include # include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" @@ -535,6 +538,21 @@ namespace detail void readDataset( BufferedGet &, adios2::IO &, adios2::Engine &, std::string const & fileName ); + /** + * @brief Define a Variable of type T within the passed IO. + * + * @param IO The adios2::IO object within which to define the + * variable. The variable can later be retrieved from + * the IO using the passed name. + * @param name As in adios2::IO::DefineVariable + * @param compressions ADIOS2 operators, including an arbitrary + * number of parameters, to be added to the + * variable upon definition. + * @param shape As in adios2::IO::DefineVariable + * @param start As in adios2::IO::DefineVariable + * @param count As in adios2::IO::DefineVariable + * @param constantDims As in adios2::IO::DefineVariable + */ static void defineVariable( adios2::IO & IO, diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 0c421b54e1..9ba5a01691 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -19,17 +19,15 @@ * If not, see . */ #include "openPMD/IO/AbstractIOHandlerHelper.hpp" -#include "openPMD/IO/DummyIOHandler.hpp" + #include "openPMD/IO/ADIOS/ADIOS1IOHandler.hpp" -#include "openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" +#include "openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp" +#include "openPMD/IO/DummyIOHandler.hpp" #include "openPMD/IO/HDF5/HDF5IOHandler.hpp" #include "openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp" #include "openPMD/IO/JSON/JSONIOHandler.hpp" - -#if openPMD_HAVE_JSON -# include -#endif +#include namespace openPMD { @@ -42,12 +40,7 @@ namespace openPMD MPI_Comm comm, std::string const & options ) { -# if openPMD_HAVE_JSON nlohmann::json optionsJson = nlohmann::json::parse( options ); -# else - if( options.size() > 0u && options != "{}" ) - throw std::runtime_error("openPMD-api built without JSON support which is required for runtime options!"); -# endif switch( format ) { case Format::HDF5: @@ -59,16 +52,11 @@ namespace openPMD throw std::runtime_error("openPMD-api built without ADIOS1 support"); # endif case Format::ADIOS2: - return std::make_shared( - path, - accessTypeBackend, - comm -# if openPMD_HAVE_JSON - , std::move(optionsJson) -# endif - ); + return std::make_shared< ADIOS2IOHandler >( + path, accessTypeBackend, comm, std::move( optionsJson ) ); default: - throw std::runtime_error("Unknown file format! Did you specify a file ending?" ); + throw std::runtime_error( + "Unknown file format! Did you specify a file ending?" ); } } #endif @@ -79,12 +67,7 @@ namespace openPMD Format format, std::string const & options ) { -#if openPMD_HAVE_JSON nlohmann::json optionsJson = nlohmann::json::parse( options ); -#else - if( options.size() > 0u && options != "{}" ) - throw std::runtime_error("openPMD-api built without JSON support which is required for runtime options!"); -#endif switch( format ) { case Format::HDF5: @@ -96,19 +79,15 @@ namespace openPMD throw std::runtime_error("openPMD-api built without ADIOS1 support"); #endif #if openPMD_HAVE_ADIOS2 - case Format::ADIOS2: - return std::make_shared< ADIOS2IOHandler >( - path, - accessType -# if openPMD_HAVE_JSON - , std::move( optionsJson ) -# endif // openPMD_HAVE_JSON - ); + case Format::ADIOS2: + return std::make_shared< ADIOS2IOHandler >( + path, accessType, std::move( optionsJson ) ); #endif // openPMD_HAVE_ADIOS2 case Format::JSON: - return std::make_shared< JSONIOHandler >(path, accessType); + return std::make_shared< JSONIOHandler >( path, accessType ); default: - throw std::runtime_error("Unknown file format! Did you specify a file ending?" ); + throw std::runtime_error( + "Unknown file format! Did you specify a file ending?" ); } } -} // openPMD + } // namespace openPMD From cd89f552a01dada2bd1ba371794918be2387e713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 18 Mar 2020 21:13:13 +0100 Subject: [PATCH 22/24] Use nlohmann::json::get to excplicitly get a string --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 845b0496a1..2e4a6548ce 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -127,7 +127,7 @@ ADIOS2IOHandlerImpl::getOperators( auxiliary::TracingJSON cfg ) ++paramIterator ) { adiosParams[ paramIterator.key() ] = - std::string( paramIterator.value() ); + paramIterator.value().get< std::string >(); } } std::unique_ptr< adios2::Operator > adiosOperator = From 422143b8d4c9e4b0bcf5f6406b29ae64c2217f95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2020 17:31:28 +0200 Subject: [PATCH 23/24] Don't run ADIOS2 configuration test if ADIOS1 is chosen per env-var Also remove some outdated todos. --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 4 ---- test/SerialIOTest.cpp | 25 +++++++++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2e4a6548ce..436812f50c 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1236,15 +1236,11 @@ namespace detail { (void)impl; std::set< std::string > alreadyConfigured; - // TODO if changing TracingJSON to return references upon operator[](), - // change this to a reference auto engineConfig = impl.config( ADIOS2Defaults::str_engine ); if( !engineConfig.json().is_null() ) { m_IO.SetEngine( impl.config( ADIOS2Defaults::str_type, engineConfig ).json() ); - // TODO if changing TracingJSON to return references upon - // operator[](), change this to a reference auto params = impl.config( ADIOS2Defaults::str_params, engineConfig ); params.declareFullyRead(); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 36fd24a664..02a86005b6 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -3,23 +3,23 @@ # define OPENPMD_private public # define OPENPMD_protected public #endif -#include "openPMD/openPMD.hpp" -#include "openPMD/auxiliary/Filesystem.hpp" - -#include - -#include #include -#include -#include -#include #include #include -#include +#include #include #include +#include #include +#include +#include #include +#include + +#include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Environment.hpp" +#include "openPMD/openPMD.hpp" +#include using namespace openPMD; @@ -2323,6 +2323,11 @@ TEST_CASE( "no_serial_adios1", "[serial][adios]") #if openPMD_HAVE_ADIOS2 TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) { + if( auxiliary::getEnvString( "OPENPMD_BP_BACKEND", "NOT_SET" ) == "ADIOS1" ) + { + // run this test for ADIOS2 only + return; + } std::string writeConfigBP3 = R"END( { "adios2": { From 00768b94f96a6acb80396bff241e88a04ae9c5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Sat, 4 Apr 2020 14:42:52 +0200 Subject: [PATCH 24/24] Apply suggestions from code review Co-Authored-By: Axel Huebl --- docs/source/details/config_layout.json | 3 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 39 ++++++-------- include/openPMD/auxiliary/JSON.hpp | 55 ++++++++++++++++++-- src/IO/ADIOS/ADIOS2IOHandler.cpp | 12 ++--- src/auxiliary/JSON.cpp | 45 +++++++++++----- test/SerialIOTest.cpp | 16 +++--- 6 files changed, 117 insertions(+), 53 deletions(-) diff --git a/docs/source/details/config_layout.json b/docs/source/details/config_layout.json index 691a6ba1ae..1e86d85876 100644 --- a/docs/source/details/config_layout.json +++ b/docs/source/details/config_layout.json @@ -1,5 +1,6 @@ { "adios": "put ADIOS config here", "adios2": "put ADIOS2 config here", - "hdf5": "put HDF5 config here" + "hdf5": "put HDF5 config here", + "json": "put JSON config here" } diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index b9233a368a..8d1f1cbbd5 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -21,6 +21,7 @@ #pragma once #include "ADIOS2FilePosition.hpp" +#include "openPMD/config.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" @@ -29,22 +30,6 @@ #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/backend/Writable.hpp" -#include -#include -#include -#include -#include // shared_ptr -#include -#include -#include // pair -#include - -#include - -#include "openPMD/config.hpp" - -#include - #if openPMD_HAVE_ADIOS2 # include # include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" @@ -54,6 +39,17 @@ # include #endif +#include + +#include +#include +#include +#include +#include // shared_ptr +#include +#include +#include // pair +#include namespace openPMD { @@ -111,10 +107,9 @@ class ADIOS2IOHandlerImpl ); - ~ADIOS2IOHandlerImpl() override = default; + ~ADIOS2IOHandlerImpl( ) override = default; - std::future< void > - flush() override; + std::future< void > flush( ) override; void createFile( Writable *, Parameter< Operation::CREATE_FILE > const & ) override; @@ -316,8 +311,6 @@ class ADIOS2IOHandlerImpl /* * The following strings are used during parsing of the JSON configuration * string for the ADIOS2 backend. - * Note that this struct's members additionally need to be defined at namespace - * scope up until C++17, see ADIOS2IOHandlerImpl.cpp */ namespace ADIOS2Defaults { @@ -757,8 +750,8 @@ friend class ADIOS2IOHandlerImpl; ADIOS2IOHandler( std::string path, AccessType, - MPI_Comm - , nlohmann::json options + MPI_Comm, + nlohmann::json options ); #endif diff --git a/include/openPMD/auxiliary/JSON.hpp b/include/openPMD/auxiliary/JSON.hpp index 81e586283b..32d9039343 100644 --- a/include/openPMD/auxiliary/JSON.hpp +++ b/include/openPMD/auxiliary/JSON.hpp @@ -1,11 +1,33 @@ -#pragma once +/* Copyright 2020 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ -#include // std::shared_ptr -#include // std::forward +#pragma once #include "openPMD/config.hpp" + #include +#include // std::shared_ptr +#include // std::forward + namespace openPMD { namespace auxiliary @@ -69,14 +91,39 @@ namespace auxiliary declareFullyRead(); private: + /** + * @brief The JSON object with which this class has been initialized. + * Shared pointer shared between all instances returned by + * operator[]() in order to avoid use-after-free situations. + * + */ std::shared_ptr< nlohmann::json > m_originalJSON; + /** + * @brief A JSON object keeping track of all accessed indices within the + * original JSON object. Initially an empty JSON object, + * gradually filled by applying each operator[]() call also to + * it. + * Shared pointer shared between all instances returned by + * operator[]() in order to avoid use-after-free situations. + * + */ std::shared_ptr< nlohmann::json > m_shadow; + /** + * @brief The sub-expression within m_originalJSON corresponding with + * the current instance. + * + */ nlohmann::json * m_positionInOriginal; + /** + * @brief The sub-expression within m_positionInOriginal corresponding + * with the current instance. + * + */ nlohmann::json * m_positionInShadow; bool m_trace = true; void - invertShadow( nlohmann::json & result, nlohmann::json & shadow ); + invertShadow( nlohmann::json & result, nlohmann::json const & shadow ); TracingJSON( std::shared_ptr< nlohmann::json > originalJSON, diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 436812f50c..5ebe86611a 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1471,8 +1471,8 @@ ADIOS2IOHandler::ADIOS2IOHandler( openPMD::AccessType at, MPI_Comm comm, nlohmann::json options ) - : AbstractIOHandler( std::move( path ), at, comm ) - , m_impl{ this, comm, std::move( options ) } + : AbstractIOHandler( std::move( path ), at, comm ), + m_impl{ this, comm, std::move( options ) } { } @@ -1482,8 +1482,8 @@ ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at, nlohmann::json options ) - : AbstractIOHandler( std::move( path ), at ) - , m_impl{ this, std::move( options ) } + : AbstractIOHandler( std::move( path ), at ), + m_impl{ this, std::move( options ) } { } @@ -1499,8 +1499,8 @@ ADIOS2IOHandler::flush() ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at, - MPI_Comm comm - , nlohmann::json + MPI_Comm comm, + nlohmann::json ) : AbstractIOHandler( std::move( path ), at, comm ) { diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index d0b10b152b..4b23c98a68 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -1,9 +1,30 @@ -#include "openPMD/auxiliary/JSON.hpp" +/* Copyright 2020 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ -#include +#include "openPMD/auxiliary/JSON.hpp" #include "openPMD/config.hpp" +#include + namespace openPMD { namespace auxiliary @@ -14,10 +35,10 @@ namespace auxiliary TracingJSON::TracingJSON( nlohmann::json originalJSON ) : m_originalJSON( - std::make_shared< nlohmann::json >( std::move( originalJSON ) ) ) - , m_shadow( std::make_shared< nlohmann::json >() ) - , m_positionInOriginal( &*m_originalJSON ) - , m_positionInShadow( &*m_shadow ) + std::make_shared< nlohmann::json >( std::move( originalJSON ) ) ), + m_shadow( std::make_shared< nlohmann::json >() ), + m_positionInOriginal( &*m_originalJSON ), + m_positionInShadow( &*m_shadow ) { } @@ -38,7 +59,7 @@ namespace auxiliary void TracingJSON::invertShadow( nlohmann::json & result, - nlohmann::json & shadow ) + nlohmann::json const & shadow ) { if( !shadow.is_object() ) { @@ -83,11 +104,11 @@ namespace auxiliary nlohmann::json * positionInOriginal, nlohmann::json * positionInShadow, bool trace ) - : m_originalJSON( std::move( originalJSON ) ) - , m_shadow( std::move( shadow ) ) - , m_positionInOriginal( positionInOriginal ) - , m_positionInShadow( positionInShadow ) - , m_trace( trace ) + : m_originalJSON( std::move( originalJSON ) ), + m_shadow( std::move( shadow ) ), + m_positionInOriginal( positionInOriginal ), + m_positionInShadow( positionInShadow ), + m_trace( trace ) { } } // namespace auxiliary diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 02a86005b6..e033fe8b2b 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -3,6 +3,13 @@ # define OPENPMD_private public # define OPENPMD_protected public #endif + +#include "openPMD/auxiliary/Environment.hpp" +#include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/openPMD.hpp" + +#include + #include #include #include @@ -16,11 +23,6 @@ #include #include -#include "openPMD/auxiliary/Filesystem.hpp" -#include "openPMD/auxiliary/Environment.hpp" -#include "openPMD/openPMD.hpp" -#include - using namespace openPMD; std::vector> getBackends() { @@ -2380,7 +2382,7 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) } } )END"; - auto write = []( std::string const & filename, + auto const write = []( std::string const & filename, std::string const & config ) { openPMD::Series series( filename, openPMD::AccessType::CREATE, config ); auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ]; @@ -2419,7 +2421,7 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) } } )END"; - auto read = []( std::string const & filename, std::string const & config ) { + auto const read = []( std::string const & filename, std::string const & config ) { openPMD::Series series( filename, openPMD::AccessType::READ_ONLY, config ); auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ];