diff --git a/CMakeLists.txt b/CMakeLists.txt index ab4d090919..fdd27320c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,6 +307,7 @@ set(CORE_SOURCE src/version.cpp src/auxiliary/Date.cpp src/auxiliary/Filesystem.cpp + src/auxiliary/JSON.cpp src/backend/Attributable.cpp src/backend/BaseRecordComponent.cpp src/backend/MeshRecordComponent.cpp diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index c247afafa2..a02eeee831 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -85,4 +85,5 @@ The example code in the :ref:`usage section ` will produce the fol when picking the JSON backend: .. literalinclude:: json_example.json + :language: json diff --git a/docs/source/details/adios2.json b/docs/source/details/adios2.json new file mode 100644 index 0000000000..9c5ccb59ba --- /dev/null +++ b/docs/source/details/adios2.json @@ -0,0 +1,19 @@ +{ + "adios2": { + "engine": { + "type": "sst", + "parameters": { + "BufferGrowthFactor": "2.0", + "QueueLimit": "2" + } + }, + "dataset": { + "operators": [ + { + "type": "bzip2", + "parameters": {} + } + ] + } + } +} diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst new file mode 100644 index 0000000000..103197950d --- /dev/null +++ b/docs/source/details/backendconfig.rst @@ -0,0 +1,62 @@ +.. _backendconfig + +Backend-Specific Configuration +============================== + +While the openPMD API intends to be a backend-*independent* implementation of the openPMD standard, it is sometimes useful to pass configuration parameters to the specific backend in use. +:ref:`For each backend `, configuration options can be passed via a JSON-formatted string or via environment variables. +A JSON option always takes precedence over an environment variable. + +The fundamental structure of this JSON configuration string is given as follows: + +.. literalinclude:: config_layout.json + :language: json + +This structure allows keeping one configuration string for several backends at once, with the concrete backend configuration being chosen upon choosing the backend itself. + +The configuration is read in a case-sensitive manner. +Generally, keys of the configuration are *lower case*. +Parameters that are directly passed through to an external library and not interpreted within openPMD API (e.g. ``adios2.engine.parameters``) are unaffected by this and follow the respective library's conventions. + +The configuration string may refer to the complete ``openPMD::Series`` or may additionally be specified per ``openPMD::Dataset``, passed in the respective constructors. +*A configuration per dataset is currently not yet implemented.* +This reflects the fact that certain backend-specific parameters may refer to the whole Series (such as storage engines and their parameters) and others refer to actual datasets (such as compression). + +For a consistent user interface, backends shall follow the following rules: + +* The configuration structures for the Series and for each dataset should be defined equivalently. +* Any setting referring to single datasets should also be applicable globally, affecting all datasets. +* If a setting is defined globally, but also for a concrete dataset, the dataset-specific setting should override the global one. +* If a setting is passed to a dataset that only makes sense globally (such as the storage engine), the setting should be ignored except for printing a warning. + Backends should define clearly which keys are applicable to datasets and which are not. + + +Configuration Structure per Backend +----------------------------------- + +ADIOS2 +^^^^^^ + +A full configuration of the ADIOS2 backend: + +.. literalinclude:: adios2.json + :language: json + +All keys found under ``adios2.dataset`` are applicable globally as well as per dataset, keys found under ``adios2.engine`` only globally. +Explanation of the single keys: + +* ``adios2.engine.type``: A string that is passed directly to ``adios2::IO:::SetEngine`` for choosing the ADIOS2 engine to be used. + Please refer to the `official ADIOS2 documentation `_ for a list of available engines. +* ``adios2.engine.type``: An associative array of string-formatted engine parameters, passed directly through to ``adios2::IO::SetParameters``. + Please refer to the official ADIOS2 documentation for the allowable engine parameters. +* ``adios2.dataset.operators``: (*currently unimplemented* – please use the ``openPMD::Dataset::compression`` for the meantime) This key contains a list of ADIOS2 `operators `_, used to enable compression or dataset transformations. + Each object in the list has three keys: + + * ``type`` supported ADIOS operator type, e.g. zfp, sz + * ``parameters`` is an associative map of string parameters for the operator (e.g. compression levels) + +Other backends +^^^^^^^^^^^^^^ + +Do currently not read the configuration string. +Please refer to the respective backends' documentations for further information on their configuration. diff --git a/docs/source/details/config_layout.json b/docs/source/details/config_layout.json new file mode 100644 index 0000000000..1e86d85876 --- /dev/null +++ b/docs/source/details/config_layout.json @@ -0,0 +1,6 @@ +{ + "adios": "put ADIOS config here", + "adios2": "put ADIOS2 config here", + "hdf5": "put HDF5 config here", + "json": "put JSON config here" +} diff --git a/docs/source/index.rst b/docs/source/index.rst index 6a91b65326..d8f187d31e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -85,6 +85,7 @@ API Details details/doxygen.rst details/python.rst details/mpi.rst + details/backendconfig.rst Utilities --------- diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index d6706df1f8..8d1f1cbbd5 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -20,27 +20,16 @@ */ #pragma once -#include "openPMD/config.hpp" - #include "ADIOS2FilePosition.hpp" +#include "openPMD/config.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IO/InvalidatableFile.hpp" +#include "openPMD/auxiliary/JSON.hpp" #include "openPMD/backend/Writable.hpp" -#include -#include -#include -#include -#include // shared_ptr -#include -#include -#include // pair -#include - - #if openPMD_HAVE_ADIOS2 # include # include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" @@ -50,6 +39,17 @@ # include #endif +#include + +#include +#include +#include +#include +#include // shared_ptr +#include +#include +#include // pair +#include namespace openPMD { @@ -95,13 +95,16 @@ class ADIOS2IOHandlerImpl #if openPMD_HAVE_MPI - ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm ); + ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm, nlohmann::json config ); MPI_Comm m_comm; #endif // openPMD_HAVE_MPI - explicit ADIOS2IOHandlerImpl( AbstractIOHandler * ); + explicit ADIOS2IOHandlerImpl( + AbstractIOHandler * + , nlohmann::json config + ); ~ADIOS2IOHandlerImpl( ) override = default; @@ -165,8 +168,6 @@ class ADIOS2IOHandlerImpl listAttributes( Writable *, Parameter< Operation::LIST_ATTS > & parameters ) override; - - /** * @brief The ADIOS2 access type to chose for Engines opened * within this instance. @@ -177,6 +178,55 @@ class ADIOS2IOHandlerImpl private: adios2::ADIOS m_ADIOS; + struct ParameterizedOperator + { + adios2::Operator const op; + adios2::Params const params; + }; + + std::vector< ParameterizedOperator > defaultOperators; + + auxiliary::TracingJSON m_config; + static auxiliary::TracingJSON nullvalue; + + void + init( nlohmann::json config ); + + template< typename Key > + auxiliary::TracingJSON + config( Key && key, auxiliary::TracingJSON & cfg ) + { + if( cfg.json().is_object() && cfg.json().contains( key ) ) + { + return cfg[ key ]; + } + else + { + return nullvalue; + } + } + + template< typename Key > + auxiliary::TracingJSON + config( Key && key ) + { + return config< Key >( std::forward< Key >( key ), m_config ); + } + + /** + * + * @param config The top-level of the ADIOS2 configuration JSON object + * with operators to be found under dataset.operators + * @return first parameter: the operators, second parameters: whether + * operators have been configured + */ + std::pair< std::vector< ParameterizedOperator >, bool > + getOperators( auxiliary::TracingJSON config ); + + // use m_config + std::pair< std::vector< ParameterizedOperator >, bool > + getOperators(); + /* * We need to give names to IO objects. These names are irrelevant * within this application, since: @@ -258,6 +308,18 @@ class ADIOS2IOHandlerImpl std::string const & var ); }; // ADIOS2IOHandlerImpl +/* + * The following strings are used during parsing of the JSON configuration + * string for the ADIOS2 backend. + */ +namespace ADIOS2Defaults +{ + using const_str = char const * const; + constexpr const_str str_engine = "engine"; + constexpr const_str str_type = "type"; + constexpr const_str str_params = "parameters"; +} + namespace detail { // Helper structs for calls to the switchType function @@ -332,17 +394,13 @@ namespace detail struct VariableDefiner { + // Parameters such as DatasetHelper< T >::defineVariable + template < typename T, typename... Params > + void operator( )( Params &&... params ); - template < typename T > - void operator( )( adios2::IO & IO, const std::string & name, - std::unique_ptr< adios2::Operator > compression, - const adios2::Dims & shape = adios2::Dims( ), - const adios2::Dims & start = adios2::Dims( ), - const adios2::Dims & count = adios2::Dims( ), - const bool constantDims = false ); - - template < int n, typename... Params > - void operator( )( adios2::IO & IO, Params &&... ); + template< int n, typename... Params > + void + operator()( Params &&... ); }; @@ -473,11 +531,31 @@ namespace detail void readDataset( BufferedGet &, adios2::IO &, adios2::Engine &, std::string const & fileName ); + /** + * @brief Define a Variable of type T within the passed IO. + * + * @param IO The adios2::IO object within which to define the + * variable. The variable can later be retrieved from + * the IO using the passed name. + * @param name As in adios2::IO::DefineVariable + * @param compressions ADIOS2 operators, including an arbitrary + * number of parameters, to be added to the + * variable upon definition. + * @param shape As in adios2::IO::DefineVariable + * @param start As in adios2::IO::DefineVariable + * @param count As in adios2::IO::DefineVariable + * @param constantDims As in adios2::IO::DefineVariable + */ static void - defineVariable( adios2::IO & IO, const std::string & name, - std::unique_ptr< adios2::Operator > compression, - const adios2::Dims & shape, const adios2::Dims & start, - const adios2::Dims & count, bool constantDims ); + defineVariable( + adios2::IO & IO, + std::string const & name, + std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > const & + compressions, + adios2::Dims const & shape = adios2::Dims(), + adios2::Dims const & start = adios2::Dims(), + adios2::Dims const & count = adios2::Dims(), + bool const constantDims = false ); void writeDataset( BufferedPut &, adios2::IO &, adios2::Engine & ); }; @@ -570,6 +648,9 @@ namespace detail ~BufferedActions( ); + void + configure_IO( ADIOS2IOHandlerImpl & impl ); + adios2::Engine & getEngine( ); template < typename BA > void enqueue( BA && ba ); @@ -666,11 +747,16 @@ friend class ADIOS2IOHandlerImpl; #if openPMD_HAVE_MPI - ADIOS2IOHandler( std::string path, AccessType, MPI_Comm ); + ADIOS2IOHandler( + std::string path, + AccessType, + MPI_Comm, + nlohmann::json options + ); #endif - ADIOS2IOHandler( std::string path, AccessType ); + ADIOS2IOHandler( std::string path, AccessType, nlohmann::json options ); std::string backendName() const override { return "ADIOS2"; } diff --git a/include/openPMD/IO/AbstractIOHandlerHelper.hpp b/include/openPMD/IO/AbstractIOHandlerHelper.hpp index deb52de92e..92e0b05380 100644 --- a/include/openPMD/IO/AbstractIOHandlerHelper.hpp +++ b/include/openPMD/IO/AbstractIOHandlerHelper.hpp @@ -34,28 +34,34 @@ namespace openPMD * @param accessType AccessType describing desired operations and permissions of the desired handler. * @param format Format describing the IO backend of the desired handler. * @param comm MPI communicator used for IO. + * @param options JSON-formatted option string, to be interpreted by + * the backend. * @return Smart pointer to created IOHandler. */ - std::shared_ptr< AbstractIOHandler > - createIOHandler( - std::string path, - AccessType accessType, - Format format, - MPI_Comm comm - ); +std::shared_ptr< AbstractIOHandler > +createIOHandler( + std::string path, + AccessType accessType, + Format format, + MPI_Comm comm, + std::string const & options = "{}" ); #endif - /** Construct an appropriate specific IOHandler for the desired IO mode. - * - * @param path Path to root folder for all operations associated with the desired handler. - * @param accessType AccessType describing desired operations and permissions of the desired handler. - * @param format Format describing the IO backend of the desired handler. - * @return Smart pointer to created IOHandler. - */ - std::shared_ptr< AbstractIOHandler > - createIOHandler( - std::string path, - AccessType accessType, - Format format - ); -} // openPMD +/** Construct an appropriate specific IOHandler for the desired IO mode. + * + * @param path Path to root folder for all operations associated with + * the desired handler. + * @param accessType AccessType describing desired operations and permissions + * of the desired handler. + * @param format Format describing the IO backend of the desired handler. + * @param options JSON-formatted option string, to be interpreted by + * the backend. + * @return Smart pointer to created IOHandler. + */ +std::shared_ptr< AbstractIOHandler > +createIOHandler( + std::string path, + AccessType accessType, + Format format, + std::string const & options = "{}" ); +} // namespace openPMD diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 358b412db3..340898974f 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -58,12 +58,17 @@ class Series : public Attributable public: #if openPMD_HAVE_MPI - Series(std::string const& filepath, - AccessType at, - MPI_Comm comm); + Series( + std::string const & filepath, + AccessType at, + MPI_Comm comm, + std::string const & options = "{}" ); #endif - Series(std::string const& filepath, - AccessType at); + + Series( + std::string const & filepath, + AccessType at, + std::string const & options = "{}" ); ~Series(); /** diff --git a/include/openPMD/auxiliary/JSON.hpp b/include/openPMD/auxiliary/JSON.hpp new file mode 100644 index 0000000000..32d9039343 --- /dev/null +++ b/include/openPMD/auxiliary/JSON.hpp @@ -0,0 +1,159 @@ +/* Copyright 2020 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + +#include "openPMD/config.hpp" + +#include + +#include // std::shared_ptr +#include // std::forward + +namespace openPMD +{ +namespace auxiliary +{ + /** + * @brief Extend nlohmann::json with tracing of which keys have been + * accessed by operator[](). + * An access is only registered if the current JSON value is a JSON object + * (not an array) and if the accessed JSON value is a leaf, i.e. anything + * but an object. This means that objects contained in arrays will not be + * traced. + * + * If working directly with the underlying JSON value (necessary since this + * class only redefines operator[]), declareFullyRead() may be used to + * declare keys read manually. + * + */ + class TracingJSON + { + public: + TracingJSON(); + TracingJSON( nlohmann::json ); + + /** + * @brief Access the underlying JSON value + * + * @return nlohmann::json& + */ + inline nlohmann::json & + json() + { + return *m_positionInOriginal; + } + + template< typename Key > + TracingJSON operator[]( Key && key ); + + /** + * @brief Get the "shadow", i.e. a copy of the original JSON value + * containing all accessed object keys. + * + * @return nlohmann::json const& + */ + nlohmann::json const & + getShadow(); + + /** + * @brief Invert the "shadow", i.e. a copy of the original JSON value + * that contains exactly those values that have not been accessed yet. + * + * @return nlohmann::json + */ + nlohmann::json + invertShadow(); + + /** + * @brief Declare all keys of the current object read. + * + */ + void + declareFullyRead(); + + private: + /** + * @brief The JSON object with which this class has been initialized. + * Shared pointer shared between all instances returned by + * operator[]() in order to avoid use-after-free situations. + * + */ + std::shared_ptr< nlohmann::json > m_originalJSON; + /** + * @brief A JSON object keeping track of all accessed indices within the + * original JSON object. Initially an empty JSON object, + * gradually filled by applying each operator[]() call also to + * it. + * Shared pointer shared between all instances returned by + * operator[]() in order to avoid use-after-free situations. + * + */ + std::shared_ptr< nlohmann::json > m_shadow; + /** + * @brief The sub-expression within m_originalJSON corresponding with + * the current instance. + * + */ + nlohmann::json * m_positionInOriginal; + /** + * @brief The sub-expression within m_positionInOriginal corresponding + * with the current instance. + * + */ + nlohmann::json * m_positionInShadow; + bool m_trace = true; + + void + invertShadow( nlohmann::json & result, nlohmann::json const & shadow ); + + TracingJSON( + std::shared_ptr< nlohmann::json > originalJSON, + std::shared_ptr< nlohmann::json > shadow, + nlohmann::json * positionInOriginal, + nlohmann::json * positionInShadow, + bool trace ); + }; + + template< typename Key > + TracingJSON TracingJSON::operator[]( Key && key ) + { + nlohmann::json * newPositionInOriginal = + &m_positionInOriginal->operator[]( key ); + // If accessing a leaf in the JSON tree from an object (not an array!) + // erase the corresponding key + static nlohmann::json nullvalue; + nlohmann::json * newPositionInShadow = &nullvalue; + if( m_trace && m_positionInOriginal->is_object() ) + { + newPositionInShadow = &m_positionInShadow->operator[]( key ); + } + bool traceFurther = newPositionInOriginal->is_object(); + return TracingJSON( + m_originalJSON, + m_shadow, + newPositionInOriginal, + newPositionInShadow, + traceFurther ); + } +} // namespace auxiliary +} // namespace openPMD + diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 99427bfee3..5ebe86611a 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -20,18 +20,20 @@ */ #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" + +#include +#include +#include +#include +#include + #include "openPMD/Datatype.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" - -#include -#include #include -#include -#include namespace openPMD @@ -58,27 +60,98 @@ namespace openPMD #if openPMD_HAVE_ADIOS2 -#if openPMD_HAVE_MPI +# if openPMD_HAVE_MPI ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler * handler, - MPI_Comm communicator ) + MPI_Comm communicator, + nlohmann::json cfg ) : AbstractIOHandlerImplCommon( handler ) , m_comm{ communicator } , m_ADIOS{ communicator, ADIOS2_DEBUG_MODE } { + init( std::move( cfg ) ); } -#endif // openPMD_HAVE_MPI +# endif // openPMD_HAVE_MPI -ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler * handler ) +ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( + AbstractIOHandler * handler, + nlohmann::json cfg ) : AbstractIOHandlerImplCommon( handler ), m_ADIOS{ ADIOS2_DEBUG_MODE } { + init( std::move( cfg ) ); +} + +void +ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) +{ + if( !cfg.contains( "adios2" ) ) + { + std::cerr << "Warning: ADIOS2 is not configured in the JSON " + "configuration. Running with default settings." + << std::endl; + return; + } + m_config = std::move( cfg[ "adios2" ] ); + defaultOperators = getOperators().first; +} + +std::pair< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator >, bool > +ADIOS2IOHandlerImpl::getOperators( auxiliary::TracingJSON cfg ) +{ + std::vector< ParameterizedOperator > res; + if( !cfg.json().contains( "dataset" ) ) + { + return std::make_pair( res, false ); + } + auto datasetConfig = cfg[ "dataset" ]; + if( !datasetConfig.json().contains( "operators" ) ) + { + return std::make_pair( res, false ); + } + auto _operators = datasetConfig[ "operators" ]; + nlohmann::json const & operators = _operators.json(); + for( auto operatorIterator = operators.begin(); + operatorIterator != operators.end(); + ++operatorIterator ) + { + nlohmann::json const & op = operatorIterator.value(); + std::string const & type = op[ "type" ]; + adios2::Params adiosParams; + if( op.contains( "parameters" ) ) + { + nlohmann::json const & params = op[ "parameters" ]; + for( auto paramIterator = params.begin(); + paramIterator != params.end(); + ++paramIterator ) + { + adiosParams[ paramIterator.key() ] = + paramIterator.value().get< std::string >(); + } + } + std::unique_ptr< adios2::Operator > adiosOperator = + getCompressionOperator( type ); + if( adiosOperator ) + { + res.emplace_back( ParameterizedOperator{ + *adiosOperator, std::move( adiosParams ) } ); + } + } + _operators.declareFullyRead(); + return std::make_pair( res, true ); +} + +std::pair< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator >, bool > +ADIOS2IOHandlerImpl::getOperators() +{ + return getOperators( m_config ); } -std::future< void > ADIOS2IOHandlerImpl::flush( ) +std::future< void > +ADIOS2IOHandlerImpl::flush() { - auto res = AbstractIOHandlerImpl::flush( ); + auto res = AbstractIOHandlerImpl::flush(); for ( auto & p : m_fileData ) { if ( m_dirty.find( p.first ) != m_dirty.end( ) ) @@ -192,17 +265,32 @@ void ADIOS2IOHandlerImpl::createDataset( * the C++11 standard * @todo replace with std::optional upon switching to C++17 */ - std::unique_ptr< adios2::Operator > compression; - if ( !parameters.compression.empty( ) ) - compression = getCompressionOperator( parameters.compression ); + + auto operators = defaultOperators; + + if( !parameters.compression.empty() ) + { + std::unique_ptr< adios2::Operator > adiosOperator = + getCompressionOperator( parameters.compression ); + if( adiosOperator ) + { + operators.push_back( ParameterizedOperator{ + *adiosOperator, + adios2::Params() } ); + } + } // cast from openPMD::Extent to adios2::Dims adios2::Dims const shape( parameters.extent.begin(), parameters.extent.end() ); auto & fileData = getFileData( file ); - switchType( parameters.dtype, detail::VariableDefiner( ), - fileData.m_IO, varName, - std::move( compression ), shape ); + switchType( + parameters.dtype, + detail::VariableDefiner(), + fileData.m_IO, + varName, + operators, + shape ); fileData.invalidateVariablesMap(); writable->written = true; m_dirty.emplace( file ); @@ -496,8 +584,7 @@ void ADIOS2IOHandlerImpl::listAttributes( } } -adios2::Mode -ADIOS2IOHandlerImpl::adios2Accesstype() +adios2::Mode ADIOS2IOHandlerImpl::adios2Accesstype( ) { switch ( m_handler->accessTypeBackend ) { @@ -515,7 +602,10 @@ ADIOS2IOHandlerImpl::adios2Accesstype() } } -std::string ADIOS2IOHandlerImpl::filePositionToString( +auxiliary::TracingJSON ADIOS2IOHandlerImpl::nullvalue = nlohmann::json(); + +std::string +ADIOS2IOHandlerImpl::filePositionToString( std::shared_ptr< ADIOS2FilePosition > filepos ) { return filepos->location; @@ -796,21 +886,20 @@ namespace detail throw std::runtime_error( "[ADIOS2] WRITE_DATASET: Invalid datatype." ); } - template < typename T > + template < typename T, typename... Params > void VariableDefiner:: - operator( )( adios2::IO & IO, const std::string & name, - std::unique_ptr< adios2::Operator > compression, - const adios2::Dims & shape, const adios2::Dims & start, - const adios2::Dims & count, const bool constantDims ) + operator( )( Params &&... params ) { - DatasetHelper< T >::defineVariable( IO, name, std::move( compression ), - shape, start, count, constantDims ); + DatasetHelper< T >::defineVariable( + std::forward< Params >( params )... ); } - template < int n, typename... Params > - void VariableDefiner::operator( )( adios2::IO &, Params &&... ) + template< int n, typename... Params > + void + VariableDefiner::operator()( Params &&... ) { - throw std::runtime_error( "[ADIOS2] Defining a variable with undefined type." ); + throw std::runtime_error( + "[ADIOS2] Defining a variable with undefined type." ); } @@ -975,13 +1064,20 @@ namespace detail engine.Get( var, ptr ); } - template < typename T > - void DatasetHelper< - T, typename std::enable_if< DatasetTypes< T >::validType >::type >:: - defineVariable( adios2::IO & IO, const std::string & name, - std::unique_ptr< adios2::Operator > compression, - const adios2::Dims & shape, const adios2::Dims & start, - const adios2::Dims & count, const bool constantDims ) + template< typename T > + void + DatasetHelper< + T, + typename std::enable_if< DatasetTypes< T >::validType >::type >:: + defineVariable( + adios2::IO & IO, + const std::string & name, + std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > const & + compressions, + const adios2::Dims & shape, + const adios2::Dims & start, + const adios2::Dims & count, + const bool constantDims ) { adios2::Variable< T > var = IO.DefineVariable< T >( name, shape, start, count, constantDims ); @@ -990,11 +1086,12 @@ namespace detail throw std::runtime_error( "[ADIOS2] Internal error: Could not create Variable '" + name + "'." ); } - // check whether the unique_ptr has an element - // and whether the held operator is valid - if ( compression && *compression ) + for( auto const & compression : compressions ) { - var.AddOperation( *compression ); + if( compression.op ) + { + var.AddOperation( compression.op, compression.params ); + } } } @@ -1101,6 +1198,7 @@ namespace detail *param.dtype = ret; } + BufferedActions::BufferedActions( ADIOS2IOHandlerImpl & impl, InvalidatableFile file ) : m_file( impl.fullPath( std::move( file ) ) ), @@ -1116,13 +1214,72 @@ namespace detail } else { - // read parameters from environment - auto const engine = auxiliary::getEnvString( "OPENPMD_ADIOS2_ENGINE", "File" ); + configure_IO(impl); + } + } + + BufferedActions::~BufferedActions() + { + // if write accessing, ensure that the engine is opened + if( !m_engine && m_mode != adios2::Mode::Read ) + { + getEngine(); + } + if( m_engine ) + { + m_engine->Close(); + } + } + + void + BufferedActions::configure_IO( ADIOS2IOHandlerImpl & impl ) + { + (void)impl; + std::set< std::string > alreadyConfigured; + auto engineConfig = impl.config( ADIOS2Defaults::str_engine ); + if( !engineConfig.json().is_null() ) + { + m_IO.SetEngine( + impl.config( ADIOS2Defaults::str_type, engineConfig ).json() ); + auto params = + impl.config( ADIOS2Defaults::str_params, engineConfig ); + params.declareFullyRead(); + if( params.json().is_object() ) + { + for( auto it = params.json().begin(); it != params.json().end(); + it++ ) + { + m_IO.SetParameter( it.key(), it.value() ); + alreadyConfigured.emplace( it.key() ); + } + } + alreadyConfigured.emplace( "Engine" ); + } + auto shadow = impl.m_config.invertShadow(); + if( shadow.size() > 0 ) + { + std::cerr << "Warning: parts of the JSON configuration for ADIOS2 " + "remain unused:\n" + << shadow << std::endl; + } + auto notYetConfigured = + [&alreadyConfigured]( std::string const & param ) { + auto it = alreadyConfigured.find( param ); + return it == alreadyConfigured.end(); + }; + + // read parameters from environment + if( notYetConfigured( "Engine" ) ) + { + auto const engine = + auxiliary::getEnvString( "OPENPMD_ADIOS2_ENGINE", "File" ); m_IO.SetEngine( engine ); + } - if ( 1 == - auxiliary::getEnvNum( - "OPENPMD_ADIOS2_HAVE_METADATA_FILE", 1 ) ) + if( notYetConfigured( "CollectiveMetadata" ) ) + { + if( 1 == + auxiliary::getEnvNum( "OPENPMD_ADIOS2_HAVE_METADATA_FILE", 1 ) ) { m_IO.SetParameter( "CollectiveMetadata", "On" ); } @@ -1130,9 +1287,13 @@ namespace detail { m_IO.SetParameter( "CollectiveMetadata", "Off" ); } - - if ( 1 == - auxiliary::getEnvNum( "OPENPMD_ADIOS2_HAVE_PROFILING", 1 ) ) + } + if( notYetConfigured( "Profile" ) ) + { + if( 1 == + auxiliary::getEnvNum( + "OPENPMD_ADIOS2_HAVE_PROFILING", 1 ) && + notYetConfigured( "Profile" ) ) { m_IO.SetParameter( "Profile", "On" ); } @@ -1140,34 +1301,22 @@ namespace detail { m_IO.SetParameter( "Profile", "Off" ); } + } #if openPMD_HAVE_MPI + { + auto num_substreams = + auxiliary::getEnvNum( "OPENPMD_ADIOS2_NUM_SUBSTREAMS", 0 ); + if( notYetConfigured( "SubStreams" ) && 0 != num_substreams ) { - auto num_substreams = - auxiliary::getEnvNum( "OPENPMD_ADIOS2_NUM_SUBSTREAMS", 0 ); - if ( 0 != num_substreams ) - { - m_IO.SetParameter( "SubStreams", - std::to_string( num_substreams ) ); - } + m_IO.SetParameter( + "SubStreams", std::to_string( num_substreams ) ); } -#endif - } - } - - BufferedActions::~BufferedActions( ) - { - // if write accessing, ensure that the engine is opened - if ( !m_engine && m_mode != adios2::Mode::Read ) - { - getEngine( ); - } - if ( m_engine ) - { - m_engine->Close( ); } +#endif } - adios2::Engine & BufferedActions::getEngine( ) + adios2::Engine & + BufferedActions::getEngine() { if ( !m_engine ) { @@ -1315,41 +1464,55 @@ namespace detail } // namespace detail -#if openPMD_HAVE_MPI - -ADIOS2IOHandler::ADIOS2IOHandler( std::string path, openPMD::AccessType at, - MPI_Comm comm ) -: AbstractIOHandler( std::move( path ), at, comm ), m_impl{this, comm +# if openPMD_HAVE_MPI - } +ADIOS2IOHandler::ADIOS2IOHandler( + std::string path, + openPMD::AccessType at, + MPI_Comm comm, + nlohmann::json options ) + : AbstractIOHandler( std::move( path ), at, comm ), + m_impl{ this, comm, std::move( options ) } { } #endif -ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at ) -: AbstractIOHandler( std::move( path ), at ), m_impl{this} +ADIOS2IOHandler::ADIOS2IOHandler( + std::string path, + AccessType at, + nlohmann::json options ) + : AbstractIOHandler( std::move( path ), at ), + m_impl{ this, std::move( options ) } { } -std::future< void > ADIOS2IOHandler::flush( ) +std::future< void > +ADIOS2IOHandler::flush() { - return m_impl.flush( ); + return m_impl.flush(); } #else // openPMD_HAVE_ADIOS2 -#if openPMD_HAVE_MPI -ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at, - MPI_Comm comm ) -: AbstractIOHandler( std::move( path ), at, comm ) +# if openPMD_HAVE_MPI +ADIOS2IOHandler::ADIOS2IOHandler( + std::string path, + AccessType at, + MPI_Comm comm, + nlohmann::json +) + : AbstractIOHandler( std::move( path ), at, comm ) { } -#endif +# endif // openPMD_HAVE_MPI -ADIOS2IOHandler::ADIOS2IOHandler( std::string path, AccessType at ) -: AbstractIOHandler( std::move( path ), at ) +ADIOS2IOHandler::ADIOS2IOHandler( + std::string path, + AccessType at, + nlohmann::json ) + : AbstractIOHandler( std::move( path ), at ) { } diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 331f9bb0fc..9ba5a01691 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -19,14 +19,15 @@ * If not, see . */ #include "openPMD/IO/AbstractIOHandlerHelper.hpp" -#include "openPMD/IO/DummyIOHandler.hpp" + #include "openPMD/IO/ADIOS/ADIOS1IOHandler.hpp" -#include "openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" +#include "openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp" +#include "openPMD/IO/DummyIOHandler.hpp" #include "openPMD/IO/HDF5/HDF5IOHandler.hpp" #include "openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp" #include "openPMD/IO/JSON/JSONIOHandler.hpp" - +#include namespace openPMD { @@ -36,9 +37,10 @@ namespace openPMD std::string path, AccessType accessTypeBackend, Format format, - MPI_Comm comm - ) + MPI_Comm comm, + std::string const & options ) { + nlohmann::json optionsJson = nlohmann::json::parse( options ); switch( format ) { case Format::HDF5: @@ -50,9 +52,11 @@ namespace openPMD throw std::runtime_error("openPMD-api built without ADIOS1 support"); # endif case Format::ADIOS2: - return std::make_shared(path, accessTypeBackend, comm); + return std::make_shared< ADIOS2IOHandler >( + path, accessTypeBackend, comm, std::move( optionsJson ) ); default: - throw std::runtime_error("Unknown file format! Did you specify a file ending?" ); + throw std::runtime_error( + "Unknown file format! Did you specify a file ending?" ); } } #endif @@ -60,9 +64,10 @@ namespace openPMD createIOHandler( std::string path, AccessType accessType, - Format format - ) + Format format, + std::string const & options ) { + nlohmann::json optionsJson = nlohmann::json::parse( options ); switch( format ) { case Format::HDF5: @@ -75,12 +80,14 @@ namespace openPMD #endif #if openPMD_HAVE_ADIOS2 case Format::ADIOS2: - return std::make_shared(path, accessType); -#endif + return std::make_shared< ADIOS2IOHandler >( + path, accessType, std::move( optionsJson ) ); +#endif // openPMD_HAVE_ADIOS2 case Format::JSON: - return std::make_shared< JSONIOHandler >(path, accessType); + return std::make_shared< JSONIOHandler >( path, accessType ); default: - throw std::runtime_error("Unknown file format! Did you specify a file ending?" ); + throw std::runtime_error( + "Unknown file format! Did you specify a file ending?" ); } } -} // openPMD + } // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index 4aea6aa7f9..ca91c05a6d 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -81,25 +81,30 @@ struct Series::ParsedInput }; //ParsedInput #if openPMD_HAVE_MPI -Series::Series(std::string const& filepath, - AccessType at, - MPI_Comm comm) - : iterations{Container< Iteration, uint64_t >()}, - m_iterationEncoding{std::make_shared< IterationEncoding >()} -{ - auto input = parseInput(filepath); - auto handler = createIOHandler(input->path, at, input->format, comm); - init(handler, std::move(input)); +Series::Series( + std::string const & filepath, + AccessType at, + MPI_Comm comm, + std::string const & options ) + : iterations{ Container< Iteration, uint64_t >() } + , m_iterationEncoding{ std::make_shared< IterationEncoding >() } +{ + auto input = parseInput( filepath ); + auto handler = + createIOHandler( input->path, at, input->format, comm, options ); + init( handler, std::move( input ) ); } #endif -Series::Series(std::string const& filepath, - AccessType at) - : iterations{Container< Iteration, uint64_t >()}, - m_iterationEncoding{std::make_shared< IterationEncoding >()} +Series::Series( + std::string const & filepath, + AccessType at, + std::string const & options ) + : iterations{ Container< Iteration, uint64_t >() } + , m_iterationEncoding{ std::make_shared< IterationEncoding >() } { - auto input = parseInput(filepath); - auto handler = createIOHandler(input->path, at, input->format); + auto input = parseInput( filepath ); + auto handler = createIOHandler( input->path, at, input->format, options ); init(handler, std::move(input)); } diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp new file mode 100644 index 0000000000..4b23c98a68 --- /dev/null +++ b/src/auxiliary/JSON.cpp @@ -0,0 +1,115 @@ +/* Copyright 2020 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/auxiliary/JSON.hpp" + +#include "openPMD/config.hpp" + +#include + +namespace openPMD +{ +namespace auxiliary +{ + TracingJSON::TracingJSON() : TracingJSON( nlohmann::json() ) + { + } + + TracingJSON::TracingJSON( nlohmann::json originalJSON ) + : m_originalJSON( + std::make_shared< nlohmann::json >( std::move( originalJSON ) ) ), + m_shadow( std::make_shared< nlohmann::json >() ), + m_positionInOriginal( &*m_originalJSON ), + m_positionInShadow( &*m_shadow ) + { + } + + nlohmann::json const & + TracingJSON::getShadow() + { + return *m_positionInShadow; + } + + nlohmann::json + TracingJSON::invertShadow() + { + nlohmann::json inverted = *m_positionInOriginal; + invertShadow( inverted, *m_positionInShadow ); + return inverted; + } + + void + TracingJSON::invertShadow( + nlohmann::json & result, + nlohmann::json const & shadow ) + { + if( !shadow.is_object() ) + { + return; + } + std::vector< std::string > toRemove; + for( auto it = shadow.begin(); it != shadow.end(); ++it ) + { + nlohmann::json & partialResult = result[ it.key() ]; + if( partialResult.is_object() ) + { + invertShadow( partialResult, it.value() ); + if( partialResult.size() == 0 ) + { + toRemove.emplace_back( it.key() ); + } + } + else + { + toRemove.emplace_back( it.key() ); + } + } + for( auto const & key : toRemove ) + { + result.erase( key ); + } + } + + void + TracingJSON::declareFullyRead() + { + if( m_trace ) + { + // copy over + *m_positionInShadow = *m_positionInOriginal; + } + } + + TracingJSON::TracingJSON( + std::shared_ptr< nlohmann::json > originalJSON, + std::shared_ptr< nlohmann::json > shadow, + nlohmann::json * positionInOriginal, + nlohmann::json * positionInShadow, + bool trace ) + : m_originalJSON( std::move( originalJSON ) ), + m_shadow( std::move( shadow ) ), + m_positionInOriginal( positionInOriginal ), + m_positionInShadow( positionInShadow ), + m_trace( trace ) + { + } +} // namespace auxiliary +} // namespace openPMD diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index af3ca74cec..03c57b7f75 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -57,10 +57,14 @@ using namespace openPMD; void init_Series(py::module &m) { py::class_(m, "Series") - .def(py::init(), - py::arg("filepath"), py::arg("access_type")) + .def(py::init(), + py::arg("filepath"), py::arg("access_type"), py::arg("options") = "{}") #if openPMD_HAVE_MPI - .def(py::init([](std::string const& filepath, AccessType at, py::object &comm){ + .def(py::init([]( + std::string const& filepath, + AccessType at, + py::object &comm, + std::string const& options){ //! @todo perform mpi4py import test and check min-version //! careful: double MPI_Init risk? only import mpi4py.MPI? //! required C-API init? probably just checks: @@ -106,9 +110,12 @@ void init_Series(py::module &m) { "(Mismatched MPI at compile vs. runtime?)"); } - return new Series(filepath, at, *mpiCommPtr); + return new Series(filepath, at, *mpiCommPtr, options); }), - py::arg("filepath"), py::arg("access_type"), py::arg("mpi_communicator") + py::arg("filepath"), + py::arg("access_type"), + py::arg("mpi_communicator"), + py::arg("options") = "{}" ) #endif diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 5220cb061a..e033fe8b2b 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -3,23 +3,25 @@ # define OPENPMD_private public # define OPENPMD_protected public #endif -#include "openPMD/openPMD.hpp" + +#include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/openPMD.hpp" #include -#include #include -#include -#include -#include #include #include -#include +#include #include #include +#include #include +#include +#include #include +#include using namespace openPMD; @@ -2320,3 +2322,119 @@ TEST_CASE( "no_serial_adios1", "[serial][adios]") } #endif +#if openPMD_HAVE_ADIOS2 +TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) +{ + if( auxiliary::getEnvString( "OPENPMD_BP_BACKEND", "NOT_SET" ) == "ADIOS1" ) + { + // run this test for ADIOS2 only + return; + } + std::string writeConfigBP3 = R"END( +{ + "adios2": { + "engine": { + "type": "bp3", + "unused": "parameter", + "parameters": { + "BufferGrowthFactor": "2.0", + "Profile": "On" + } + }, + "unused": "as well", + "dataset": { + "operators": [ + { + "type": "blosc", + "parameters": { + "clevel": "1", + "doshuffle": "BLOSC_BITSHUFFLE" + } + } + ] + } + } +} +)END"; + std::string writeConfigBP4 = R"END( +{ + "adios2": { + "engine": { + "type": "bp4", + "unused": "parameter", + "parameters": { + "BufferGrowthFactor": "2.0", + "Profile": "On" + } + }, + "unused": "as well", + "dataset": { + "operators": [ + { + "type": "blosc", + "parameters": { + "clevel": "1", + "doshuffle": "BLOSC_BITSHUFFLE" + } + } + ] + } + } +} +)END"; + auto const write = []( std::string const & filename, + std::string const & config ) { + openPMD::Series series( filename, openPMD::AccessType::CREATE, config ); + auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ]; + E_x.resetDataset( + openPMD::Dataset( openPMD::Datatype::INT, { 1000 } ) ); + std::vector< int > data( 1000, 0 ); + E_x.storeChunk( data, { 0 }, { 1000 } ); + series.flush(); + }; + write( "../samples/jsonConfiguredBP4.bp", writeConfigBP4 ); + write( "../samples/jsonConfiguredBP3.bp", writeConfigBP3 ); + + // BP3 engine writes files, BP4 writes directories + REQUIRE( + openPMD::auxiliary::file_exists( "../samples/jsonConfiguredBP3.bp" ) ); + REQUIRE( openPMD::auxiliary::directory_exists( + "../samples/jsonConfiguredBP4.bp" ) ); + + std::string readConfigBP3 = R"END( +{ + "adios2": { + "engine": { + "type": "bp3", + "unused": "parameter" + } + } +} +)END"; + std::string readConfigBP4 = R"END( +{ + "adios2": { + "engine": { + "type": "bp4", + "unused": "parameter" + } + } +} +)END"; + auto const read = []( std::string const & filename, std::string const & config ) { + openPMD::Series series( + filename, openPMD::AccessType::READ_ONLY, config ); + auto E_x = series.iterations[ 0 ].meshes[ "E" ][ "x" ]; + REQUIRE( E_x.getDimensionality() == 1 ); + REQUIRE( E_x.getExtent()[ 0 ] == 1000 ); + auto chunk = E_x.loadChunk< int >( { 0 }, { 1000 } ); + series.flush(); + for( size_t i = 0; i < 1000; ++i ) + { + REQUIRE( chunk.get()[ i ] == 0 ); + } + }; + read( "../samples/jsonConfiguredBP3.bp", readConfigBP3 ); + read( "../samples/jsonConfiguredBP4.bp", readConfigBP4 ); +} +#endif