Advanced backend configuration via JSON

CMakeLists.txt

@@ -307,6 +307,7 @@ set(CORE_SOURCE
         src/version.cpp
         src/auxiliary/Date.cpp
         src/auxiliary/Filesystem.cpp
+        src/auxiliary/JSON.cpp
         src/backend/Attributable.cpp
         src/backend/BaseRecordComponent.cpp
         src/backend/MeshRecordComponent.cpp

docs/source/backends/json.rst

@@ -85,4 +85,5 @@ The example code in the :ref:`usage section <usage-serial>` will produce the fol
 when picking the JSON backend:
 
 .. literalinclude:: json_example.json
+   :language: json
 

docs/source/details/adios2.json

@@ -0,0 +1,19 @@
+{
+  "adios2": {
+    "engine": {
+      "type": "sst",
+      "parameters": {
+        "BufferGrowthFactor": "2.0",
+        "QueueLimit": "2"
+      }
+    },
+    "dataset": {
+      "operators": [ 
+        {
+          "type": "bzip2",
+          "parameters": {}
+        }
+      ]
+    }
+  }
+}

docs/source/details/backendconfig.rst

@@ -0,0 +1,62 @@
+.. _backendconfig
+
+Backend-Specific Configuration
+==============================
+
+While the openPMD API intends to be a backend-*independent* implementation of the openPMD standard, it is sometimes useful to pass configuration parameters to the specific backend in use.
+:ref:`For each backend <backends-overview>`, configuration options can be passed via a JSON-formatted string or via environment variables.
+A JSON option always takes precedence over an environment variable.
+
+The fundamental structure of this JSON configuration string is given as follows:
+
+.. literalinclude:: config_layout.json
+   :language: json
+
+This structure allows keeping one configuration string for several backends at once, with the concrete backend configuration being chosen upon choosing the backend itself.
+
+The configuration is read in a case-sensitive manner.
+Generally, keys of the configuration are *lower case*.
+Parameters that are directly passed through to an external library and not interpreted within openPMD API (e.g. ``adios2.engine.parameters``) are unaffected by this and follow the respective library's conventions.
+
+The configuration string may refer to the complete ``openPMD::Series`` or may additionally be specified per ``openPMD::Dataset``, passed in the respective constructors.
+*A configuration per dataset is currently not yet implemented.*
+This reflects the fact that certain backend-specific parameters may refer to the whole Series (such as storage engines and their parameters) and others refer to actual datasets (such as compression).
+
+For a consistent user interface, backends shall follow the following rules:
+
+* The configuration structures for the Series and for each dataset should be defined equivalently.
+* Any setting referring to single datasets should also be applicable globally, affecting all datasets.
+* If a setting is defined globally, but also for a concrete dataset, the dataset-specific setting should override the global one.
+* If a setting is passed to a dataset that only makes sense globally (such as the storage engine), the setting should be ignored except for printing a warning.
+  Backends should define clearly which keys are applicable to datasets and which are not.
+
+
+Configuration Structure per Backend
+-----------------------------------
+
+ADIOS2
+^^^^^^
+
+A full configuration of the ADIOS2 backend:
+
+.. literalinclude:: adios2.json
+   :language: json
+
+All keys found under ``adios2.dataset`` are applicable globally as well as per dataset, keys found under ``adios2.engine`` only globally.
+Explanation of the single keys:
+
+* ``adios2.engine.type``: A string that is passed directly to ``adios2::IO:::SetEngine`` for choosing the ADIOS2 engine to be used.
+  Please refer to the `official ADIOS2 documentation <https://adios2.readthedocs.io/en/latest/engines/engines.html>`_ for a list of available engines.
+* ``adios2.engine.type``: An associative array of string-formatted engine parameters, passed directly through to ``adios2::IO::SetParameters``.
+  Please refer to the official ADIOS2 documentation for the allowable engine parameters.
+* ``adios2.dataset.operators``: (*currently unimplemented* – please use the ``openPMD::Dataset::compression`` for the meantime) This key contains a list of ADIOS2 `operators <https://adios2.readthedocs.io/en/latest/components/components.html#operator>`_, used to enable compression or dataset transformations.
+  Each object in the list has three keys:
+
+  * ``type`` supported ADIOS operator type, e.g. zfp, sz
+  * ``parameters`` is an associative map of string parameters for the operator (e.g. compression levels)
+
+Other backends
+^^^^^^^^^^^^^^
+
+Do currently not read the configuration string.
+Please refer to the respective backends' documentations for further information on their configuration.

docs/source/details/config_layout.json

@@ -0,0 +1,6 @@
+{
+    "adios": "put ADIOS config here",
+    "adios2": "put ADIOS2 config here",
+    "hdf5": "put HDF5 config here",
+    "json": "put JSON config here"
+}

docs/source/index.rst

@@ -85,6 +85,7 @@ API Details
    details/doxygen.rst
    details/python.rst
    details/mpi.rst
+   details/backendconfig.rst
 
 Utilities
 ---------

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -20,27 +20,16 @@
  */
 #pragma once
 
-#include "openPMD/config.hpp"
-
 #include "ADIOS2FilePosition.hpp"
+#include "openPMD/config.hpp"
 #include "openPMD/IO/AbstractIOHandler.hpp"
 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
 #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp"
 #include "openPMD/IO/IOTask.hpp"
 #include "openPMD/IO/InvalidatableFile.hpp"
+#include "openPMD/auxiliary/JSON.hpp"
 #include "openPMD/backend/Writable.hpp"
 
-#include <array>
-#include <exception>
-#include <future>
-#include <iostream>
-#include <memory> // shared_ptr
-#include <string>
-#include <unordered_map>
-#include <utility> // pair
-#include <vector>
-
-
 #if openPMD_HAVE_ADIOS2
 #   include <adios2.h>
 #   include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
@@ -50,6 +39,17 @@
 #   include <mpi.h>
 #endif
 
+#include <nlohmann/json.hpp>
+
+#include <array>
+#include <exception>
+#include <future>
+#include <iostream>
+#include <memory> // shared_ptr
+#include <string>
+#include <unordered_map>
+#include <utility> // pair
+#include <vector>
 
 namespace openPMD
 {
@@ -95,13 +95,16 @@ class ADIOS2IOHandlerImpl
 
 #if openPMD_HAVE_MPI
 
-    ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm );
+    ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm, nlohmann::json config );
 
     MPI_Comm m_comm;
 
 #endif // openPMD_HAVE_MPI
 
-    explicit ADIOS2IOHandlerImpl( AbstractIOHandler * );
+    explicit ADIOS2IOHandlerImpl(
+        AbstractIOHandler *
+        , nlohmann::json config
+    );
 
 
     ~ADIOS2IOHandlerImpl( ) override = default;
@@ -165,8 +168,6 @@ class ADIOS2IOHandlerImpl
     listAttributes( Writable *,
                     Parameter< Operation::LIST_ATTS > & parameters ) override;
 
-
-
     /**
      * @brief The ADIOS2 access type to chose for Engines opened
      * within this instance.
@@ -177,6 +178,55 @@ class ADIOS2IOHandlerImpl
 private:
     adios2::ADIOS m_ADIOS;
 
+    struct ParameterizedOperator
+    {
+        adios2::Operator const op;
+        adios2::Params const params;
+    };
+
+    std::vector< ParameterizedOperator > defaultOperators;
+
+    auxiliary::TracingJSON m_config;
+    static auxiliary::TracingJSON nullvalue;
+
+    void
+    init( nlohmann::json config );
+
+    template< typename Key >
+    auxiliary::TracingJSON
+    config( Key && key, auxiliary::TracingJSON & cfg )
+    {
+        if( cfg.json().is_object() && cfg.json().contains( key ) )
+        {
+            return cfg[ key ];
+        }
+        else
+        {
+            return nullvalue;
+        }
+    }
+
+    template< typename Key >
+    auxiliary::TracingJSON
+    config( Key && key )
+    {
+        return config< Key >( std::forward< Key >( key ), m_config );
+    }
+
+    /**
+     *
+     * @param config The top-level of the ADIOS2 configuration JSON object
+     * with operators to be found under dataset.operators
+     * @return first parameter: the operators, second parameters: whether
+     * operators have been configured
+     */
+    std::pair< std::vector< ParameterizedOperator >, bool >
+    getOperators( auxiliary::TracingJSON config );
+
+    // use m_config
+    std::pair< std::vector< ParameterizedOperator >, bool >
+    getOperators();
+
     /*
      * We need to give names to IO objects. These names are irrelevant
      * within this application, since:
@@ -258,6 +308,18 @@ class ADIOS2IOHandlerImpl
                                          std::string const & var );
 }; // ADIOS2IOHandlerImpl
 
+/*
+ * The following strings are used during parsing of the JSON configuration
+ * string for the ADIOS2 backend.
+ */
+namespace ADIOS2Defaults
+{
+    using const_str = char const * const;
+    constexpr const_str str_engine = "engine";
+    constexpr const_str str_type = "type";
+    constexpr const_str str_params = "parameters";
+}
+
 namespace detail
 {
     // Helper structs for calls to the switchType function
@@ -332,17 +394,13 @@ namespace detail
 
     struct VariableDefiner
     {
+        // Parameters such as DatasetHelper< T >::defineVariable
+        template < typename T, typename... Params >
+        void operator( )( Params &&... params );
 
-        template < typename T >
-        void operator( )( adios2::IO & IO, const std::string & name,
-                          std::unique_ptr< adios2::Operator > compression,
-                          const adios2::Dims & shape = adios2::Dims( ),
-                          const adios2::Dims & start = adios2::Dims( ),
-                          const adios2::Dims & count = adios2::Dims( ),
-                          const bool constantDims = false );
-
-        template < int n, typename... Params >
-        void operator( )( adios2::IO & IO, Params &&... );
+        template< int n, typename... Params >
+        void
+        operator()( Params &&... );
     };
 
 
@@ -473,11 +531,31 @@ namespace detail
         void readDataset( BufferedGet &, adios2::IO &, adios2::Engine &,
                           std::string const & fileName );
 
+        /**
+         * @brief Define a Variable of type T within the passed IO.
+         * 
+         * @param IO The adios2::IO object within which to define the
+         *           variable. The variable can later be retrieved from
+         *           the IO using the passed name.
+         * @param name As in adios2::IO::DefineVariable
+         * @param compressions ADIOS2 operators, including an arbitrary
+         *                     number of parameters, to be added to the
+         *                     variable upon definition.
+         * @param shape As in adios2::IO::DefineVariable
+         * @param start As in adios2::IO::DefineVariable
+         * @param count As in adios2::IO::DefineVariable
+         * @param constantDims As in adios2::IO::DefineVariable
+         */
         static void
-        defineVariable( adios2::IO & IO, const std::string & name,
-                        std::unique_ptr< adios2::Operator > compression,
-                        const adios2::Dims & shape, const adios2::Dims & start,
-                        const adios2::Dims & count, bool constantDims );
+        defineVariable(
+            adios2::IO & IO,
+            std::string const & name,
+            std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > const &
+                compressions,
+            adios2::Dims const & shape = adios2::Dims(),
+            adios2::Dims const & start = adios2::Dims(),
+            adios2::Dims const & count = adios2::Dims(),
+            bool const constantDims = false );
 
         void writeDataset( BufferedPut &, adios2::IO &, adios2::Engine & );
     };
@@ -570,6 +648,9 @@ namespace detail
 
         ~BufferedActions( );
 
+        void
+        configure_IO( ADIOS2IOHandlerImpl & impl );
+
         adios2::Engine & getEngine( );
 
         template < typename BA > void enqueue( BA && ba );
@@ -666,11 +747,16 @@ friend class ADIOS2IOHandlerImpl;
 
 #if openPMD_HAVE_MPI
 
-    ADIOS2IOHandler( std::string path, AccessType, MPI_Comm );
+    ADIOS2IOHandler(
+        std::string path,
+        AccessType,
+        MPI_Comm,
+        nlohmann::json options
+    );
 
 #endif
 
-    ADIOS2IOHandler( std::string path, AccessType );
+    ADIOS2IOHandler( std::string path, AccessType, nlohmann::json options );
 
     std::string backendName() const override { return "ADIOS2"; }