lazy parsing of iterations

docs/source/details/backendconfig.rst

@@ -1,7 +1,7 @@
 .. _backendconfig:
 
-Backend-Specific Configuration
-==============================
+JSON configuration
+==================
 
 While the openPMD API intends to be a backend-*independent* implementation of the openPMD standard, it is sometimes useful to pass configuration parameters to the specific backend in use.
 :ref:`For each backend <backends-overview>`, configuration options can be passed via a JSON-formatted string or via environment variables.
@@ -33,6 +33,19 @@ For a consistent user interface, backends shall follow the following rules:
   Backends should define clearly which keys are applicable to datasets and which are not.
 
 
+Backend-independent JSON configuration
+--------------------------------------
+
+The key ``defer_iteration_parsing`` can be used to optimize the process of opening an openPMD Series.
+By default, a Series is parsed eagerly, i.e. opening a Series implies reading all available iterations.
+Especially when a Series has many iterations, this can be a costly operation and users may wish to defer parsing of iterations to a later point adding ``{"defer_iteration_parsing": true}`` to their JSON configuration.
+
+When parsing non-eagerly, each iteration needs to be explicitly opened with ``Iteration::open()`` before accessing.
+(Notice that ``Iteration::open()`` is generally recommended to be used in parallel contexts to avoid parallel file accessing hazards).
+Using the Streaming API (i.e. ``SeriesImpl::readIteration()``) will do this automatically.
+Parsing eagerly might be very expensive for a Series with many iterations, but will avoid bugs by forgotten calls to ``Iteration::open()``.
+In complex environments, calling ``Iteration::open()`` on an already open environment does no harm (and does not incur additional runtime cost for additional ``open()`` calls).
+
 Configuration Structure per Backend
 -----------------------------------
 

include/openPMD/IO/AbstractIOHandlerHelper.hpp

@@ -20,10 +20,10 @@
  */
 #pragma once
 
+
 #include "openPMD/config.hpp"
 #include "openPMD/IO/AbstractIOHandler.hpp"
 
-
 namespace openPMD
 {
 #if openPMD_HAVE_MPI
@@ -36,15 +36,18 @@ namespace openPMD
      * @param   comm        MPI communicator used for IO.
      * @param   options     JSON-formatted option string, to be interpreted by
      *                      the backend.
+     * @tparam  JSON        Substitute for nlohmann::json. Templated to avoid
+                            including nlohmann::json in a .hpp file.
      * @return  Smart pointer to created IOHandler.
      */
+template< typename JSON >
 std::shared_ptr< AbstractIOHandler >
 createIOHandler(
     std::string path,
     Access access,
     Format format,
     MPI_Comm comm,
-    std::string const & options = "{}" );
+    JSON options );
 #endif
 
 /** Construct an appropriate specific IOHandler for the desired IO mode.
@@ -56,12 +59,22 @@ createIOHandler(
  * @param   format      Format describing the IO backend of the desired handler.
  * @param   options     JSON-formatted option string, to be interpreted by
  *                      the backend.
+ * @tparam  JSON        Substitute for nlohmann::json. Templated to avoid
+                        including nlohmann::json in a .hpp file.
  * @return  Smart pointer to created IOHandler.
  */
+template< typename JSON >
 std::shared_ptr< AbstractIOHandler >
 createIOHandler(
     std::string path,
     Access access,
     Format format,
-    std::string const & options = "{}" );
+    JSON options = JSON() );
+
+// version without configuration to use in AuxiliaryTest
+std::shared_ptr< AbstractIOHandler >
+createIOHandler(
+    std::string path,
+    Access access,
+    Format format );
 } // namespace openPMD

include/openPMD/Iteration.hpp

@@ -20,6 +20,7 @@
  */
 #pragma once
 
+#include "openPMD/auxiliary/Option.hpp"
 #include "openPMD/auxiliary/Variant.hpp"
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/Container.hpp"
@@ -153,22 +154,48 @@ class Iteration : public LegacyAttributable
 private:
     Iteration();
 
+    struct DeferredParseAccess
+    {
+        std::string index;
+        bool fileBased = false;
+        std::string filename;
+    };
+
     void flushFileBased(std::string const&, uint64_t);
     void flushGroupBased(uint64_t);
     void flush();
+    void deferParseAccess( DeferredParseAccess );
+    /*
+     * Control flow for read(), readFileBased(), readGroupBased() and
+     * read_impl():
+     * read() is called as the entry point. File-based and group-based
+     * iteration layouts need to be parsed slightly differently:
+     * In file-based iteration layout, each iteration's file also contains
+     * attributes for the /data group. In group-based layout, those have
+     * already been parsed during opening of the Series.
+     * Hence, read() will call either readFileBased() or readGroupBased() to
+     * allow for those different control flows.
+     * Finally, read_impl() is called which contains the common parsing
+     * logic for an iteration.
+     *
+     */
     void read();
+    void readFileBased( std::string filePath, std::string const & groupPath );
+    void readGroupBased( std::string const & groupPath );
+    void read_impl( std::string const & groupPath );
 
     /**
      * @brief Whether an iteration has been closed yet.
      *
      */
     enum class CloseStatus
     {
+        ParseAccessDeferred, //!< The reader has not yet parsed this iteration
         Open,             //!< Iteration has not been closed
         ClosedInFrontend, /*!< Iteration has been closed, but task has not yet
                                been propagated to the backend */
-        ClosedInBackend,  /*!< Iteration has been closed and task has been
-                               propagated to the backend */
+        ClosedInBackend, /*!< Iteration has been closed and task has been
+                              propagated to the backend */
         ClosedTemporarily /*!< Iteration has been closed internally and may
                                be reopened later */
     };
@@ -194,6 +221,11 @@ class Iteration : public LegacyAttributable
     std::shared_ptr< StepStatus > m_stepStatus =
         std::make_shared< StepStatus >( StepStatus::NoStep );
 
+    std::shared_ptr< auxiliary::Option< DeferredParseAccess > >
+        m_deferredParseAccess =
+            std::make_shared< auxiliary::Option< DeferredParseAccess > >(
+                auxiliary::Option< DeferredParseAccess >() );
+
     /**
      * @brief Begin an IO step on the IO file (or file-like object)
      *        containing this iteration. In case of group-based iteration
@@ -252,6 +284,13 @@ class Iteration : public LegacyAttributable
      * @param w The Writable representing the parent.
      */
     virtual void linkHierarchy(Writable& w);
+
+    /**
+     * @brief Access an iteration in read mode that has potentially not been
+     *      parsed yet.
+     * 
+     */
+    void runDeferredParseAccess();
 };  // Iteration
 
 extern template

include/openPMD/Series.hpp

@@ -66,10 +66,6 @@ namespace internal
  */
 class SeriesData : public AttributableData
 {
-    friend class openPMD::SeriesImpl;
-    friend class openPMD::Iteration;
-    friend class openPMD::Series;
-
 public:
     explicit SeriesData() = default;
 
@@ -83,8 +79,8 @@ class SeriesData : public AttributableData
 
     Container< Iteration, uint64_t > iterations{};
 
-OPENPMD_private :
     auxiliary::Option< WriteIterations > m_writeIterations;
+    auxiliary::Option< std::string > m_overrideFilebasedFilename;
     std::string m_name;
     std::string m_filenamePrefix;
     std::string m_filenamePostfix;
@@ -99,6 +95,7 @@ OPENPMD_private :
      * one among both flags.
      */
     StepStatus m_stepStatus = StepStatus::NoStep;
+    bool m_parseLazily = false;
 }; // SeriesData
 
 class SeriesInternal;
@@ -337,6 +334,7 @@ class SeriesImpl : public AttributableImpl
     void flushMeshesPath();
     void flushParticlesPath();
     void readFileBased( );
+    void readOneIterationFileBased( std::string const & filePath );
     /**
      * Note on re-parsing of a Series:
      * If init == false, the parsing process will seek for new
@@ -347,7 +345,6 @@ class SeriesImpl : public AttributableImpl
      */
     void readGroupBased( bool init = true );
     void readBase();
-    void read();
     std::string iterationFilename( uint64_t i );
     void openIteration( uint64_t index, Iteration iteration );
 
@@ -386,6 +383,9 @@ namespace internal
 class SeriesInternal : public SeriesData, public SeriesImpl
 {
     friend struct SeriesShared;
+    friend class openPMD::Iteration;
+    friend class openPMD::Series;
+    friend class openPMD::Writable;
 
 public:
 #if openPMD_HAVE_MPI
@@ -409,6 +409,9 @@ class SeriesInternal : public SeriesData, public SeriesImpl
  *
  * Entry point and common link between all iterations of particle and mesh data.
  *
+ * An instance can be created either directly via the given constructors or via
+ * the SeriesBuilder class.
+ *
  * @see https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#hierarchy-of-the-data-file
  * @see https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#iterations-and-time-series
  */
@@ -426,6 +429,15 @@ class Series : public SeriesImpl
         std::string const & options = "{}" );
 #endif
 
+    /**
+     * @brief Construct a new Series
+     *
+     * @param filepath The backend will be determined by the filepath extension.
+     * @param at Access mode.
+     * @param options Advanced backend configuration via JSON.
+     *      May be specified as a JSON-formatted string directly, or as a path
+     *      to a JSON textfile, prepended by an at sign '@'.
+     */
     Series(
         std::string const & filepath,
         Access at,

src/IO/AbstractIOHandlerHelper.cpp

@@ -32,15 +32,16 @@
 namespace openPMD
 {
 #if openPMD_HAVE_MPI
+    template<>
     std::shared_ptr< AbstractIOHandler >
-    createIOHandler(
+    createIOHandler< nlohmann::json >(
         std::string path,
         Access access,
         Format format,
         MPI_Comm comm,
-        std::string const & options )
+        nlohmann::json options )
     {
-        nlohmann::json optionsJson = auxiliary::parseOptions( options, comm );
+        (void) options;
         switch( format )
         {
             case Format::HDF5:
@@ -53,28 +54,29 @@ namespace openPMD
 #   endif
             case Format::ADIOS2:
                 return std::make_shared< ADIOS2IOHandler >(
-                    path, access, comm, std::move( optionsJson ), "bp4" );
+                    path, access, comm, std::move( options ), "bp4" );
             case Format::ADIOS2_SST:
                 return std::make_shared< ADIOS2IOHandler >(
-                    path, access, comm, std::move( optionsJson ), "sst" );
+                    path, access, comm, std::move( options ), "sst" );
             case Format::ADIOS2_SSC:
                 return std::make_shared< ADIOS2IOHandler >(
-                    path, access, comm, std::move( optionsJson ), "ssc" );
+                    path, access, comm, std::move( options ), "ssc" );
             default:
                 throw std::runtime_error(
                     "Unknown file format! Did you specify a file ending?" );
         }
     }
 #endif
 
+    template<>
     std::shared_ptr< AbstractIOHandler >
-    createIOHandler(
+    createIOHandler< nlohmann::json >(
         std::string path,
         Access access,
         Format format,
-        std::string const & options )
+        nlohmann::json options )
     {
-        nlohmann::json optionsJson = auxiliary::parseOptions( options );
+        (void) options;
         switch( format )
         {
             case Format::HDF5:
@@ -88,13 +90,13 @@ namespace openPMD
 #if openPMD_HAVE_ADIOS2
             case Format::ADIOS2:
                 return std::make_shared< ADIOS2IOHandler >(
-                    path, access, std::move( optionsJson ), "bp4" );
+                    path, access, std::move( options ), "bp4" );
             case Format::ADIOS2_SST:
                 return std::make_shared< ADIOS2IOHandler >(
-                    path, access, std::move( optionsJson ), "sst" );
+                    path, access, std::move( options ), "sst" );
             case Format::ADIOS2_SSC:
                 return std::make_shared< ADIOS2IOHandler >(
-                    path, access, std::move( optionsJson ), "ssc" );
+                    path, access, std::move( options ), "ssc" );
 #endif // openPMD_HAVE_ADIOS2
             case Format::JSON:
                 return std::make_shared< JSONIOHandler >( path, access );
@@ -103,4 +105,11 @@ namespace openPMD
                     "Unknown file format! Did you specify a file ending?" );
         }
     }
+
+    std::shared_ptr< AbstractIOHandler >
+    createIOHandler( std::string path, Access access, Format format )
+    {
+        return createIOHandler(
+            std::move( path ), access, format, nlohmann::json::object() );
+    }
 } // namespace openPMD