diff --git a/CMakeLists.txt b/CMakeLists.txt index 8cca88b188..699b4b9f06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -469,10 +469,12 @@ set(CORE_SOURCE src/benchmark/mpi/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp) set(IO_SOURCE + src/IO/AbstractIOHandler.cpp src/IO/AbstractIOHandlerImpl.cpp src/IO/AbstractIOHandlerHelper.cpp src/IO/DummyIOHandler.cpp src/IO/IOTask.cpp + src/IO/FlushParams.cpp src/IO/HDF5/HDF5IOHandler.cpp src/IO/HDF5/ParallelHDF5IOHandler.cpp src/IO/HDF5/HDF5Auxiliary.cpp diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index 8996466544..34450bd2c0 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -130,12 +130,18 @@ This buffer is drained to storage only at specific times: The usage pattern of openPMD, especially the choice of iteration encoding influences the memory use of ADIOS2. The following graphs are created from a real-world application using openPMD (PIConGPU) using KDE Heaptrack. -Ignore the 30GB initialization phases. + +BP4 file engine +*************** + +The internal data structure of BP4 is one large buffer that holds all data written by a process. +It is drained to the disk upon ending a step or closing the engine (in parallel applications, data will usually be aggregated at the node-level before this). +This approach enables a very high IO performance by requiring only very few, very large IO operations, at the cost of a high memory consumption and some common usage pitfalls as detailed below: * **file-based iteration encoding:** A new ADIOS2 engine is opened for each iteration and closed upon ``Iteration::close()``. Each iteration has its own buffer: -.. image:: ./memory_filebased.png +.. figure:: https://user-images.githubusercontent.com/14241876/181477396-746ee21d-6efe-450b-bb2f-f53d49945fb9.png :alt: Memory usage of file-based iteration encoding * **variable-based iteration encoding and group-based iteration encoding with steps**: @@ -147,19 +153,100 @@ Ignore the 30GB initialization phases. These memory spikes can easily lead to out-of-memory (OOM) situations, motivating that the ``InitialBufferSize`` should not be chosen too small. Both behaviors are depicted in the following two pictures: -.. image:: ./memory_variablebased.png +.. figure:: https://user-images.githubusercontent.com/14241876/181477405-0439b017-256b-48d6-a169-014b3fe3aeb3.png :alt: Memory usage of variable-based iteration encoding -.. image:: ./memory_variablebased_initialization.png +.. figure:: https://user-images.githubusercontent.com/14241876/181477406-f6e2a173-2ec1-48df-a417-0cb97a160c91.png :alt: Memory usage of variable-based iteration encoding with bad ``InitialBufferSize`` * **group-based iteration encoding without steps:** This encoding **should be avoided** in ADIOS2. No data will be written to disk before closing the ``Series``, leading to a continuous buildup of memory, and most likely to an OOM situation: -.. image:: ./memory_groupbased_nosteps.png +.. figure:: https://user-images.githubusercontent.com/14241876/181477397-4d923061-7051-48c4-ae3a-a9efa10dcac7.png :alt: Memory usage of group-based iteration without using steps +SST staging engine +****************** + +Like the BP4 engine, the SST engine uses one large buffer as an internal data structure. + +Unlike the BP4 engine, however, a new buffer is allocated for each IO step, leading to a memory profile with clearly distinct IO steps: + +.. figure:: https://user-images.githubusercontent.com/14241876/181477403-7ed7810b-dedf-48b8-b17b-8ce89fd3c34a.png + :alt: Ideal memory usage of the SST engine + +The SST engine performs all IO asynchronously in the background and releases memory only as soon as the reader is done interacting with an IO step. +With slow readers, this can lead to a buildup of past IO steps in memory and subsequently to an out-of-memory condition: + +.. figure:: https://user-images.githubusercontent.com/14241876/181477400-f342135f-612e-464f-b0e7-c1978ef47a94.png + :alt: Memory congestion in SST due to a slow reader + +This can be avoided by specifying the `ADIOS2 parameter `_ ``QueueLimit``: + +.. code:: cpp + + std::string const adios2Config = R"( + {"adios2": {"engine": {"parameters": {"QueueLimit": 1}}}} + )"; + Series series("simData.sst", Access::CREATE, adios2Config); + +By default, the openPMD-api configures a queue limit of 2. +Depending on the value of the ADIOS2 parameter ``QueueFullPolicy``, the SST engine will either ``"Discard"`` steps or ``"Block"`` the writer upon reaching the queue limit. + +BP5 file engine +*************** + +The BP5 file engine internally uses a linked list of equally-sized buffers. +The size of each buffer can be specified up to a maximum of 2GB with the `ADIOS2 parameter `_ ``BufferChunkSize``: + +.. code:: cpp + + std::string const adios2Config = R"( + {"adios2": {"engine": {"parameters": {"BufferChunkSize": 2147381248}}}} + )"; + Series series("simData.bp5", Access::CREATE, adios2Config); + +This approach implies a sligthly lower IO performance due to more frequent and smaller writes, but it lets users control memory usage better and avoids out-of-memory issues when configuring ADIOS2 incorrectly. + +The buffer is drained upon closing a step or the engine, but draining to the filesystem can also be triggered manually. +In the openPMD-api, this can be done by specifying backend-specific parameters to the ``Series::flush()`` or ``Attributable::seriesFlush()`` calls: + +.. code:: cpp + + series.flush(R"({"adios2": {"preferred_flush_target": "disk"}})") + +The memory consumption of this approach shows that the 2GB buffer is first drained and then recreated after each ``flush()``: + +.. figure:: https://user-images.githubusercontent.com/14241876/181477392-7eff2020-7bfb-4ddb-b31c-27b9937e088a.png + :alt: Memory usage of BP5 when flushing directly to disk + +.. note:: + + KDE Heaptrack tracks the **virtual memory** consumption. + While the BP4 engine uses ``std::vector`` for its internal buffer, BP5 uses plain ``malloc()`` (hence the 2GB limit), which does not initialize memory. + Memory pages will only be allocated to physical memory upon writing. + In applications with small IO sizes on systems with virtual memory, the physical memory usage will stay well below 2GB even if specifying the BufferChunkSize as 2GB. + + **=> Specifying the buffer chunk size as 2GB as shown above is a good idea in most cases.** + +Alternatively, data can be flushed to the buffer. +Note that this involves data copies that can be avoided by either flushing directly to disk or by entirely avoiding to flush until ``Iteration::close()``: + +.. code:: cpp + + series.flush(R"({"adios2": {"preferred_flush_target": "buffer"}})") + +With this strategy, the BP5 engine will slowly build up its buffer until ending the step. +Rather than by reallocation as in BP4, this is done by appending a new chunk, leading to a clearly more acceptable memory profile: + +.. figure:: https://user-images.githubusercontent.com/14241876/181477384-ce4ea8ab-3bde-4210-991b-2e627dfcc7c9.png + :alt: Memory usage of BP5 when flushing to the engine buffer + +The default is to flush to disk, but the default ``preferred_flush_target`` can also be specified via JSON/TOML at the ``Series`` level. + + + Known Issues ------------ diff --git a/docs/source/backends/memory_filebased.png b/docs/source/backends/memory_filebased.png deleted file mode 100644 index a7fbc02d0d..0000000000 Binary files a/docs/source/backends/memory_filebased.png and /dev/null differ diff --git a/docs/source/backends/memory_groupbased_nosteps.png b/docs/source/backends/memory_groupbased_nosteps.png deleted file mode 100644 index ad2567f6a9..0000000000 Binary files a/docs/source/backends/memory_groupbased_nosteps.png and /dev/null differ diff --git a/docs/source/backends/memory_variablebased.png b/docs/source/backends/memory_variablebased.png deleted file mode 100644 index 3da406d978..0000000000 Binary files a/docs/source/backends/memory_variablebased.png and /dev/null differ diff --git a/docs/source/backends/memory_variablebased_initialization.png b/docs/source/backends/memory_variablebased_initialization.png deleted file mode 100644 index 15f1540ad7..0000000000 Binary files a/docs/source/backends/memory_variablebased_initialization.png and /dev/null differ diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index 536090a650..cef1e4d6f9 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -37,7 +37,10 @@ The configuration string may refer to the complete ``openPMD::Series`` or may ad This reflects the fact that certain backend-specific parameters may refer to the whole Series (such as storage engines and their parameters) and others refer to actual datasets (such as compression). Dataset-specific configurations are (currently) only available during dataset creation, but not when reading datasets. -A JSON/TOML configuration may either be specified as an inline string that can be parsed as a JSON/TOML object, or alternatively as a path to a JSON/TOML-formatted text file (only in the constructor of ``openPMD::Series``): +Additionally, some backends may provide different implementations to the ``Series::flush()`` and ``Attributable::flushSeries()`` calls. +JSON/TOML strings may be passed to these calls as optional parameters. + +A JSON/TOML configuration may either be specified as an inline string that can be parsed as a JSON/TOML object, or alternatively as a path to a JSON/TOML-formatted text file (only in the constructor of ``openPMD::Series``, all other API calls that accept a JSON/TOML specification require in-line datasets): * File paths are distinguished by prepending them with an at-sign ``@``. JSON and TOML are then distinguished by the filename extension ``.json`` or ``.toml``. @@ -119,6 +122,16 @@ Explanation of the single keys: Please refer to the `official ADIOS2 documentation `_ for the available engine parameters. The openPMD-api does not interpret these values and instead simply forwards them to ADIOS2. * ``adios2.engine.usesteps``: Described more closely in the documentation for the :ref:`ADIOS2 backend` (usesteps). +* ``adios2.engine.preferred_flush_target`` Only relevant for BP5 engine, possible values are ``"disk"`` and ``"buffer"`` (default: ``"disk"``). + + * If ``"disk"``, data will be moved to disk on every flush. + * If ``"buffer"``, then only upon ending an IO step or closing an engine. + + This behavior can be overridden on a per-flush basis by specifying this JSON/TOML key as an optional parameter to the ``Series::flush()`` or ``Attributable::seriesFlush()`` methods. + + Additionally, specifying ``"disk_override"`` or ``"buffer_override"`` will take precedence over options specified without the ``_override`` suffix, allowing to invert the normal precedence order. + This way, a data producing code can hardcode the preferred flush target per ``flush()`` call, but users can e.g. still entirely deactivate flushing to disk in the ``Series`` constructor by specifying ``preferred_flush_target = buffer_override``. + This is useful when applying the asynchronous IO capabilities of the BP5 engine. * ``adios2.dataset.operators``: This key contains a list of ADIOS2 `operators `_, used to enable compression or dataset transformations. Each object in the list has two keys: diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index 7cf0e232d7..ac5398b4cf 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -63,3 +63,12 @@ Attributes are (currently) unaffected by this: * In writing, attributes are stored internally by value and can afterwards not be accessed by the user. * In reading, attributes are parsed upon opening the Series / an iteration and are available to read right-away. + +.. attention:: + + Note that the concrete implementation of ``Series::flush()`` and ``Attributable::seriesFlush()`` is backend-specific. + Using these calls does neither guarantee that data is moved to storage/transport nor that it can be accessed by independent readers at this point. + + Some backends (e.g. the BP5 engine of ADIOS2) have multiple implementations for the openPMD-api-level guarantees of flush points. + For user-guided selection of such implementations, ``Series::flush`` and ``Attributable::seriesFlush()`` take an optional JSON/TOML string as a parameter. + See the section on :ref:`backend-specific configuration ` for details. diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp index a967433cf4..bef874c607 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp @@ -50,7 +50,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "ADIOS1"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; void enqueue(IOTask const &) override; @@ -72,7 +72,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "DUMMY_ADIOS1"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 18a4aad192..af79d72d3d 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" +#include "openPMD/IO/FlushParametersInternal.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IO/InvalidatableFile.hpp" #include "openPMD/IterationEncoding.hpp" @@ -140,7 +141,7 @@ class ADIOS2IOHandlerImpl ~ADIOS2IOHandlerImpl() override; - std::future flush(internal::FlushParams const &); + std::future flush(internal::ParsedFlushParams &); void createFile(Writable *, Parameter const &) override; @@ -209,6 +210,16 @@ class ADIOS2IOHandlerImpl */ adios2::Mode adios2AccessMode(std::string const &fullPath); + enum class FlushTarget : unsigned char + { + Buffer, + Buffer_Override, + Disk, + Disk_Override + }; + + FlushTarget m_flushTarget = FlushTarget::Disk; + private: adios2::ADIOS m_ADIOS; /* @@ -412,6 +423,7 @@ namespace ADIOS2Defaults constexpr const_str str_type = "type"; constexpr const_str str_params = "parameters"; constexpr const_str str_usesteps = "usesteps"; + constexpr const_str str_flushtarget = "preferred_flush_target"; constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps"; constexpr const_str str_adios2Schema = "__openPMD_internal/openPMD2_adios2_schema"; @@ -927,6 +939,8 @@ namespace detail friend struct BufferedGet; friend struct BufferedPut; + using FlushTarget = ADIOS2IOHandlerImpl::FlushTarget; + BufferedActions(BufferedActions const &) = delete; /** @@ -1039,10 +1053,26 @@ namespace detail template void enqueue(BA &&ba, decltype(m_buffer) &); + struct ADIOS2FlushParams + { + /* + * Only execute performPutsGets if UserFlush. + */ + FlushLevel level; + FlushTarget flushTarget = FlushTarget::Disk; + + ADIOS2FlushParams(FlushLevel level_in) : level(level_in) + {} + + ADIOS2FlushParams(FlushLevel level_in, FlushTarget flushTarget_in) + : level(level_in), flushTarget(flushTarget_in) + {} + }; + /** * Flush deferred IO actions. * - * @param level Flush Level. Only execute performPutsGets if UserFlush. + * @param flushParams Flush level and target. * @param performPutsGets A functor that takes as parameters (1) *this * and (2) the ADIOS2 engine. * Its task is to ensure that ADIOS2 performs Put/Get operations. @@ -1057,7 +1087,7 @@ namespace detail */ template void flush( - FlushLevel level, + ADIOS2FlushParams flushParams, F &&performPutsGets, bool writeAttributes, bool flushUnconditionally); @@ -1067,7 +1097,7 @@ namespace detail * and does not flush unconditionally. * */ - void flush(FlushLevel, bool writeAttributes = false); + void flush(ADIOS2FlushParams, bool writeAttributes = false); /** * @brief Begin or end an ADIOS step. @@ -1265,7 +1295,8 @@ class ADIOS2IOHandler : public AbstractIOHandler // we must not throw in a destructor try { - this->flush(internal::defaultFlushParams); + auto params = internal::defaultParsedFlushParams; + this->flush(params); } catch (std::exception const &ex) { @@ -1304,6 +1335,6 @@ class ADIOS2IOHandler : public AbstractIOHandler return "ADIOS2"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; }; // ADIOS2IOHandler } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp index eefac95854..e28122582b 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp @@ -54,7 +54,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandler : public AbstractIOHandler return "MPI_ADIOS1"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; #if openPMD_HAVE_ADIOS1 void enqueue(IOTask const &) override; #endif diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 80616e62bc..4f6916ae55 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -103,12 +103,24 @@ namespace internal struct FlushParams { FlushLevel flushLevel = FlushLevel::InternalFlush; + std::string backendConfig = "{}"; + + explicit FlushParams() + {} + FlushParams(FlushLevel flushLevel_in) : flushLevel(flushLevel_in) + {} + FlushParams(FlushLevel flushLevel_in, std::string backendConfig_in) + : flushLevel(flushLevel_in) + , backendConfig{std::move(backendConfig_in)} + {} }; /* * To be used for reading */ - constexpr FlushParams defaultFlushParams{}; + FlushParams const defaultFlushParams{}; + + struct ParsedFlushParams; } // namespace internal /** Interface for communicating between logical and physically persistent data. @@ -164,7 +176,14 @@ class AbstractIOHandler * @return Future indicating the completion state of the operation for * backends that decide to implement this operation asynchronously. */ - virtual std::future flush(internal::FlushParams const &) = 0; + std::future flush(internal::FlushParams const &); + + /** Process operations in queue according to FIFO. + * + * @return Future indicating the completion state of the operation for + * backends that decide to implement this operation asynchronously. + */ + virtual std::future flush(internal::ParsedFlushParams &) = 0; /** The currently used backend */ virtual std::string backendName() const = 0; diff --git a/include/openPMD/IO/DummyIOHandler.hpp b/include/openPMD/IO/DummyIOHandler.hpp index 9a4f3c3852..7cd4123699 100644 --- a/include/openPMD/IO/DummyIOHandler.hpp +++ b/include/openPMD/IO/DummyIOHandler.hpp @@ -44,6 +44,6 @@ class DummyIOHandler : public AbstractIOHandler /** No-op consistent with the IOHandler interface to enable library use * without IO. */ - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; }; // DummyIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/FlushParametersInternal.hpp b/include/openPMD/IO/FlushParametersInternal.hpp new file mode 100644 index 0000000000..2233ec002d --- /dev/null +++ b/include/openPMD/IO/FlushParametersInternal.hpp @@ -0,0 +1,40 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" + +#include + +namespace openPMD::internal +{ +struct ParsedFlushParams +{ + ParsedFlushParams(FlushParams const &); + + FlushLevel flushLevel = FlushLevel::InternalFlush; + json::TracingJSON backendConfig; +}; + +ParsedFlushParams const defaultParsedFlushParams{defaultFlushParams}; +} // namespace openPMD::internal diff --git a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp index 85d6ab9d40..e81996b389 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp @@ -42,7 +42,7 @@ class HDF5IOHandler : public AbstractIOHandler return "HDF5"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp index e1c8d52257..18d43c93ab 100644 --- a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp @@ -48,7 +48,7 @@ class ParallelHDF5IOHandler : public AbstractIOHandler return "MPI_HDF5"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 37b00fa165..0cdc6f3c36 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -38,7 +38,7 @@ class JSONIOHandler : public AbstractIOHandler return "JSON"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; private: JSONIOHandlerImpl m_impl; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index a5af2be601..39da51057b 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -442,8 +442,13 @@ class Series : public Attributable std::string backend() const; /** Execute all required remaining IO operations to write or read data. + * + * @param backendConfig Further backend-specific instructions on how to + * implement this flush call. + * Must be provided in-line, configuration is not read + * from files. */ - void flush(); + void flush(std::string backendConfig = "{}"); /** * @brief Entry point to the reading end of the streaming API. diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index bba3eb409c..1c9dda8429 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -238,8 +238,13 @@ class Attributable * of parents. This method will walk up the parent list until it reaches * an object that has no parent, which is the Series object, and flush()-es * it. + * + * @param backendConfig Further backend-specific instructions on how to + * implement this flush call. + * Must be provided in-line, configuration is not read + * from files. */ - void seriesFlush(); + void seriesFlush(std::string backendConfig = "{}"); /** String serialization to describe an Attributable * diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 1ed7054a6f..81d83955f9 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -109,7 +109,7 @@ class Writable final * an object that has no parent, which is the Series object, and flush()-es * it. */ - void seriesFlush(); + void seriesFlush(std::string backendConfig = "{}"); // clang-format off OPENPMD_private diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 14ea1aeaaa..92166ca259 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -336,7 +336,7 @@ ADIOS1IOHandler::ADIOS1IOHandler( ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush(internal::FlushParams const &) +std::future ADIOS1IOHandler::flush(internal::ParsedFlushParams &) { return m_impl->flush(); } @@ -438,7 +438,7 @@ ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at, json::TracingJSON) ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush(internal::FlushParams const &) +std::future ADIOS1IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2e53700bea..f0e7976f16 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -253,16 +253,121 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const } } +using FlushTarget = ADIOS2IOHandlerImpl::FlushTarget; +static FlushTarget flushTargetFromString(std::string const &str) +{ + if (str == "buffer") + { + return FlushTarget::Buffer; + } + else if (str == "disk") + { + return FlushTarget::Disk; + } + else if (str == "buffer_override") + { + return FlushTarget::Buffer_Override; + } + else if (str == "disk_override") + { + return FlushTarget::Disk_Override; + } + else + { + throw error::BackendConfigSchema( + {"adios2", "engine", ADIOS2Defaults::str_flushtarget}, + "Flush target must be either 'disk' or 'buffer', but " + "was " + + str + "."); + } +} + +static FlushTarget & +overrideFlushTarget(FlushTarget &inplace, FlushTarget new_val) +{ + auto allowsOverride = [](FlushTarget ft) { + switch (ft) + { + case FlushTarget::Buffer: + case FlushTarget::Disk: + return true; + case FlushTarget::Buffer_Override: + case FlushTarget::Disk_Override: + return false; + } + return true; + }; + + if (allowsOverride(inplace)) + { + inplace = new_val; + } + else + { + if (!allowsOverride(new_val)) + { + inplace = new_val; + } + // else { keep the old value, no-op } + } + return inplace; +} + std::future -ADIOS2IOHandlerImpl::flush(internal::FlushParams const &flushParams) +ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams) { auto res = AbstractIOHandlerImpl::flush(); + + detail::BufferedActions::ADIOS2FlushParams adios2FlushParams{ + flushParams.flushLevel, m_flushTarget}; + if (flushParams.backendConfig.json().contains("adios2")) + { + auto adios2Config = flushParams.backendConfig["adios2"]; + if (adios2Config.json().contains("engine")) + { + auto engineConfig = adios2Config["engine"]; + if (engineConfig.json().contains(ADIOS2Defaults::str_flushtarget)) + { + auto target = json::asLowerCaseStringDynamic( + engineConfig[ADIOS2Defaults::str_flushtarget].json()); + if (!target.has_value()) + { + throw error::BackendConfigSchema( + {"adios2", "engine", ADIOS2Defaults::str_flushtarget}, + "Flush target must be either 'disk' or 'buffer', but " + "was non-literal type."); + } + overrideFlushTarget( + adios2FlushParams.flushTarget, + flushTargetFromString(target.value())); + } + } + + if (auto shadow = adios2Config.invertShadow(); shadow.size() > 0) + { + switch (adios2Config.originallySpecifiedAs) + { + case json::SupportedLanguages::JSON: + std::cerr << "Warning: parts of the backend configuration for " + "ADIOS2 remain unused:\n" + << shadow << std::endl; + break; + case json::SupportedLanguages::TOML: { + auto asToml = json::jsonToToml(shadow); + std::cerr << "Warning: parts of the backend configuration for " + "ADIOS2 remain unused:\n" + << asToml << std::endl; + break; + } + } + } + } + for (auto &p : m_fileData) { if (m_dirty.find(p.first) != m_dirty.end()) { - p.second->flush( - flushParams.flushLevel, /* writeAttributes = */ false); + p.second->flush(adios2FlushParams, /* writeAttributes = */ false); } else { @@ -2310,6 +2415,22 @@ namespace detail streamStatus = bool(tmp) ? StreamStatus::OutsideOfStep : StreamStatus::NoStream; } + + if (engineConfig.json().contains(ADIOS2Defaults::str_flushtarget)) + { + auto target = json::asLowerCaseStringDynamic( + engineConfig[ADIOS2Defaults::str_flushtarget].json()); + if (!target.has_value()) + { + throw error::BackendConfigSchema( + {"adios2", "engine", ADIOS2Defaults::str_flushtarget}, + "Flush target must be either 'disk' or 'buffer', but " + "was non-literal type."); + } + overrideFlushTarget( + m_impl->m_flushTarget, + flushTargetFromString(target.value())); + } } auto shadow = impl.m_config.invertShadow(); @@ -2581,11 +2702,12 @@ namespace detail template void BufferedActions::flush( - FlushLevel level, + ADIOS2FlushParams flushParams, F &&performPutGets, bool writeAttributes, bool flushUnconditionally) { + auto level = flushParams.level; if (streamStatus == StreamStatus::StreamOver) { if (flushUnconditionally) @@ -2681,16 +2803,52 @@ namespace detail } } - void BufferedActions::flush(FlushLevel level, bool writeAttributes) + void + BufferedActions::flush(ADIOS2FlushParams flushParams, bool writeAttributes) { + auto decideFlushAPICall = [this, flushTarget = flushParams.flushTarget]( + adios2::Engine &engine) { +#if ADIOS2_VERSION_MAJOR * 1000000000 + ADIOS2_VERSION_MINOR * 100000000 + \ + ADIOS2_VERSION_PATCH * 1000000 + ADIOS2_VERSION_TWEAK >= \ + 2701001223 + bool performDataWrite{}; + switch (flushTarget) + { + case FlushTarget::Disk: + case FlushTarget::Disk_Override: + performDataWrite = true; + break; + case FlushTarget::Buffer: + case FlushTarget::Buffer_Override: + performDataWrite = false; + break; + } + performDataWrite = performDataWrite && m_engineType == "bp5"; + + if (performDataWrite) + { + engine.PerformDataWrite(); + } + else + { + engine.PerformPuts(); + } +#else + (void)this; + (void)flushTarget; + engine.PerformPuts(); +#endif + }; + flush( - level, - [](BufferedActions &ba, adios2::Engine &eng) { + flushParams, + [decideFlushAPICall = std::move(decideFlushAPICall)]( + BufferedActions &ba, adios2::Engine &eng) { switch (ba.m_mode) { case adios2::Mode::Write: case adios2::Mode::Append: - eng.PerformPuts(); + decideFlushAPICall(eng); break; case adios2::Mode::Read: eng.PerformGets(); @@ -2717,7 +2875,7 @@ namespace detail { m_IO.DefineAttribute( ADIOS2Defaults::str_usesstepsAttribute, 0); - flush(FlushLevel::UserFlush, /* writeAttributes = */ false); + flush({FlushLevel::UserFlush}, /* writeAttributes = */ false); return AdvanceStatus::OK; } @@ -2757,7 +2915,7 @@ namespace detail } } flush( - FlushLevel::UserFlush, + {FlushLevel::UserFlush}, [](BufferedActions &, adios2::Engine &eng) { eng.EndStep(); }, /* writeAttributes = */ true, /* flushUnconditionally = */ true); @@ -2917,7 +3075,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( {} std::future -ADIOS2IOHandler::flush(internal::FlushParams const &flushParams) +ADIOS2IOHandler::flush(internal::ParsedFlushParams &flushParams) { return m_impl.flush(flushParams); } @@ -2937,7 +3095,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( : AbstractIOHandler(std::move(path), at) {} -std::future ADIOS2IOHandler::flush(internal::FlushParams const &) +std::future ADIOS2IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index 3fbc9aa395..a451dd2c15 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -355,7 +355,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) +std::future ParallelADIOS1IOHandler::flush(internal::ParsedFlushParams &) { return m_impl->flush(); } @@ -478,7 +478,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) +std::future ParallelADIOS1IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/AbstractIOHandler.cpp b/src/IO/AbstractIOHandler.cpp new file mode 100644 index 0000000000..25adae581e --- /dev/null +++ b/src/IO/AbstractIOHandler.cpp @@ -0,0 +1,35 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IO/AbstractIOHandler.hpp" + +#include "openPMD/IO/FlushParametersInternal.hpp" + +namespace openPMD +{ +std::future AbstractIOHandler::flush(internal::FlushParams const ¶ms) +{ + internal::ParsedFlushParams parsedParams{params}; + auto future = this->flush(parsedParams); + json::warnGlobalUnusedOptions(parsedParams.backendConfig); + return future; +} +} // namespace openPMD diff --git a/src/IO/DummyIOHandler.cpp b/src/IO/DummyIOHandler.cpp index 308f584ce4..6bc6ec4d64 100644 --- a/src/IO/DummyIOHandler.cpp +++ b/src/IO/DummyIOHandler.cpp @@ -32,7 +32,7 @@ DummyIOHandler::DummyIOHandler(std::string path, Access at) void DummyIOHandler::enqueue(IOTask const &) {} -std::future DummyIOHandler::flush(internal::FlushParams const &) +std::future DummyIOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/FlushParams.cpp b/src/IO/FlushParams.cpp new file mode 100644 index 0000000000..d09018ef4f --- /dev/null +++ b/src/IO/FlushParams.cpp @@ -0,0 +1,31 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IO/FlushParametersInternal.hpp" + +namespace openPMD::internal +{ +ParsedFlushParams::ParsedFlushParams(FlushParams const &flushParams) + : flushLevel(flushParams.flushLevel) + , backendConfig(json::parseOptions( + flushParams.backendConfig, /* considerFiles = */ false)) +{} +} // namespace openPMD::internal diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index ae4e6588aa..b663740077 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -2371,7 +2371,7 @@ HDF5IOHandler::HDF5IOHandler( HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush(internal::FlushParams const &) +std::future HDF5IOHandler::flush(internal::ParsedFlushParams &) { return m_impl->flush(); } @@ -2385,7 +2385,7 @@ HDF5IOHandler::HDF5IOHandler( HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush(internal::FlushParams const &) +std::future HDF5IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index e0a17ed980..369ee2e317 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -54,7 +54,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) +std::future ParallelHDF5IOHandler::flush(internal::ParsedFlushParams &) { return m_impl->flush(); } @@ -196,7 +196,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) +std::future ParallelHDF5IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 15d18194c7..10ffce9927 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -29,7 +29,7 @@ JSONIOHandler::JSONIOHandler(std::string path, Access at) : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}} {} -std::future JSONIOHandler::flush(internal::FlushParams const &) +std::future JSONIOHandler::flush(internal::ParsedFlushParams &) { return m_impl.flush(); } diff --git a/src/Series.cpp b/src/Series.cpp index 36a7bd90d6..a68d1270fa 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -372,13 +372,13 @@ std::string Series::backend() const return IOHandler()->backendName(); } -void Series::flush() +void Series::flush(std::string backendConfig) { auto &series = get(); flush_impl( series.iterations.begin(), series.iterations.end(), - {FlushLevel::UserFlush}); + {FlushLevel::UserFlush, std::move(backendConfig)}); } std::unique_ptr Series::parseInput(std::string filepath) @@ -1354,7 +1354,7 @@ AdvanceStatus Series::advance( iterations_iterator begin, Iteration &iteration) { - constexpr internal::FlushParams flushParams = {FlushLevel::UserFlush}; + internal::FlushParams const flushParams = {FlushLevel::UserFlush}; auto &series = get(); auto end = begin; ++end; diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index aa2f600da1..0c0b715e28 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -106,9 +106,9 @@ Attributable &Attributable::setComment(std::string const &c) return *this; } -void Attributable::seriesFlush() +void Attributable::seriesFlush(std::string backendConfig) { - writable().seriesFlush(); + writable().seriesFlush(std::move(backendConfig)); } Series Attributable::retrieveSeries() const diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index fae7630795..32d4cd9816 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -27,9 +27,9 @@ namespace openPMD Writable::Writable(internal::AttributableData *a) : attributable{a} {} -void Writable::seriesFlush() +void Writable::seriesFlush(std::string backendConfig) { - seriesFlush({FlushLevel::UserFlush}); + seriesFlush({FlushLevel::UserFlush, std::move(backendConfig)}); } void Writable::seriesFlush(internal::FlushParams flushParams) diff --git a/src/binding/python/Attributable.cpp b/src/binding/python/Attributable.cpp index 69708fc948..b60bd8cb48 100644 --- a/src/binding/python/Attributable.cpp +++ b/src/binding/python/Attributable.cpp @@ -371,7 +371,10 @@ void init_Attributable(py::module &m) return ""; }) - .def("series_flush", py::overload_cast<>(&Attributable::seriesFlush)) + .def( + "series_flush", + py::overload_cast(&Attributable::seriesFlush), + py::arg("backend_config") = "{}") .def_property_readonly( "attributes", diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index e95ed1bfbb..1c8ddd3df7 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -197,7 +197,7 @@ void init_Series(py::module &m) &Series::iterationFormat, &Series::setIterationFormat) .def_property("name", &Series::name, &Series::setName) - .def("flush", &Series::flush) + .def("flush", &Series::flush, py::arg("backend_config") = "{}") .def_property_readonly("backend", &Series::backend) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 6846ee0f14..c3543fcb7b 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -30,6 +30,14 @@ #include #include +#ifdef __unix__ +#include +#include +#include +#include +#include +#endif + using namespace openPMD; struct BackendSelection @@ -3945,6 +3953,274 @@ TEST_CASE("git_adios2_early_chunk_query", "[serial][adios2]") R"({"backend": "adios2"})"); } +/* + * Require __unix__ since we need all that filestat stuff for this test. + */ +#if defined(__unix__) && defined(ADIOS2_HAVE_BP5) + +enum class FlushDuringStep +{ + Never, + Default_No, + Default_Yes, + Always +}; + +void adios2_bp5_flush(std::string const &cfg, FlushDuringStep flushDuringStep) +{ + constexpr size_t size = 1024 * 1024; + + auto getsize = []() { + off_t res = 0; + int dirfd = open("../samples/bp5_flush.bp", O_RDONLY); + if (dirfd < 0) + { + throw std::system_error( + std::error_code(errno, std::system_category())); + } + DIR *directory = fdopendir(dirfd); + if (!directory) + { + close(dirfd); + throw std::system_error( + std::error_code(errno, std::system_category())); + } + dirent *entry; + struct stat statbuf; + while ((entry = readdir(directory)) != nullptr) + { + if (strcmp(entry->d_name, ".") == 0 || + strcmp(entry->d_name, "..") == 0) + { + continue; + } + int err = fstatat(dirfd, entry->d_name, &statbuf, 0); + if (err != 0) + { + closedir(directory); + close(dirfd); + throw std::system_error( + std::error_code(errno, std::system_category())); + } + res += statbuf.st_size; + } + closedir(directory); + close(dirfd); + return res; + }; + std::vector data(size, 10); + { + Series write("../samples/bp5_flush.bp", Access::CREATE, cfg); + + { + auto component = + write.writeIterations()[0] + .meshes["e_chargeDensity"][RecordComponent::SCALAR]; + component.resetDataset({Datatype::INT, {size}}); + component.storeChunk(data, {0}, {size}); + // component.seriesFlush(FlushMode::NonCollective); + component.seriesFlush(); + } + + auto currentSize = getsize(); + if (flushDuringStep == FlushDuringStep::Default_Yes || + flushDuringStep == FlushDuringStep::Always) + { + // should be roughly within 1% of 4Mb + REQUIRE(std::abs(1 - double(currentSize) / (4 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + + { + auto component = + write.writeIterations()[0] + .meshes["i_chargeDensity"][RecordComponent::SCALAR]; + component.resetDataset({Datatype::INT, {size}}); + component.storeChunk(data, {0}, {size}); + } + + currentSize = getsize(); + if (flushDuringStep == FlushDuringStep::Default_Yes || + flushDuringStep == FlushDuringStep::Always) + { + // should still be roughly within 1% of 4Mb + REQUIRE(std::abs(1 - double(currentSize) / (4 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + + write.flush(); + currentSize = getsize(); + if (flushDuringStep == FlushDuringStep::Default_Yes || + flushDuringStep == FlushDuringStep::Always) + { + // should now be roughly within 1% of 8Mb + REQUIRE(std::abs(1 - double(currentSize) / (8 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + + { + auto component = + write.writeIterations()[0] + .meshes["temperature"][RecordComponent::SCALAR]; + component.resetDataset({Datatype::INT, {size}}); + component.storeChunk(data, {0}, {size}); + // component.seriesFlush(FlushMode::NonCollective); + component.seriesFlush( + "adios2.engine.preferred_flush_target = \"buffer\""); + } + currentSize = getsize(); + if (flushDuringStep == FlushDuringStep::Always) + { + // should now be roughly within 1% of 12Mb + REQUIRE(std::abs(1 - double(currentSize) / (12 * size)) <= 0.01); + } + else if (flushDuringStep == FlushDuringStep::Default_Yes) + { + // should now be roughly within 1% of 8Mb + REQUIRE(std::abs(1 - double(currentSize) / (8 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + + { + auto component = + write.writeIterations()[0] + .meshes["temperature"][RecordComponent::SCALAR]; + component.resetDataset({Datatype::INT, {size}}); + component.storeChunk(data, {0}, {size}); + // component.seriesFlush(FlushMode::NonCollective); + component.seriesFlush( + "adios2.engine.preferred_flush_target = \"disk\""); + } + currentSize = getsize(); + if (flushDuringStep != FlushDuringStep::Never) + { + // should now be roughly within 1% of 16Mb + REQUIRE(std::abs(1 - double(currentSize) / (16 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + } + auto currentSize = getsize(); + // should now be indiscriminately be roughly within 1% of 8Mb after + // closing the Series + REQUIRE(std::abs(1 - double(currentSize) / (16 * size)) <= 0.01); +} + +TEST_CASE("adios2_bp5_flush", "[serial][adios2]") +{ + std::string cfg1 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +preferred_flush_target = "disk" + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush( + cfg1, /* flushDuringStep = */ FlushDuringStep::Default_Yes); + + std::string cfg2 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +preferred_flush_target = "buffer" + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush(cfg2, /* flushDuringStep = */ FlushDuringStep::Default_No); + + std::string cfg3 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +# preferred_flush_target = + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush( + cfg3, /* flushDuringStep = */ FlushDuringStep::Default_Yes); + + std::string cfg4 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +preferred_flush_target = "buffer_override" + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush(cfg4, /* flushDuringStep = */ FlushDuringStep::Never); + + std::string cfg5 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +preferred_flush_target = "disk_override" + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush(cfg5, /* flushDuringStep = */ FlushDuringStep::Always); +} +#endif + TEST_CASE("serial_adios2_backend_config", "[serial][adios2]") { if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") == "ADIOS1")