From a1cbeaa22034995a71202fcd48b5cb910e7e22d0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 20 Mar 2023 19:10:37 -0700 Subject: [PATCH 01/11] [Draft] Release: 0.15.0 Update version and write announcement text in changelog. --- CHANGELOG.rst | 2 +- CITATION.cff | 2 +- NEWS.rst | 2 ++ docs/source/conf.py | 2 +- docs/source/index.rst | 2 +- include/openPMD/version.hpp | 2 +- setup.py | 2 +- 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 86a08b082c..b4c74fc212 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ Changelog 0.15.0 ------ -**Date:** TBA +**Date:** 2023-03-XY [Title] diff --git a/CITATION.cff b/CITATION.cff index 6dbf0c1728..ebb82b34d7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -25,7 +25,7 @@ contact: orcid: https://orcid.org/0000-0003-1943-7141 email: axelhuebl@lbl.gov title: "openPMD-api: C++ & Python API for Scientific I/O with openPMD" -version: 0.15.0-dev +version: 0.15.0 repository-code: https://github.com/openPMD/openPMD-api doi: 10.14278/rodare.27 license: LGPL-3.0-or-later diff --git a/NEWS.rst b/NEWS.rst index 707d1cb712..784e361df7 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -18,6 +18,8 @@ The following backend-specific members of the ``Dataset`` class have been remove They are replaced by backend-specific options in the JSON-based backend configuration. This can be passed in ``Dataset::options``. +XY + 0.14.0 ------ diff --git a/docs/source/conf.py b/docs/source/conf.py index ca3f41e65a..03d8d7bbf7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -87,7 +87,7 @@ # The short X.Y version. version = u'0.15.0' # The full version, including alpha/beta/rc tags. -release = u'0.15.0-dev' +release = u'0.15.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/index.rst b/docs/source/index.rst index 1abdadebaf..c28dacb6c6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,7 +42,7 @@ openPMD-api version supported openPMD standard versions ======================== =================================== ``2.0.0+`` ``2.0.0+`` (not released yet) ``1.0.0+`` ``1.0.1-1.1.0`` (not released yet) -``0.13.1-0.14.0`` (beta) ``1.0.0-1.1.0`` +``0.13.1-0.15.0`` (beta) ``1.0.0-1.1.0`` ``0.1.0-0.12.0`` (alpha) ``1.0.0-1.1.0`` ======================== =================================== diff --git a/include/openPMD/version.hpp b/include/openPMD/version.hpp index 031c5085e7..29a4f7d5ba 100644 --- a/include/openPMD/version.hpp +++ b/include/openPMD/version.hpp @@ -30,7 +30,7 @@ #define OPENPMDAPI_VERSION_MAJOR 0 #define OPENPMDAPI_VERSION_MINOR 15 #define OPENPMDAPI_VERSION_PATCH 0 -#define OPENPMDAPI_VERSION_LABEL "dev" +#define OPENPMDAPI_VERSION_LABEL "" /** @} */ /** maximum supported version of the openPMD standard (read & write, diff --git a/setup.py b/setup.py index c73dd3c024..cea00f5805 100644 --- a/setup.py +++ b/setup.py @@ -168,7 +168,7 @@ def build_extension(self, ext): setup( name='openPMD-api', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version='0.15.0.dev', + version='0.15.0', author='Axel Huebl, Franz Poeschel, Fabian Koller, Junmin Gu', author_email='axelhuebl@lbl.gov, f.poeschel@hzdr.de', maintainer='Axel Huebl', From df779a07f48ca962c2a642c020b252c140c5775d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 21 Mar 2023 14:38:00 +0100 Subject: [PATCH 02/11] Changelog: Until 93befd7476974d6bfa1a5f69d228c133cadf4ef6 --- CHANGELOG.rst | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b4c74fc212..44369ba75a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,20 +17,92 @@ Changes to "0.14.0" Features """""""" -- Python: support of 3.10 and 3.11, removal of 3.6 #1323 +- Python: support of 3.10 and 3.11, removal of 3.6 #1323 #1139 - include internally shipped toml11 v3.7.1 #1148 #1227 - pybind11: require version 2.10.1+ #1220 #1322 +- Switch to C++17 #1128 #1140 #1157 +- Error-recovery during parsing #1150 +- Extensive update for JSON/TOML configuration #1043 + - TOML as an alternative to JSON #1146 + - compression configuration via JSON# 1043 + - case insensitivity #1043 + - datatype conversion for string values #1043 + - ``json::merge`` public function #1043 + - better warnings for unused values #1043 + - new JSON options: ``backend`` and ``iteration_encoding`` #1043 + - ADIOS1 compression configuration via JSON #1043 #1162 +- Header for openPMD-defined error types #1080 +- Support for ADIOS2 BP5 engine #1119 +- I/O optimizations for HDF5 #1129 #1133 +- Support for empty string attributes #1087 +- Support for inconsistent padding of filenames in file-based encoding #1118 +- ADIOS2: Automatic (de)activation of span API depending on compression configuration #1155 Bug Fixes """"""""" +- HDF5 + + - Support attribute reads from HDF5 Vlen Strings #1084 + +- ADIOS2 + + - Don't apply compression operators multiple times #1152 + - Fix logic for associating openPMD objects to files and paths therein #1073 + - Fix precedence of environment variable vs. JSON configuration - CMake: - MPI: prefer HDF5 in Config package, too #1340 + - ADIOS1: do not include as -isystem #1076 +- Don't forget closing unmodified files #1083 +- Diverse relaxations on attribute type conversions #1085 #1096 #1137 +- Performance bug: Don't reread iterations that are already parsed #1089 +- Avoid object slicing in Series class #1107 + +Breaking Changes +"""""""""""""""" + +- Deprecations + + - Iteration.closed attribute #1088 +- Redesign of public class structure + + - Apply frontend redesign to Container and deriving classes #1115 #1159 +- Removal of APIs + - Dataset::transform, Dataset::compression and Dataset::chunksize #1043 Other """"" - Catch2: updated to 2.13.10 #1299 #... +- Tests & Examples: + + - Test: Interleaved Write and Close #1073 #1078 + - Extend and fix examples 8a and 8b (bench write/read parallel) #1131 #1144 + - Extensive Python example for Streaming API #1141 +- CI + + - URL Check for broken links #1086 + - CI savings (abort prior push, draft skips most) #1116 + - Appveyor fixes for Python Executable #1127 + - Pre-commit and clang-format #1142 +- CMake + + - Disable in-source builds #1079 + - Fixes for NVCC #1102 + - Set RPATHs on installed targets #1105 #1103 + +- Docs + + - More easily findable documentation for ``-DPython_EXECUTABLE`` #1104 and lazy parsing #1111 + - HDF5 performance tuning and known issues #1129 #1132 + - HDF5: Document HDF5_USE_FILE_LOCKING #1106 + - SST/libfabric installation notes for Cray systems #1134 + - Doc: OMPI_MCA_io Control #1114 +- Enable use of Series::setName() and Series::setIterationEncoding() in combination with file-based encoding 1081 +- Remove DATATYPE, HIGHEST_DATATYPE AND LOWEST_DATATYPE from Datatype enumeration #1100 +- Check for undefined datatypes in dataset definitions #1099 +- Include StringManip header into public headers #1124 +Add default constructor for DynamicMemoryView class #1156 0.14.3 From ff4cd7374c544b8ad85ed88cbc36341e0cc7a0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 21 Mar 2023 18:03:07 +0100 Subject: [PATCH 03/11] Add remaining items to changelog TODO: Cleanup, sorting, grouping --- CHANGELOG.rst | 92 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 77 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 44369ba75a..890cb6a86f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,23 +20,37 @@ Features - Python: support of 3.10 and 3.11, removal of 3.6 #1323 #1139 - include internally shipped toml11 v3.7.1 #1148 #1227 - pybind11: require version 2.10.1+ #1220 #1322 -- Switch to C++17 #1128 #1140 #1157 -- Error-recovery during parsing #1150 +- Switch to C++17 #1128 #1140 #1157 #1164 #1183 #1185 +- Error-recovery during parsing #1150 #1179 #1237 - Extensive update for JSON/TOML configuration #1043 - TOML as an alternative to JSON #1146 - compression configuration via JSON# 1043 - case insensitivity #1043 - datatype conversion for string values #1043 - - ``json::merge`` public function #1043 + - ``json::merge`` public function #1043 #1333 - better warnings for unused values #1043 - new JSON options: ``backend`` and ``iteration_encoding`` #1043 - ADIOS1 compression configuration via JSON #1043 #1162 -- Header for openPMD-defined error types #1080 -- Support for ADIOS2 BP5 engine #1119 -- I/O optimizations for HDF5 #1129 #1133 -- Support for empty string attributes #1087 -- Support for inconsistent padding of filenames in file-based encoding #1118 +New access types: + - ``APPEND``: Add new iterations without reading, supports ADIOS2 Append mode #1007 #1302 + - ``READ_LINEAR``: For reading through ADIOS2 steps, for full support of ADIOS2 BP5 #1291 #1379 +- Support for all char types (CHAR SCHAR UCHAR) #1275 #1378 +- Header for openPMD-defined error types #1080 #1355 +- Support for ADIOS2 BP5 engine #1119 #1215 #1258 #1262 #1291 +- Support for selecting flush targets (buffer/disk) in ADIOS2 BP5 for more fine-grained memory control #1226 #1207 +- Add file extensions for ADIOS2: ``.bp4``, ``.bp5`` and furthers, make them behave more as expected #1218 +- Add ``Series::close()`` API call #1324 +- Optionally explicitly map ADIOS2 steps to openPMD iterations via modifiable attributes (only supported in experimental ADIOS2 modes) #949 +- Support for ADIOS 2.8 and newer #1166 +- I/O optimizations for HDF5 #1129 #1133 #1192 +- Support for empty string attributes #1087 #1223 #1338 +- Support for inconsistent padding of filenames in file-based encoding #1118 #1253 - ADIOS2: Automatic (de)activation of span API depending on compression configuration #1155 +- ADIOS2: Support for operator specification at read time #1191 +- Support for array specializations of C++ smart pointer types #1296 +- Direct support for raw pointer types in ``store/loadChunk()`` API, replacing former ``shareRaw()`` #1229 +- Support and backend optimizations (ADIOS2 BP5) based on unique pointer types in ``store/loadChunk()`` #1294 +- Use C++ ``std::optional`` types in public Attribute API (``Attribute::getOptional()``) for dynamic attribute type conversion #1278 Bug Fixes """"""""" @@ -44,20 +58,38 @@ Bug Fixes - HDF5 - Support attribute reads from HDF5 Vlen Strings #1084 + - Close HFD5 handles in availableChunks task #1386 +- ADIOS1 + - Fix use-after-free issue #1224 - ADIOS2 - Don't apply compression operators multiple times #1152 - Fix logic for associating openPMD objects to files and paths therein #1073 - Fix precedence of environment variable vs. JSON configuration + - Detect changing datatypes and warn/fail accordingly #1356 - CMake: - MPI: prefer HDF5 in Config package, too #1340 - ADIOS1: do not include as -isystem #1076 + - Fix ``-Wsign-compare`` #1202 + - Remove caching of global CMake variables #1313 + - Fix Build & Install Option Names #1326 + - Prefer parallel HDF5 in find_package in downstream use #1340 + - CMake: Multi-Config Generator #1384 +Python: + + - Fix ``__repr__`` (time and Iteration) #1242 #1149 + - Python Tests: Fix ``long`` Numpy Type #1348 + - use ``double`` as standard for attributes #1290 #1369kk + - Fix dtype_from_numpy #1357 - Don't forget closing unmodified files #1083 - Diverse relaxations on attribute type conversions #1085 #1096 #1137 - Performance bug: Don't reread iterations that are already parsed #1089 +- Performance bug: Don't flush prematurely #1264 - Avoid object slicing in Series class #1107 +- Avoid use-after-free in Python bindings #1225 +- Logical fixes for opening iterations #1239 Breaking Changes """""""""""""""" @@ -65,6 +97,8 @@ Breaking Changes - Deprecations - Iteration.closed attribute #1088 + - ``shareRaw`` (replaced with raw- and unique-ptr overloads, see features section) #1229 + - ADIOS1 backend (deprecation notice has hints on upgrading to ADIOS2) #1314 - Redesign of public class structure - Apply frontend redesign to Container and deriving classes #1115 #1159 @@ -73,36 +107,64 @@ Breaking Changes Other """"" -- Catch2: updated to 2.13.10 #1299 #... +- Catch2: updated to 2.13.10 #1299 #1344 - Tests & Examples: - Test: Interleaved Write and Close #1073 #1078 - - Extend and fix examples 8a and 8b (bench write/read parallel) #1131 #1144 + - Extend and fix examples 8a and 8b (bench write/read parallel) #1131 #1144 #1231 #1359 + - GPU support in example 8a #1240 - Extensive Python example for Streaming API #1141 + - General overhaul of examples to newest API standards #1371 - CI - URL Check for broken links #1086 - CI savings (abort prior push, draft skips most) #1116 - Appveyor fixes for Python Executable #1127 - - Pre-commit and clang-format #1142 + - Pre-commit and clang-format #1142 #1175 #1178 #1032 #1222 #1370 + - ADIOS1: Fix Serial Builds, CI: Clang 10->12 #1167 + - Upgrade NVHPC Apt repository #1241 + - Spack upgrade to v0.17.1 and further fixes #1244 + - Update CUDA repository key #1256 + - Switch from Conda to Mamba #1261 + - Remove ``-Wno-deprecated-declarations`` where possible #1246 + - Expand read-only permission tests #1272 + - Ensure that the CI also build against ADIOS2 v2.7.1 #1271 + - Build(deps): Bump s-weigand/setup-conda from 1.1.0 to 1.1.1 #1284 + - Style w/ Ubuntu 22.04 #1346 + - Add CodeQL workflow for GitHub code scanning #1345 + - Cache Action v3 #1358 #1362 - CMake + - Extra CMake Arg Control in setup.py #1199 + - Do not strip Python symbols in Debug #1219 - Disable in-source builds #1079 - - Fixes for NVCC #1102 + - Fixes for NVCC #1102 ##1184 - Set RPATHs on installed targets #1105 #1103 - + - CMake 3.22+: Policy CMP0127 #1165 + - Warning Flags First in CXXFLAGS #1172 - Docs - More easily findable documentation for ``-DPython_EXECUTABLE`` #1104 and lazy parsing #1111 - HDF5 performance tuning and known issues #1129 #1132 - HDF5: Document HDF5_USE_FILE_LOCKING #1106 - SST/libfabric installation notes for Cray systems #1134 - - Doc: OMPI_MCA_io Control #1114 + - OMPI_MCA_io Control #1114 + - Update Citation & Add BibTeX (#1168) + - Fix CLI Highlighting #1171 + - HDF5 versions that support collective metadata #1250 + - Recommend Static Build for Superbuilds #1325 + - Latest Sphinx, Docutils, RTD #1341 +- Tooling + - openpmd-pipe: better optional support for MPI #1186 #1336 - Enable use of Series::setName() and Series::setIterationEncoding() in combination with file-based encoding 1081 - Remove DATATYPE, HIGHEST_DATATYPE AND LOWEST_DATATYPE from Datatype enumeration #1100 - Check for undefined datatypes in dataset definitions #1099 - Include StringManip header into public headers #1124 -Add default constructor for DynamicMemoryView class #1156 +- Add default constructor for DynamicMemoryView class #1156 +- Helpful error message upon wrong backend specification #1214 +- Helpful error message for errors in ``loadChunk`` API #1373 +- No warning when opening a single file of a file-based Series #1368 +- Add ``IterationIndex_t`` type alias #1285 0.14.3 From 37e659e359febd73b2aa51667f1db9c311d8808c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 22 Mar 2023 11:45:59 +0100 Subject: [PATCH 04/11] Group changelog items --- CHANGELOG.rst | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 890cb6a86f..ae1e068fa9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -34,23 +34,32 @@ Features New access types: - ``APPEND``: Add new iterations without reading, supports ADIOS2 Append mode #1007 #1302 - ``READ_LINEAR``: For reading through ADIOS2 steps, for full support of ADIOS2 BP5 #1291 #1379 -- Support for all char types (CHAR SCHAR UCHAR) #1275 #1378 -- Header for openPMD-defined error types #1080 #1355 -- Support for ADIOS2 BP5 engine #1119 #1215 #1258 #1262 #1291 -- Support for selecting flush targets (buffer/disk) in ADIOS2 BP5 for more fine-grained memory control #1226 #1207 -- Add file extensions for ADIOS2: ``.bp4``, ``.bp5`` and furthers, make them behave more as expected #1218 -- Add ``Series::close()`` API call #1324 -- Optionally explicitly map ADIOS2 steps to openPMD iterations via modifiable attributes (only supported in experimental ADIOS2 modes) #949 -- Support for ADIOS 2.8 and newer #1166 -- I/O optimizations for HDF5 #1129 #1133 #1192 +- ADIOS2: + + - Support for ADIOS 2.8 and newer #1166 + - Support for ADIOS2 BP5 engine #1119 #1215 #1258 #1262 #1291 + - Support for selecting flush targets (buffer/disk) in ADIOS2 BP5 for more fine-grained memory control #1226 #1207 + - Add file extensions for ADIOS2: ``.bp4``, ``.bp5`` and furthers, make them behave more as expected #1218 + - ADIOS2: Support for operator specification at read time #1191 + - ADIOS2: Automatic (de)activation of span API depending on compression configuration #1155 + - Optionally explicitly map ADIOS2 steps to openPMD iterations via modifiable attributes (only supported in experimental ADIOS2 modes) #949 + +- HDF5: + + - I/O optimizations for HDF5 #1129 #1133 #1192 + +- Miscellaneous API additions: + + - Support for all char types (CHAR SCHAR UCHAR) #1275 #1378 + - Header for openPMD-defined error types #1080 #1355 + - Add ``Series::close()`` API call #1324 + - Support for array specializations of C++ smart pointer types #1296 + - Direct support for raw pointer types in ``store/loadChunk()`` API, replacing former ``shareRaw()`` #1229 + - Support for and backend optimizations (ADIOS2 BP5) based on unique pointer types in ``store/loadChunk()`` #1294 + - Use C++ ``std::optional`` types in public Attribute API (``Attribute::getOptional()``) for dynamic attribute type conversion #1278 + - Support for empty string attributes #1087 #1223 #1338 - Support for inconsistent padding of filenames in file-based encoding #1118 #1253 -- ADIOS2: Automatic (de)activation of span API depending on compression configuration #1155 -- ADIOS2: Support for operator specification at read time #1191 -- Support for array specializations of C++ smart pointer types #1296 -- Direct support for raw pointer types in ``store/loadChunk()`` API, replacing former ``shareRaw()`` #1229 -- Support and backend optimizations (ADIOS2 BP5) based on unique pointer types in ``store/loadChunk()`` #1294 -- Use C++ ``std::optional`` types in public Attribute API (``Attribute::getOptional()``) for dynamic attribute type conversion #1278 Bug Fixes """"""""" @@ -60,8 +69,8 @@ Bug Fixes - Support attribute reads from HDF5 Vlen Strings #1084 - Close HFD5 handles in availableChunks task #1386 - ADIOS1 - - Fix use-after-free issue #1224 + - Fix use-after-free issue #1224 - ADIOS2 - Don't apply compression operators multiple times #1152 From 7117be333beec7433906cade0d9ae7a132f365d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 22 Mar 2023 11:54:25 +0100 Subject: [PATCH 05/11] Fix formatting --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ae1e068fa9..d14a9b2e2b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -31,7 +31,7 @@ Features - better warnings for unused values #1043 - new JSON options: ``backend`` and ``iteration_encoding`` #1043 - ADIOS1 compression configuration via JSON #1043 #1162 -New access types: +- New access types: - ``APPEND``: Add new iterations without reading, supports ADIOS2 Append mode #1007 #1302 - ``READ_LINEAR``: For reading through ADIOS2 steps, for full support of ADIOS2 BP5 #1291 #1379 - ADIOS2: From f85d632c8f45924dfedcb442e7379e636c1ce157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 22 Mar 2023 14:22:34 +0100 Subject: [PATCH 06/11] Upgrading guide --- CHANGELOG.rst | 8 +++-- NEWS.rst | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d14a9b2e2b..636ae8c993 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -86,7 +86,7 @@ Bug Fixes - Fix Build & Install Option Names #1326 - Prefer parallel HDF5 in find_package in downstream use #1340 - CMake: Multi-Config Generator #1384 -Python: +- Python: - Fix ``__repr__`` (time and Iteration) #1242 #1149 - Python Tests: Fix ``long`` Numpy Type #1348 @@ -105,7 +105,7 @@ Breaking Changes - Deprecations - - Iteration.closed attribute #1088 + - ``Iteration::closedByWriter()`` attribute #1088 - ``shareRaw`` (replaced with raw- and unique-ptr overloads, see features section) #1229 - ADIOS1 backend (deprecation notice has hints on upgrading to ADIOS2) #1314 - Redesign of public class structure @@ -114,6 +114,10 @@ Breaking Changes - Removal of APIs - Dataset::transform, Dataset::compression and Dataset::chunksize #1043 +.. note:: + + See :ref:`NEWS.rst ` for a more detailed upgrade guide. + Other """"" - Catch2: updated to 2.13.10 #1299 #1344 diff --git a/NEWS.rst b/NEWS.rst index 784e361df7..40246be635 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -17,8 +17,92 @@ Catch2 2.13.10 is now the minimally supported version for tests. The following backend-specific members of the ``Dataset`` class have been removed: ``Dataset::setChunkSize()``, ``Dataset::setCompression()``, ``Dataset::setCustomTransform()``, ``Dataset::chunkSize``, ``Dataset::compression``, ``Dataset::transform``. They are replaced by backend-specific options in the JSON-based backend configuration. This can be passed in ``Dataset::options``. +The following configuration shows a compression configuration for ADIOS1 and ADIOS2: -XY +.. code-block:: json + + { + "adios1": { + "dataset": { + "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" + } + }, + "adios2": { + "dataset": { + "operators": [ + { + "type": "zlib", + "parameters": { + "clevel": 9 + } + } + ] + } + } + } + +Or alternatively, in TOML: + +.. code-block:: toml + + [adios1.dataset] + transform = "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" + + [[adios2.dataset.operators]] + type = "zlib" + parameters.clevel = 9 + + +The helper function ``shareRaw`` of the C++ API has been deprecated. +In its stead, there are now new API calls ``RecordComponent::storeChunkRaw()`` and ``RecordComponent::loadChunkRaw``. + +The **ADIOS1 backend** is now deprecated, to be replaced fully with ADIOS2. +Now is a good time to check if ADIOS2 is able to read old ADIOS1 datasets that you might have. Otherwise, ``openpmd-pipe`` can be used for conversion: + +.. code-block:: bash + + openpmd-pipe --infile adios1_dataset_%T.bp --inconfig 'backend = "adios1"' --outfile adios2_dataset_%T.bp --outconfig 'backend = "adios2"' + +The class structure of ``Container`` and deriving classes has been reworked. +Usage of the API generally stays the same, but code that relies on the concrete class structure might break. + +The ``Iteration::closedByWriter()`` attribute has been deprecated as a leftover from the early streaming implementation. + +Old: + +.. code-block:: cpp + + double const * data; + recordComponent.storeChunk(shareRaw(data), offset, extent); + +New: + +.. code-block:: cpp + + double const * data; + recordComponent.storeChunkRaw(data, offset, extent); + +Additionally, ``determineDatatype`` now accepts pointer types (raw and smart pointers): + +Old: + +.. code-block:: cpp + + std::vector data; + Datatype dt = determineDatatype(shareRaw(data)); + +New: + +.. code-block:: cpp + + std::vector data; + Datatype dt = determineDatatype(data.data()); + +.. note:: + + ``determineDatatype`` does not directly accept ``determineDatatype(data)``, since it's unclear if the result from that call would be ``Datatype::DOUBLE`` or ``Datatype::VEC_DOUBLE``. + + In order to get the direct mapping between C++ type and openPMD datatype, use the template parameter of ``determineDatatype``: ``determineDatatype()`` or ``determineDatatype>()``. 0.14.0 From 9b9a37bc31318aa676e6f93f37ef0d9992fb69cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 22 Mar 2023 15:35:30 +0100 Subject: [PATCH 07/11] Add summary --- CHANGELOG.rst | 11 +++++++++-- examples/7_extended_write_serial.cpp | 28 ++++++++-------------------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 636ae8c993..7698fdfc33 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,9 +7,16 @@ Changelog ------ **Date:** 2023-03-XY -[Title] +C++17, Parsing error recovery, Support for ADIOS2 BP5 engine, Append and Read-Linear access modes, performance and memory optimizations -[Summary] +This release adds error recovery mechanisms, in order to access erroneous datasets, created e.g. by crashing simulations. +The BP5 engine of ADIOS2 v2.9 is fully supported by this release, including access to its various features for more fine-grained control of memory usage. +Various I/O performance improvements for HDF5 are activated by default. +Runtime configuration of openPMD and its backends, e.g. selection of backends and compression, is now consistently done via JSON, and alternatively via TOML for better readability. +The data storage/retrieval API now consistently supports all common C++ pointer types (raw and smart pointers), implementing automatic memory optimizations for ADIOS2 BP5 if using unique pointers. + +The miminum required C++ version is now C++17. +Supported Python versions are Python 3.10 and 3.11. Changes to "0.14.0" ^^^^^^^^^^^^^^^^^^^ diff --git a/examples/7_extended_write_serial.cpp b/examples/7_extended_write_serial.cpp index bfb64e1fff..f6780afa8e 100644 --- a/examples/7_extended_write_serial.cpp +++ b/examples/7_extended_write_serial.cpp @@ -9,7 +9,7 @@ int main() { auto f = - io::Series("working/directory/2D_simData.h5", io::Access::CREATE); + io::Series("working/directory/2D_simData.bp", io::Access::CREATE); // all required openPMD attributes will be set to reasonable default // values (all ones, all zeros, empty strings,...) manually setting them @@ -105,25 +105,13 @@ int main() io::Datatype dtype = io::determineDatatype(partial_mesh); auto d = io::Dataset(dtype, io::Extent{2, 5}); std::string datasetConfig = R"END( -{ - "adios1": { - "dataset": { - "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" - } - }, - "adios2": { - "dataset": { - "operators": [ - { - "type": "zlib", - "parameters": { - "clevel": 9 - } - } - ] - } - } -})END"; +[adios1.dataset] +transform = "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" + +[[adios2.dataset.operatorors]] +type = "zlib" +parameters.clevel = 9 +)END"; d.options = datasetConfig; mesh["x"].resetDataset(d); From 3c4086a15524c82e4498f3382bf1d0514030768a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 22 Mar 2023 16:12:46 +0100 Subject: [PATCH 08/11] Cleanup --- CHANGELOG.rst | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7698fdfc33..e7960b7880 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,7 +27,7 @@ Features - Python: support of 3.10 and 3.11, removal of 3.6 #1323 #1139 - include internally shipped toml11 v3.7.1 #1148 #1227 - pybind11: require version 2.10.1+ #1220 #1322 -- Switch to C++17 #1128 #1140 #1157 #1164 #1183 #1185 +- Switch to C++17 #1103 #1128 #1140 #1157 #1164 #1183 #1185 - Error-recovery during parsing #1150 #1179 #1237 - Extensive update for JSON/TOML configuration #1043 - TOML as an alternative to JSON #1146 @@ -81,13 +81,13 @@ Bug Fixes - ADIOS2 - Don't apply compression operators multiple times #1152 - - Fix logic for associating openPMD objects to files and paths therein #1073 + - Fix logic for associating openPMD objects to files and paths therein (needed for interleaved write and close) #1073 - Fix precedence of environment variable vs. JSON configuration - Detect changing datatypes and warn/fail accordingly #1356 - CMake: - MPI: prefer HDF5 in Config package, too #1340 - - ADIOS1: do not include as -isystem #1076 + - ADIOS1: do not include as ``-isystem`` #1076 - Fix ``-Wsign-compare`` #1202 - Remove caching of global CMake variables #1313 - Fix Build & Install Option Names #1326 @@ -98,7 +98,7 @@ Bug Fixes - Fix ``__repr__`` (time and Iteration) #1242 #1149 - Python Tests: Fix ``long`` Numpy Type #1348 - use ``double`` as standard for attributes #1290 #1369kk - - Fix dtype_from_numpy #1357 + - Fix ``dtype_from_numpy`` #1357 - Don't forget closing unmodified files #1083 - Diverse relaxations on attribute type conversions #1085 #1096 #1137 - Performance bug: Don't reread iterations that are already parsed #1089 @@ -119,7 +119,7 @@ Breaking Changes - Apply frontend redesign to Container and deriving classes #1115 #1159 - Removal of APIs - - Dataset::transform, Dataset::compression and Dataset::chunksize #1043 + - ``Dataset::transform``, ``Dataset::compression`` and ``Dataset::chunksize`` #1043 .. note:: @@ -131,8 +131,10 @@ Other - Tests & Examples: - Test: Interleaved Write and Close #1073 #1078 - - Extend and fix examples 8a and 8b (bench write/read parallel) #1131 #1144 #1231 #1359 - - GPU support in example 8a #1240 + - Extend and fix examples 8a and 8b (bench write/read parallel) #1131 #1144 #1231 #1359 #1240 + - support variable encoding #1131 + - block located at top left corner was mistaken to read a block in the center #1131 + - GPU support in example 8a #1240 - Extensive Python example for Streaming API #1141 - General overhaul of examples to newest API standards #1371 - CI @@ -153,34 +155,36 @@ Other - Style w/ Ubuntu 22.04 #1346 - Add CodeQL workflow for GitHub code scanning #1345 - Cache Action v3 #1358 #1362 + - Spack: No More ``load -r`` #1125 - CMake - - Extra CMake Arg Control in setup.py #1199 + - Extra CMake Arg Control in ``setup.py`` #1199 - Do not strip Python symbols in Debug #1219 - Disable in-source builds #1079 - - Fixes for NVCC #1102 ##1184 - - Set RPATHs on installed targets #1105 #1103 - - CMake 3.22+: Policy CMP0127 #1165 - - Warning Flags First in CXXFLAGS #1172 + - Fixes for NVCC #1102 #1103 #1184 + - Set RPATHs on installed targets #1105 + - CMake 3.22+: Policy ``CMP0127`` #1165 + - Warning Flags First in ``CXXFLAGS`` #1172 - Docs - More easily findable documentation for ``-DPython_EXECUTABLE`` #1104 and lazy parsing #1111 - HDF5 performance tuning and known issues #1129 #1132 - - HDF5: Document HDF5_USE_FILE_LOCKING #1106 + - HDF5: Document ``HDF5_USE_FILE_LOCKING`` #1106 - SST/libfabric installation notes for Cray systems #1134 - - OMPI_MCA_io Control #1114 + - OpenMPI: Document ``OMPI_MCA_io`` Control #1114 - Update Citation & Add BibTeX (#1168) - Fix CLI Highlighting #1171 - HDF5 versions that support collective metadata #1250 - Recommend Static Build for Superbuilds #1325 - Latest Sphinx, Docutils, RTD #1341 - Tooling - - openpmd-pipe: better optional support for MPI #1186 #1336 -- Enable use of Series::setName() and Series::setIterationEncoding() in combination with file-based encoding 1081 -- Remove DATATYPE, HIGHEST_DATATYPE AND LOWEST_DATATYPE from Datatype enumeration #1100 + - ``openpmd-pipe``: better optional support for MPI #1186 #1336 + - ``openpmd-ls``: use lazy parsing #1111 +- Enable use of ``Series::setName()`` and ``Series::setIterationEncoding()`` in combination with file-based encoding 1081 +- Remove ``DATATYPE``, ``HIGHEST_DATATYPE`` AND ``LOWEST_DATATYPE`` from Datatype enumeration #1100 - Check for undefined datatypes in dataset definitions #1099 -- Include StringManip header into public headers #1124 -- Add default constructor for DynamicMemoryView class #1156 +- Include ``StringManip`` header into public headers #1124 +- Add default constructor for ``DynamicMemoryView`` class #1156 - Helpful error message upon wrong backend specification #1214 - Helpful error message for errors in ``loadChunk`` API #1373 - No warning when opening a single file of a file-based Series #1368 From a0f7abba2ebf01ede62c8521ca4fc4adf1c0d27a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 22 Mar 2023 16:31:12 +0100 Subject: [PATCH 09/11] Add 0.14.4 and 0.14.5 changelog --- CHANGELOG.rst | 137 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 133 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e7960b7880..b39da15dcd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -54,6 +54,7 @@ Features - HDF5: - I/O optimizations for HDF5 #1129 #1133 #1192 + - Improve write time by disabling fill #1192 - Miscellaneous API additions: @@ -66,7 +67,7 @@ Features - Use C++ ``std::optional`` types in public Attribute API (``Attribute::getOptional()``) for dynamic attribute type conversion #1278 - Support for empty string attributes #1087 #1223 #1338 -- Support for inconsistent padding of filenames in file-based encoding #1118 #1253 +- Support for inconsistent and overflowing padding of filenames in file-based encoding #1118 #1173 #1253 Bug Fixes """"""""" @@ -77,34 +78,45 @@ Bug Fixes - Close HFD5 handles in availableChunks task #1386 - ADIOS1 - - Fix use-after-free issue #1224 + - Fix use-after-free issue in ``ADIOS1IOHandler`` #1224 - ADIOS2 - Don't apply compression operators multiple times #1152 - Fix logic for associating openPMD objects to files and paths therein (needed for interleaved write and close) #1073 - Fix precedence of environment variable vs. JSON configuration - Detect changing datatypes and warn/fail accordingly #1356 + - Remove deprecated debug parameter in ADIOS2 #1269 +- HDF5 + + - missing HDF5 include #1236 - CMake: - MPI: prefer HDF5 in Config package, too #1340 - ADIOS1: do not include as ``-isystem`` #1076 - - Fix ``-Wsign-compare`` #1202 - Remove caching of global CMake variables #1313 - Fix Build & Install Option Names #1326 - Prefer parallel HDF5 in find_package in downstream use #1340 - CMake: Multi-Config Generator #1384 +- Warnings: + + - Avoid copying std::string in for loop #1268 + - SerialIOTest: Fix GCC Pragma Check #1213 #1260 + - Fix ``-Wsign-compare`` #1202 - Python: - Fix ``__repr__`` (time and Iteration) #1242 #1149 - Python Tests: Fix ``long`` Numpy Type #1348 - use ``double`` as standard for attributes #1290 #1369kk - Fix ``dtype_from_numpy`` #1357 + - Wheels: Fix macOS arm64 (M1) builds #1233 + - Avoid use-after-free in Python bindings #1225 + - Patch MSVC pybind11 debug bug #1209 + - sign compare warning #1198 - Don't forget closing unmodified files #1083 - Diverse relaxations on attribute type conversions #1085 #1096 #1137 - Performance bug: Don't reread iterations that are already parsed #1089 - Performance bug: Don't flush prematurely #1264 - Avoid object slicing in Series class #1107 -- Avoid use-after-free in Python bindings #1225 - Logical fixes for opening iterations #1239 Breaking Changes @@ -191,6 +203,123 @@ Other - Add ``IterationIndex_t`` type alias #1285 +0.14.5 +------ +**Date:** 2022-06-07 + +Improve Series Parsing, Python & Fix Backend Bugs + +This release improves reading back iterations that overflow the specified zero-pattern. +ADIOS1, ADIOS2 and HDF5 backend stability and performance were improved. +Python bindings got additional wheel platform support and various smaller issues were fixed. + +Changes to "0.14.4" +^^^^^^^^^^^^^^^^^^^ + +Bug Fixes +""""""""" + +- Series and iterations: + + - fix read of overflowing zero patterns #1173 #1253 + - fix for opening an iteration #1239 +- ADIOS1: + + - fix use-after-free in ``ADIOS1IOHandler`` #1224 + - Remove task from IO queue if it fails with exception #1179 +- ADIOS2: + + - Remove deprecated debug parameter in ADIOS2 #1269 + - Add memory leak suppression: ``ps_make_timer_name_`` #1235 + - Don't safeguard empty strings while reading #1223 +- HDF5: + + - missing HDF5 include #1236 +- Python: + + - Wheels: Fix macOS arm64 (M1) builds #1233 + - Python Iteration: Fix ``__repr__`` (time) #1242 + - Increase reference count also in other ``load_chunk`` overload #1225 + - Do Not Strip Symbols In Debug #1219 + - Patch MSVC pybind11 debug bug #1209 + +Other +""""" + +- HDF5: + + - Improve write time by disabling fill #1192 + - Update documented HDF5 versions with collective metadata issues #1250 +- Print warning if mpi4py is not found in ``openpmd-pipe`` #1186 +- Pass-through flushing parameters #1226 +- Clang-Format #1032 #1222 +- Warnings: + + - Avoid copying std::string in for loop #1268 + - SerialIOTest: Fix GCC Pragma Check #1213 #1260 + - Fix ``-Wsign-compare`` #1202 +- CI: + + - Fix Conda Build - <3 Mamba #1261 + - Fix Spack #1244 + - Update CUDA repo key #1256 + - NVHPC New Apt Repo #1241 +- Python: + + - ``setup.py``: Extra CMake Arg Control #1199 + - sign compare warning #1198 + + +0.14.4 +------ +**Date:** 2022-01-21 + +Increased Compatibility & Python Install Bug + +This release fixes various read/parsing bugs and increases compatibility with upcoming versions of ADIOS and old releases of Intel ``icpc``. +An installation issue for pip-based installs from source in the last release was fixed and Python 3.10 support added. +Various documentation and installation warnings have been fixed. + +Changes to "0.14.3" +^^^^^^^^^^^^^^^^^^^ + +Bug Fixes +""""""""" + +- ADIOS2: + + - automatically deactivate ``span`` based ``Put`` API when operators are present #1155 + - solve incompatibilities w/ post-``2.7.1`` ``master``-branch #1166 +- ICC 19.1.2: C++17 work-arounds (``variant``) #1157 +- Don't apply compression operators multiple times in variable-based iteration encoding #1152 +- Reading/parsing: + + - remove invalid records from data structures entirely #1150 + - fix grid spacing with type long double #1137 +- Python: + + - fix ``Iteration`` ``__repr__`` typo #1149 + - add ``cmake/`` to ``MANIFEST.in`` #1140 + +Other +""""" + +- add simple ``.pre-commit-config.yaml`` +- Python: + + - support Python 3.10 #1139 +- CMake: + + - warning flags first in ``CXXFLAGS`` #1172 + - add policy CMP0127 (v3.22+) #1165 +- Docs: + + - fix CLI highlighting #1171 + - update citation & add BibTeX #1168 + - fix HDF5 JSON File #1169 + - minor warnings #1170 + + 0.14.3 ------ **Date:** 2021-11-03 From 2d594ca7a4576af91fd0b3063181d5ab7fbf26b0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 25 Mar 2023 00:36:38 -0700 Subject: [PATCH 10/11] Shorter Release Title, Date --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b39da15dcd..c08be42008 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,9 +5,9 @@ Changelog 0.15.0 ------ -**Date:** 2023-03-XY +**Date:** 2023-03-25 -C++17, Parsing error recovery, Support for ADIOS2 BP5 engine, Append and Read-Linear access modes, performance and memory optimizations +C++17, Error Recovery, ADIOS2 BP5, Append & Read-Linear Modes, Performance & Memory This release adds error recovery mechanisms, in order to access erroneous datasets, created e.g. by crashing simulations. The BP5 engine of ADIOS2 v2.9 is fully supported by this release, including access to its various features for more fine-grained control of memory usage. From fc1cda81a336828842fbd1c0a075fd26fa38cb34 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 25 Mar 2023 01:07:24 -0700 Subject: [PATCH 11/11] Revert: `examples/7_extended_write_serial.cpp` --- examples/7_extended_write_serial.cpp | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/examples/7_extended_write_serial.cpp b/examples/7_extended_write_serial.cpp index f6780afa8e..bfb64e1fff 100644 --- a/examples/7_extended_write_serial.cpp +++ b/examples/7_extended_write_serial.cpp @@ -9,7 +9,7 @@ int main() { auto f = - io::Series("working/directory/2D_simData.bp", io::Access::CREATE); + io::Series("working/directory/2D_simData.h5", io::Access::CREATE); // all required openPMD attributes will be set to reasonable default // values (all ones, all zeros, empty strings,...) manually setting them @@ -105,13 +105,25 @@ int main() io::Datatype dtype = io::determineDatatype(partial_mesh); auto d = io::Dataset(dtype, io::Extent{2, 5}); std::string datasetConfig = R"END( -[adios1.dataset] -transform = "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" - -[[adios2.dataset.operatorors]] -type = "zlib" -parameters.clevel = 9 -)END"; +{ + "adios1": { + "dataset": { + "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" + } + }, + "adios2": { + "dataset": { + "operators": [ + { + "type": "zlib", + "parameters": { + "clevel": 9 + } + } + ] + } + } +})END"; d.options = datasetConfig; mesh["x"].resetDataset(d);