From f8a9ca03ef180de3611500bc9c0900572479d53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 28 Mar 2025 11:37:58 +0100 Subject: [PATCH 1/7] Emplace scalar tuple only when needed --- include/openPMD/backend/BaseRecord.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index edb8ae5f8a..874004335e 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -28,6 +28,7 @@ #include "openPMD/backend/Container.hpp" #include +#include #include #include #include // std::remove_reference_t From 91166ef3de799e29b0934c5e2e643fe6f5586156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 28 Mar 2025 12:17:42 +0100 Subject: [PATCH 2/7] Flush only dirty meshes/species TODO: Maybe do this in more places, and also when reading --- src/Iteration.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 39ae03fdca..ec5cb086cd 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -364,7 +364,12 @@ void Iteration::flush(internal::FlushParams const &flushParams) } meshes.flush(s.meshesPath(), flushParams); for (auto &m : meshes) - m.second.flush(m.first, flushParams); + { + if (m.second.dirtyRecursive()) + { + m.second.flush(m.first, flushParams); + } + } } else { @@ -380,7 +385,12 @@ void Iteration::flush(internal::FlushParams const &flushParams) } particles.flush(s.particlesPath(), flushParams); for (auto &species : particles) - species.second.flush(species.first, flushParams); + { + if (species.second.dirtyRecursive()) + { + species.second.flush(species.first, flushParams); + } + } } else { From 684eb9b412a708f882aad2584fe9b143e854eb33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 28 Mar 2025 12:28:02 +0100 Subject: [PATCH 3/7] Improved specification of standards --- src/ParticleSpecies.cpp | 15 +++++++++------ src/Record.cpp | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 4debca137c..1932170f62 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -168,12 +168,15 @@ void ParticleSpecies::flush( } else { - auto it = find("position"); - if (it != end()) - it->second.setUnitDimension({{UnitDimension::L, 1}}); - it = find("positionOffset"); - if (it != end()) - it->second.setUnitDimension({{UnitDimension::L, 1}}); + if (flushParams.flushLevel != FlushLevel::SkeletonOnly) + { + auto it = find("position"); + if (it != end()) + it->second.setUnitDimension({{UnitDimension::L, 1}}); + it = find("positionOffset"); + if (it != end()) + it->second.setUnitDimension({{UnitDimension::L, 1}}); + } Container::flush(path, flushParams); diff --git a/src/Record.cpp b/src/Record.cpp index 7d41fce5c2..5be926d0e0 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -45,8 +45,8 @@ Record &Record::setUnitDimension(unit_representations::AsMap const &udim) } Record &Record::setUnitDimension(unit_representations::AsArray const &udim) { - return setUnitDimension( - unit_representations::asMap(udim, /* skip_zeros = */ false)); + setAttribute("unitDimension", udim); + return *this; } void Record::flush_impl( From 65372120e2cc6103642cc23993746db1e7a978bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 31 Mar 2025 18:42:20 +0200 Subject: [PATCH 4/7] Don't forward empty Span operations to backend --- include/openPMD/RecordComponent.tpp | 9 +++++++++ include/openPMD/Span.hpp | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 923e3e2e09..2796ed1ae3 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -114,6 +114,15 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) dCreate.name = rc.m_name; IOHandler()->enqueue(IOTask(this, dCreate)); } + + if (size == 0) + { + // Don't forward this operation to the backend as it might create ugly + // zero-blocks in ADIOS2 + setDirtyRecursive(true); + return DynamicMemoryView(); + } + Parameter getBufferView; getBufferView.offset = o; getBufferView.extent = e; diff --git a/include/openPMD/Span.hpp b/include/openPMD/Span.hpp index 93bc0d24f7..85570fd148 100644 --- a/include/openPMD/Span.hpp +++ b/include/openPMD/Span.hpp @@ -113,7 +113,10 @@ class DynamicMemoryView } public: - explicit DynamicMemoryView() = default; + explicit DynamicMemoryView() + { + m_param.out->backendManagedBuffer = false; + } /** * @brief Acquire the underlying buffer at its current position in memory. From 16200fc18e2f46485bede99129ecf441092b8be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 16 Jul 2025 15:30:42 +0200 Subject: [PATCH 5/7] Verify that Iterations are not reparsed --- src/Iteration.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index ec5cb086cd..f8ad51f78f 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -480,6 +480,11 @@ void Iteration::readGorVBased( void Iteration::read_impl(std::string const &groupPath) { + if (!get().m_deferredParseAccess.has_value()) + { + throw error::Internal( + "Attempted reparsing an Iteration that is already parsed."); + } Parameter pOpen; pOpen.path = groupPath; IOHandler()->enqueue(IOTask(this, pOpen)); From 79d25b78974933b618f4f96a625f7a8fd11125b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 16 Jul 2025 16:15:08 +0200 Subject: [PATCH 6/7] Check for dirtyRecursive more fine-grained --- src/Iteration.cpp | 12 ++++++------ src/Mesh.cpp | 4 ++++ src/ParticleSpecies.cpp | 4 ++++ src/Record.cpp | 4 ++++ src/RecordComponent.cpp | 4 ++++ src/backend/BaseRecord.cpp | 5 +++++ src/backend/MeshRecordComponent.cpp | 4 ++++ src/backend/PatchRecord.cpp | 4 ++++ 8 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index f8ad51f78f..35fbf15247 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -362,10 +362,10 @@ void Iteration::flush(internal::FlushParams const &flushParams) s.setMeshesPath("meshes/"); s.flushMeshesPath(); } - meshes.flush(s.meshesPath(), flushParams); - for (auto &m : meshes) + if (meshes.dirtyRecursive()) { - if (m.second.dirtyRecursive()) + meshes.flush(s.meshesPath(), flushParams); + for (auto &m : meshes) { m.second.flush(m.first, flushParams); } @@ -383,10 +383,10 @@ void Iteration::flush(internal::FlushParams const &flushParams) s.setParticlesPath("particles/"); s.flushParticlesPath(); } - particles.flush(s.particlesPath(), flushParams); - for (auto &species : particles) + if (particles.dirtyRecursive()) { - if (species.second.dirtyRecursive()) + particles.flush(s.particlesPath(), flushParams); + for (auto &species : particles) { species.second.flush(species.first, flushParams); } diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 227a8764a3..cfd1735898 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -366,6 +366,10 @@ template Mesh &Mesh::setTimeOffset(float); void Mesh::flush_impl( std::string const &name, internal::FlushParams const &flushParams) { + if (!dirtyRecursive()) + { + return; + } if (access::readOnly(IOHandler()->m_frontendAccess)) { auto &m = get(); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 1932170f62..99ec6dfe6d 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -155,6 +155,10 @@ namespace void ParticleSpecies::flush( std::string const &path, internal::FlushParams const &flushParams) { + if (!dirtyRecursive()) + { + return; + } if (access::readOnly(IOHandler()->m_frontendAccess)) { for (auto &record : *this) diff --git a/src/Record.cpp b/src/Record.cpp index 5be926d0e0..c6baedec7c 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -52,6 +52,10 @@ Record &Record::setUnitDimension(unit_representations::AsArray const &udim) void Record::flush_impl( std::string const &name, internal::FlushParams const &flushParams) { + if (!dirtyRecursive()) + { + return; + } if (access::readOnly(IOHandler()->m_frontendAccess)) { if (scalar()) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 49febc68e8..f2a67a2f60 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -255,6 +255,10 @@ bool RecordComponent::empty() const void RecordComponent::flush( std::string const &name, internal::FlushParams const &flushParams) { + if (!dirtyRecursive()) + { + return; + } auto &rc = get(); if (flushParams.flushLevel == FlushLevel::SkeletonOnly) { diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index 111cdab61b..9467701a2a 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -819,6 +819,11 @@ template inline void BaseRecord::flush( std::string const &name, internal::FlushParams const &flushParams) { + if (!this->dirtyRecursive()) + { + return; + } + if (!this->written() && this->empty() && !this->datasetDefined()) throw std::runtime_error( "A Record can not be written without any contained " diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index 827102ad32..947db30f7c 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -72,6 +72,10 @@ void MeshRecordComponent::read() void MeshRecordComponent::flush( std::string const &name, internal::FlushParams const ¶ms) { + if (!dirtyRecursive()) + { + return; + } if (access::write(IOHandler()->m_frontendAccess) && !containsAttribute("position")) { diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 2a6581e56d..e674828181 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -41,6 +41,10 @@ PatchRecord::setUnitDimension(std::map const &udim) void PatchRecord::flush_impl( std::string const &path, internal::FlushParams const &flushParams) { + if (!dirtyRecursive()) + { + return; + } if (!this->datasetDefined()) { if (IOHandler()->m_frontendAccess != Access::READ_ONLY) From d1ef6f5faec454ddfb2c260798fe22e892644024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 30 Sep 2025 20:11:09 +0200 Subject: [PATCH 7/7] Update include/openPMD/backend/BaseRecord.hpp --- include/openPMD/backend/BaseRecord.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 874004335e..edb8ae5f8a 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -28,7 +28,6 @@ #include "openPMD/backend/Container.hpp" #include -#include #include #include #include // std::remove_reference_t