From afd83a761e9f14116bff112e42dc49a2bdbe0dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 11 Mar 2026 18:43:11 +0100 Subject: [PATCH 01/58] Flush structural info of resetDataset() to backend immediately This makes it easier to keep MPI processes in sync --- src/IO/AbstractIOHandlerImpl.cpp | 7 ++++++- src/RecordComponent.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 4f93ff1a5b..e6d489400d 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -457,7 +457,12 @@ std::future AbstractIOHandlerImpl::flush() auto ¶meter = deref_dynamic_cast>( i.parameter.get()); writeToStderr( - "[", i.writable->parent, "->", i.writable, "] SET_WRITTEN"); + "[", + i.writable->parent, + "->", + i.writable, + "] SET_WRITTEN ", + parameter.target_status ? "true" : "false"); setWritten(i.writable, parameter); break; } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index ec254d3b99..cac0118401 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -215,9 +215,36 @@ RecordComponent &RecordComponent::setUnitSI(double usi) return *this; } +namespace +{ + template + struct defer_type + { + F functor; + ~defer_type() + { + std::move(functor)(); + } + }; + + template + auto defer(F &&functor) -> defer_type> + { + return defer_type>{std::forward(functor)}; + } +} // namespace + RecordComponent &RecordComponent::resetDataset(Dataset d) { auto &rc = get(); + auto cleanup = defer([&rc, this]() { + if (rc.m_dataset.has_value() && + rc.m_dataset->dtype != Datatype::UNDEFINED) + { + seriesFlush_impl( + {FlushLevel::SkeletonOnly}); + } + }); if (written()) { if (!rc.m_dataset.has_value()) From 15b4c7e3303d1be0d1b9d5c237508c4cf78e9c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 11 Mar 2026 18:44:51 +0100 Subject: [PATCH 02/58] Erase flushMeshes/ParticlesPath These were unnecessary, but they snuck WRITE_ATT tasks into the skeleton flush. --- include/openPMD/Series.hpp | 2 -- src/Iteration.cpp | 2 -- src/Series.cpp | 20 -------------------- 3 files changed, 24 deletions(-) diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 93dfe333b4..3a63b5b7c7 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -900,8 +900,6 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); void flushRankTable(); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 67a28c51bc..9cc58ec0b9 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -408,7 +408,6 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!s.containsAttribute("meshesPath")) { s.setMeshesPath("meshes/"); - s.flushMeshesPath(); } if (meshes.dirtyRecursive()) { @@ -429,7 +428,6 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!s.containsAttribute("particlesPath")) { s.setParticlesPath("particles/"); - s.flushParticlesPath(); } if (particles.dirtyRecursive()) { diff --git a/src/Series.cpp b/src/Series.cpp index 855178c18d..0d47b91152 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1688,26 +1688,6 @@ void Series::flushGorVBased( } } -void Series::flushMeshesPath() -{ - Parameter aWrite; - aWrite.name = "meshesPath"; - Attribute a = getAttribute("meshesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - -void Series::flushParticlesPath() -{ - Parameter aWrite; - aWrite.name = "particlesPath"; - Attribute a = getAttribute("particlesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - void Series::readFileBased( std::optional read_only_this_single_iteration) { From 41dfd58b7ffcaf793024ea211fc753898ef0c1bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 11 Mar 2026 19:01:54 +0100 Subject: [PATCH 03/58] Move flushing from storeChunk to resetDataset --- include/openPMD/RecordComponent.tpp | 1 + include/openPMD/backend/Attributable.hpp | 2 +- include/openPMD/backend/Writable.hpp | 5 +++-- src/RecordComponent.cpp | 10 ++++++++-- src/backend/Attributable.cpp | 13 +++++++------ src/backend/BaseRecord.cpp | 1 + src/backend/Writable.cpp | 24 ++++++++++++++---------- 7 files changed, 35 insertions(+), 21 deletions(-) diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index b796ab1a93..0492dff297 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -90,6 +90,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) { size *= ext; } + /* * Flushing the skeleton does not create datasets, * so we might need to do it now. diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index f05cc8d15b..81dc5ddb24 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -478,7 +478,7 @@ OPENPMD_protected /** @} */ template - void seriesFlush_impl(internal::FlushParams const &); + void seriesFlush_impl(internal::FlushParams const &, bool flush_io_handler); void flushAttributes(internal::FlushParams const &); diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index a58af82f9e..3623d0b39c 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -128,14 +128,15 @@ class Writable final * it. */ template - void seriesFlush(std::string backendConfig = "{}"); + void + seriesFlush(std::string backendConfig = "{}", bool flush_io_handler = true); // clang-format off OPENPMD_private // clang-format on template - void seriesFlush(internal::FlushParams const &); + void seriesFlush(internal::FlushParams const &, bool flush_io_handler); /* * These members need to be shared pointers since distinct instances of * Writable may share them. diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index cac0118401..e84b3492de 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -22,6 +22,7 @@ #include "openPMD/Dataset.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/Format.hpp" #include "openPMD/Series.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -239,10 +240,15 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) auto &rc = get(); auto cleanup = defer([&rc, this]() { if (rc.m_dataset.has_value() && - rc.m_dataset->dtype != Datatype::UNDEFINED) + rc.m_dataset->dtype != Datatype::UNDEFINED && + IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) { + // TODO: try getting flush_io_handler = false to run seriesFlush_impl( - {FlushLevel::SkeletonOnly}); + {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); + Parameter dCreate(rc.m_dataset.value()); + dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; + IOHandler()->enqueue(IOTask(this, dCreate)); } }); if (written()) diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index d19fa31a00..45c5f5aa59 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -344,14 +344,15 @@ OpenpmdStandard Attributable::openPMDStandard() const } template -void Attributable::seriesFlush_impl(internal::FlushParams const &flushParams) +void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler) { - writable().seriesFlush(flushParams); + writable().seriesFlush(flushParams, flush_io_handler); } -template void -Attributable::seriesFlush_impl(internal::FlushParams const &flushParams); -template void -Attributable::seriesFlush_impl(internal::FlushParams const &flushParams); +template void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler); +template void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler); void Attributable::flushAttributes(internal::FlushParams const &flushParams) { diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index c4eab2318f..ba5564ccf6 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" #include "openPMD/backend/scientific_defaults/ConfigAttribute.hpp" diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index ea6e56b9c5..54e2a60b4a 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -52,16 +52,20 @@ Writable::~Writable() } template -void Writable::seriesFlush(std::string backendConfig) +void Writable::seriesFlush(std::string backendConfig, bool flush_io_handler) { seriesFlush( - internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)}); + internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)}, + flush_io_handler); } -template void Writable::seriesFlush(std::string backendConfig); -template void Writable::seriesFlush(std::string backendConfig); +template void +Writable::seriesFlush(std::string backendConfig, bool flush_io_handler); +template void +Writable::seriesFlush(std::string backendConfig, bool flush_io_handler); template -void Writable::seriesFlush(internal::FlushParams const &flushParams) +void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler) { Attributable impl; impl.setData({attributable, [](auto const *) {}}); @@ -103,10 +107,10 @@ void Writable::seriesFlush(internal::FlushParams const &flushParams) return {series.iterations.begin(), series.iterations.end()}; } }(); - series.flush_impl(begin, end, flushParams); + series.flush_impl(begin, end, flushParams, flush_io_handler); } -template void -Writable::seriesFlush(internal::FlushParams const &flushParams); -template void -Writable::seriesFlush(internal::FlushParams const &flushParams); +template void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler); +template void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler); } // namespace openPMD From 08e1db02a78dbef8fd15d5302a674f57d47cbeed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2026 12:52:17 +0100 Subject: [PATCH 04/58] Hmm, move CREATE_DATASET task back to storeChunk not so great, but lets keep that for now --- src/RecordComponent.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index e84b3492de..2e2adc4604 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -243,12 +243,8 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) rc.m_dataset->dtype != Datatype::UNDEFINED && IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) { - // TODO: try getting flush_io_handler = false to run seriesFlush_impl( {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); - Parameter dCreate(rc.m_dataset.value()); - dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; - IOHandler()->enqueue(IOTask(this, dCreate)); } }); if (written()) From fc0e85ac128e7db4a5b72eea21a0e96a52ebd182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2026 14:29:03 +0100 Subject: [PATCH 05/58] Fix attribute flushing logic --- src/Iteration.cpp | 6 ++++-- src/Series.cpp | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 9cc58ec0b9..a18d5eb656 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -405,7 +405,8 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!meshes.empty() || s.containsAttribute("meshesPath")) { - if (!s.containsAttribute("meshesPath")) + if (!s.containsAttribute("meshesPath") && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) { s.setMeshesPath("meshes/"); } @@ -425,7 +426,8 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!particles.empty() || s.containsAttribute("particlesPath")) { - if (!s.containsAttribute("particlesPath")) + if (!s.containsAttribute("particlesPath") && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) { s.setParticlesPath("particles/"); } diff --git a/src/Series.cpp b/src/Series.cpp index 0d47b91152..83192188ef 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1546,8 +1546,7 @@ void Series::flushFileBased( */ setDirty(allDirty); } - setDirty(false); - + determineUnsetDirty(flushParams.flushLevel); // Phase 3 if (flushIOHandler) { From 82edbbe1fb280dbc093dd17524e85fa6fd03f96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 14 Jan 2026 15:31:41 +0100 Subject: [PATCH 06/58] flush mode helpers --- include/openPMD/IO/AbstractIOHandler.hpp | 58 ++++++++++++++++++++++++ src/IO/ADIOS/ADIOS2File.cpp | 18 ++------ src/Iteration.cpp | 26 ++--------- src/backend/Attributable.cpp | 8 +--- 4 files changed, 68 insertions(+), 42 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 9b7735b5ba..02f4b9c6ae 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -81,6 +81,64 @@ enum class FlushLevel CreateOrOpenFiles }; +namespace flush_level +{ + inline constexpr auto global_flushpoint(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + // same as global_flushpoint for now, but we will soon introduce + // immediate_flush + inline constexpr auto write_datasets(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto write_attributes(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + return true; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto flush_hierarchy(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + return true; + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } +} // namespace flush_level + enum class OpenpmdStandard { v_1_0_0, diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 1d181033eb..0260836136 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -1049,25 +1049,16 @@ void ADIOS2File::flush_impl( drainedUniquePtrPuts.swap(m_uniquePtrPuts); } - if (readOnly(m_mode)) + if (readOnly(m_mode) || flush_level::write_datasets(level)) { - level = FlushLevel::UserFlush; - } - - switch (level) - { - case FlushLevel::UserFlush: performPutGets(*this, eng); m_updateSpans.clear(); m_buffer.clear(); m_alreadyEnqueued.clear(); drainedUniquePtrPuts.clear(); - - break; - - case FlushLevel::InternalFlush: - case FlushLevel::SkeletonOnly: - case FlushLevel::CreateOrOpenFiles: + } + else + { /* * Tasks have been given to ADIOS2, but we don't flush them * yet. So, move everything to m_alreadyEnqueued to avoid @@ -1084,7 +1075,6 @@ void ADIOS2File::flush_impl( "wrong time."); } m_buffer.clear(); - break; } } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index a18d5eb656..dcbf5b995f 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -306,15 +306,9 @@ void Iteration::flushFileBased( s.openIteration(i, *this); } - switch (flushParams.flushLevel) + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: - break; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: flush(flushParams); - break; } } @@ -329,15 +323,9 @@ void Iteration::flushGroupBased( IOHandler()->enqueue(IOTask(this, pCreate)); } - switch (flushParams.flushLevel) + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: - break; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: flush(flushParams); - break; } } @@ -352,17 +340,13 @@ void Iteration::flushVariableBased( IOHandler()->enqueue(IOTask(this, pOpen)); } - switch (flushParams.flushLevel) + if (!flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: return; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: - flush(flushParams); - break; } + flush(flushParams); + if (!written()) { /* create iteration path */ diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 45c5f5aa59..a39aa3cdb7 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -356,15 +356,9 @@ template void Attributable::seriesFlush_impl( void Attributable::flushAttributes(internal::FlushParams const &flushParams) { - switch (flushParams.flushLevel) + if (!flush_level::write_attributes(flushParams.flushLevel)) { - case FlushLevel::SkeletonOnly: - case FlushLevel::CreateOrOpenFiles: return; - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: - // pass - break; } if (dirty()) { From fef5b0934011aca4c358ab217dc8ac0c7e3da1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 14 Jan 2026 17:12:26 +0100 Subject: [PATCH 07/58] Fix dirty handling --- include/openPMD/backend/Attributable.hpp | 21 +++++++++++++++++++++ src/Iteration.cpp | 8 ++++---- src/ParticleSpecies.cpp | 7 ++----- src/RecordComponent.cpp | 5 +---- src/backend/Attributable.cpp | 5 +---- src/backend/BaseRecord.cpp | 5 +---- src/backend/PatchRecord.cpp | 5 +---- 7 files changed, 31 insertions(+), 25 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 81dc5ddb24..33b28c6ea2 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -606,6 +606,27 @@ OPENPMD_protected { return writable().dirtyRecursive; } + void determineUnsetDirty(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + setDirty(false); + break; + case FlushLevel::InternalFlush: + // Used for parsing + if (IOHandler()->m_seriesStatus == internal::SeriesStatus::Parsing) + { + throw error::Internal( + "Parsing procedures should directly unset dirty."); + } + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + // noop + break; + } + } void setDirty(bool dirty_in) { auto &w = writable(); diff --git a/src/Iteration.cpp b/src/Iteration.cpp index dcbf5b995f..5d6923d338 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -379,7 +379,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) m.second.flush(m.first, flushParams); for (auto &species : particles) species.second.flush(species.first, flushParams); - setDirty(false); + determineUnsetDirty(flushParams.flushLevel); } else { @@ -433,9 +433,9 @@ void Iteration::flush(internal::FlushParams const &flushParams) } if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { - setDirty(false); - meshes.setDirty(false); - particles.setDirty(false); + determineUnsetDirty(flushParams.flushLevel); + meshes.determineUnsetDirty(flushParams.flushLevel); + particles.determineUnsetDirty(flushParams.flushLevel); } } diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 8a2b9b58f7..718d3e847a 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -197,11 +197,8 @@ void ParticleSpecies::flush( patch.second.flush(patch.first, flushParams); } } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - particlePatches.setDirty(false); - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); + particlePatches.determineUnsetDirty(flushParams.flushLevel); } void ParticleSpecies::scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 2e2adc4604..bce3da6aeb 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -554,10 +554,7 @@ void RecordComponent::flush( flushAttributes(flushParams); } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void RecordComponent::read() diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index a39aa3cdb7..16f69c6cbb 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -372,10 +372,7 @@ void Attributable::flushAttributes(internal::FlushParams const &flushParams) } } // Do this outside the if branch to also setDirty to dirtyRecursive - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void Attributable::readAttributes(ReadMode mode) diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index ba5564ccf6..89777af8d6 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -799,10 +799,7 @@ inline void BaseRecord::flush( } this->flush_impl(name, flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - this->setDirty(false); - } + this->determineUnsetDirty(flushParams.flushLevel); // flush_impl must take care to correctly set the dirty() flag so this // method doesn't do it } diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 740f44cc51..7d68b16035 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -70,10 +70,7 @@ void PatchRecord::flush_impl( } else T_RecordComponent::flush(path, flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void PatchRecord::read() From 8fba090c9347008cf17147da330a179bd368b3da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2026 15:57:24 +0100 Subject: [PATCH 08/58] Add TODO comment --- src/Series.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Series.cpp b/src/Series.cpp index 83192188ef..6d9d6c6914 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1543,6 +1543,8 @@ void Series::flushFileBased( } /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes + * TODO: Ideally, we would skip this in SkeletonOnly flush mode, but + * for some reason, this leads to hanging parallel tests..? */ setDirty(allDirty); } From 8ae4881511946ba9cfd9a673b68f594a361e9291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Mar 2026 16:55:55 +0100 Subject: [PATCH 09/58] WIP Runtime verification of flush level --- include/openPMD/IO/AbstractIOHandler.hpp | 3 + include/openPMD/IO/AbstractIOHandlerImpl.hpp | 2 +- include/openPMD/IO/IOTask.hpp | 3 + include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 2 +- include/openPMD/Series.hpp | 2 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 2 +- src/IO/AbstractIOHandler.cpp | 23 +++++ src/IO/AbstractIOHandlerImpl.cpp | 66 +++++++++++++- src/IO/HDF5/HDF5IOHandler.cpp | 2 +- src/IO/IOTask.cpp | 91 +++++++++++++++++++ src/IO/JSON/JSONIOHandler.cpp | 4 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 5 +- src/Iteration.cpp | 2 +- src/Series.cpp | 12 ++- 14 files changed, 204 insertions(+), 15 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 02f4b9c6ae..a660676648 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/config.hpp" #include "openPMD/version.hpp" +#include #if openPMD_HAVE_MPI #include @@ -81,6 +82,8 @@ enum class FlushLevel CreateOrOpenFiles }; +std::ostream &operator<<(std::ostream &, FlushLevel); + namespace flush_level { inline constexpr auto global_flushpoint(FlushLevel fl) diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index d45ce1bdcc..fdb8af3599 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -39,7 +39,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - std::future flush(); + std::future flush(FlushLevel); /** * Close the file corresponding with the writable and release file handles. diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 25e0d6ad54..a8efab571e 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ }; // note: if you change the enum members here, please update // docs/source/dev/design.rst +std::ostream &operator<<(std::ostream &os, Operation op); + namespace internal { /* diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 6df0c60ced..3e0758aee2 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void touch(Writable *, Parameter const &) override; - std::future flush(); + std::future flush(internal::ParsedFlushParams ¶ms); private: #if openPMD_HAVE_MPI diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 3a63b5b7c7..de45d63042 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -984,7 +984,7 @@ OPENPMD_private * * @param doFlush If true, flush the IO handler. */ - void flushStep(bool doFlush); + void flushStep(bool doFlush, FlushLevel l); /* * setIterationEncoding() should only be called by users of our public API, diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2e8084848d..036b4f6098 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -582,7 +582,7 @@ overrideFlushTarget(FlushTarget &inplace, FlushTarget new_val) std::future ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams) { - auto res = AbstractIOHandlerImpl::flush(); + auto res = AbstractIOHandlerImpl::flush(flushParams.flushLevel); detail::ADIOS2File::ADIOS2FlushParams adios2FlushParams{ flushParams.flushLevel, m_flushTarget}; diff --git a/src/IO/AbstractIOHandler.cpp b/src/IO/AbstractIOHandler.cpp index 5f2bdeb2f5..564eebba01 100644 --- a/src/IO/AbstractIOHandler.cpp +++ b/src/IO/AbstractIOHandler.cpp @@ -27,6 +27,29 @@ #include +namespace openPMD +{ +std::ostream &operator<<(std::ostream &os, FlushLevel l) +{ + switch (l) + { + case FlushLevel::UserFlush: + os << "UserFlush"; + break; + case FlushLevel::InternalFlush: + os << "InternalFlush"; + break; + case FlushLevel::SkeletonOnly: + os << "SkeletonOnly"; + break; + case FlushLevel::CreateOrOpenFiles: + os << "CreateOrOpenFiles"; + break; + } + return os; +} +} // namespace openPMD + namespace openPMD::auxiliary { using pair_t = std::pair; diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index e6d489400d..1b94da00b1 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -21,6 +21,8 @@ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -87,13 +89,74 @@ void AbstractIOHandlerImpl::writeToStderr([[maybe_unused]] Args &&...args) const } } -std::future AbstractIOHandlerImpl::flush() +namespace +{ + void verifyFlushType(Operation op, FlushLevel l) + { + auto do_throw = [&](char const *least_flush_level) { + std::stringstream err; + err << "Operation " << op << " is not allowed below flush level " + << least_flush_level << ", but flush level was " << l << "."; + throw error::Internal(err.str()); + }; + switch (op) + { + case Operation::ADVANCE: + case Operation::CREATE_FILE: + case Operation::CHECK_FILE: + case Operation::OPEN_FILE: + case Operation::CLOSE_FILE: + case Operation::DELETE_FILE: + case Operation::DEREGISTER: + case Operation::TOUCH: + case Operation::LIST_ATTS: + case Operation::LIST_PATHS: + case Operation::OPEN_PATH: + case Operation::SET_WRITTEN: + case Operation::CREATE_PATH: + break; + case Operation::CLOSE_PATH: + case Operation::DELETE_PATH: + case Operation::CREATE_DATASET: + case Operation::EXTEND_DATASET: + case Operation::OPEN_DATASET: + case Operation::DELETE_DATASET: + case Operation::LIST_DATASETS: + if (!flush_level::flush_hierarchy(l)) + { + do_throw("SkeletonOnly (hierarchy operations)"); + } + break; + case Operation::GET_BUFFER_VIEW: + case Operation::DELETE_ATT: + case Operation::WRITE_ATT: + case Operation::READ_ATT: + case Operation::READ_ATT_ALLSTEPS: + case Operation::AVAILABLE_CHUNKS: + if (!flush_level::write_attributes(l)) + { + do_throw("InternalFlush (metadata operations)"); + } + break; + case Operation::WRITE_DATASET: + case Operation::READ_DATASET: + if (!flush_level::write_datasets(l)) + { + do_throw("UserFlush (flushpoint operations)"); + } + break; + } + } +} // namespace + +std::future AbstractIOHandlerImpl::flush(FlushLevel l) { using namespace auxiliary; while (!(*m_handler).m_work.empty()) { IOTask &i = (*m_handler).m_work.front(); + // verifyFlushType(i.operation, l); try { switch (i.operation) @@ -467,6 +530,7 @@ std::future AbstractIOHandlerImpl::flush() break; } } + verifyFlushType(i.operation, l); } catch (...) { diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 714a69b9d3..3096b2297c 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -3562,7 +3562,7 @@ auto HDF5IOHandlerImpl::requireFile( std::future HDF5IOHandlerImpl::flush(internal::ParsedFlushParams ¶ms) { - auto res = AbstractIOHandlerImpl::flush(); + auto res = AbstractIOHandlerImpl::flush(params.flushLevel); if (params.backendConfig.json().contains("hdf5")) { diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 26010ead52..af12ff766c 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -35,6 +35,97 @@ Writable *getWritable(Attributable *a) return &a->writable(); } +std::ostream &operator<<(std::ostream &os, Operation op) +{ + switch (op) + { + case Operation::CREATE_FILE: + os << "CREATE_FILE"; + break; + case Operation::CHECK_FILE: + os << "CHECK_FILE"; + break; + case Operation::OPEN_FILE: + os << "OPEN_FILE"; + break; + case Operation::CLOSE_FILE: + os << "CLOSE_FILE"; + break; + case Operation::DELETE_FILE: + os << "DELETE_FILE"; + break; + case Operation::CREATE_PATH: + os << "CREATE_PATH"; + break; + case Operation::CLOSE_PATH: + os << "CLOSE_PATH"; + break; + case Operation::OPEN_PATH: + os << "OPEN_PATH"; + break; + case Operation::DELETE_PATH: + os << "DELETE_PATH"; + break; + case Operation::LIST_PATHS: + os << "LIST_PATHS"; + break; + case Operation::CREATE_DATASET: + os << "CREATE_DATASET"; + break; + case Operation::EXTEND_DATASET: + os << "EXTEND_DATASET"; + break; + case Operation::OPEN_DATASET: + os << "OPEN_DATASET"; + break; + case Operation::DELETE_DATASET: + os << "DELETE_DATASET"; + break; + case Operation::WRITE_DATASET: + os << "WRITE_DATASET"; + break; + case Operation::READ_DATASET: + os << "READ_DATASET"; + break; + case Operation::LIST_DATASETS: + os << "LIST_DATASETS"; + break; + case Operation::GET_BUFFER_VIEW: + os << "GET_BUFFER_VIEW"; + break; + case Operation::DELETE_ATT: + os << "DELETE_ATT"; + break; + case Operation::WRITE_ATT: + os << "WRITE_ATT"; + break; + case Operation::READ_ATT: + os << "READ_ATT"; + break; + case Operation::READ_ATT_ALLSTEPS: + os << "READ_ATT_ALLSTEPS"; + break; + case Operation::LIST_ATTS: + os << "LIST_ATTS"; + break; + case Operation::ADVANCE: + os << "ADVANCE"; + break; + case Operation::AVAILABLE_CHUNKS: + os << "AVAILABLE_CHUNKS"; + break; + case Operation::DEREGISTER: + os << "DEREGISTER"; + break; + case Operation::TOUCH: + os << "TOUCH"; + break; + case Operation::SET_WRITTEN: + os << "SET_WRITTEN"; + break; + } + return os; +} template <> void AbstractParameter::warnUnusedParameters( json::TracingJSON &config, diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index c531aabb00..9af9a06728 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -53,8 +53,8 @@ JSONIOHandler::JSONIOHandler( {} #endif -std::future JSONIOHandler::flush(internal::ParsedFlushParams &) +std::future JSONIOHandler::flush(internal::ParsedFlushParams ¶ms) { - return m_impl.flush(); + return m_impl.flush(params); } } // namespace openPMD diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 7647746d74..0660d1fe41 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -24,6 +24,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/FlushParametersInternal.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSONMatcher.hpp" @@ -444,9 +445,9 @@ void JSONIOHandlerImpl::init(openPMD::json::TracingJSON config) JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; -std::future JSONIOHandlerImpl::flush() +std::future JSONIOHandlerImpl::flush(internal::ParsedFlushParams ¶ms) { - AbstractIOHandlerImpl::flush(); + AbstractIOHandlerImpl::flush(params.flushLevel); if (access::readOnly(m_handler->m_backendAccess) && !m_dirty.empty()) { throw error::Internal( diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 5d6923d338..a4646f6a4a 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -347,7 +347,7 @@ void Iteration::flushVariableBased( flush(flushParams); - if (!written()) + if (!written() && flush_level::write_datasets(flushParams.flushLevel)) { /* create iteration path */ Parameter pOpen; diff --git a/src/Series.cpp b/src/Series.cpp index 6d9d6c6914..b0286e4d54 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2681,7 +2681,7 @@ AdvanceStatus Series::advance( if (mode == AdvanceMode::ENDSTEP) { - flushStep(/* doFlush = */ false); + flushStep(/* doFlush = */ false, FlushLevel::UserFlush); } Parameter param; @@ -2787,7 +2787,7 @@ AdvanceStatus Series::advance(AdvanceMode mode) if (mode == AdvanceMode::ENDSTEP) { - flushStep(/* doFlush = */ false); + flushStep(/* doFlush = */ false, FlushLevel::UserFlush); } Parameter param; @@ -2810,8 +2810,12 @@ AdvanceStatus Series::advance(AdvanceMode mode) return *param.status; } -void Series::flushStep(bool doFlush) +void Series::flushStep(bool doFlush, FlushLevel l) { + if (!flush_level::write_datasets(l)) + { + return; + } auto &series = get(); if (!series.m_currentlyActiveIterations.empty() && access::write(IOHandler()->m_frontendAccess)) @@ -3315,7 +3319,7 @@ namespace internal */ if (impl.iterationEncoding() != IterationEncoding::fileBased) { - impl.flushStep(/* doFlush = */ true); + impl.flushStep(/* doFlush = */ true, FlushLevel::UserFlush); } } // Not strictly necessary, but clear the map of iterations From b6dec7ad9e6ff45029b94f13156ecf5981c451b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Mar 2026 13:56:21 +0100 Subject: [PATCH 10/58] dont flush to IO handler yet in resetDataset --- src/RecordComponent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index bce3da6aeb..89c2b1e5a3 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -244,7 +244,7 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) { seriesFlush_impl( - {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); + {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ false); } }); if (written()) From 9e816425ad6f4fa46a8f22bc7a3b81e086935db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Mar 2026 16:00:08 +0100 Subject: [PATCH 11/58] Revert "dont flush to IO handler yet in resetDataset" This reverts commit 6a5c9f55feee37184b46c2ef8d2d86c920ed6931. --- src/RecordComponent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 89c2b1e5a3..bce3da6aeb 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -244,7 +244,7 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) { seriesFlush_impl( - {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ false); + {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); } }); if (written()) From f988440926cb2f1a07865084ba71cad83b3093ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Mar 2026 18:34:42 +0100 Subject: [PATCH 12/58] Continue fixing and breaking things.. --- include/openPMD/IO/AbstractIOHandler.hpp | 1 + include/openPMD/Series.hpp | 2 +- src/Iteration.cpp | 2 +- src/Series.cpp | 10 +++++++--- test/SerialIOTest.cpp | 4 ++++ 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index a660676648..b54b661c78 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -182,6 +182,7 @@ namespace internal * To be used for reading */ FlushParams const defaultFlushParams{}; + FlushParams const publicFlush{FlushLevel::UserFlush}; struct ParsedFlushParams; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index de45d63042..f1b5b3872c 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -900,7 +900,7 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushRankTable(); + void flushRankTable(FlushLevel); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. */ diff --git a/src/Iteration.cpp b/src/Iteration.cpp index a4646f6a4a..6092bd2579 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -288,7 +288,7 @@ void Iteration::flushFileBased( s.get() .m_rankTable.m_attributable.get() .m_writable.abstractFilePosition.reset(); - s.flushRankTable(); + s.flushRankTable(flushParams.flushLevel); /* create basePath */ Parameter pCreate; diff --git a/src/Series.cpp b/src/Series.cpp index b0286e4d54..b125cdcb85 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -405,7 +405,7 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) readDataset.data = get; IOHandler()->enqueue(IOTask(&rankTable.m_attributable, readDataset)); - IOHandler()->flush(internal::defaultFlushParams); + IOHandler()->flush(internal::publicFlush); }; #if openPMD_HAVE_MPI @@ -464,8 +464,12 @@ Series &Series::setRankTable(const std::string &myRankInfo) return *this; } -void Series::flushRankTable() +void Series::flushRankTable(FlushLevel l) { + if (!flush_level::global_flushpoint(l)) + { + return; + } auto &series = get(); auto &rankTable = series.m_rankTable; auto maybeMyRankInfo = std::visit( @@ -1634,7 +1638,7 @@ void Series::flushGorVBased( fCreate.name = series.m_name; IOHandler()->enqueue(IOTask(this, fCreate)); - flushRankTable(); + flushRankTable(flushParams.flushLevel); } series.iterations.flush( diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 49aab1db18..50481ade62 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1972,6 +1972,10 @@ inline void fileBased_write_test(const std::string &backend) "\\", "/")); + // TODO: somehow make the rank table appear in iteration 1 + o.iterations[1]; + o.flush(); + ParticleSpecies &e_1 = o.iterations[1].particles["e"]; std::vector position_global(4); From 848e573c6decef99f5d5265bacd3cacdbf7056f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Mar 2026 11:01:47 +0100 Subject: [PATCH 13/58] Fix API call after rebase --- include/openPMD/RecordComponent.tpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 0492dff297..523f4f1e41 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -130,7 +130,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) * actual data yet. */ seriesFlush_impl( - {FlushLevel::SkeletonOnly}); + {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false); Parameter dCreate(rc.m_dataset.value()); dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); From 8739cc1ceb9a8d65f932aa0959709b31280366b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Mar 2026 14:03:32 +0100 Subject: [PATCH 14/58] Fix dirty handling filebased --- src/Iteration.cpp | 33 +++++++++++++++++++++------------ src/Series.cpp | 26 +++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 6092bd2579..b3cf165ff5 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -387,16 +387,28 @@ void Iteration::flush(internal::FlushParams const &flushParams) * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); - if (!meshes.empty() || s.containsAttribute("meshesPath")) - { - if (!s.containsAttribute("meshesPath") && - flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + auto set_and_get_mp_path = + [&](char const *attrName, + char const *defaultVal, + Series &(Series::*set)(std::string const &)) -> std::string { + if (s.containsAttribute(attrName)) + { + return s.getAttribute(attrName).get(); + } + else { - s.setMeshesPath("meshes/"); + (s.*set)(defaultVal); + return defaultVal; } + }; + + if (!meshes.empty() || s.containsAttribute("meshesPath")) + { + auto meshesPath = set_and_get_mp_path( + "meshesPath", "meshes/", &Series::setMeshesPath); if (meshes.dirtyRecursive()) { - meshes.flush(s.meshesPath(), flushParams); + meshes.flush(meshesPath, flushParams); for (auto &m : meshes) { m.second.flush(m.first, flushParams); @@ -410,14 +422,11 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!particles.empty() || s.containsAttribute("particlesPath")) { - if (!s.containsAttribute("particlesPath") && - flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) - { - s.setParticlesPath("particles/"); - } + auto particlesPath = set_and_get_mp_path( + "particlesPath", "particles/", &Series::setParticlesPath); if (particles.dirtyRecursive()) { - particles.flush(s.particlesPath(), flushParams); + particles.flush(particlesPath, flushParams); for (auto &species : particles) { species.second.flush(species.first, flushParams); diff --git a/src/Series.cpp b/src/Series.cpp index b125cdcb85..38d5f4066b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1498,6 +1498,12 @@ void Series::flushFileBased( case Access::APPEND_RANDOM_ACCESS: case Access::APPEND_LINEAR: { bool allDirty = dirty(); + // In flush level SkeletonOnly, we might need to set some attributes + // (especially: particlesPath, meshesPath), but cannot flush them yet + // (as writing attributes is only permissible at higher flush levels). + // This flag records if the Series became dirty during this flush. If + // yes, we set the Series back to dirty at the end of flushing. + bool hasBecomeDirty = false; for (auto it = begin; it != end; ++it) { // Phase 1 @@ -1550,9 +1556,27 @@ void Series::flushFileBased( * TODO: Ideally, we would skip this in SkeletonOnly flush mode, but * for some reason, this leads to hanging parallel tests..? */ + if (flushParams.flushLevel == FlushLevel::SkeletonOnly) + { + if (allDirty && !dirty()) + { + throw error::Internal( + "Flush mode SkeletonOnly must not unset dirty flags."); + } + hasBecomeDirty |= + flushParams.flushLevel == FlushLevel::SkeletonOnly && + !allDirty && dirty(); + } setDirty(allDirty); } - determineUnsetDirty(flushParams.flushLevel); + if (!hasBecomeDirty) + { + determineUnsetDirty(flushParams.flushLevel); + } + else + { + setDirty(true); + } // Phase 3 if (flushIOHandler) { From fd20fdcd76bf28ed328549a0e236c0308b7f963e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Mar 2026 14:29:19 +0100 Subject: [PATCH 15/58] TMP REVERT ME: deactivate span table tests --- test/ParallelIOTest.cpp | 16 ++++++++-------- test/SerialIOTest.cpp | 7 ++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 9c28f52945..6ec217413f 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -619,7 +619,7 @@ TEST_CASE("adios_write_test", "[parallel][adios]") Access::READ_LINEAR, MPI_COMM_WORLD); i.parseBase(); - REQUIRE(i.rankTable(/* collective = */ true) == compare); + // REQUIRE(i.rankTable(/* collective = */ true) == compare); } { Series i( @@ -627,21 +627,21 @@ TEST_CASE("adios_write_test", "[parallel][adios]") Access::READ_LINEAR, MPI_COMM_WORLD); i.parseBase(); - REQUIRE(i.rankTable(/* collective = */ false) == compare); + // REQUIRE(i.rankTable(/* collective = */ false) == compare); } { Series i( "../samples/parallel_write.bp", Access::READ_RANDOM_ACCESS, MPI_COMM_WORLD); - REQUIRE(i.rankTable(/* collective = */ true) == compare); + // REQUIRE(i.rankTable(/* collective = */ true) == compare); } { Series i( "../samples/parallel_write.bp", Access::READ_RANDOM_ACCESS, MPI_COMM_WORLD); - REQUIRE(i.rankTable(/* collective = */ false) == compare); + // REQUIRE(i.rankTable(/* collective = */ false) == compare); } } @@ -877,8 +877,8 @@ void close_iteration_test(std::string const &file_ending) } // Need this in file-based iteration encoding i.iterations.begin()->second.open(); - REQUIRE( - i.rankTable(/* collective = */ read_collectively) == compare); + // REQUIRE( + // i.rankTable(/* collective = */ read_collectively) == compare); } } } @@ -2553,7 +2553,7 @@ void run_test() * are running on the same nodes. */ auto rankMetaIn = series.rankTable(/* collective = */ true); - OPENPMD_REQUIRE_GUARD_WINDOWS(rankMetaIn == writingRanksHostnames); + // OPENPMD_REQUIRE_GUARD_WINDOWS(rankMetaIn == writingRanksHostnames); auto E_x = series.iterations[0].meshes["E"]["x"]; /* @@ -2721,7 +2721,7 @@ void run_test() TEST_CASE("adios2_chunk_distribution", "[parallel][adios2]") { - adios2_chunk_distribution::run_test(); + // adios2_chunk_distribution::run_test(); } #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 50481ade62..a2a93248cf 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1764,7 +1764,7 @@ inline void write_test( #ifndef _WIN32 if (test_rank_table) { - REQUIRE(read.rankTable(/* collective = */ false) == compare); + // REQUIRE(read.rankTable(/* collective = */ false) == compare); } #endif } @@ -2340,7 +2340,8 @@ inline void fileBased_write_test(const std::string &backend) std::string fullPath = std::string("../samples/subdir/") + entry->d_name; Series single_file(fullPath, Access::READ_ONLY); - REQUIRE(single_file.rankTable(/* collective = */ false) == compare); + // REQUIRE(single_file.rankTable(/* collective = */ false) == + // compare); } closedir(directory); close(dirfd); @@ -5339,7 +5340,7 @@ void serial_iterator(std::string const &file) std::cout << "POST Rank '" << rank << "' written from host '" << host << "'\n"; } - REQUIRE(rank_table.size() == 1); + // REQUIRE(rank_table.size() == 1); } #endif REQUIRE(last_iteration_index == 9); From 7a97533d8b88dc2b44d5eca9f36f595e6945bd6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Mar 2026 14:54:18 +0100 Subject: [PATCH 16/58] TMP REVERT ME take out hanging parallel test --- .../iterate_nonstreaming_series.cpp | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp index 8ddadfc40a..fe72ef1698 100644 --- a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp +++ b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp @@ -181,11 +181,12 @@ auto iterate_nonstreaming_series() -> void { for (auto const &backend : testedBackends()) { - run_test( - "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + - backend.extension, - false, - backend.jsonBaseConfig()); + std::cout << "TESTING BACKEND " << backend.extension << std::endl; + // run_test( + // "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + + // backend.extension, + // false, + // backend.jsonBaseConfig()); run_test( "../samples/iterate_nonstreaming_series/parallel_groupbased." + backend.extension, @@ -194,13 +195,14 @@ auto iterate_nonstreaming_series() -> void #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_ADIOS2_BP5 if (backend.extension == "bp") { - run_test( - "../samples/iterate_nonstreaming_series/" - "parallel_filebased_bp5_%T." + - backend.extension, - false, - json::merge( - backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); + // run_test( + // "../samples/iterate_nonstreaming_series/" + // "parallel_filebased_bp5_%T." + + // backend.extension, + // false, + // json::merge( + // backend.jsonBaseConfig(), "adios2.engine.type = + // \"bp5\"")); run_test( "../samples/iterate_nonstreaming_series/" "parallel_groupbased_bp5." + From c256c9394ea495164ca7c48fb98bc35dbb307133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 14:57:59 +0100 Subject: [PATCH 17/58] Fix ranktable logic --- src/IO/AbstractIOHandlerImpl.cpp | 3 ++ src/Iteration.cpp | 26 ++++++++------ src/Series.cpp | 3 ++ test/ParallelIOTest.cpp | 16 ++++----- test/SerialIOTest.cpp | 58 ++++++++++++++++---------------- 5 files changed, 59 insertions(+), 47 deletions(-) diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 1b94da00b1..473b140aef 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -153,6 +153,8 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) { using namespace auxiliary; + writeToStderr("\nFLUSHING"); + while (!(*m_handler).m_work.empty()) { IOTask &i = (*m_handler).m_work.front(); @@ -575,6 +577,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) } (*m_handler).m_work.pop(); } + writeToStderr("FLUSHED\n"); return std::future(); } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index b3cf165ff5..caac558986 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -272,6 +272,10 @@ void Iteration::flushFileBased( /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); + auto &series = s.get(); + + bool do_flush_rank_table = + !series.m_rankTable.m_attributable.written() || !this->written(); if (!written()) { @@ -280,16 +284,6 @@ void Iteration::flushFileBased( fCreate.name = filename; IOHandler()->enqueue(IOTask(&s.writable(), fCreate)); - /* - * If it was written before, then in the context of another iteration. - */ - auto &attr = s.get().m_rankTable.m_attributable; - attr.setWritten(false, Attributable::EnqueueAsynchronously::Both); - s.get() - .m_rankTable.m_attributable.get() - .m_writable.abstractFilePosition.reset(); - s.flushRankTable(flushParams.flushLevel); - /* create basePath */ Parameter pCreate; pCreate.path = auxiliary::replace_first(s.basePath(), "%T/", ""); @@ -306,6 +300,18 @@ void Iteration::flushFileBased( s.openIteration(i, *this); } + if (do_flush_rank_table) + { + /* + * If it was written before, then in the context of another iteration. + */ + auto &attr = series.m_rankTable.m_attributable; + attr.setWritten(false, Attributable::EnqueueAsynchronously::Both); + attr.get().m_writable.abstractFilePosition.reset(); + + s.flushRankTable(flushParams.flushLevel); + } + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { flush(flushParams); diff --git a/src/Series.cpp b/src/Series.cpp index 38d5f4066b..fbd2151df3 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1661,7 +1661,10 @@ void Series::flushGorVBased( Parameter fCreate; fCreate.name = series.m_name; IOHandler()->enqueue(IOTask(this, fCreate)); + } + if (!series.m_rankTable.m_attributable.written()) + { flushRankTable(flushParams.flushLevel); } diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 6ec217413f..9c28f52945 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -619,7 +619,7 @@ TEST_CASE("adios_write_test", "[parallel][adios]") Access::READ_LINEAR, MPI_COMM_WORLD); i.parseBase(); - // REQUIRE(i.rankTable(/* collective = */ true) == compare); + REQUIRE(i.rankTable(/* collective = */ true) == compare); } { Series i( @@ -627,21 +627,21 @@ TEST_CASE("adios_write_test", "[parallel][adios]") Access::READ_LINEAR, MPI_COMM_WORLD); i.parseBase(); - // REQUIRE(i.rankTable(/* collective = */ false) == compare); + REQUIRE(i.rankTable(/* collective = */ false) == compare); } { Series i( "../samples/parallel_write.bp", Access::READ_RANDOM_ACCESS, MPI_COMM_WORLD); - // REQUIRE(i.rankTable(/* collective = */ true) == compare); + REQUIRE(i.rankTable(/* collective = */ true) == compare); } { Series i( "../samples/parallel_write.bp", Access::READ_RANDOM_ACCESS, MPI_COMM_WORLD); - // REQUIRE(i.rankTable(/* collective = */ false) == compare); + REQUIRE(i.rankTable(/* collective = */ false) == compare); } } @@ -877,8 +877,8 @@ void close_iteration_test(std::string const &file_ending) } // Need this in file-based iteration encoding i.iterations.begin()->second.open(); - // REQUIRE( - // i.rankTable(/* collective = */ read_collectively) == compare); + REQUIRE( + i.rankTable(/* collective = */ read_collectively) == compare); } } } @@ -2553,7 +2553,7 @@ void run_test() * are running on the same nodes. */ auto rankMetaIn = series.rankTable(/* collective = */ true); - // OPENPMD_REQUIRE_GUARD_WINDOWS(rankMetaIn == writingRanksHostnames); + OPENPMD_REQUIRE_GUARD_WINDOWS(rankMetaIn == writingRanksHostnames); auto E_x = series.iterations[0].meshes["E"]["x"]; /* @@ -2721,7 +2721,7 @@ void run_test() TEST_CASE("adios2_chunk_distribution", "[parallel][adios2]") { - // adios2_chunk_distribution::run_test(); + adios2_chunk_distribution::run_test(); } #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index a2a93248cf..52836ae903 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1764,31 +1764,31 @@ inline void write_test( #ifndef _WIN32 if (test_rank_table) { - // REQUIRE(read.rankTable(/* collective = */ false) == compare); + REQUIRE(read.rankTable(/* collective = */ false) == compare); } #endif } TEST_CASE("write_test", "[serial]") { - for (auto const &t : testedFileExtensions()) + for (auto const &t : {std::string("json")}) { if (t == "json") { - write_test( - "template." + t, - R"( -{ - "json": { - "dataset": { - "mode": "template" - }, - "attribute": { - "mode": "short" - } - } -})", - false); + // write_test( + // "template." + t, + // R"( + // { + // "json": { + // "dataset": { + // "mode": "template" + // }, + // "attribute": { + // "mode": "short" + // } + // } + // })", + // false); write_test( t, R"( @@ -2340,8 +2340,7 @@ inline void fileBased_write_test(const std::string &backend) std::string fullPath = std::string("../samples/subdir/") + entry->d_name; Series single_file(fullPath, Access::READ_ONLY); - // REQUIRE(single_file.rankTable(/* collective = */ false) == - // compare); + REQUIRE(single_file.rankTable(/* collective = */ false) == compare); } closedir(directory); close(dirfd); @@ -2354,19 +2353,20 @@ inline void fileBased_write_test(const std::string &backend) TEST_CASE("fileBased_write_test", "[serial]") { - for (auto const &t : testedFileExtensions()) + for (auto const &t : {"bp5"}) { fileBased_write_test(t); } - if (auto extensions = getFileExtensions(); - std::find(extensions.begin(), extensions.end(), "toml") != - extensions.end()) - { /* - * TOML backend is not generally tested for performance reasons, opt in to - * testing it here. - */ - fileBased_write_test("toml"); - } + // if (auto extensions = getFileExtensions(); + // std::find(extensions.begin(), extensions.end(), "toml") != + // extensions.end()) + // { /* + // * TOML backend is not generally tested for performance reasons, opt in + // to + // * testing it here. + // */ + // fileBased_write_test("toml"); + // } } inline void sample_write_thetaMode(std::string const &file_ending) @@ -5340,7 +5340,7 @@ void serial_iterator(std::string const &file) std::cout << "POST Rank '" << rank << "' written from host '" << host << "'\n"; } - // REQUIRE(rank_table.size() == 1); + REQUIRE(rank_table.size() == 1); } #endif REQUIRE(last_iteration_index == 9); From 5a24a8afbeac962bef71ef67215966a8307d66e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 17:06:42 +0100 Subject: [PATCH 18/58] Take out the next hanging parallel test --- test/ParallelIOTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 9c28f52945..787a560b8d 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1004,6 +1004,7 @@ void file_based_write_read(std::string const &file_ending) TEST_CASE("file_based_write_read", "[parallel]") { + return; for (auto const &t : getBackends()) { file_based_write_read(t); From 98685f459ef4e7481ae6a7be087f21314c4fa369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 19:19:45 +0100 Subject: [PATCH 19/58] Separate MPI tests by MPI barriers --- test/ParallelIOTest.cpp | 79 +++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 787a560b8d..59f310a912 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -36,6 +36,23 @@ #include #include +#define PARALLEL_TEST_CASE(name, tags) \ + static void openPMD_parallel_##name(); \ + TEST_CASE(#name, tags) \ + { \ + MPI_Barrier(MPI_COMM_WORLD); \ + int rank; \ + MPI_Comm_rank(MPI_COMM_WORLD, &rank); \ + if (rank == 0) \ + { \ + std::cout << "\nStarting test '" << #name << "'.\n" << std::endl; \ + } \ + MPI_Barrier(MPI_COMM_WORLD); \ + openPMD_parallel_##name(); \ + MPI_Barrier(MPI_COMM_WORLD); \ + } \ + static void openPMD_parallel_##name() + #if !openPMD_HAVE_MPI TEST_CASE("none", "[parallel]") {} @@ -80,7 +97,7 @@ TEST_CASE("none", "[parallel]") using namespace openPMD; -TEST_CASE("parallel_multi_series_test", "[parallel]") +PARALLEL_TEST_CASE(parallel_multi_series_test, "[parallel]") { std::list allSeries; @@ -223,7 +240,7 @@ void write_test_zero_extent( #endif #if openPMD_HAVE_HDF5 && openPMD_HAVE_MPI -TEST_CASE("git_hdf5_sample_content_test", "[parallel][hdf5]") +PARALLEL_TEST_CASE(git_hdf5_sample_content_test, "[parallel][hdf5]") { int mpi_rank{-1}; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); @@ -307,7 +324,7 @@ TEST_CASE("git_hdf5_sample_content_test", "[parallel][hdf5]") } } -TEST_CASE("hdf5_write_test", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test, "[parallel][hdf5]") { int mpi_s{-1}; int mpi_r{-1}; @@ -377,13 +394,13 @@ TEST_CASE("hdf5_write_test", "[parallel][hdf5]") o.flush("hdf5.independent_stores = false"); } -TEST_CASE("hdf5_write_test_zero_extent", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_zero_extent, "[parallel][hdf5]") { write_test_zero_extent(false, "h5", true, true); write_test_zero_extent(true, "h5", true, true); } -TEST_CASE("hdf5_write_test_skip_chunk", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_skip_chunk, "[parallel][hdf5]") { //! @todo add via JSON option instead of environment read auto const hdf5_collective = @@ -397,7 +414,7 @@ TEST_CASE("hdf5_write_test_skip_chunk", "[parallel][hdf5]") REQUIRE(true); } -TEST_CASE("hdf5_write_test_skip_declare", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_skip_declare, "[parallel][hdf5]") { //! @todo add via JSON option instead of environment read auto const hdf5_collective = @@ -413,7 +430,7 @@ TEST_CASE("hdf5_write_test_skip_declare", "[parallel][hdf5]") #else -TEST_CASE("no_parallel_hdf5", "[parallel][hdf5]") +PARALLEL_TEST_CASE(no_parallel_hdf5, "[parallel][hdf5]") { REQUIRE(true); } @@ -495,7 +512,7 @@ void available_chunks_test(std::string const &file_ending) } } -TEST_CASE("available_chunks_test", "[parallel][adios]") +PARALLEL_TEST_CASE(available_chunks_test, "[parallel][adios]") { available_chunks_test("bp"); } @@ -550,14 +567,14 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig) } } -TEST_CASE("extend_dataset", "[parallel]") +PARALLEL_TEST_CASE(extend_dataset, "[parallel]") { extendDataset("bp", R"({"backend": "adios2"})"); } #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -TEST_CASE("adios_write_test", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test, "[parallel][adios]") { Series o = Series( "../samples/parallel_write.bp", @@ -645,25 +662,25 @@ TEST_CASE("adios_write_test", "[parallel][adios]") } } -TEST_CASE("adios_write_test_zero_extent", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_zero_extent, "[parallel][adios]") { write_test_zero_extent(false, "bp", true, true); write_test_zero_extent(true, "bp", true, true); } -TEST_CASE("adios_write_test_skip_chunk", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_skip_chunk, "[parallel][adios]") { write_test_zero_extent(false, "bp", false, true); write_test_zero_extent(true, "bp", false, true); } -TEST_CASE("adios_write_test_skip_declare", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_skip_declare, "[parallel][adios]") { write_test_zero_extent(false, "bp", false, false); write_test_zero_extent(true, "bp", false, false); } -TEST_CASE("hzdr_adios_sample_content_test", "[parallel][adios2][bp3]") +PARALLEL_TEST_CASE(hzdr_adios_sample_content_test, "[parallel][adios2][bp3]") { int mpi_rank{-1}; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); @@ -743,7 +760,7 @@ void write_4D_test(std::string const &file_ending) o.flush(); } -TEST_CASE("write_4D_test", "[parallel]") +PARALLEL_TEST_CASE(write_4D_test, "[parallel]") { for (auto const &t : getBackends()) { @@ -776,7 +793,7 @@ void write_makeconst_some(std::string const &file_ending) E_x.makeConstant(42); } -TEST_CASE("write_makeconst_some", "[parallel]") +PARALLEL_TEST_CASE(write_makeconst_some, "[parallel]") { for (auto const &t : getBackends()) { @@ -883,7 +900,7 @@ void close_iteration_test(std::string const &file_ending) } } -TEST_CASE("close_iteration_test", "[parallel]") +PARALLEL_TEST_CASE(close_iteration_test, "[parallel]") { for (auto const &t : getBackends()) { @@ -1002,7 +1019,7 @@ void file_based_write_read(std::string const &file_ending) } } -TEST_CASE("file_based_write_read", "[parallel]") +PARALLEL_TEST_CASE(file_based_write_read, "[parallel]") { return; for (auto const &t : getBackends()) @@ -1182,7 +1199,7 @@ void hipace_like_write(std::string const &file_ending) } } -TEST_CASE("hipace_like_write", "[parallel]") +PARALLEL_TEST_CASE(hipace_like_write, "[parallel]") { for (auto const &t : getBackends()) { @@ -1192,7 +1209,7 @@ TEST_CASE("hipace_like_write", "[parallel]") #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -TEST_CASE("independent_write_with_collective_flush", "[parallel]") +PARALLEL_TEST_CASE(independent_write_with_collective_flush, "[parallel]") { Series write( "../samples/independent_write_with_collective_flush.bp5", @@ -1226,7 +1243,7 @@ TEST_CASE("independent_write_with_collective_flush", "[parallel]") #endif #if openPMD_HAVE_MPI -TEST_CASE("unavailable_backend", "[core][parallel]") +PARALLEL_TEST_CASE(unavailable_backend, "[core][parallel]") { #if !openPMD_HAVE_ADIOS2 { @@ -1374,7 +1391,7 @@ void adios2_streaming(bool variableBasedLayout) } } -TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") +PARALLEL_TEST_CASE(adios2_streaming, "[pseudoserial][adios2]") { #if HAS_ADIOS_2_9 adios2_streaming(true); @@ -1382,7 +1399,7 @@ TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") adios2_streaming(false); } -TEST_CASE("parallel_adios2_json_config", "[parallel][adios2]") +PARALLEL_TEST_CASE(parallel_adios2_json_config, "[parallel][adios2]") { int size{-1}; int rank{-1}; @@ -1593,7 +1610,7 @@ void adios2_ssc() } } -TEST_CASE("adios2_ssc", "[parallel][adios2]") +PARALLEL_TEST_CASE(adios2_ssc, "[parallel][adios2]") { adios2_ssc(); } @@ -1919,7 +1936,7 @@ void append_mode( #endif } -TEST_CASE("append_mode", "[serial]") +PARALLEL_TEST_CASE(append_mode, "[serial]") { for (auto const &t : testedFileExtensions()) { @@ -2122,7 +2139,7 @@ void joined_dim(std::string const &ext) } } -TEST_CASE("joined_dim", "[parallel]") +PARALLEL_TEST_CASE(joined_dim, "[parallel]") { #if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ @@ -2147,7 +2164,7 @@ TEST_CASE("joined_dim", "[parallel]") #if openPMD_HAVE_ADIOS2_BP5 // Parallel version of the same test from SerialIOTest.cpp -TEST_CASE("adios2_flush_via_step") +PARALLEL_TEST_CASE(adios2_flush_via_step, "[parallel]") { int size_i(0), rank_i(0); MPI_Comm_rank(MPI_COMM_WORLD, &rank_i); @@ -2254,12 +2271,12 @@ TEST_CASE("adios2_flush_via_step") } #endif -TEST_CASE("read_variablebased_randomaccess") +PARALLEL_TEST_CASE(read_variablebased_randomaccess, "[parallel]") { read_variablebased_randomaccess::read_variablebased_randomaccess(); } -TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") +PARALLEL_TEST_CASE(iterate_nonstreaming_series, "[parallel][adios2]") { iterate_nonstreaming_series::iterate_nonstreaming_series(); } @@ -2720,14 +2737,14 @@ void run_test() } } // namespace adios2_chunk_distribution -TEST_CASE("adios2_chunk_distribution", "[parallel][adios2]") +PARALLEL_TEST_CASE(adios2_chunk_distribution, "[parallel][adios2]") { adios2_chunk_distribution::run_test(); } #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI #if openPMD_HAVE_MPI -TEST_CASE("bug_1655_bp5_writer_hangup", "[parallel]") +PARALLEL_TEST_CASE(bug_1655_bp5_writer_hangup, "[parallel]") { bug_1655_bp5_writer_hangup::bug_1655_bp5_writer_hangup(); } From 944a1e2ba52debf0bcd7e1554436fb2591c9e394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 22:03:57 +0100 Subject: [PATCH 20/58] Fix wrong MPI_COMM_WORLD --- src/auxiliary/Mpi.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/auxiliary/Mpi.cpp b/src/auxiliary/Mpi.cpp index ef899e4207..5e8379ff41 100644 --- a/src/auxiliary/Mpi.cpp +++ b/src/auxiliary/Mpi.cpp @@ -49,7 +49,7 @@ StringMatrix collectStringsAsMatrixTo( 1, MPI_INT, destRank, - MPI_COMM_WORLD); + communicator); int maxLength = std::accumulate( recvcounts.begin(), recvcounts.end(), 0, [](int a, int b) { return std::max(a, b); @@ -78,7 +78,7 @@ StringMatrix collectStringsAsMatrixTo( displs.data(), MPI_CHAR, destRank, - MPI_COMM_WORLD); + communicator); return res; } @@ -95,7 +95,7 @@ std::vector distributeStringsToAllRanks( int *displs = new int[size]; MPI_Allgather( - &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, MPI_COMM_WORLD); + &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, communicator); char *namesBuffer; { @@ -116,7 +116,7 @@ std::vector distributeStringsToAllRanks( sizesBuffer, displs, MPI_CHAR, - MPI_COMM_WORLD); + communicator); std::vector hostnames(size); for (int i = 0; i < size; ++i) From 3a81d3b651ed31a550f63e63f1a5563b503d1fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 27 Mar 2026 22:04:14 +0100 Subject: [PATCH 21/58] wip: debugging state --- src/RecordComponent.cpp | 18 +++++++++--------- src/auxiliary/Mpi.cpp | 2 ++ test/CatchRunner.cpp | 5 +++-- test/ParallelIOTest.cpp | 5 ++++- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index bce3da6aeb..7ba00c04f8 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -238,15 +238,15 @@ namespace RecordComponent &RecordComponent::resetDataset(Dataset d) { auto &rc = get(); - auto cleanup = defer([&rc, this]() { - if (rc.m_dataset.has_value() && - rc.m_dataset->dtype != Datatype::UNDEFINED && - IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) - { - seriesFlush_impl( - {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); - } - }); + // auto cleanup = defer([&rc, this]() { + // if (rc.m_dataset.has_value() && + // rc.m_dataset->dtype != Datatype::UNDEFINED && + // IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) + // { + // seriesFlush_impl( + // {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); + // } + // }); if (written()) { if (!rc.m_dataset.has_value()) diff --git a/src/auxiliary/Mpi.cpp b/src/auxiliary/Mpi.cpp index 5e8379ff41..b92055889b 100644 --- a/src/auxiliary/Mpi.cpp +++ b/src/auxiliary/Mpi.cpp @@ -94,6 +94,8 @@ std::vector distributeStringsToAllRanks( int *sizesBuffer = new int[size]; int *displs = new int[size]; + MPI_Barrier(communicator); + MPI_Allgather( &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, communicator); diff --git a/test/CatchRunner.cpp b/test/CatchRunner.cpp index 107ec2e46e..81a6abf7c4 100644 --- a/test/CatchRunner.cpp +++ b/test/CatchRunner.cpp @@ -29,8 +29,9 @@ int main(int argc, char *argv[]) Catch::Session session; // session.configData().runOrder = Catch::TestRunOrder::Declared; - MPI_Bcast( - &session.configData().rngSeed, 1, MPI_UINT32_T, 0, MPI_COMM_WORLD); + session.configData().rngSeed = 3036063643; + // MPI_Bcast( + // &session.configData().rngSeed, 1, MPI_UINT32_T, 0, MPI_COMM_WORLD); int result = session.applyCommandLine(argc, argv); if (result == 0) { diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 59f310a912..7e5831752c 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -2559,10 +2559,13 @@ void run_test() E_x.storeChunk(data, {unsigned(mpi_rank * 2), 0}, {1, 10}); E_x.storeChunk(data, {unsigned(mpi_rank * 2 + 1), 0}, {1, 10}); series.flush(); + series.close(); } + MPI_Barrier(MPI_COMM_WORLD); { - Series series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); + Series series( + filename, openPMD::Access::READ_ONLY /* , MPI_COMM_WORLD */); /* * Inquire the writing application's "MPI rank -> hostname" mapping. * The reading application needs to know about its own mapping. From 87f86fbf5b4af6e832a4f52bac296c23540e061a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 12:53:20 +0200 Subject: [PATCH 22/58] deactivate malicious tests --- test/ParallelIOTest.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 7e5831752c..f54417f16b 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -40,6 +40,12 @@ static void openPMD_parallel_##name(); \ TEST_CASE(#name, tags) \ { \ + int flag; \ + MPI_Initialized(&flag); \ + if (!flag) \ + { \ + throw std::runtime_error("MPI IS NOT INITIALIZED"); \ + } \ MPI_Barrier(MPI_COMM_WORLD); \ int rank; \ MPI_Comm_rank(MPI_COMM_WORLD, &rank); \ @@ -53,6 +59,14 @@ } \ static void openPMD_parallel_##name() +#define DEACTIVATE_TEST_CASE(name, tags) \ + static void openPMD_parallel_##name(); \ + TEST_CASE(#name, tags) \ + { \ + return; \ + } \ + static void openPMD_parallel_##name() + #if !openPMD_HAVE_MPI TEST_CASE("none", "[parallel]") {} @@ -512,7 +526,7 @@ void available_chunks_test(std::string const &file_ending) } } -PARALLEL_TEST_CASE(available_chunks_test, "[parallel][adios]") +DEACTIVATE_TEST_CASE(available_chunks_test, "[parallel][adios]") { available_chunks_test("bp"); } @@ -574,7 +588,7 @@ PARALLEL_TEST_CASE(extend_dataset, "[parallel]") #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -PARALLEL_TEST_CASE(adios_write_test, "[parallel][adios]") +DEACTIVATE_TEST_CASE(adios_write_test, "[parallel][adios]") { Series o = Series( "../samples/parallel_write.bp", From 4f50fa2fb25235584b47a70831c5f55f384cb84a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 15:46:01 +0200 Subject: [PATCH 23/58] Revert "deactivate malicious tests" This reverts commit 246609ff5fbe5edb68119b7f3b29a40e7bf23d2d. --- test/ParallelIOTest.cpp | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index f54417f16b..7e5831752c 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -40,12 +40,6 @@ static void openPMD_parallel_##name(); \ TEST_CASE(#name, tags) \ { \ - int flag; \ - MPI_Initialized(&flag); \ - if (!flag) \ - { \ - throw std::runtime_error("MPI IS NOT INITIALIZED"); \ - } \ MPI_Barrier(MPI_COMM_WORLD); \ int rank; \ MPI_Comm_rank(MPI_COMM_WORLD, &rank); \ @@ -59,14 +53,6 @@ } \ static void openPMD_parallel_##name() -#define DEACTIVATE_TEST_CASE(name, tags) \ - static void openPMD_parallel_##name(); \ - TEST_CASE(#name, tags) \ - { \ - return; \ - } \ - static void openPMD_parallel_##name() - #if !openPMD_HAVE_MPI TEST_CASE("none", "[parallel]") {} @@ -526,7 +512,7 @@ void available_chunks_test(std::string const &file_ending) } } -DEACTIVATE_TEST_CASE(available_chunks_test, "[parallel][adios]") +PARALLEL_TEST_CASE(available_chunks_test, "[parallel][adios]") { available_chunks_test("bp"); } @@ -588,7 +574,7 @@ PARALLEL_TEST_CASE(extend_dataset, "[parallel]") #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -DEACTIVATE_TEST_CASE(adios_write_test, "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test, "[parallel][adios]") { Series o = Series( "../samples/parallel_write.bp", From 2720e7f49d849e74d20c594a8eb0ba71dbe1955e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 15:46:47 +0200 Subject: [PATCH 24/58] Revert "wip: debugging state" This reverts commit 2df195749b53b0a54832585899bd469d35f81d6d. --- src/RecordComponent.cpp | 18 +++++++++--------- src/auxiliary/Mpi.cpp | 2 -- test/CatchRunner.cpp | 5 ++--- test/ParallelIOTest.cpp | 5 +---- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 7ba00c04f8..bce3da6aeb 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -238,15 +238,15 @@ namespace RecordComponent &RecordComponent::resetDataset(Dataset d) { auto &rc = get(); - // auto cleanup = defer([&rc, this]() { - // if (rc.m_dataset.has_value() && - // rc.m_dataset->dtype != Datatype::UNDEFINED && - // IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) - // { - // seriesFlush_impl( - // {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); - // } - // }); + auto cleanup = defer([&rc, this]() { + if (rc.m_dataset.has_value() && + rc.m_dataset->dtype != Datatype::UNDEFINED && + IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) + { + seriesFlush_impl( + {FlushLevel::SkeletonOnly}, /* flush_io_handler = */ true); + } + }); if (written()) { if (!rc.m_dataset.has_value()) diff --git a/src/auxiliary/Mpi.cpp b/src/auxiliary/Mpi.cpp index b92055889b..5e8379ff41 100644 --- a/src/auxiliary/Mpi.cpp +++ b/src/auxiliary/Mpi.cpp @@ -94,8 +94,6 @@ std::vector distributeStringsToAllRanks( int *sizesBuffer = new int[size]; int *displs = new int[size]; - MPI_Barrier(communicator); - MPI_Allgather( &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, communicator); diff --git a/test/CatchRunner.cpp b/test/CatchRunner.cpp index 81a6abf7c4..107ec2e46e 100644 --- a/test/CatchRunner.cpp +++ b/test/CatchRunner.cpp @@ -29,9 +29,8 @@ int main(int argc, char *argv[]) Catch::Session session; // session.configData().runOrder = Catch::TestRunOrder::Declared; - session.configData().rngSeed = 3036063643; - // MPI_Bcast( - // &session.configData().rngSeed, 1, MPI_UINT32_T, 0, MPI_COMM_WORLD); + MPI_Bcast( + &session.configData().rngSeed, 1, MPI_UINT32_T, 0, MPI_COMM_WORLD); int result = session.applyCommandLine(argc, argv); if (result == 0) { diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 7e5831752c..59f310a912 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -2559,13 +2559,10 @@ void run_test() E_x.storeChunk(data, {unsigned(mpi_rank * 2), 0}, {1, 10}); E_x.storeChunk(data, {unsigned(mpi_rank * 2 + 1), 0}, {1, 10}); series.flush(); - series.close(); } - MPI_Barrier(MPI_COMM_WORLD); { - Series series( - filename, openPMD::Access::READ_ONLY /* , MPI_COMM_WORLD */); + Series series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); /* * Inquire the writing application's "MPI rank -> hostname" mapping. * The reading application needs to know about its own mapping. From 560b64e523e4f3420766a0289f5f137d46e1797c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 16:44:10 +0200 Subject: [PATCH 25/58] Revert "TMP REVERT ME take out hanging parallel test" This reverts commit 07fb558431acc7b3fa1d69f5a258be8164dad227. --- .../iterate_nonstreaming_series.cpp | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp index fe72ef1698..8ddadfc40a 100644 --- a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp +++ b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp @@ -181,12 +181,11 @@ auto iterate_nonstreaming_series() -> void { for (auto const &backend : testedBackends()) { - std::cout << "TESTING BACKEND " << backend.extension << std::endl; - // run_test( - // "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + - // backend.extension, - // false, - // backend.jsonBaseConfig()); + run_test( + "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + + backend.extension, + false, + backend.jsonBaseConfig()); run_test( "../samples/iterate_nonstreaming_series/parallel_groupbased." + backend.extension, @@ -195,14 +194,13 @@ auto iterate_nonstreaming_series() -> void #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_ADIOS2_BP5 if (backend.extension == "bp") { - // run_test( - // "../samples/iterate_nonstreaming_series/" - // "parallel_filebased_bp5_%T." + - // backend.extension, - // false, - // json::merge( - // backend.jsonBaseConfig(), "adios2.engine.type = - // \"bp5\"")); + run_test( + "../samples/iterate_nonstreaming_series/" + "parallel_filebased_bp5_%T." + + backend.extension, + false, + json::merge( + backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); run_test( "../samples/iterate_nonstreaming_series/" "parallel_groupbased_bp5." + From 0a195f6c0ab4d39760fc13bb2d8eddf3fade0c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 30 Mar 2026 18:20:23 +0200 Subject: [PATCH 26/58] Revert some WIPs --- test/SerialIOTest.cpp | 55 ++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 52836ae903..49aab1db18 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1771,24 +1771,24 @@ inline void write_test( TEST_CASE("write_test", "[serial]") { - for (auto const &t : {std::string("json")}) + for (auto const &t : testedFileExtensions()) { if (t == "json") { - // write_test( - // "template." + t, - // R"( - // { - // "json": { - // "dataset": { - // "mode": "template" - // }, - // "attribute": { - // "mode": "short" - // } - // } - // })", - // false); + write_test( + "template." + t, + R"( +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})", + false); write_test( t, R"( @@ -1972,10 +1972,6 @@ inline void fileBased_write_test(const std::string &backend) "\\", "/")); - // TODO: somehow make the rank table appear in iteration 1 - o.iterations[1]; - o.flush(); - ParticleSpecies &e_1 = o.iterations[1].particles["e"]; std::vector position_global(4); @@ -2353,20 +2349,19 @@ inline void fileBased_write_test(const std::string &backend) TEST_CASE("fileBased_write_test", "[serial]") { - for (auto const &t : {"bp5"}) + for (auto const &t : testedFileExtensions()) { fileBased_write_test(t); } - // if (auto extensions = getFileExtensions(); - // std::find(extensions.begin(), extensions.end(), "toml") != - // extensions.end()) - // { /* - // * TOML backend is not generally tested for performance reasons, opt in - // to - // * testing it here. - // */ - // fileBased_write_test("toml"); - // } + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + fileBased_write_test("toml"); + } } inline void sample_write_thetaMode(std::string const &file_ending) From ff4788fd1a484ea0ec6bc2882318f976059a2f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 19 May 2026 19:51:12 +0200 Subject: [PATCH 27/58] Use an Attributable per Iteration for rankTable in filebased encoding needs some documentation still --- include/openPMD/Iteration.hpp | 10 +-- include/openPMD/Series.hpp | 15 ++-- include/openPMD/backend/PerIterationData.hpp | 45 +++++++++++ src/Iteration.cpp | 28 +++---- src/Series.cpp | 82 +++++++++++--------- 5 files changed, 107 insertions(+), 73 deletions(-) create mode 100644 include/openPMD/backend/PerIterationData.hpp diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 0892627f2d..aaf199fdb8 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -28,6 +28,7 @@ #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/PerIterationData.hpp" #include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include @@ -122,14 +123,7 @@ namespace internal */ bool allow_reopening_implicitly = false; - /** - * Whether a step is currently active for this iteration. - * Used for file-based iteration layout, see Series.hpp for - * group-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. - */ - StepStatus m_stepStatus = StepStatus::NoStep; + PerIterationData m_perIterationData; /** * Cached copy of the key under which this Iteration lives in diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index f1b5b3872c..43d29cbd11 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -33,6 +33,7 @@ #include "openPMD/backend/Container.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" #include "openPMD/backend/ParsePreference.hpp" +#include "openPMD/backend/PerIterationData.hpp" #include "openPMD/config.hpp" #include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/version.hpp" @@ -205,14 +206,9 @@ namespace internal * Detected IO format (backend). */ Format m_format; - /** - * Whether a step is currently active for this iteration. - * Used for group-based iteration layout, see SeriesData.hpp for - * iteration-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. - */ - StepStatus m_stepStatus = StepStatus::NoStep; + + PerIterationData m_perIterationData; + /** * True if a user opts into lazy parsing. */ @@ -261,7 +257,6 @@ namespace internal struct RankTableData { - Attributable m_attributable; std::variant< NoSourceSpecified, SourceSpecifiedViaJSON, @@ -900,7 +895,7 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushRankTable(FlushLevel); + void flushRankTable(FlushLevel, Attributable &attributable); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. */ diff --git a/include/openPMD/backend/PerIterationData.hpp b/include/openPMD/backend/PerIterationData.hpp new file mode 100644 index 0000000000..cf3c347437 --- /dev/null +++ b/include/openPMD/backend/PerIterationData.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include "openPMD/ChunkInfo.hpp" +#include "openPMD/Streaming.hpp" +#include "openPMD/backend/Attributable.hpp" + +#include + +namespace openPMD::internal +{ +struct NoSourceSpecified +{}; +struct SourceSpecifiedViaJSON +{ + std::string value; +}; +struct SourceSpecifiedManually +{ + std::string value; +}; + +struct RankTableData +{ + Attributable m_attributable; + std::variant< + NoSourceSpecified, + SourceSpecifiedViaJSON, + SourceSpecifiedManually> + m_rankTableSource; + std::optional m_bufferedRead; +}; + +struct PerIterationData +{ + /** + * Whether a step is currently active for this iteration. + * Used for group-based iteration layout, see SeriesData.hpp for + * iteration-based layout. + * Access via stepStatus() method to automatically select the correct + * one among both flags. + */ + StepStatus m_stepStatus = StepStatus::NoStep; + Attributable m_rankTableAttributable; +}; +} // namespace openPMD::internal diff --git a/src/Iteration.cpp b/src/Iteration.cpp index caac558986..7e708ca4e4 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -272,10 +272,6 @@ void Iteration::flushFileBased( /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); - auto &series = s.get(); - - bool do_flush_rank_table = - !series.m_rankTable.m_attributable.written() || !this->written(); if (!written()) { @@ -300,16 +296,11 @@ void Iteration::flushFileBased( s.openIteration(i, *this); } - if (do_flush_rank_table) + auto &rankTableAttributable = + get().m_perIterationData.m_rankTableAttributable; + if (!rankTableAttributable.written()) { - /* - * If it was written before, then in the context of another iteration. - */ - auto &attr = series.m_rankTable.m_attributable; - attr.setWritten(false, Attributable::EnqueueAsynchronously::Both); - attr.get().m_writable.abstractFilePosition.reset(); - - s.flushRankTable(flushParams.flushLevel); + s.flushRankTable(flushParams.flushLevel, rankTableAttributable); } if (flush_level::flush_hierarchy(flushParams.flushLevel)) @@ -784,7 +775,7 @@ auto Iteration::beginStep( } else { - series.get().m_stepStatus = StepStatus::DuringStep; + series.get().m_perIterationData.m_stepStatus = StepStatus::DuringStep; status = series.advance(AdvanceMode::BEGINSTEP); } @@ -887,10 +878,10 @@ StepStatus Iteration::getStepStatus() { using IE = IterationEncoding; case IE::fileBased: - return get().m_stepStatus; + return get().m_perIterationData.m_stepStatus; case IE::groupBased: case IE::variableBased: - return s.get().m_stepStatus; + return s.get().m_perIterationData.m_stepStatus; default: throw std::runtime_error("[Iteration] unreachable"); } @@ -903,11 +894,11 @@ void Iteration::setStepStatus(StepStatus status) { using IE = IterationEncoding; case IE::fileBased: - get().m_stepStatus = status; + get().m_perIterationData.m_stepStatus = status; break; case IE::groupBased: case IE::variableBased: - s.get().m_stepStatus = status; + s.get().m_perIterationData.m_stepStatus = status; break; default: throw std::runtime_error("[Iteration] unreachable"); @@ -919,6 +910,7 @@ void Iteration::linkHierarchy(Writable &w) Attributable::linkHierarchy(w); meshes.linkHierarchy(this->writable()); particles.linkHierarchy(this->writable()); + get().m_perIterationData.m_rankTableAttributable.linkHierarchy(*w.parent); } void Iteration::runDeferredParseAccess() diff --git a/src/Series.cpp b/src/Series.cpp index fbd2151df3..199ce705c1 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -355,6 +355,12 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) IOHandler()->enqueue(IOTask(this, openFile)); #endif } + Attributable &attributable = + iterationEncoding() == IterationEncoding::fileBased + ? iterations.begin() + ->second.get() + .m_perIterationData.m_rankTableAttributable + : series.m_perIterationData.m_rankTableAttributable; auto datasets = availableDatasets(); if (std::find(datasets.begin(), datasets.end(), "rankTable") == datasets.end()) @@ -364,7 +370,7 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } Parameter openDataset; openDataset.name = "rankTable"; - IOHandler()->enqueue(IOTask(&rankTable.m_attributable, openDataset)); + IOHandler()->enqueue(IOTask(&attributable, openDataset)); IOHandler()->flush(internal::defaultFlushParams); if (openDataset.extent->size() != 2) @@ -394,19 +400,20 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) new char[writerRanks * lineWidth], [](char const *ptr) { delete[] ptr; }}; - auto doReadDataset = [&openDataset, this, &get, &rankTable]() { - Parameter readDataset; - // read the whole thing - readDataset.offset.resize(2); - readDataset.extent = *openDataset.extent; - // @todo better cross-platform support by switching over - // *openDataset.dtype - readDataset.dtype = Datatype::CHAR; - readDataset.data = get; - - IOHandler()->enqueue(IOTask(&rankTable.m_attributable, readDataset)); - IOHandler()->flush(internal::publicFlush); - }; + auto doReadDataset = + [&openDataset, this, &get, &rankTable, &attributable]() { + Parameter readDataset; + // read the whole thing + readDataset.offset.resize(2); + readDataset.extent = *openDataset.extent; + // @todo better cross-platform support by switching over + // *openDataset.dtype + readDataset.dtype = Datatype::CHAR; + readDataset.data = get; + + IOHandler()->enqueue(IOTask(&attributable, readDataset)); + IOHandler()->flush(internal::publicFlush); + }; #if openPMD_HAVE_MPI if (collective && series.m_communicator.has_value()) @@ -464,7 +471,7 @@ Series &Series::setRankTable(const std::string &myRankInfo) return *this; } -void Series::flushRankTable(FlushLevel l) +void Series::flushRankTable(FlushLevel l, Attributable &attributable) { if (!flush_level::global_flushpoint(l)) { @@ -512,29 +519,28 @@ void Series::flushRankTable(FlushLevel l) int rank{0}, size{1}; unsigned long long maxSize = mySize; - auto createRankTable = [&size, &maxSize, &rankTable, this]() { - if (rankTable.m_attributable.written()) - { - return; - } - Parameter param( - AbstractParameter::I_dont_want_to_use_joined_dimensions); - param.name = "rankTable"; - param.dtype = Datatype::CHAR; - param.extent = {uint64_t(size), uint64_t(maxSize)}; - IOHandler()->enqueue( - IOTask(&rankTable.m_attributable, std::move(param))); - }; + auto createRankTable = + [&size, &maxSize, &rankTable, this, &attributable]() { + if (attributable.written()) + { + return; + } + Parameter param( + AbstractParameter::I_dont_want_to_use_joined_dimensions); + param.name = "rankTable"; + param.dtype = Datatype::CHAR; + param.extent = {uint64_t(size), uint64_t(maxSize)}; + IOHandler()->enqueue(IOTask(&attributable, std::move(param))); + }; - auto writeDataset = [&rank, &maxSize, this, &rankTable]( + auto writeDataset = [&rank, &maxSize, this, &rankTable, &attributable]( std::shared_ptr put, size_t num_lines = 1) { Parameter chunk; chunk.dtype = Datatype::CHAR; chunk.offset = {uint64_t(rank), 0}; chunk.extent = {num_lines, maxSize}; chunk.data = std::move(put); - IOHandler()->enqueue( - IOTask(&rankTable.m_attributable, std::move(chunk))); + IOHandler()->enqueue(IOTask(&attributable, std::move(chunk))); }; #if openPMD_HAVE_MPI @@ -578,8 +584,7 @@ void Series::flushRankTable(FlushLevel l) // Must ensure that the Writable is consistently set to written on all // ranks - series.m_rankTable.m_attributable.setWritten( - true, EnqueueAsynchronously::OnlyAsync); + attributable.setWritten(true, EnqueueAsynchronously::OnlyAsync); return; } #endif @@ -985,7 +990,8 @@ void Series::init( std::make_unique(parsed_directory, at)); auto &series = get(); series.iterations.linkHierarchy(writable()); - series.m_rankTable.m_attributable.linkHierarchy(writable()); + series.m_perIterationData.m_rankTableAttributable.linkHierarchy( + writable()); series.m_deferred_initialization = [called_this_already = false, filepath, @@ -1211,7 +1217,7 @@ void Series::initSeries( series.iterations.linkHierarchy(writable); series.iterations.writable().ownKeyWithinParent = "data"; - series.m_rankTable.m_attributable.linkHierarchy(writable); + series.m_perIterationData.m_rankTableAttributable.linkHierarchy(writable); series.m_name = input->name; @@ -1663,9 +1669,11 @@ void Series::flushGorVBased( IOHandler()->enqueue(IOTask(this, fCreate)); } - if (!series.m_rankTable.m_attributable.written()) + if (!series.m_perIterationData.m_rankTableAttributable.written()) { - flushRankTable(flushParams.flushLevel); + flushRankTable( + flushParams.flushLevel, + series.m_perIterationData.m_rankTableAttributable); } series.iterations.flush( From 937030726805498e5bb0925964e261f5396461b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 May 2026 12:12:58 +0200 Subject: [PATCH 28/58] fix nompi builds --- test/ParallelIOTest.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 59f310a912..37a6d45ee9 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -36,6 +36,16 @@ #include #include +#if !openPMD_HAVE_MPI +#define PARALLEL_TEST_CASE(name, tags) TEST_CASE(#name, tags) + +PARALLEL_TEST_CASE(none, "[parallel]") +{} + +#else + +#include + #define PARALLEL_TEST_CASE(name, tags) \ static void openPMD_parallel_##name(); \ TEST_CASE(#name, tags) \ @@ -53,14 +63,6 @@ } \ static void openPMD_parallel_##name() -#if !openPMD_HAVE_MPI -TEST_CASE("none", "[parallel]") -{} - -#else - -#include - #if openPMD_HAVE_ADIOS2 #include #define HAS_ADIOS_2_8 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208) From 4ea4e61caebaf16ddfcc2fb4257f3fb8cd0e2527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 May 2026 14:07:14 +0200 Subject: [PATCH 29/58] CI fixes --- include/openPMD/Iteration.hpp | 9 +++ include/openPMD/Series.hpp | 10 +++ include/openPMD/backend/PerIterationData.hpp | 9 +++ src/Series.cpp | 69 +++++++++++--------- 4 files changed, 65 insertions(+), 32 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index aaf199fdb8..0a3c1dfcb0 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -123,6 +123,15 @@ namespace internal */ bool allow_reopening_implicitly = false; + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. + */ PerIterationData m_perIterationData; /** diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 43d29cbd11..503eec0ace 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -207,6 +207,15 @@ namespace internal */ Format m_format; + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. + */ PerIterationData m_perIterationData; /** @@ -978,6 +987,7 @@ OPENPMD_private * least one step was written. * * @param doFlush If true, flush the IO handler. + * @param l This operation must only run at flush level write_datasets */ void flushStep(bool doFlush, FlushLevel l); diff --git a/include/openPMD/backend/PerIterationData.hpp b/include/openPMD/backend/PerIterationData.hpp index cf3c347437..6cea297bf2 100644 --- a/include/openPMD/backend/PerIterationData.hpp +++ b/include/openPMD/backend/PerIterationData.hpp @@ -30,6 +30,15 @@ struct RankTableData std::optional m_bufferedRead; }; +/* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each Iteration. + * Access must be distinguished by iteration encoding. + */ struct PerIterationData { /** diff --git a/src/Series.cpp b/src/Series.cpp index 199ce705c1..99fe227582 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -339,10 +339,11 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } if (iterationEncoding() == IterationEncoding::fileBased) { - std::cerr << "[Series] Use rank table in file-based iteration encoding " - "at your own risk. Make sure to have an iteration open " - "before calling this." - << std::endl; + std::cerr + << "[Series] Use rank table in file-based iteration encoding " + "at your own risk. Make sure to have the first iteration open " + "before calling this." + << std::endl; if (iterations.empty()) { return {}; @@ -357,6 +358,12 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } Attributable &attributable = iterationEncoding() == IterationEncoding::fileBased + /* + * Only second class support for file encoding. We indiscriminately use + * the first Iteration for this operation. It is on the user to ensure + * that this Iteration is actually open. The warning printed above + * informs about this. + */ ? iterations.begin() ->second.get() .m_perIterationData.m_rankTableAttributable @@ -400,20 +407,19 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) new char[writerRanks * lineWidth], [](char const *ptr) { delete[] ptr; }}; - auto doReadDataset = - [&openDataset, this, &get, &rankTable, &attributable]() { - Parameter readDataset; - // read the whole thing - readDataset.offset.resize(2); - readDataset.extent = *openDataset.extent; - // @todo better cross-platform support by switching over - // *openDataset.dtype - readDataset.dtype = Datatype::CHAR; - readDataset.data = get; - - IOHandler()->enqueue(IOTask(&attributable, readDataset)); - IOHandler()->flush(internal::publicFlush); - }; + auto doReadDataset = [&openDataset, this, &get, &attributable]() { + Parameter readDataset; + // read the whole thing + readDataset.offset.resize(2); + readDataset.extent = *openDataset.extent; + // @todo better cross-platform support by switching over + // *openDataset.dtype + readDataset.dtype = Datatype::CHAR; + readDataset.data = get; + + IOHandler()->enqueue(IOTask(&attributable, readDataset)); + IOHandler()->flush(internal::publicFlush); + }; #if openPMD_HAVE_MPI if (collective && series.m_communicator.has_value()) @@ -519,21 +525,20 @@ void Series::flushRankTable(FlushLevel l, Attributable &attributable) int rank{0}, size{1}; unsigned long long maxSize = mySize; - auto createRankTable = - [&size, &maxSize, &rankTable, this, &attributable]() { - if (attributable.written()) - { - return; - } - Parameter param( - AbstractParameter::I_dont_want_to_use_joined_dimensions); - param.name = "rankTable"; - param.dtype = Datatype::CHAR; - param.extent = {uint64_t(size), uint64_t(maxSize)}; - IOHandler()->enqueue(IOTask(&attributable, std::move(param))); - }; + auto createRankTable = [&size, &maxSize, this, &attributable]() { + if (attributable.written()) + { + return; + } + Parameter param( + AbstractParameter::I_dont_want_to_use_joined_dimensions); + param.name = "rankTable"; + param.dtype = Datatype::CHAR; + param.extent = {uint64_t(size), uint64_t(maxSize)}; + IOHandler()->enqueue(IOTask(&attributable, std::move(param))); + }; - auto writeDataset = [&rank, &maxSize, this, &rankTable, &attributable]( + auto writeDataset = [&rank, &maxSize, this, &attributable]( std::shared_ptr put, size_t num_lines = 1) { Parameter chunk; chunk.dtype = Datatype::CHAR; From 0e70a8ed082bb5ba7e24eacb1953dd1ccecbecf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 May 2026 17:34:17 +0200 Subject: [PATCH 30/58] Activate test again --- test/ParallelIOTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 37a6d45ee9..d32d31253b 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1023,7 +1023,6 @@ void file_based_write_read(std::string const &file_ending) PARALLEL_TEST_CASE(file_based_write_read, "[parallel]") { - return; for (auto const &t : getBackends()) { file_based_write_read(t); From 46f2b9fe892cd426b36bb2e2c51f518acdb4974d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 30 Apr 2026 20:01:05 +0200 Subject: [PATCH 31/58] Keep a list of children per Attributable --- include/openPMD/auxiliary/TypeTraits.hpp | 3 + include/openPMD/backend/Attributable.hpp | 15 ++- include/openPMD/backend/BaseRecord.hpp | 6 +- include/openPMD/backend/Container.hpp | 122 +++++++++++++++++++++- include/openPMD/backend/ContainerImpl.tpp | 104 ++++++++++-------- src/Iteration.cpp | 4 +- src/ParticlePatches.cpp | 6 +- src/ParticleSpecies.cpp | 8 +- src/Record.cpp | 6 +- src/Series.cpp | 12 ++- src/backend/Attributable.cpp | 4 +- src/backend/BaseRecord.cpp | 58 ++++++---- src/backend/PatchRecord.cpp | 3 +- src/snapshots/StatefulIterator.cpp | 9 +- 14 files changed, 272 insertions(+), 88 deletions(-) diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index 8d617aad3f..b395c199e1 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -163,6 +163,9 @@ using ScalarType_t = typename detail::ScalarType::type; template using VectorType_t = std::vector>; +template +using dependent_const = std::conditional_t; + /** Emulate in the C++ concept ContiguousContainer * * Users can implement this trait for a type to signal it can be used as diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 33b28c6ea2..7c81065ed2 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -60,6 +60,7 @@ namespace internal struct HomogenizeExtents; struct ConfigAttribute; class ScientificDefaults; + class AttributableData; class SharedAttributableData { @@ -87,6 +88,15 @@ namespace internal * The attributes defined by this Attributable. */ A_MAP m_attributes; + + // Using shared_ptr in here because + // AttributableData is non-copyable and non-movable, but we need + // movability for map handling + // Disallowing move in AttributableData is only a measure for code + // discipline anyway. + using children_map_t = + std::map>; + children_map_t m_children; }; /* @@ -112,6 +122,7 @@ namespace internal using SharedData_t = std::shared_ptr; using A_MAP = SharedData_t::element_type::A_MAP; + using parent_t = std::shared_ptr; public: AttributableData(); @@ -125,7 +136,7 @@ namespace internal // Make copies explicit, only to be used under the conditions described // above - void cloneFrom(AttributableData const &other); + void cloneFrom(parent_t const &other); template T asInternalCopyOf() @@ -751,6 +762,6 @@ Attributable::readVectorFloatingpoint(std::string const &key) const std::is_floating_point::value, "Type of attribute must be floating point"); - return getAttribute(key).get >(); + return getAttribute(key).get>(); } } // namespace openPMD diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 295b12e909..0c528011a9 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -409,10 +409,12 @@ template auto BaseRecord::emplace(Args &&...args) -> std::pair { detail::verifyNonscalar(this); - auto res = this->container().emplace(std::forward(args)...); + auto res = this->syncInsertResult( + this->container_front().emplace(std::forward(args)...)); if (res.first->first == RecordComponent::SCALAR) { - this->container().erase(res.first); + this->container_back().erase(res.first->first); + this->container_front().erase(res.first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 839f6452c9..f8fe871756 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -121,6 +121,59 @@ class Container : virtual public Attributable using ContainerData = internal::ContainerData; using InternalContainer = T_container; + using stringify_t = std::conditional_t< + std::is_same_v, + std::string const &, + std::string>; + static auto key_as_string(T_key const &key) -> stringify_t + { + if constexpr (std::is_same_v) + { + return key; + } + else + { + return std::to_string(key); + } + } + + template + struct SynchronizedContainers + { + using front_t = auxiliary::dependent_const; + using back_t = auxiliary::dependent_const< + const_, + std::map< + std::string, + std::shared_ptr>>; + + front_t *front; + back_t *back; + + template + inline auto for_both(Functor &&f) + { + f(*this->front); + return f(*this->back); + } + + template + inline auto for_both_to_string(Functor &&f) + { + f(*this->front, [](auto key) { return key; }); + if constexpr (std::is_same_v) + { + return f(*this->back, [](auto key) { return key; }); + } + else + { + return f( + *this->back, + static_cast(&std::to_string)); + } + } + }; + std::shared_ptr m_containerData; inline void setData(std::shared_ptr containerData) @@ -129,16 +182,62 @@ class Container : virtual public Attributable Attributable::setData(m_containerData); } - inline InternalContainer const &container() const + inline SynchronizedContainers container() const + { +#ifndef NDEBUG + auto size_front = container_front().size(); + auto size_back = container_back().size(); + if (size_front > size_back) + { + throw std::runtime_error( + "Invalid container state: " + std::to_string(size_front) + + " != " + std::to_string(size_back) + "."); + } +#endif + return {&container_front(), &container_back()}; + } + + inline SynchronizedContainers container() + { +#ifndef NDEBUG + auto size_front = container_front().size(); + auto size_back = container_back().size(); + if (size_front > size_back) + { + throw std::runtime_error( + "Invalid container state: " + std::to_string(size_front) + + " != " + std::to_string(size_back) + "."); + } +#endif + return {&container_front(), &container_back()}; + } + + inline auto container_front() const -> + typename SynchronizedContainers::front_t & { return m_containerData->m_container; } - inline InternalContainer &container() + inline auto container_front() -> + typename SynchronizedContainers::front_t & { return m_containerData->m_container; } + inline auto container_back() const -> + typename SynchronizedContainers::back_t & + { + traits::DeferredInitPolicy::call(*this); + return Attributable::get().m_children; + } + + inline auto container_back() -> + typename SynchronizedContainers::back_t & + { + traits::DeferredInitPolicy::call(*this); + return Attributable::get().m_children; + } + public: using key_type = typename InternalContainer::key_type; using mapped_type = typename InternalContainer::mapped_type; @@ -260,7 +359,8 @@ class Container : virtual public Attributable auto emplace(Args &&...args) -> decltype(InternalContainer().emplace(std::forward(args)...)) { - return container().emplace(std::forward(args)...); + return syncInsertResult( + container_front().emplace(std::forward(args)...)); } template @@ -289,6 +389,22 @@ OPENPMD_protected Container(NoInit); + auto syncInsertResult(std::pair res) + -> std::pair + { + if (res.second) + { + syncInsertResult(res.first); + } + return res; + } + auto syncInsertResult(iterator res) -> iterator + { + container_back().emplace( + key_as_string(res->first), *res->second.m_attri); + return res; + } + public: /* * Need to define these manually due to the virtual inheritance from diff --git a/include/openPMD/backend/ContainerImpl.tpp b/include/openPMD/backend/ContainerImpl.tpp index de11c91f14..35704cdd0e 100644 --- a/include/openPMD/backend/ContainerImpl.tpp +++ b/include/openPMD/backend/ContainerImpl.tpp @@ -19,6 +19,7 @@ * If not, see . */ +#include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" /* @@ -33,101 +34,101 @@ namespace openPMD template auto Container::begin() noexcept -> iterator { - return container().begin(); + return container_front().begin(); } template auto Container::begin() const noexcept -> const_iterator { - return container().begin(); + return container_front().begin(); } template auto Container::cbegin() const noexcept -> const_iterator { - return container().cbegin(); + return container_front().cbegin(); } template auto Container::end() noexcept -> iterator { - return container().end(); + return container_front().end(); } template auto Container::end() const noexcept -> const_iterator { - return container().end(); + return container_front().end(); } template auto Container::cend() const noexcept -> const_iterator { - return container().cend(); + return container_front().cend(); } template auto Container::rbegin() noexcept -> reverse_iterator { - return container().rbegin(); + return container_front().rbegin(); } template auto Container::rbegin() const noexcept -> const_reverse_iterator { - return container().rbegin(); + return container_front().rbegin(); } template auto Container::crbegin() const noexcept -> const_reverse_iterator { - return container().crbegin(); + return container_front().crbegin(); } template auto Container::rend() noexcept -> reverse_iterator { - return container().rend(); + return container_front().rend(); } template auto Container::rend() const noexcept -> const_reverse_iterator { - return container().rend(); + return container_front().rend(); } template auto Container::crend() const noexcept -> const_reverse_iterator { - return container().crend(); + return container_front().crend(); } template auto Container::empty() const noexcept -> bool { - return container().empty(); + return container_front().empty(); } template auto Container::size() const noexcept -> size_type { - return container().size(); + return container_front().size(); } template auto Container::at(key_type const &key) -> mapped_type & { - return container().at(key); + return container_front().at(key); } template auto Container::at(key_type const &key) const -> mapped_type const & { - return container().at(key); + return container_front().at(key); } template auto Container::operator[](key_type const &key) -> mapped_type & { - auto it = container().find(key); - if (it != container().end()) + auto it = container_front().find(key); + if (it != container_front().end()) return it->second; else { @@ -140,7 +141,9 @@ auto Container::operator[](key_type const &key) T t = T(); t.linkHierarchy(writable()); - auto inserted_iterator = container().insert({key, std::move(t)}).first; + auto inserted_iterator = + syncInsertResult(container_front().insert({key, std::move(t)})) + .first; auto &ret = inserted_iterator->second; if constexpr (std::is_same_v) { @@ -159,8 +162,8 @@ template auto Container::operator[](key_type &&key) -> mapped_type & { - auto it = container().find(key); - if (it != container().end()) + auto it = container_front().find(key); + if (it != container_front().end()) return it->second; else { @@ -173,7 +176,9 @@ auto Container::operator[](key_type &&key) T t = T(); t.linkHierarchy(writable()); - auto inserted_iterator = container().insert({key, std::move(t)}).first; + auto inserted_iterator = + syncInsertResult(container_front().insert({key, std::move(t)})) + .first; auto &ret = inserted_iterator->second; if constexpr (std::is_same_v) { @@ -203,49 +208,60 @@ template auto Container::insert(value_type const &value) -> std::pair { - return container().insert(value); + return syncInsertResult(container_front().insert(value)); } template auto Container::insert(value_type &&value) -> std::pair { - return container().insert(value); + return syncInsertResult(container_front().insert(value)); } template auto Container::insert( const_iterator hint, value_type const &value) -> iterator { - return container().insert(hint, value); + return syncInsertResult(container_front().insert(hint, value)); } template auto Container::insert( const_iterator hint, value_type &&value) -> iterator { - return container().insert(hint, value); + return syncInsertResult(container_front().insert(hint, value)); } template auto Container::insert( std::initializer_list ilist) -> void { - container().insert(ilist); + std::vector + internal_insert_list; + internal_insert_list.reserve(ilist.size()); + for (auto &v : ilist) + { + internal_insert_list.emplace_back( + key_as_string(v.first), *v.second.m_attri); + } + container_front().insert(std::move(ilist)); + container_back().insert( + internal_insert_list.begin(), internal_insert_list.end()); } template auto Container::swap(Container &other) -> void { - container().swap(other.container()); + container_front().swap(other.container_front()); + container_back().swap(other.container_back()); } template auto Container::find(key_type const &key) -> iterator { - return container().find(key); + return container_front().find(key); } template auto Container::find(key_type const &key) const -> const_iterator { - return container().find(key); + return container_front().find(key); } /** This returns either 1 if the key is found in the container of 0 if not. @@ -257,7 +273,7 @@ template auto Container::count(key_type const &key) const -> size_type { - return container().count(key); + return container_front().count(key); } /** Checks if there is an element with a key equivalent to an exiting key in @@ -270,7 +286,7 @@ template auto Container::contains(key_type const &key) const -> bool { - return container().find(key) != container().end(); + return container_front().find(key) != container_front().end(); } template @@ -280,15 +296,17 @@ auto Container::erase(key_type const &key) -> size_type throw std::runtime_error( "Can not erase from a container in a read-only Series."); - auto res = container().find(key); - if (res != container().end() && res->second.written()) + auto res = container_front().find(key); + if (res != container_front().end() && res->second.written()) { Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); IOHandler()->flush(internal::defaultFlushParams); } - return container().erase(key); + return container().for_both_to_string([&key](auto &map, auto &&to_string) { + return map.erase(to_string(key)); + }); } template @@ -298,14 +316,15 @@ auto Container::erase(iterator res) -> iterator throw std::runtime_error( "Can not erase from a container in a read-only Series."); - if (res != container().end() && res->second.written()) + if (res != container_front().end() && res->second.written()) { Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); IOHandler()->flush(internal::defaultFlushParams); } - return container().erase(res); + container_back().erase(key_as_string(res->first)); + return container_front().erase(res); } template @@ -315,7 +334,7 @@ auto Container::clear_unchecked() -> void throw std::runtime_error( "Clearing a written container not (yet) implemented."); - container().clear(); + container().for_both([](auto &map) { map.clear(); }); } template @@ -421,12 +440,12 @@ namespace internal template EraseStaleEntries::~EraseStaleEntries() { - auto &map = m_originalContainer.container(); + auto map = m_originalContainer.container(); using iterator_t = typename Container_t::InternalContainer::const_iterator; std::vector deleteMe; - deleteMe.reserve(map.size() - m_accessedKeys.size()); - for (iterator_t it = map.begin(); it != map.end(); ++it) + deleteMe.reserve(map.front->size() - m_accessedKeys.size()); + for (iterator_t it = map.front->begin(); it != map.front->end(); ++it) { auto lookup = m_accessedKeys.find(it->first); if (lookup == m_accessedKeys.end()) @@ -436,7 +455,8 @@ namespace internal } for (auto &it : deleteMe) { - map.erase(it); + map.back->erase(it->first); + map.front->erase(it); } } } // namespace internal diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 7e708ca4e4..1b2eb28873 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -567,7 +567,7 @@ void Iteration::read_impl(std::string const &groupPath) std::cerr << "Cannot read meshes in iteration " << groupPath << " and will skip them due to read error:\n" << err.what() << std::endl; - meshes.container().clear(); + meshes.container().for_both([](auto &map) { map.clear(); }); } } meshes.setDirty(false); @@ -583,7 +583,7 @@ void Iteration::read_impl(std::string const &groupPath) std::cerr << "Cannot read particles in iteration " << groupPath << " and will skip them due to read error:\n" << err.what() << std::endl; - particles.container().clear(); + particles.container().for_both([](auto &map) { map.clear(); }); } } particles.setDirty(false); diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index e27208e596..1803a576ad 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -61,7 +61,8 @@ void ParticlePatches::read() std::cerr << "Cannot read patch record '" << record_name << "' due to read error and will skip it:" << err.what() << std::endl; - this->container().erase(record_name); + this->container().for_both( + [&record_name](auto &map) { map.erase(record_name); }); } } @@ -113,7 +114,8 @@ void ParticlePatches::read() << "Cannot read record component '" << component_name << "' in particle patch and will skip it due to read error:\n" << err.what() << std::endl; - Container::container().erase(component_name); + Container::container().for_both( + [&component_name](auto &map) { map.erase(component_name); }); } } setDirty(false); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 718d3e847a..f43bfa7d18 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -113,9 +113,11 @@ void ParticleSpecies::read() if (!hasParticlePatches) { - auto &container = particlePatches.container(); - container.erase("numParticles"); - container.erase("numParticlesOffset"); + auto container = particlePatches.container(); + container.for_both([](auto &map_) { + map_.erase("numParticles"); + map_.erase("numParticlesOffset"); + }); particlePatches.setDirty(false); } diff --git a/src/Record.cpp b/src/Record.cpp index f99f26f015..8f7a7736a1 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -161,7 +161,8 @@ auto Record::read() -> internal::HomogenizeExtents std::cerr << "Cannot read record component '" << component << "' and will skip it due to read error:\n" << err.what() << std::endl; - this->container().erase(component); + this->container().for_both( + [&component](auto &map) { map.erase(component); }); continue; } check_extent(rc); @@ -190,7 +191,8 @@ auto Record::read() -> internal::HomogenizeExtents std::cerr << "Cannot read record component '" << component << "' and will skip it due to read error:\n" << err.what() << std::endl; - this->container().erase(component); + this->container().for_both( + [&component](auto &map) { map.erase(component); }); continue; } check_extent(rc); diff --git a/src/Series.cpp b/src/Series.cpp index 99fe227582..76344dbe08 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1943,7 +1943,10 @@ void Series::readFileBased( for (auto index : unparseableIterations) { - series.iterations.container().erase(index); + series.iterations.container().for_both_to_string( + [index](auto &map, auto &&to_string) { + map.erase(to_string(index)); + }); } if (padding > 0) @@ -2279,7 +2282,10 @@ creating new iterations. std::cerr << "Cannot read iteration '" << index << "' and will skip it due to read error:\n" << err.what() << std::endl; - series.iterations.container().erase(index); + series.iterations.container().for_both_to_string( + [index](auto &map, auto &&to_string) { + map.erase(to_string(index)); + }); return {err}; } i.get().m_closed = internal::CloseStatus::Open; @@ -3368,7 +3374,7 @@ namespace internal } // Not strictly necessary, but clear the map of iterations // This releases the openPMD hierarchy - iterations.container().clear(); + iterations.container().for_both([](auto &map) { map.clear(); }); // Release the IO Handler if (IOHandler) { diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 16f69c6cbb..08829be970 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -53,12 +53,10 @@ namespace internal : SharedData_t({raw_ptr, [](auto const *) {}}) {} - void AttributableData::cloneFrom(AttributableData const &other) + void AttributableData::cloneFrom(parent_t const &other) { - using parent_t = std::shared_ptr; static_cast(*this) = static_cast(other); } - } // namespace internal Attributable::Attributable() diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index 89777af8d6..5d42f06ea6 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -282,7 +282,7 @@ auto BaseRecord::rbegin() -> reverse_iterator } else { - return makeReverseIterator(this->container().rbegin()); + return makeReverseIterator(this->container_front().rbegin()); } } @@ -295,7 +295,7 @@ auto BaseRecord::rbegin() const -> const_reverse_iterator } else { - return makeReverseIterator(this->container().rbegin()); + return makeReverseIterator(this->container_front().rbegin()); } } @@ -308,7 +308,7 @@ auto BaseRecord::crbegin() const -> const_reverse_iterator } else { - return makeReverseIterator(this->container().crbegin()); + return makeReverseIterator(this->container_front().crbegin()); } } @@ -321,7 +321,7 @@ auto BaseRecord::rend() -> reverse_iterator } else { - return makeReverseIterator(this->container().rend()); + return makeReverseIterator(this->container_front().rend()); } } @@ -334,7 +334,7 @@ auto BaseRecord::rend() const -> const_reverse_iterator } else { - return makeReverseIterator(this->container().rend()); + return makeReverseIterator(this->container_front().rend()); } } @@ -347,7 +347,7 @@ auto BaseRecord::crend() const -> const_reverse_iterator } else { - return makeReverseIterator(this->container().crend()); + return makeReverseIterator(this->container_front().crend()); } } @@ -527,7 +527,7 @@ auto BaseRecord::find(key_type const &key) -> iterator } else { - return makeIterator(r.m_container.find(key)); + return makeIterator(this->container_front().find(key)); } } @@ -552,7 +552,7 @@ auto BaseRecord::find(key_type const &key) const -> const_iterator } else { - return makeIterator(r.m_container.find(key)); + return makeIterator(this->container_front().find(key)); } } @@ -626,10 +626,12 @@ auto BaseRecord::insert(value_type const &value) -> std::pair { detail::verifyNonscalar(this); - auto res = this->container().insert(value); + auto res = this->syncInsertResult(this->container_front().insert(value)); if (res.first->first == RecordComponent::SCALAR) { - this->container().erase(res.first); + // this->container().erase(res.first); + this->container_front().erase(res.first); + this->container_back().erase(res.first->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; @@ -639,10 +641,12 @@ template auto BaseRecord::insert(value_type &&value) -> std::pair { detail::verifyNonscalar(this); - auto res = this->container().insert(std::move(value)); + auto res = this->syncInsertResult( + this->container_front().insert(std::move(value))); if (res.first->first == RecordComponent::SCALAR) { - this->container().erase(res.first); + this->container_front().erase(res.first); + this->container_back().erase(res.first->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; @@ -659,13 +663,15 @@ auto BaseRecord::insert(const_iterator hint, value_type const &value) [this](typename const_iterator::Right) { return static_cast const *>(this) ->container() - .begin(); + .front->begin(); }}, hint.m_iterator); - auto res = this->container().insert(base_hint, value); + auto res = this->syncInsertResult( + this->container_front().insert(base_hint, value)); if (res->first == RecordComponent::SCALAR) { - this->container().erase(res); + this->container_front().erase(res); + this->container_back().erase(res->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return makeIterator(res); @@ -682,13 +688,15 @@ auto BaseRecord::insert(const_iterator hint, value_type &&value) [this](typename const_iterator::Right) { return static_cast const *>(this) ->container() - .begin(); + .front->begin(); }}, hint.m_iterator); - auto res = this->container().insert(base_hint, std::move(value)); + auto res = this->syncInsertResult( + this->container_front().insert(base_hint, std::move(value))); if (res->first == RecordComponent::SCALAR) { - this->container().erase(res); + this->container_front().erase(res); + this->container_back().erase(res->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return makeIterator(res); @@ -718,7 +726,16 @@ template auto BaseRecord::insert(std::initializer_list ilist) -> void { detail::verifyNonscalar(this); - this->container().insert(std::move(ilist)); + std::vector + internal_insert_list; + internal_insert_list.reserve(ilist.size()); + for (auto &v : ilist) + { + internal_insert_list.emplace_back(v.first, *v.second.m_attri); + } + this->container_front().insert(std::move(ilist)); + this->container_back().insert( + internal_insert_list.begin(), internal_insert_list.end()); /* * We skip this check as it changes the runtime of this call from * O(last-first) to O(container().size()). @@ -738,7 +755,8 @@ auto BaseRecord::swap(BaseRecord &other) noexcept -> void { detail::verifyNonscalar(this); detail::verifyNonscalar(&other); - this->container().swap(other.container()); + this->container_front().swap(other.container_front()); + this->container_back().swap(other.container_back()); } template diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 7d68b16035..8fe89380e4 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -100,7 +100,8 @@ void PatchRecord::read() << component_name << "' and will skip it due to read error:" << err.what() << std::endl; - this->container().erase(component_name); + this->container().for_both( + [&component_name](auto &map) { map.erase(component_name); }); } } diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 2a7f994873..57bdf82d32 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -657,8 +657,10 @@ std::optional StatefulIterator::loopBody(Seek const &seek) else if ( series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) { - data.series.iterations.container().erase( - *maybe_current_iteration); + data.series.iterations.container().for_both_to_string( + [&maybe_current_iteration](auto &map, auto &&to_string) { + map.erase(to_string(*maybe_current_iteration)); + }); } } } @@ -853,7 +855,8 @@ void StatefulIterator::deactivateDeadIteration(iteration_index_t index) } break; } - data.series.iterations.container().erase(index); + data.series.iterations.container().for_both_to_string( + [index](auto &map, auto &&to_string) { map.erase(to_string(index)); }); } StatefulIterator &StatefulIterator::operator++() From 331941e9278925ec3a2ad9b1f247f7a33869f00e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 5 May 2026 19:03:21 +0200 Subject: [PATCH 32/58] Steal CustomHierarchy.hpp from old branch Most of the .cpp file currently commented out --- CMakeLists.txt | 1 + include/openPMD/CustomHierarchy.hpp | 127 ++++++ src/CustomHierarchy.cpp | 674 ++++++++++++++++++++++++++++ 3 files changed, 802 insertions(+) create mode 100644 include/openPMD/CustomHierarchy.hpp create mode 100644 src/CustomHierarchy.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 21ac656880..7a24803011 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -402,6 +402,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) set(CORE_SOURCE src/config.cpp src/ChunkInfo.cpp + src/CustomHierarchy.cpp src/Dataset.cpp src/Datatype.cpp src/Error.cpp diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp new file mode 100644 index 0000000000..d210af4eaa --- /dev/null +++ b/include/openPMD/CustomHierarchy.hpp @@ -0,0 +1,127 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Container.hpp" + +#include +#include +#include +#include + +namespace openPMD +{ +class CustomHierarchy; +namespace internal +{ + using CustomHierarchyData = ContainerData; +} // namespace internal + +/* + * This is its own class, so the return value of asContainerOf() is also + * convsersible again. + */ +template +class ConvertibleContainer : public Container +{ + template + friend class ConversibleContainer; + +protected: + using Container_t = Container; + using Data_t = internal::ContainerData; + static_assert( + std::is_base_of_v); + + using Container_t::Container_t; + +public: + template + auto asContainerOf() -> ConvertibleContainer + { + if constexpr ( + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) + { + // TODO: If Mesh or ParticleSpecies, create Container on the fly by + // evaluating the meshes/particles path. If RecordComponent, maybe + // use dynamic casting to check if children are datasets? + throw std::runtime_error("UNIMPLEMENTED"); + } + else + { + static_assert( + auxiliary::dependent_false_v, + "[CustomHierarchy::asContainerOf] Type parameter must be " + "one of: CustomHierarchy, RecordComponent, Mesh, " + "ParticleSpecies."); + } + } +}; + +// TODO: Use Container internally, but otherwise override members +// such that we have: +// +// operator[](key) -> CustomHierarchy +// +// Or find a better solution for having this automatically.. +class CustomHierarchy : public ConversibleContainer +{ + friend class Iteration; + friend class Container; + +private: + using Parent_t = ConversibleContainer; + using Container_t = typename Parent_t::Container_t; + using Data_t = typename Parent_t::Data_t; + +protected: + CustomHierarchy(); + CustomHierarchy(NoInit); + + void read(); + void read(std::vector ¤tPath); + + void flush_internal( + internal::FlushParams const &, std::vector currentPath); + void flush(std::string const &path, internal::FlushParams const &) override; + + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w) override; + +public: + CustomHierarchy(CustomHierarchy const &other) = default; + CustomHierarchy(CustomHierarchy &&other) = default; + + CustomHierarchy &operator=(CustomHierarchy const &) = default; + CustomHierarchy &operator=(CustomHierarchy &&) = default; +}; +} // namespace openPMD diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp new file mode 100644 index 0000000000..1e76572cfc --- /dev/null +++ b/src/CustomHierarchy.cpp @@ -0,0 +1,674 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/CustomHierarchy.hpp" + +#if 0 +#include "openPMD/Dataset.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/IO/IOTask.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/Writable.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// @todo add handselected choice of [:punct:] characters to this +// using a macro here to make string interpolation simpler +#define OPENPMD_LEGAL_IDENTIFIER_CHARS "[:alnum:]_" +#define OPENPMD_SINGLE_GLOBBING_CHAR "%" +#define OPENPMD_DOUBLE_GLOBBING_CHAR "%%" + +namespace +{ +template +std::string +concatWithSep(Iterator &&begin, Iterator const &end, std::string const &sep) +{ + if (begin == end) + { + return ""; + } + std::stringstream res; + res << *(begin++); + for (; begin != end; ++begin) + { + res << sep << *begin; + } + return res.str(); +} + +std::string +concatWithSep(std::vector const &v, std::string const &sep) +{ + return concatWithSep(v.begin(), v.end(), sep); +} + +// Not specifying std::regex_constants::optimize here, only using it where +// it makes sense to. +constexpr std::regex_constants::syntax_option_type regex_flags = + std::regex_constants::egrep; + +template +void setDefaultMeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles, + OutParam &writeTarget) +{ + std::regex is_default_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + constexpr char const *default_default_mesh = "meshes"; + constexpr char const *default_default_particle = "particles"; + for (auto [vec, defaultPath, default_default] : + {std::make_tuple( + &meshes, &writeTarget.m_defaultMeshesPath, default_default_mesh), + std::make_tuple( + &particles, + &writeTarget.m_defaultParticlesPath, + default_default_particle)}) + { + bool set_default = true; + /* + * The first eligible path in meshesPath/particlesPath is used as + * the default, "meshes"/"particles" otherwise. + */ + for (auto const &path : *vec) + { + if (std::regex_match(path, is_default_path_specification)) + { + *defaultPath = openPMD::auxiliary::replace_last(path, "/", ""); + set_default = false; + break; + } + } + if (set_default) + { + *defaultPath = default_default; + } + } +} + +bool anyPathRegexMatches( + std::regex const ®ex, std::vector const &path) +{ + std::string pathToMatch = '/' + concatWithSep(path, "/") + '/'; + return std::regex_match(pathToMatch, regex); +} +} // namespace + +namespace openPMD +{ +namespace internal +{ + namespace + { + std::string globToRegexLongForm(std::string const &glob) + { + return auxiliary::replace_all( + auxiliary::replace_all( + glob, + OPENPMD_DOUBLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*)"), + OPENPMD_SINGLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "]*)"); + } + + std::string globToRegexShortForm(std::string const &glob) + { + return "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*/" + glob; + } + } // namespace + + MeshesParticlesPath::MeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles) + { + /* + * /group/meshes/E is a mesh if the meshes path contains: + * + * 1) '/group/meshes/' (absolute path to mesh container) + * 2) 'meshes/' (relative path to mesh container) + * + * All this analogously for particles path. + */ + + // regex for detecting option 1) + // e.g. '/path/to/meshes/': The path to the meshes. Mandatory slashes at + // beginning and end, possibly slashes in + // between. Mandatory slash at beginning might + // be replaced with '%%' to enable paths like + // '%%/path/to/meshes'. + // resolves to: `(/|%%)[[:alnum:]_%/]+/` + std::regex is_legal_long_path_specification( + "(/|" OPENPMD_DOUBLE_GLOBBING_CHAR + ")[" OPENPMD_LEGAL_IDENTIFIER_CHARS OPENPMD_SINGLE_GLOBBING_CHAR + "/]+/", + regex_flags | std::regex_constants::optimize); + + // Regex for detecting option 2) + // e.g. 'meshes/': The name without path. One single mandatory slash + // at the end, no slashes otherwise. + // resolves to `[[:alnum:]_]+/` + std::regex is_legal_short_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + + for (auto [target_regex, vec] : + {std::make_tuple(&this->meshRegex, &meshes), + std::make_tuple(&this->particleRegex, &particles)}) + { + std::stringstream build_regex; + // neutral element: empty language, regex doesn't match anything + build_regex << "(a^)"; + for (auto const &entry : *vec) + { + if (std::regex_match(entry, is_legal_short_path_specification)) + { + build_regex << "|(" << globToRegexShortForm(entry) << ')'; + } + else if (std::regex_match( + entry, is_legal_long_path_specification)) + { + build_regex << "|(" << globToRegexLongForm(entry) << ')'; + } + else + { + std::cerr + << "[WARNING] Not a legal meshes-/particles-path: '" + << entry << "'. Will skip." << std::endl; + } + } + auto regex_string = build_regex.str(); + // std::cout << "Using regex string: " << regex_string << std::endl; + *target_regex = std::regex( + regex_string, regex_flags | std::regex_constants::optimize); + } + setDefaultMeshesParticlesPath(meshes, particles, *this); + } + + ContainedType MeshesParticlesPath::determineType( + std::vector const &path) const + { + if (isMeshContainer(path)) + { + return ContainedType::Mesh; + } + else if (isParticleContainer(path)) + { + return ContainedType::Particle; + } + else + { + return ContainedType::Group; + } + } + + bool MeshesParticlesPath::isParticleContainer( + std::vector const &path) const + { + return anyPathRegexMatches(particleRegex, path); + } + bool MeshesParticlesPath::isMeshContainer( + std::vector const &path) const + { + return anyPathRegexMatches(meshRegex, path); + } + + CustomHierarchyData::CustomHierarchyData() + { + syncAttributables(); + } + + void CustomHierarchyData::syncAttributables() + { + /* + * m_embeddeddatasets and its friends should point to the same instance + * of Attributable. + * Not strictly necessary to do this explicitly due to virtual + * inheritance (all Attributable instances are the same anyway), + * but let's be explicit about this. + */ + for (auto p : std::initializer_list{ + static_cast *>(this), + static_cast *>(this), + static_cast *>(this), + static_cast *>(this)}) + { + p->asSharedPtrOfAttributable() = this->asSharedPtrOfAttributable(); + } + } +} // namespace internal + +// template +// class ConversibleContainer; + +CustomHierarchy::CustomHierarchy() : ConversibleContainer(NoInit{}) +{ + setData(std::make_shared()); +} +CustomHierarchy::CustomHierarchy(NoInit) : ConversibleContainer(NoInit{}) +{} + +void CustomHierarchy::readNonscalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter aList; + + Mesh &m = map[mesh_name]; + + pOpen.path = mesh_name; + aList.attributes->clear(); + IOHandler()->enqueue(IOTask(&m, pOpen)); + IOHandler()->enqueue(IOTask(&m, aList)); + IOHandler()->flush(internal::defaultFlushParams); + + // Find constant scalar meshes. shape generally required for meshes, + // shape also required for scalars. + // https://github.com/openPMD/openPMD-standard/pull/289 + auto att_begin = aList.attributes->begin(); + auto att_end = aList.attributes->end(); + auto value = std::find(att_begin, att_end, "value"); + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) + { + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.get().m_isConstant = true; + } + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readScalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter pList; + + Parameter dOpen; + Mesh &m = map[mesh_name]; + dOpen.name = mesh_name; + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); + mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readParticleSpecies( + EraseStaleParticles &map, std::string const &species_name) +{ + Parameter pOpen; + Parameter pList; + + ParticleSpecies &p = map[species_name]; + pOpen.path = species_name; + IOHandler()->enqueue(IOTask(&p, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + try + { + p.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species with name '" << species_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(species_name); + } +} + +void CustomHierarchy::read(internal::MeshesParticlesPath const &mpp) +{ + std::vector currentPath; + read(mpp, currentPath); +} + +void CustomHierarchy::read( + internal::MeshesParticlesPath const &mpp, + std::vector ¤tPath) +{ + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); + + Attributable::readAttributes(ReadMode::FullyReread); + Parameter dList; + IOHandler()->enqueue(IOTask(this, dList)); + IOHandler()->flush(internal::defaultFlushParams); + + std::deque constantComponentsPushback; + auto &data = get(); + auto embeddedMeshes = data.embeddedMeshesWrapped(); + auto embeddedParticles = data.embeddedParticlesWrapped(); + EraseStaleMeshes meshesMap(embeddedMeshes); + EraseStaleParticles particlesMap(embeddedParticles); + for (auto const &path : *pList.paths) + { + switch (mpp.determineType(currentPath)) + { + case internal::ContainedType::Group: { + Parameter pOpen; + pOpen.path = path; + auto &subpath = this->operator[](path); + IOHandler()->enqueue(IOTask(&subpath, pOpen)); + currentPath.emplace_back(path); + try + { + subpath.read(mpp, currentPath); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read subgroup '" << path << "' at path '" + << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + container().erase(path); + } + currentPath.pop_back(); + if (subpath.size() == 0 && subpath.containsAttribute("shape") && + subpath.containsAttribute("value")) + { + // This is not a group, but a constant record component + // Writable::~Writable() will deal with removing this from the + // backend again. + constantComponentsPushback.push_back(path); + container().erase(path); + } + break; + } + case internal::ContainedType::Mesh: { + try + { + readNonscalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + case internal::ContainedType::Particle: { + try + { + readParticleSpecies(particlesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + particlesMap.forget(path); + } + break; + } + } + } + for (auto const &path : *dList.datasets) + { + switch (mpp.determineType(currentPath)) + { + + case internal::ContainedType::Particle: + std::cerr << "[Warning] Dataset found at '" + << (concatWithSep(currentPath, "/") + "/" + path) + << "' inside the particles path. A particle species is " + "always a group, never a dataset. Will parse as a " + "custom dataset. Storing custom datasets inside the " + "particles path is discouraged." + << std::endl; + [[fallthrough]]; + // Group is a bit of an internal misnomer here, it just means that + // it matches neither meshes nor particles path + case internal::ContainedType::Group: { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + Parameter dOpen; + dOpen.name = path; + IOHandler()->enqueue(IOTask(&rc, dOpen)); + try + { + IOHandler()->flush(internal::defaultFlushParams); + rc.setWritten(false, Attributable::EnqueueAsynchronously::No); + rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + rc.setWritten(true, Attributable::EnqueueAsynchronously::No); + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read contained custom dataset '" << path + << "' at path '" << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + break; + } + case internal::ContainedType::Mesh: + try + { + readScalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read scalar mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + } + + for (auto const &path : constantComponentsPushback) + { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + try + { + Parameter pOpen; + pOpen.path = path; + IOHandler()->enqueue(IOTask(&rc, pOpen)); + rc.get().m_isConstant = true; + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read dataset at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + } + setDirty(false); +} + +void CustomHierarchy::flush_internal( + internal::FlushParams const &flushParams, + internal::MeshesParticlesPath &mpp, + std::vector currentPath) +{ + if (!dirtyRecursive()) + { + return; + } + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + // No need to do anything in access::readOnly since meshes and particles + // are initialized as aliases for subgroups at parsing time + auto &data = get(); + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } + + Parameter pCreate; + for (auto &[name, subpath] : *this) + { + if (!subpath.written()) + { + pCreate.path = name; + IOHandler()->enqueue(IOTask(&subpath, pCreate)); + } + currentPath.emplace_back(name); + subpath.flush_internal(flushParams, mpp, currentPath); + currentPath.pop_back(); + } + for (auto &[name, mesh] : data.embeddedMeshesInternal()) + { + if (!mpp.isMeshContainer(currentPath)) + { + std::string extend_meshes_path; + // Check if this can be covered by shorthand notation + // (e.g. meshesPath == "meshes/") + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultMeshesPath) + { + extend_meshes_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_meshes_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + } + mpp.collectNewMeshesPaths.emplace(std::move(extend_meshes_path)); + } + mesh.flush(name, flushParams); + } + for (auto &[name, particleSpecies] : data.embeddedParticlesInternal()) + { + if (!mpp.isParticleContainer(currentPath)) + { + std::string extend_particles_path; + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultParticlesPath) + { + // Check if this can be covered by shorthand notation + // (e.g. particlesPath == "particles/") + extend_particles_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_particles_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + ; + } + mpp.collectNewParticlesPaths.emplace( + std::move(extend_particles_path)); + } + particleSpecies.flush(name, flushParams); + } + for (auto &[name, dataset] : get().embeddedDatasetsInternal()) + { + dataset.flush(name, flushParams, /* set_defaults = */ false); + } + + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + setDirty(false); + } +} + +void CustomHierarchy::flush( + std::string const & /* path */, internal::FlushParams const &) +{ + throw std::runtime_error( + "[CustomHierarchy::flush()] Don't use this method. Flushing should be " + "triggered via Iteration class."); +} + +void CustomHierarchy::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); +} +} // namespace openPMD + +#undef OPENPMD_LEGAL_IDENTIFIER_CHARS +#undef OPENPMD_SINGLE_GLOBBING_CHAR +#undef OPENPMD_DOUBLE_GLOBBING_CHAR +#endif From 825d3f61edb1dbbc0fbd438bacf811e7d9a55463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 6 May 2026 11:56:59 +0200 Subject: [PATCH 33/58] Add DeferredInitPolicy --- include/openPMD/backend/Container.hpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index f8fe871756..8bb3fa83ad 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -57,6 +57,15 @@ namespace traits void operator()(T &) {} }; + + template + struct DeferredInitPolicy + { + static void call(Container_t &) + {} + static void call(Container_t const &) + {} + }; } // namespace traits namespace internal @@ -117,6 +126,9 @@ class Container : virtual public Attributable friend class internal::EraseStaleEntries; friend class StatefulIterator; + using Self_t = Container; + friend class traits::DeferredInitPolicy; + protected: using ContainerData = internal::ContainerData; using InternalContainer = T_container; @@ -215,12 +227,14 @@ class Container : virtual public Attributable inline auto container_front() const -> typename SynchronizedContainers::front_t & { + traits::DeferredInitPolicy::call(*this); return m_containerData->m_container; } inline auto container_front() -> typename SynchronizedContainers::front_t & { + traits::DeferredInitPolicy::call(*this); return m_containerData->m_container; } From f98b238b13e61bfc5fd2e3115efd73a227fa2229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 6 May 2026 14:09:16 +0200 Subject: [PATCH 34/58] Derive CustomHierarchy from Container --- include/openPMD/CustomHierarchy.hpp | 9 +++++++-- include/openPMD/backend/Container.hpp | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index d210af4eaa..d67f59f52f 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -39,6 +39,8 @@ namespace internal using CustomHierarchyData = ContainerData; } // namespace internal +class CustomHierarchy; + /* * This is its own class, so the return value of asContainerOf() is also * convsersible again. @@ -57,6 +59,9 @@ class ConvertibleContainer : public Container using Container_t::Container_t; +private: + explicit ConvertibleContainer() = default; + public: template auto asContainerOf() -> ConvertibleContainer @@ -89,13 +94,13 @@ class ConvertibleContainer : public Container // operator[](key) -> CustomHierarchy // // Or find a better solution for having this automatically.. -class CustomHierarchy : public ConversibleContainer +class CustomHierarchy : public ConvertibleContainer { friend class Iteration; friend class Container; private: - using Parent_t = ConversibleContainer; + using Parent_t = ConvertibleContainer; using Container_t = typename Parent_t::Container_t; using Data_t = typename Parent_t::Data_t; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 8bb3fa83ad..7f5d74c92d 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -68,12 +68,26 @@ namespace traits }; } // namespace traits +class CustomHierarchy; + namespace internal { class SeriesData; template class EraseStaleEntries; + template + constexpr inline bool isDerivedFromAttributable = + std::is_base_of_v; + + /* + * Opt out from this check due to the recursive definition of + * class CustomHierarchy : public Container{ ... }; + * Cannot check this while CustomHierarchy is still an incomplete type. + */ + template <> + constexpr inline bool isDerivedFromAttributable = true; + template < typename T, typename T_key = std::string, @@ -114,7 +128,7 @@ template < class Container : virtual public Attributable { static_assert( - std::is_base_of::value, + internal::isDerivedFromAttributable, "Type of container element must be derived from Writable"); friend class Iteration; @@ -127,7 +141,7 @@ class Container : virtual public Attributable friend class StatefulIterator; using Self_t = Container; - friend class traits::DeferredInitPolicy; + friend struct traits::DeferredInitPolicy; protected: using ContainerData = internal::ContainerData; From 025b4312e95f9eb852709447132e32fb1ac7667a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 6 May 2026 15:11:19 +0200 Subject: [PATCH 35/58] Untested: reopening as custom hierarchy might work now now custom flushing and reading yet --- include/openPMD/CustomHierarchy.hpp | 16 +++++ include/openPMD/backend/Attributable.hpp | 20 ++++++ include/openPMD/backend/Container.hpp | 2 + src/CustomHierarchy.cpp | 80 ++++++++++++++++++++++++ src/backend/Attributable.cpp | 6 ++ src/backend/Container.cpp | 2 + 6 files changed, 126 insertions(+) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index d67f59f52f..70688a6b96 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -24,6 +24,7 @@ #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" #include @@ -50,6 +51,7 @@ class ConvertibleContainer : public Container { template friend class ConversibleContainer; + friend class CustomHierarchy; protected: using Container_t = Container; @@ -88,6 +90,16 @@ class ConvertibleContainer : public Container } }; +namespace traits +{ + template <> + struct DeferredInitPolicy> + { + template + static void call(Container_const_or_not &); + }; +} // namespace traits + // TODO: Use Container internally, but otherwise override members // such that we have: // @@ -98,6 +110,8 @@ class CustomHierarchy : public ConvertibleContainer { friend class Iteration; friend class Container; + friend class Attributable; + friend struct traits::DeferredInitPolicy>; private: using Parent_t = ConvertibleContainer; @@ -107,6 +121,8 @@ class CustomHierarchy : public ConvertibleContainer protected: CustomHierarchy(); CustomHierarchy(NoInit); + CustomHierarchy(std::shared_ptr other); + CustomHierarchy(Attributable const &other); void read(); void read(std::vector ¤tPath); diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 7c81065ed2..b1c098a692 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -52,6 +52,7 @@ class AbstractFilePosition; class Attributable; class Iteration; class Series; +class CustomHierarchy; namespace internal { @@ -123,6 +124,7 @@ namespace internal using SharedData_t = std::shared_ptr; using A_MAP = SharedData_t::element_type::A_MAP; using parent_t = std::shared_ptr; + friend class openPMD::CustomHierarchy; public: AttributableData(); @@ -131,6 +133,18 @@ namespace internal AttributableData(AttributableData &&) = delete; virtual ~AttributableData() = default; + inline auto asSharedPtrOfAttributable() + -> std::shared_ptr & + { + return *this; + } + + [[nodiscard]] inline auto asSharedPtrOfAttributable() const + -> std::shared_ptr const & + { + return *this; + } + AttributableData &operator=(AttributableData const &) = delete; AttributableData &operator=(AttributableData &&) = delete; @@ -260,6 +274,7 @@ class Attributable friend struct internal::HomogenizeExtents; friend struct internal::ConfigAttribute; friend class internal::ScientificDefaults; + friend class CustomHierarchy; protected: // tag for internal constructor @@ -468,6 +483,11 @@ class Attributable [[nodiscard]] OpenpmdStandard openPMDStandard() const; + // TODO: Add parse? parameter, dont parse, parse this object, parse + // recursively + // Alternatively, parse upon deferred initialization? + auto customHierarchies() -> CustomHierarchy; + // clang-format off OPENPMD_protected // clang-format on diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 7f5d74c92d..d1dc332ea1 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -220,6 +220,7 @@ class Container : virtual public Attributable " != " + std::to_string(size_back) + "."); } #endif + traits::DeferredInitPolicy::call(*this); return {&container_front(), &container_back()}; } @@ -235,6 +236,7 @@ class Container : virtual public Attributable " != " + std::to_string(size_back) + "."); } #endif + traits::DeferredInitPolicy::call(*this); return {&container_front(), &container_back()}; } diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 1e76572cfc..087203f49d 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -21,6 +21,86 @@ #include "openPMD/CustomHierarchy.hpp" +namespace openPMD +{ +namespace traits +{ + template + void DeferredInitPolicy>::call( + Container_const_or_not &container) + { + auto &container_front = container.m_containerData->m_container; + auto &container_back = container.Attributable::get().m_children; + + auto size_front = container_front.size(); + auto size_back = container_back.size(); + + if (size_front == size_back) + { + return; + } + else if (size_front > size_back) + { + throw error::Internal("CustomHierarchy went into illegal state?"); + } + // Need to sync backend objects into the CustomHierarchy instance + // Need to be a bit sneaky, we must modify my_container&, but this + // method might be called as const. shared_ptr<>s implement interior + // mutability, so use that here. + + // auto &my_container = container.container_front(); + auto it = container_front.begin(); + auto end = container_front.end(); + for (auto const &[key, attributable] : container_back) + { + if (it == end || it->first != key) + { + // under the invariant that the front container contains no + // elements that are not present in the back container, it now + // points to an entry past the to-be-inserted key + it = container_front.emplace_hint( + it, key, CustomHierarchy(attributable)); + } + } + } + template void DeferredInitPolicy>::call( + Container &); + template void DeferredInitPolicy>::call( + Container const &); +} // namespace traits +CustomHierarchy::CustomHierarchy() : ConvertibleContainer(NoInit{}) +{ + setData(std::make_shared()); +} + +CustomHierarchy::CustomHierarchy(NoInit) : ConvertibleContainer(NoInit{}) +{} + +CustomHierarchy::CustomHierarchy( + std::shared_ptr other) + : ConvertibleContainer(NoInit{}) +{ + auto data = std::make_shared(); + data->asSharedPtrOfAttributable() = std::move(other); + setData(std::move(data)); +} + +CustomHierarchy::CustomHierarchy(Attributable const &other) + : CustomHierarchy(other.m_attri->asSharedPtrOfAttributable()) +{} + +void CustomHierarchy::flush( + std::string const & /* path */, internal::FlushParams const &) +{ + throw std::runtime_error("Unimplemented!"); +} + +void CustomHierarchy::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); +} +} // namespace openPMD + #if 0 #include "openPMD/Dataset.hpp" #include "openPMD/Error.hpp" diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 08829be970..c6e8831a04 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/backend/Attributable.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/Iteration.hpp" @@ -341,6 +342,11 @@ OpenpmdStandard Attributable::openPMDStandard() const return IOHandler()->m_standard; } +auto Attributable::customHierarchies() -> CustomHierarchy +{ + return CustomHierarchy{*this}; +} + template void Attributable::seriesFlush_impl( internal::FlushParams const &flushParams, bool flush_io_handler) diff --git a/src/backend/Container.cpp b/src/backend/Container.cpp index fc0785d2c1..c39ffc3e91 100644 --- a/src/backend/Container.cpp +++ b/src/backend/Container.cpp @@ -21,6 +21,7 @@ #include "openPMD/backend/ContainerImpl.tpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" @@ -42,6 +43,7 @@ OPENPMD_INSTANTIATE(PatchRecord) OPENPMD_INSTANTIATE(PatchRecordComponent) OPENPMD_INSTANTIATE(Record) OPENPMD_INSTANTIATE(RecordComponent) +OPENPMD_INSTANTIATE(CustomHierarchy) OPENPMD_INSTANTIATE(Iteration OPENPMD_COMMA Iteration::IterationIndex_t) #undef OPENPMD_INSTANTIATE #undef OPENPMD_COMMA From d42af3507591a55041465e2be29d34279f10b546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 6 May 2026 17:28:12 +0200 Subject: [PATCH 36/58] little test --- examples/2_read_serial.cpp | 2 ++ include/openPMD/CustomHierarchy.hpp | 5 ++++ include/openPMD/backend/Writable.hpp | 1 + include/openPMD/openPMD.hpp | 1 + src/CustomHierarchy.cpp | 38 ++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+) diff --git a/examples/2_read_serial.cpp b/examples/2_read_serial.cpp index 4eaac05ea2..2cc6838042 100644 --- a/examples/2_read_serial.cpp +++ b/examples/2_read_serial.cpp @@ -99,6 +99,8 @@ int main() auto all_data = E_x.loadChunk(); + series.customHierarchies().printRecursively(); + // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. i.close(); diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index 70688a6b96..f313b46bd5 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -144,5 +144,10 @@ class CustomHierarchy : public ConvertibleContainer CustomHierarchy &operator=(CustomHierarchy const &) = default; CustomHierarchy &operator=(CustomHierarchy &&) = default; + + void printRecursively(); + +private: + void printRecursively(std::string indent); }; } // namespace openPMD diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 3623d0b39c..7c4989eb49 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -108,6 +108,7 @@ class Writable final friend struct Parameter; friend struct Parameter; friend class internal::ScientificDefaults; + friend class CustomHierarchy; private: Writable(internal::AttributableData *); diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index 91e5f79195..e5da821699 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -26,6 +26,7 @@ namespace openPMD {} // IWYU pragma: begin_exports +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/Error.hpp" diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 087203f49d..f383834c8c 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -99,6 +99,44 @@ void CustomHierarchy::linkHierarchy(Writable &w) { Attributable::linkHierarchy(w); } + +void CustomHierarchy::printRecursively() +{ + if (empty()) + { + std::cout << "\n"; + return; + } + else + { + std::cout << writable().ownKeyWithinParent << '\n'; + printRecursively(""); + std::cout.flush(); + } +} + +void CustomHierarchy::printRecursively(std::string indent) +{ + auto print_indent = [&indent]() { std::cout << indent; }; + auto it = begin(); + auto end_ = end(); + if (it == end_) + { + return; + } + auto prev = it; + ++it; + auto next_indent = indent + "│ "; + for (; it != end_; prev = it, ++it) + { + print_indent(); + std::cout << "├─" << prev->first << '\n'; + prev->second.printRecursively(next_indent); + } + print_indent(); + std::cout << "└─" << prev->first << '\n'; + prev->second.printRecursively(indent + " "); +} } // namespace openPMD #if 0 From b38e8d7995c231134c8fda8bd5ba20c5a04219ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 6 May 2026 18:08:01 +0200 Subject: [PATCH 37/58] Emplace custom classes into backend hierarchy --- src/Iteration.cpp | 3 +++ src/ParticleSpecies.cpp | 1 + src/Series.cpp | 3 +++ 3 files changed, 7 insertions(+) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 1b2eb28873..392db6c353 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -69,6 +69,9 @@ Iteration::Iteration() : Attributable(NoInit()) setData(std::make_shared()); meshes.writable().ownKeyWithinParent = "meshes"; particles.writable().ownKeyWithinParent = "particles"; + auto &container_back = Attributable::get().m_children; + container_back["meshes"] = *meshes.m_attri; + container_back["particles"] = *particles.m_attri; } uint64_t Iteration::getCachedIterationIndex() const diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index f43bfa7d18..2ec1529f36 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -39,6 +39,7 @@ void ParticleSpecies::visitHierarchy(HierarchyVisitor &v, bool recursive) ParticleSpecies::ParticleSpecies() { particlePatches.writable().ownKeyWithinParent = "particlePatches"; + container_back()["particlePatches"] = *particlePatches.m_attri; } void ParticleSpecies::read() diff --git a/src/Series.cpp b/src/Series.cpp index 76344dbe08..c807c07128 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1222,6 +1222,9 @@ void Series::initSeries( series.iterations.linkHierarchy(writable); series.iterations.writable().ownKeyWithinParent = "data"; + series->m_writable.ownKeyWithinParent = "ROOT"; + auto &container_back = series->m_children; + container_back["data"] = *series.iterations.m_attri; series.m_perIterationData.m_rankTableAttributable.linkHierarchy(writable); series.m_name = input->name; From d61cd1e72bde3818ed577f641133659953d4a953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 7 May 2026 15:17:09 +0200 Subject: [PATCH 38/58] wip: reading --- examples/2_read_serial.cpp | 5 ++- include/openPMD/CustomHierarchy.hpp | 4 +- include/openPMD/RecordComponent.hpp | 1 + src/CustomHierarchy.cpp | 68 +++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) diff --git a/examples/2_read_serial.cpp b/examples/2_read_serial.cpp index 2cc6838042..6da5c83372 100644 --- a/examples/2_read_serial.cpp +++ b/examples/2_read_serial.cpp @@ -99,7 +99,10 @@ int main() auto all_data = E_x.loadChunk(); - series.customHierarchies().printRecursively(); + auto ch = series.customHierarchies(); + ch.printRecursively(); + ch["data"]["200"]["meshes"].read(); + ch.printRecursively(); // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index f313b46bd5..67adba2a03 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -124,7 +124,6 @@ class CustomHierarchy : public ConvertibleContainer CustomHierarchy(std::shared_ptr other); CustomHierarchy(Attributable const &other); - void read(); void read(std::vector ¤tPath); void flush_internal( @@ -145,6 +144,9 @@ class CustomHierarchy : public ConvertibleContainer CustomHierarchy &operator=(CustomHierarchy const &) = default; CustomHierarchy &operator=(CustomHierarchy &&) = default; + // TODO maybe make this automatic somehow + void read(); + void printRecursively(); private: diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 3def700f71..425eab273b 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -134,6 +134,7 @@ class RecordComponent friend T &internal::makeOwning(T &self, Series); friend class internal::ScientificDefaults; friend class Attributable; + friend class CustomHierarchy; public: enum class Allocation diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index f383834c8c..e1a904ccea 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -89,6 +89,74 @@ CustomHierarchy::CustomHierarchy(Attributable const &other) : CustomHierarchy(other.m_attri->asSharedPtrOfAttributable()) {} +void CustomHierarchy::read() +{ + if (!writable().written) + { + throw error::WrongAPIUsage( + "Cannot read contents of CustomHierarchy path '" + + myPath().openPMDPath() + + "', since the backend does not yet / no longer know about this " + "object. Ensure that the object is open (e.g. by opening its " + "containing Iteration or by reading parent paths first)"); + } + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); + + Attributable::readAttributes(ReadMode::FullyReread); + Parameter dList; + IOHandler()->enqueue(IOTask(this, dList)); + IOHandler()->flush(internal::defaultFlushParams); + + std::deque constantComponentsPushback; + auto &container_back_ = container_back(); + for (auto const &path : *pList.paths) + { + if (container_back_.find(path) != container_back_.end()) + { + // Is already known + continue; + } + Parameter pOpen; + pOpen.path = path; + auto &subpath = this->operator[](path); + subpath.linkHierarchy(this->writable()); + IOHandler()->enqueue(IOTask(&subpath, pOpen)); + } + + for (auto const &path : *dList.datasets) + { + if (container_back_.find(path) != container_back_.end()) + { + // Is already known + continue; + } + Parameter dOpen; + dOpen.name = path; + + // TODO uhhm i think this wont work, but lets see when we get there + RecordComponent rc; + auto [it, emplaced] = this->emplace( + path, CustomHierarchy(static_cast(rc))); + if (!emplaced) + { + throw error::Internal( + "Control flow error / internal container state error."); + } + auto &subpath = it->second; + subpath.linkHierarchy(this->writable()); + IOHandler()->enqueue(IOTask(&subpath, dOpen)); + } + + setDirty(false); +} + void CustomHierarchy::flush( std::string const & /* path */, internal::FlushParams const &) { From 590e033731c56f8b159f71a29957e764ad042356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 7 May 2026 23:47:39 +0200 Subject: [PATCH 39/58] Seem like CustomHierarchy::read() is now somewhat working lmao --- examples/2_read_serial.cpp | 2 +- include/openPMD/Series.hpp | 3 ++ include/openPMD/auxiliary/Defer.hpp | 3 +- include/openPMD/backend/Container.hpp | 4 +- src/CustomHierarchy.cpp | 72 +++++++++++++++++++++++---- src/Iteration.cpp | 10 ++-- src/Series.cpp | 60 ++++++++++------------ 7 files changed, 106 insertions(+), 48 deletions(-) diff --git a/examples/2_read_serial.cpp b/examples/2_read_serial.cpp index 6da5c83372..effd752fda 100644 --- a/examples/2_read_serial.cpp +++ b/examples/2_read_serial.cpp @@ -101,7 +101,7 @@ int main() auto ch = series.customHierarchies(); ch.printRecursively(); - ch["data"]["200"]["meshes"].read(); + ch["data"]["200"]["fields"].read(); ch.printRecursively(); // The iteration can be closed in order to help free up resources. diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 503eec0ace..f713165ab9 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -1000,6 +1000,9 @@ OPENPMD_private Series &setIterationEncoding_internal( IterationEncoding iterationEncoding, internal::default_or_explicit); + Series &setParticlesPath_internal(std::string const &particlesPath); + Series &setMeshesPath_internal(std::string const &meshesPath); + /* * Returns the current content of the /data/snapshot attribute. * (We could also add this to the public API some time) diff --git a/include/openPMD/auxiliary/Defer.hpp b/include/openPMD/auxiliary/Defer.hpp index c6bc4e0533..505f5fcd39 100644 --- a/include/openPMD/auxiliary/Defer.hpp +++ b/include/openPMD/auxiliary/Defer.hpp @@ -21,7 +21,8 @@ struct defer_type std::move(functor)(); } - explicit defer_type() = default; + explicit defer_type() : do_run_this(false) + {} struct forwarding_tag {}; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index d1dc332ea1..e349e502ac 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -430,7 +430,9 @@ OPENPMD_protected } auto syncInsertResult(iterator res) -> iterator { - container_back().emplace( + // container_back() expects the invariant that we are currently about to + // fulfill so we cannot use it just yet + Attributable::get().m_children.emplace( key_as_string(res->first), *res->second.m_attri); return res; } diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index e1a904ccea..e33b258c05 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -20,6 +20,7 @@ */ #include "openPMD/CustomHierarchy.hpp" +#include "openPMD/auxiliary/Defer.hpp" namespace openPMD { @@ -41,7 +42,32 @@ namespace traits } else if (size_front > size_back) { - throw error::Internal("CustomHierarchy went into illegal state?"); + std::stringstream error; + auto print = [&error](auto const &map) -> std::stringstream & { + if (map.empty()) + { + error << "[]"; + } + else + { + error << '['; + auto it = map.begin(); + error << (it++)->first; + auto end = map.end(); + for (; it != end; ++it) + { + error << ", " << it->first; + } + error << ']'; + } + return error; + }; + error << "CustomHierarchy went into illegal state at '" + << container.myPath().openPMDPath() + << "':\nfront container: "; + print(container_front) << "\nback container: "; + print(container_back) << '\n'; + throw error::Internal(error.str()); } // Need to sync backend objects into the CustomHierarchy instance // Need to be a bit sneaky, we must modify my_container&, but this @@ -91,14 +117,39 @@ CustomHierarchy::CustomHierarchy(Attributable const &other) void CustomHierarchy::read() { + auxiliary::opaque_defer_type reset_parsing_status; + if (IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) + { + IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; + reset_parsing_status = auxiliary::defer([&]() { + IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; + }); + } if (!writable().written) { - throw error::WrongAPIUsage( - "Cannot read contents of CustomHierarchy path '" + - myPath().openPMDPath() + - "', since the backend does not yet / no longer know about this " - "object. Ensure that the object is open (e.g. by opening its " - "containing Iteration or by reading parent paths first)"); + auto do_throw = [&]() { + throw error::WrongAPIUsage( + "Cannot read contents of CustomHierarchy path '" + + myPath().openPMDPath() + + "', since the backend does not yet / no longer know about this " + "object. Ensure that the object is open (e.g. by opening its " + "containing Iteration or by reading parent paths first)"); + }; + auto parent = writable().parent; + if (!parent) + { + do_throw(); + } + Attributable parent_attributable(NoInit{}); + parent_attributable.setData( + std::shared_ptr{ + parent->attributable, [](auto const *) {}}); + CustomHierarchy parent_(parent_attributable); + parent_.read(); + if (!writable() /* still not */.written) + { + do_throw(); + } } /* * Convention for CustomHierarchy::flush and CustomHierarchy::read: @@ -118,7 +169,8 @@ void CustomHierarchy::read() auto &container_back_ = container_back(); for (auto const &path : *pList.paths) { - if (container_back_.find(path) != container_back_.end()) + if (auto it = container_back_.find(path); + it != container_back_.end() && it->second->m_writable.written) { // Is already known continue; @@ -132,7 +184,8 @@ void CustomHierarchy::read() for (auto const &path : *dList.datasets) { - if (container_back_.find(path) != container_back_.end()) + if (auto it = container_back_.find(path); + it != container_back_.end() && it->second->m_writable.written) { // Is already known continue; @@ -155,6 +208,7 @@ void CustomHierarchy::read() } setDirty(false); + IOHandler()->flush(internal::defaultFlushParams); } void CustomHierarchy::flush( diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 392db6c353..5d0b675008 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -69,9 +69,6 @@ Iteration::Iteration() : Attributable(NoInit()) setData(std::make_shared()); meshes.writable().ownKeyWithinParent = "meshes"; particles.writable().ownKeyWithinParent = "particles"; - auto &container_back = Attributable::get().m_children; - container_back["meshes"] = *meshes.m_attri; - container_back["particles"] = *particles.m_attri; } uint64_t Iteration::getCachedIterationIndex() const @@ -914,6 +911,13 @@ void Iteration::linkHierarchy(Writable &w) meshes.linkHierarchy(this->writable()); particles.linkHierarchy(this->writable()); get().m_perIterationData.m_rankTableAttributable.linkHierarchy(*w.parent); + + auto &container_back = Attributable::get().m_children; + auto s = retrieveSeries(); + container_back[auxiliary::replace_all_nonrecursively( + s.meshesPath(), "/", "")] = *meshes.m_attri; + container_back[auxiliary::replace_all_nonrecursively( + s.particlesPath(), "/", "")] = *particles.m_attri; } void Iteration::runDeferredParseAccess() diff --git a/src/Series.cpp b/src/Series.cpp index c807c07128..bc2fc2fdc3 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -285,12 +285,7 @@ Series &Series::setMeshesPath(std::string const &mp) "A files meshesPath can not (yet) be changed after it has been " "written."); - if (auxiliary::ends_with(mp, '/')) - setAttribute("meshesPath", mp); - else - setAttribute("meshesPath", mp + "/"); - setDirty(true); - return *this; + return setMeshesPath_internal(mp); } std::vector Series::availableDatasets() @@ -623,12 +618,7 @@ Series &Series::setParticlesPath(std::string const &pp) "A files particlesPath can not (yet) be changed after it has been " "written."); - if (auxiliary::ends_with(pp, '/')) - setAttribute("particlesPath", pp); - else - setAttribute("particlesPath", pp + "/"); - setDirty(true); - return *this; + return setParticlesPath_internal(pp); } std::string Series::author() const @@ -2450,7 +2440,6 @@ creating new iterations. void Series::readBase() { - auto &series = get(); Parameter aRead; aRead.name = "openPMD"; @@ -2536,16 +2525,9 @@ void Series::readBase() .getOptional(); val.has_value()) { - /* allow setting the meshes path after completed IO */ - for (auto &it : series.iterations) - it.second.meshes.setWritten( - false, Attributable::EnqueueAsynchronously::No); - - setMeshesPath(val.value()); - - for (auto &it : series.iterations) - it.second.meshes.setWritten( - true, Attributable::EnqueueAsynchronously::No); + /* use internal api to allow setting the meshes path after completed + * IO */ + setMeshesPath_internal(val.value()); } else throw error::ReadError( @@ -2578,16 +2560,9 @@ void Series::readBase() .getOptional(); val.has_value()) { - /* allow setting the meshes path after completed IO */ - for (auto &it : series.iterations) - it.second.particles.setWritten( - false, Attributable::EnqueueAsynchronously::No); - - setParticlesPath(val.value()); - - for (auto &it : series.iterations) - it.second.particles.setWritten( - true, Attributable::EnqueueAsynchronously::No); + /* use internal api to allow setting the meshes path after completed + * IO */ + setParticlesPath_internal(val.value()); } else throw error::ReadError( @@ -2951,6 +2926,25 @@ Series &Series::setIterationEncoding_internal( return *this; } +Series &Series::setParticlesPath_internal(std::string const &pp) +{ + if (auxiliary::ends_with(pp, '/')) + setAttribute("particlesPath", pp); + else + setAttribute("particlesPath", pp + "/"); + setDirty(true); + return *this; +} + +Series &Series::setMeshesPath_internal(std::string const &mp) +{ + if (auxiliary::ends_with(mp, '/')) + setAttribute("meshesPath", mp); + else + setAttribute("meshesPath", mp + "/"); + setDirty(true); + return *this; +} auto Series::openIterationIfDirty(IterationIndex_t index, Iteration &iteration) -> IterationOpened { From 842fbb7f20627d403ad6ce4a10005daa0994701b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 8 May 2026 14:30:23 +0200 Subject: [PATCH 40/58] Sync state more carefully --- include/openPMD/backend/BaseRecord.hpp | 2 +- include/openPMD/backend/Container.hpp | 40 ++++++++++++++++------- include/openPMD/backend/ContainerImpl.tpp | 23 +++++++++---- src/CustomHierarchy.cpp | 5 ++- src/ParticleSpecies.cpp | 3 +- src/backend/BaseRecord.cpp | 28 +++++++++++----- 6 files changed, 72 insertions(+), 29 deletions(-) diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 0c528011a9..b8940fcf29 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -413,7 +413,7 @@ auto BaseRecord::emplace(Args &&...args) -> std::pair this->container_front().emplace(std::forward(args)...)); if (res.first->first == RecordComponent::SCALAR) { - this->container_back().erase(res.first->first); + this->container_back(/* verify = */ true).erase(res.first->first); this->container_front().erase(res.first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index e349e502ac..a2162babd9 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -212,7 +212,7 @@ class Container : virtual public Attributable { #ifndef NDEBUG auto size_front = container_front().size(); - auto size_back = container_back().size(); + auto size_back = container_back(/* verify = */ true).size(); if (size_front > size_back) { throw std::runtime_error( @@ -221,14 +221,14 @@ class Container : virtual public Attributable } #endif traits::DeferredInitPolicy::call(*this); - return {&container_front(), &container_back()}; + return {&container_front(), &container_back(/* verify = */ true)}; } inline SynchronizedContainers container() { #ifndef NDEBUG auto size_front = container_front().size(); - auto size_back = container_back().size(); + auto size_back = container_back(/* verify = */ true).size(); if (size_front > size_back) { throw std::runtime_error( @@ -237,7 +237,7 @@ class Container : virtual public Attributable } #endif traits::DeferredInitPolicy::call(*this); - return {&container_front(), &container_back()}; + return {&container_front(), &container_back(/* verify = */ true)}; } inline auto container_front() const -> @@ -254,17 +254,23 @@ class Container : virtual public Attributable return m_containerData->m_container; } - inline auto container_back() const -> + inline auto container_back(bool verify) const -> typename SynchronizedContainers::back_t & { - traits::DeferredInitPolicy::call(*this); + if (verify) + { + traits::DeferredInitPolicy::call(*this); + } return Attributable::get().m_children; } - inline auto container_back() -> + inline auto container_back(bool verify) -> typename SynchronizedContainers::back_t & { - traits::DeferredInitPolicy::call(*this); + if (verify) + { + traits::DeferredInitPolicy::call(*this); + } return Attributable::get().m_children; } @@ -430,10 +436,20 @@ OPENPMD_protected } auto syncInsertResult(iterator res) -> iterator { - // container_back() expects the invariant that we are currently about to - // fulfill so we cannot use it just yet - Attributable::get().m_children.emplace( - key_as_string(res->first), *res->second.m_attri); + auto &cont = container_back(/* verify = */ false); + decltype(auto) key = key_as_string(res->first); + auto it = cont.find(key); + if (it == cont.end()) + { + cont.emplace(key_as_string(res->first), *res->second.m_attri); + } + else + { + // uhhm this might cause edge cases + // backend value is older, so it gets seniority + res->second.m_attri->asSharedPtrOfAttributable() = it->second; + res->second.preferCurrentBackpointer(); + } return res; } diff --git a/include/openPMD/backend/ContainerImpl.tpp b/include/openPMD/backend/ContainerImpl.tpp index 35704cdd0e..c4f7caa8d9 100644 --- a/include/openPMD/backend/ContainerImpl.tpp +++ b/include/openPMD/backend/ContainerImpl.tpp @@ -235,21 +235,32 @@ auto Container::insert( std::vector internal_insert_list; internal_insert_list.reserve(ilist.size()); + auto &cont = container_back(/* verify = */ false); for (auto &v : ilist) { - internal_insert_list.emplace_back( - key_as_string(v.first), *v.second.m_attri); + decltype(auto) key = key_as_string(v.first); + auto it = cont.find(key); + if (it == cont.end()) + { + internal_insert_list.emplace_back(key, *v.second.m_attri); + } + else + { + // backend value is older, so it gets seniority + v.second.m_attri->asSharedPtrOfAttributable() = it->second; + v.second.preferCurrentBackpointer(); + } } container_front().insert(std::move(ilist)); - container_back().insert( - internal_insert_list.begin(), internal_insert_list.end()); + cont.insert(internal_insert_list.begin(), internal_insert_list.end()); } template auto Container::swap(Container &other) -> void { container_front().swap(other.container_front()); - container_back().swap(other.container_back()); + container_back(/* verify = */ true) + .swap(other.container_back(/* verify = */ true)); } template @@ -323,7 +334,7 @@ auto Container::erase(iterator res) -> iterator IOHandler()->enqueue(IOTask(&res->second, pDelete)); IOHandler()->flush(internal::defaultFlushParams); } - container_back().erase(key_as_string(res->first)); + container_back(/* verify = */ true).erase(key_as_string(res->first)); return container_front().erase(res); } diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index e33b258c05..9a1e909a1d 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -166,7 +166,7 @@ void CustomHierarchy::read() IOHandler()->flush(internal::defaultFlushParams); std::deque constantComponentsPushback; - auto &container_back_ = container_back(); + auto &container_back_ = container_back(/* verify = */ true); for (auto const &path : *pList.paths) { if (auto it = container_back_.find(path); @@ -224,6 +224,7 @@ void CustomHierarchy::linkHierarchy(Writable &w) void CustomHierarchy::printRecursively() { + std::cout << &writable() << " "; if (empty()) { std::cout << "\n"; @@ -251,10 +252,12 @@ void CustomHierarchy::printRecursively(std::string indent) auto next_indent = indent + "│ "; for (; it != end_; prev = it, ++it) { + std::cout << &prev->second.writable() << " "; print_indent(); std::cout << "├─" << prev->first << '\n'; prev->second.printRecursively(next_indent); } + std::cout << &prev->second.writable() << " "; print_indent(); std::cout << "└─" << prev->first << '\n'; prev->second.printRecursively(indent + " "); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 2ec1529f36..21cb43ab2e 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -39,7 +39,8 @@ void ParticleSpecies::visitHierarchy(HierarchyVisitor &v, bool recursive) ParticleSpecies::ParticleSpecies() { particlePatches.writable().ownKeyWithinParent = "particlePatches"; - container_back()["particlePatches"] = *particlePatches.m_attri; + container_back(/* verify = */ true)["particlePatches"] = + *particlePatches.m_attri; } void ParticleSpecies::read() diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index 5d42f06ea6..7743b2fe2a 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -631,7 +631,7 @@ auto BaseRecord::insert(value_type const &value) { // this->container().erase(res.first); this->container_front().erase(res.first); - this->container_back().erase(res.first->first); + this->container_back(/* verify = */ true).erase(res.first->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; @@ -646,7 +646,7 @@ auto BaseRecord::insert(value_type &&value) -> std::pair if (res.first->first == RecordComponent::SCALAR) { this->container_front().erase(res.first); - this->container_back().erase(res.first->first); + this->container_back(/* verify = */ true).erase(res.first->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; @@ -671,7 +671,7 @@ auto BaseRecord::insert(const_iterator hint, value_type const &value) if (res->first == RecordComponent::SCALAR) { this->container_front().erase(res); - this->container_back().erase(res->first); + this->container_back(/* verify = */ true).erase(res->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return makeIterator(res); @@ -696,7 +696,7 @@ auto BaseRecord::insert(const_iterator hint, value_type &&value) if (res->first == RecordComponent::SCALAR) { this->container_front().erase(res); - this->container_back().erase(res->first); + this->container_back(/* verify = */ true).erase(res->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return makeIterator(res); @@ -729,13 +729,24 @@ auto BaseRecord::insert(std::initializer_list ilist) -> void std::vector internal_insert_list; internal_insert_list.reserve(ilist.size()); + auto &cont = this->container_back(/* verify = */ false); for (auto &v : ilist) { - internal_insert_list.emplace_back(v.first, *v.second.m_attri); + decltype(auto) key = this->key_as_string(v.first); + auto it = cont.find(key); + if (it == cont.end()) + { + internal_insert_list.emplace_back(key, *v.second.m_attri); + } + else + { + // backend value is older, so it gets seniority + v.second.m_attri->asSharedPtrOfAttributable() = it->second; + v.second.preferCurrentBackpointer(); + } } this->container_front().insert(std::move(ilist)); - this->container_back().insert( - internal_insert_list.begin(), internal_insert_list.end()); + cont.insert(internal_insert_list.begin(), internal_insert_list.end()); /* * We skip this check as it changes the runtime of this call from * O(last-first) to O(container().size()). @@ -756,7 +767,8 @@ auto BaseRecord::swap(BaseRecord &other) noexcept -> void detail::verifyNonscalar(this); detail::verifyNonscalar(&other); this->container_front().swap(other.container_front()); - this->container_back().swap(other.container_back()); + this->container_back(/* verify = */ true) + .swap(other.container_back(/* verify = */ true)); } template From 9a6e2314e02a32ea13c1775e5cd61fc156da620e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 8 May 2026 17:57:42 +0200 Subject: [PATCH 41/58] Recursive reading --- include/openPMD/CustomHierarchy.hpp | 6 +++- include/openPMD/backend/Container.hpp | 4 +-- include/openPMD/backend/Writable.hpp | 11 +++++++ src/CustomHierarchy.cpp | 46 +++++++++++++++++++-------- src/IO/AbstractIOHandlerImpl.cpp | 5 +++ 5 files changed, 54 insertions(+), 18 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index 67adba2a03..dc3cd9e6e1 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -145,7 +145,11 @@ class CustomHierarchy : public ConvertibleContainer CustomHierarchy &operator=(CustomHierarchy &&) = default; // TODO maybe make this automatic somehow - void read(); + // set max_recursion_depth = 0 for infinite cycling + // recursion depth includes the current object + // recursion will not continue expanding into regions that are already known + // (hence not transitively expand into unknown subregions of known regions) + void read(size_t max_recursion_depth = 1); void printRecursively(); diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index a2162babd9..c501a2ce22 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -169,9 +169,7 @@ class Container : virtual public Attributable using front_t = auxiliary::dependent_const; using back_t = auxiliary::dependent_const< const_, - std::map< - std::string, - std::shared_ptr>>; + internal::SharedAttributableData::children_map_t>; front_t *front; back_t *back; diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 7c4989eb49..d7c5313d4f 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -63,6 +63,14 @@ namespace debug void printDirty(Series const &); } +enum class ObjectType : std::uint8_t +{ + Group, + Dataset + // Attributes do not get their own objects but are attached to their + // respective group or dataset +}; + /** @brief Layer to mirror structure of logical data and persistent data in * file. * @@ -156,6 +164,7 @@ OPENPMD_private * If multiple Attributables share the same Writable, then the creating one. * (See SharedAttributableData) */ + // TODO turn this into a weak pointer internal::AttributableData *attributable = nullptr; Writable *parent = nullptr; @@ -198,5 +207,7 @@ OPENPMD_private * */ bool written = false; + + ObjectType objectType = ObjectType::Group; }; } // namespace openPMD diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 9a1e909a1d..2987e7f76e 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -115,7 +115,7 @@ CustomHierarchy::CustomHierarchy(Attributable const &other) : CustomHierarchy(other.m_attri->asSharedPtrOfAttributable()) {} -void CustomHierarchy::read() +void CustomHierarchy::read(size_t const max_recursion_depth) { auxiliary::opaque_defer_type reset_parsing_status; if (IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) @@ -145,7 +145,7 @@ void CustomHierarchy::read() std::shared_ptr{ parent->attributable, [](auto const *) {}}); CustomHierarchy parent_(parent_attributable); - parent_.read(); + parent_.read(1); if (!writable() /* still not */.written) { do_throw(); @@ -157,15 +157,40 @@ void CustomHierarchy::read() * to create/open path for contained subpaths. */ + Attributable::readAttributes(ReadMode::FullyReread); + + switch (writable().objectType) + { + case ObjectType::Group: + break; + case ObjectType::Dataset: + return; + } + + auto do_recurse = [this, max_recursion_depth](CustomHierarchy &child) { + switch (max_recursion_depth) + { + case 0: + IOHandler()->flush(internal::defaultFlushParams); + child.read(0); + break; + case 1: + break; + default: + IOHandler()->flush(internal::defaultFlushParams); + child.read(max_recursion_depth - 1); + break; + } + }; + Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - Attributable::readAttributes(ReadMode::FullyReread); Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); + IOHandler()->flush(internal::defaultFlushParams); - std::deque constantComponentsPushback; auto &container_back_ = container_back(/* verify = */ true); for (auto const &path : *pList.paths) { @@ -180,6 +205,7 @@ void CustomHierarchy::read() auto &subpath = this->operator[](path); subpath.linkHierarchy(this->writable()); IOHandler()->enqueue(IOTask(&subpath, pOpen)); + do_recurse(subpath); } for (auto const &path : *dList.datasets) @@ -193,18 +219,10 @@ void CustomHierarchy::read() Parameter dOpen; dOpen.name = path; - // TODO uhhm i think this wont work, but lets see when we get there - RecordComponent rc; - auto [it, emplaced] = this->emplace( - path, CustomHierarchy(static_cast(rc))); - if (!emplaced) - { - throw error::Internal( - "Control flow error / internal container state error."); - } - auto &subpath = it->second; + auto &subpath = this->operator[](path); subpath.linkHierarchy(this->writable()); IOHandler()->enqueue(IOTask(&subpath, dOpen)); + do_recurse(subpath); } setDirty(false); diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 473b140aef..b6dfd50a50 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -217,6 +217,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) ", extent=", [¶meter]() { return vec_as_string(parameter.extent); }); createDataset(i.writable, parameter); + i.writable->objectType = ObjectType::Dataset; break; } case O::EXTEND_DATASET: { @@ -230,6 +231,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) i.writable, "] EXTEND_DATASET"); extendDataset(i.writable, parameter); + i.writable->objectType = ObjectType::Dataset; break; } case O::OPEN_FILE: { @@ -286,6 +288,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) "] OPEN_DATASET: ", parameter.name); openDataset(i.writable, parameter); + i.writable->objectType = ObjectType::Dataset; break; } case O::DELETE_FILE: { @@ -345,6 +348,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) ", extent=", [¶meter]() { return vec_as_string(parameter.extent); }); writeDataset(i.writable, parameter); + i.writable->objectType = ObjectType::Dataset; break; } case O::WRITE_ATT: { @@ -376,6 +380,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) ", extent=", [¶meter]() { return vec_as_string(parameter.extent); }); readDataset(i.writable, parameter); + i.writable->objectType = ObjectType::Dataset; break; } case O::GET_BUFFER_VIEW: { From 37892a7fc6af5eb9233cd7c86f9b823f407737c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 8 May 2026 18:02:25 +0200 Subject: [PATCH 42/58] TODO comments --- include/openPMD/backend/Attributable.hpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index b1c098a692..6eb239596a 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -486,6 +486,18 @@ class Attributable // TODO: Add parse? parameter, dont parse, parse this object, parse // recursively // Alternatively, parse upon deferred initialization? + // TODO CustomHierarchy is ephemeral, and i dont even know what that word + // means. but i want to say that things are created on the spot. if we go to + // the same place in the openPMD file using different paths, the objects + // will be different, except for the backend data stored in + // sharedattributabledata and writable classes. even when later turning a + // customhierarchy object into a recordcomponent, this will not be same + // recordcomponent object as in the usual openpmd hierarchy. + // TODO if objects are *created* by customhierarchies, the pointer + // writable.attributable will point to that object as the first instance. so + // i guess we will still need to store the objects somewhere and cannot keep + // them fully ephemeral. likely a resource container in the parent (to break + // reference cycles). auto customHierarchies() -> CustomHierarchy; // clang-format off From 36f5bcbc85f023119daf2e778469a9e3094cc950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 11 May 2026 12:28:41 +0200 Subject: [PATCH 43/58] object storage for custom hierarchies --- include/openPMD/CustomHierarchy.hpp | 20 +++++++++++++++++++- include/openPMD/backend/Attributable.hpp | 10 ++++++++++ include/openPMD/backend/Writable.hpp | 8 ++++++++ src/CustomHierarchy.cpp | 5 ++++- 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index dc3cd9e6e1..176eda2ab0 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -98,6 +98,23 @@ namespace traits template static void call(Container_const_or_not &); }; + + template <> + struct GenerationPolicy + { + template + void operator()(Iterator &it) + { + if (it->second.writable().attributable == it->second.m_attri.get()) + { + // throw std::runtime_error( + // "Unimplemented: CustomHierarchy must not (yet) be the " + // "first instance of an object in the openPMD hierarchy."); + it->second.Attributable::get() + .m_children_object_storage[it->first] = it->second; + } + } + }; } // namespace traits // TODO: Use Container internally, but otherwise override members @@ -119,7 +136,6 @@ class CustomHierarchy : public ConvertibleContainer using Data_t = typename Parent_t::Data_t; protected: - CustomHierarchy(); CustomHierarchy(NoInit); CustomHierarchy(std::shared_ptr other); CustomHierarchy(Attributable const &other); @@ -138,6 +154,8 @@ class CustomHierarchy : public ConvertibleContainer void linkHierarchy(Writable &w) override; public: + CustomHierarchy(); + CustomHierarchy(CustomHierarchy const &other) = default; CustomHierarchy(CustomHierarchy &&other) = default; diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 6eb239596a..c44593c089 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -98,6 +98,16 @@ namespace internal using children_map_t = std::map>; children_map_t m_children; + + // Attributable::customHierarchies() creates objects of type + // CustomHierarchy ephemerally on the spot. If that object is the first + // object for its associated SharedAttributableData instance, it must be + // stored somewhere still, because the first instance is back-referenced + // by Writable class (TODO: turn that back-reference into a weak_ptr?). + // Store these objects in the parent to avoid reference cycles. + using children_object_storage_t = + std::map; + children_object_storage_t m_children_object_storage; }; /* diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index d7c5313d4f..7c04f5f6aa 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -46,6 +46,12 @@ template class Span; class Series; +namespace traits +{ + template + struct GenerationPolicy; +} // namespace traits + namespace internal { class SharedAttributableData; @@ -117,6 +123,8 @@ class Writable final friend struct Parameter; friend class internal::ScientificDefaults; friend class CustomHierarchy; + template + friend struct traits::GenerationPolicy; private: Writable(internal::AttributableData *); diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 2987e7f76e..991c32f545 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -21,6 +21,7 @@ #include "openPMD/CustomHierarchy.hpp" #include "openPMD/auxiliary/Defer.hpp" +#include "openPMD/backend/Container.hpp" namespace openPMD { @@ -74,7 +75,8 @@ namespace traits // method might be called as const. shared_ptr<>s implement interior // mutability, so use that here. - // auto &my_container = container.container_front(); + GenerationPolicy gen; + auto it = container_front.begin(); auto end = container_front.end(); for (auto const &[key, attributable] : container_back) @@ -86,6 +88,7 @@ namespace traits // points to an entry past the to-be-inserted key it = container_front.emplace_hint( it, key, CustomHierarchy(attributable)); + gen(it); } } } From 1bbd04cfa65f10fba65d8974925d876cd4568a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 11 May 2026 13:43:39 +0200 Subject: [PATCH 44/58] Fix object storage --- include/openPMD/CustomHierarchy.hpp | 44 ++++++++++++++++++----- include/openPMD/Iteration.hpp | 4 +-- include/openPMD/ParticleSpecies.hpp | 4 +-- include/openPMD/backend/Container.hpp | 4 +-- include/openPMD/backend/ContainerImpl.tpp | 4 +-- src/CustomHierarchy.cpp | 2 +- 6 files changed, 45 insertions(+), 17 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index 176eda2ab0..2d0a22d5ca 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -102,16 +102,44 @@ namespace traits template <> struct GenerationPolicy { - template - void operator()(Iterator &it) + constexpr static bool is_noop = false; + template + void operator()(Container &cont, Iterator &it) { - if (it->second.writable().attributable == it->second.m_attri.get()) + auto &writable = it->second.writable(); + + // These should be different + auto child_shared_data = &it->second.Attributable::get(); + auto parent_shared_data = &cont.Attributable::get(); + if (child_shared_data == parent_shared_data) + { + throw std::runtime_error( + "Trying to emplace object as its own child"); + } + + // These might be different, but might also be the same + // + // For an explanation, ref. the documentation of + // Writable::attributable: This is a pointer back to the first + // created Attributable instance linking this Writable. There might + // be multiple Attributable objects linking the same backend + // Writable object when opening multiple "views" on the same backend + // object, e.g. when a scalar Record is at the same time a + // RecordComponent, or when reopening an object as a + // CustomHierarchy. + // + // Since CustomHierarchy performs no memory management by default, + // we must ensure that the backpointer in Writable::attributable + // remains valid when the frontend instance pointed by + // Writable::attributable *is* the CustomHierarchy instance (happens + // when it is the first frontend object created for that backend + // object). + auto backpointer = writable.attributable; + auto emplaced_pointer = it->second.m_attri.get(); + if (backpointer == emplaced_pointer) { - // throw std::runtime_error( - // "Unimplemented: CustomHierarchy must not (yet) be the " - // "first instance of an object in the openPMD hierarchy."); - it->second.Attributable::get() - .m_children_object_storage[it->first] = it->second; + (**cont.m_attri).m_children_object_storage[it->first] = + it->second; } } }; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 0a3c1dfcb0..ccaf21d1ae 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -486,8 +486,8 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = false; - template - void operator()(Iterator &it) + template + void operator()(Container &, Iterator &it) { it->second.get().m_iterationIndex = it->first; } diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index 1ec1ff8d9c..7b13a39784 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -74,8 +74,8 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = false; - template - void operator()(T &it) + template + void operator()(Container &, T &it) { it->second.particlePatches.linkHierarchy(it->second.writable()); } diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index c501a2ce22..5e7bf1040f 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -53,8 +53,8 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = true; - template - void operator()(T &) + template + void operator()(Container &, T &) {} }; diff --git a/include/openPMD/backend/ContainerImpl.tpp b/include/openPMD/backend/ContainerImpl.tpp index c4f7caa8d9..6144b050a5 100644 --- a/include/openPMD/backend/ContainerImpl.tpp +++ b/include/openPMD/backend/ContainerImpl.tpp @@ -154,7 +154,7 @@ auto Container::operator[](key_type const &key) ret.writable().ownKeyWithinParent = std::to_string(key); } traits::GenerationPolicy gen; - gen(inserted_iterator); + gen(*this, inserted_iterator); return ret; } } @@ -189,7 +189,7 @@ auto Container::operator[](key_type &&key) ret.writable().ownKeyWithinParent = std::to_string(std::move(key)); } traits::GenerationPolicy gen; - gen(inserted_iterator); + gen(*this, inserted_iterator); return ret; } } diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 991c32f545..443c4b2d53 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -88,7 +88,7 @@ namespace traits // points to an entry past the to-be-inserted key it = container_front.emplace_hint( it, key, CustomHierarchy(attributable)); - gen(it); + gen(container, it); } } } From c400f3309746b1d006f9a086995a71200f46ca98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 11 May 2026 15:44:21 +0200 Subject: [PATCH 45/58] Add preferCurrentBackpointer --- include/openPMD/backend/Attributable.hpp | 2 ++ src/backend/Attributable.cpp | 40 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index c44593c089..6cafc0e179 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -635,6 +635,8 @@ OPENPMD_protected return (*m_attri)->m_writable; } + void preferCurrentBackpointer() const; + inline void setData(std::shared_ptr attri) { m_attri = std::move(attri); diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index c6e8831a04..7748b81af6 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -30,6 +30,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/HierarchyVisitorImpl.hpp" +#include "openPMD/backend/Writable.hpp" #include #include @@ -588,6 +589,45 @@ void Attributable::readAttributes(ReadMode mode) setDirty(false); } +void Attributable::preferCurrentBackpointer() const +{ + auto this_as_custom_hierarchy = dynamic_cast(this); + if (this_as_custom_hierarchy) + { + return; + } + + auto &shareddata = **m_attri; + auto &w = shareddata.m_writable; + + auto backpointer_as_custom_hierarchy = + dynamic_cast(w.attributable); + if (!backpointer_as_custom_hierarchy) + { + return; + } + + w.attributable = m_attri.get(); + + if (!w.parent) + { + throw error::Internal( + "CustomHierarchy object was created without parent. Why?"); + } + auto count_of_erased_elements = + (*w.parent->attributable) + ->m_children_object_storage.erase(w.ownKeyWithinParent); + if (count_of_erased_elements != 1) + { + throw error::Internal( + "Unexpected state: Expected to erase 1 element from internal " + "object storage, found " + + std::to_string(count_of_erased_elements) + " instead."); + } + + // std::cout << "REWIRED '" << myPath().openPMDPath() << "'." << std::endl; +} + void Attributable::setWritten(bool val, EnqueueAsynchronously ea) { switch (ea) From 758100eb6abab3b35c42fb652ceb856eb3abaac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 11 May 2026 15:44:43 +0200 Subject: [PATCH 46/58] TMP: example changes --- examples/2_read_serial.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/examples/2_read_serial.cpp b/examples/2_read_serial.cpp index effd752fda..2a0826a5eb 100644 --- a/examples/2_read_serial.cpp +++ b/examples/2_read_serial.cpp @@ -31,7 +31,8 @@ using namespace openPMD; int main() { Series series = Series( - "../samples/git-sample/data%T.h5", + // "../samples/git-sample/data%T.h5", + "data.h5", Access::READ_ONLY, R"({"defer_iteration_parsing": true})"); cout << "Read a Series with openPMD standard version " << series.openPMD() @@ -101,12 +102,22 @@ int main() auto ch = series.customHierarchies(); ch.printRecursively(); + std::cout << "READING 200/fields" << std::endl; ch["data"]["200"]["fields"].read(); + std::cout << "READING 200/particles" << std::endl; + ch["data"]["200"]["particles"].read(); + std::cout << "READING 300" << std::endl; + ch["data"]["300"].read(0); ch.printRecursively(); // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. i.close(); + std::cout << "OPENING 200" << std::endl; + i = series.snapshots()[200].open(); + ch.printRecursively(); + series.snapshots()[300].open(); + cout << "Full E/x starts with:\n\t{"; for (size_t col = 0; col < extent[1] && col < 5; ++col) cout << all_data.get()[col] << ", "; From 3422d0c6bdb76b171953301c61cad7c57eeeb70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 11 May 2026 20:02:58 +0200 Subject: [PATCH 47/58] WIP customHierarchyFlush --- include/openPMD/backend/Attributable.hpp | 2 + src/CustomHierarchy.cpp | 2 +- src/Iteration.cpp | 3 ++ src/backend/Attributable.cpp | 54 ++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 6cafc0e179..221452bbfc 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -399,6 +399,8 @@ class Attributable */ void iterationFlush(std::string backendConfig = "{}"); + void customHierarchyFlush(internal::FlushParams const &); + /** String serialization to describe an Attributable * * This object contains the Series data path as well as the openPMD object diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 443c4b2d53..6bfcb3b90c 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -235,7 +235,7 @@ void CustomHierarchy::read(size_t const max_recursion_depth) void CustomHierarchy::flush( std::string const & /* path */, internal::FlushParams const &) { - throw std::runtime_error("Unimplemented!"); + throw std::runtime_error("Use Attributable::customHierarchyFlush instead!"); } void CustomHierarchy::linkHierarchy(Writable &w) diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 5d0b675008..cd7c55d714 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -437,6 +437,9 @@ void Iteration::flush(internal::FlushParams const &flushParams) flushAttributes(flushParams); } + + customHierarchyFlush(flushParams); + if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { determineUnsetDirty(flushParams.flushLevel); diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 7748b81af6..79268b8804 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -179,6 +179,55 @@ void Attributable::iterationFlush(std::string backendConfig) std::move(backendConfig)); } +void Attributable::customHierarchyFlush( + internal::FlushParams const &flushParams) +{ + if (!dirtyRecursive()) + { + return; + } + + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + // No need to do anything in access::readOnly since meshes and particles + // are initialized as aliases for subgroups at parsing time + auto &data = get(); + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } + + Parameter pCreate; + for (auto &[name, subpath] : data.m_children_object_storage) + { + auto backpointer = subpath.writable().attributable; + auto casted_backpointer = + dynamic_cast(backpointer); + if (!casted_backpointer) + { + throw error::Internal( + "SharedAttributableData::m_children_object_storage contained " + "an object that should be flushed conventionally."); + } + if (!subpath.written()) + { + pCreate.path = name; + IOHandler()->enqueue(IOTask(&subpath, pCreate)); + } + subpath.flush(name, flushParams); + } + + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + setDirty(false); + } +} + Series Attributable::retrieveSeries() const { Writable const *findSeries = &writable(); @@ -345,6 +394,11 @@ OpenpmdStandard Attributable::openPMDStandard() const auto Attributable::customHierarchies() -> CustomHierarchy { + // No need to emplace this in + // SharedAttributableData::m_children_object_storage. Only those instances + // of CustomHierarchy need to be emplaced that do not have a counter-object + // inside the openPMD hierarchy keeping it alive, e.g. children created or + // read by the returned instance outside the openPMD hierarchy. return CustomHierarchy{*this}; } From 27c13c3dffbeb1f5da979d06cd24b2fb4629a7b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 12 May 2026 17:02:49 +0200 Subject: [PATCH 48/58] Guard against unset meshes/particles path --- include/openPMD/Series.hpp | 10 ++++++++++ .../openPMD/auxiliary/MonadicOperations.hpp | 18 ++++++++++++++++++ include/openPMD/backend/Attributable.hpp | 9 +++++++++ src/Iteration.cpp | 19 +++++++++++++++---- src/Series.cpp | 15 +++++++++++++++ src/backend/Attributable.cpp | 14 +++++++++++++- 6 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 include/openPMD/auxiliary/MonadicOperations.hpp diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index f713165ab9..0031f2be5b 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -451,6 +451,11 @@ class Series : public Attributable * basePath. */ std::string meshesPath() const; + /** + * @return String representing the path to mesh records, relative(!) to + * basePath. + */ + std::optional meshesPathOptional() const; /** Set the path to mesh * records, relative(!) to basePath. @@ -506,6 +511,11 @@ class Series : public Attributable * basePath. */ std::string particlesPath() const; + /** + * @return String representing the path to particle species, relative(!) to + * basePath. + */ + std::optional particlesPathOptional() const; /** Set the path to groups for each particle * species, relative(!) to basePath. diff --git a/include/openPMD/auxiliary/MonadicOperations.hpp b/include/openPMD/auxiliary/MonadicOperations.hpp new file mode 100644 index 0000000000..c96f682bc8 --- /dev/null +++ b/include/openPMD/auxiliary/MonadicOperations.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace openPMD::auxiliary +{ +template +auto optional_and_then(Optional &&arg, F &&then) + -> std::invoke_result_t())> +{ + if (!arg.has_value()) + { + return std::nullopt; + } + return std::forward(then)(*std::forward(arg)); +} +} // namespace openPMD::auxiliary diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 221452bbfc..d6339bdff6 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -330,6 +330,15 @@ class Attributable */ Attribute getAttribute(std::string const &key) const; + /** Retrieve value of Attribute stored with provided key. + * + * @throw no_such_attribute_error If no Attribute is currently stored with + * the provided key. + * @param key Key (i.e. name) of the Attribute to retrieve value for. + * @return If found, the stored Attribute in Variant form. + */ + std::optional getAttributeOptional(std::string const &key) const; + /** Remove Attribute of provided value both logically and physically. * * @param key Key (i.e. name) of the Attribute to remove. diff --git a/src/Iteration.cpp b/src/Iteration.cpp index cd7c55d714..46b9f91bde 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -917,10 +917,21 @@ void Iteration::linkHierarchy(Writable &w) auto &container_back = Attributable::get().m_children; auto s = retrieveSeries(); - container_back[auxiliary::replace_all_nonrecursively( - s.meshesPath(), "/", "")] = *meshes.m_attri; - container_back[auxiliary::replace_all_nonrecursively( - s.particlesPath(), "/", "")] = *particles.m_attri; + auto link_mp = [&](auto &meshes_or_particles, + std::optional const &mp_path, + char const *default_) { + if (mp_path) + { + container_back[auxiliary::replace_all_nonrecursively( + *mp_path, "/", "")] = *meshes_or_particles.m_attri; + } + else + { + container_back[default_] = *meshes_or_particles.m_attri; + } + }; + link_mp(meshes, s.meshesPathOptional(), "meshes"); + link_mp(particles, s.particlesPathOptional(), "particles"); } void Iteration::runDeferredParseAccess() diff --git a/src/Series.cpp b/src/Series.cpp index bc2fc2fdc3..7e22d8b90e 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -36,6 +36,7 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/auxiliary/MonadicOperations.hpp" #include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" @@ -272,6 +273,13 @@ std::string Series::meshesPath() const return getAttribute("meshesPath").get(); } +std::optional Series::meshesPathOptional() const +{ + return auxiliary::optional_and_then( + getAttributeOptional("meshesPath"), + [](Attribute const &attr) { return attr.getOptional(); }); +} + Series &Series::setMeshesPath(std::string const &mp) { auto &series = get(); @@ -605,6 +613,13 @@ std::string Series::particlesPath() const return getAttribute("particlesPath").get(); } +std::optional Series::particlesPathOptional() const +{ + return auxiliary::optional_and_then( + getAttributeOptional("particlesPath"), + [](Attribute const &attr) { return attr.getOptional(); }); +} + Series &Series::setParticlesPath(std::string const &pp) { auto &series = get(); diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 79268b8804..39fe128ec5 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -105,13 +105,25 @@ bool Attributable::setAttribute(std::string const &key, Attribute attribute) } Attribute Attributable::getAttribute(std::string const &key) const +{ + auto attribute = getAttributeOptional(key); + if (attribute.has_value()) + { + return *attribute; + } + + throw no_such_attribute_error(key); +} + +std::optional +Attributable::getAttributeOptional(std::string const &key) const { auto &attri = get(); auto it = attri.m_attributes.find(key); if (it != attri.m_attributes.cend()) return it->second; - throw no_such_attribute_error(key); + return std::nullopt; } bool Attributable::deleteAttribute(std::string const &key) From 4dfc00521c51704b217065eef62ad990dcd0315f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 18 May 2026 11:56:23 +0200 Subject: [PATCH 49/58] Fix custom hierarchy flushing --- examples/10_streaming_write.cpp | 2 ++ include/openPMD/backend/Attributable.hpp | 2 +- src/CustomHierarchy.cpp | 9 ++++++--- src/Iteration.cpp | 5 +++-- src/backend/Attributable.cpp | 8 +++++--- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 01bd2fcb33..e38ab3daa9 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -82,6 +82,8 @@ int main() pos.resetDataset(dataset); pos.storeChunk(local_data, Offset{0}, global_extent); } + auto ch = iteration.customHierarchies(); + ch["rabimmel"].setAttribute("rabammel", "rabumm"); iteration.close(); } diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index d6339bdff6..054627dbfa 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -408,7 +408,7 @@ class Attributable */ void iterationFlush(std::string backendConfig = "{}"); - void customHierarchyFlush(internal::FlushParams const &); + void customHierarchyFlush(internal::FlushParams const &, bool unset_dirty); /** String serialization to describe an Attributable * diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index 6bfcb3b90c..c6c84b0c0f 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -245,7 +245,8 @@ void CustomHierarchy::linkHierarchy(Writable &w) void CustomHierarchy::printRecursively() { - std::cout << &writable() << " "; + std::cout << &writable() << " [" << dirty() << "," << dirtyRecursive() + << "] "; if (empty()) { std::cout << "\n"; @@ -273,12 +274,14 @@ void CustomHierarchy::printRecursively(std::string indent) auto next_indent = indent + "│ "; for (; it != end_; prev = it, ++it) { - std::cout << &prev->second.writable() << " "; + std::cout << &prev->second.writable() << " [" << prev->second.dirty() + << "," << prev->second.dirtyRecursive() << "] "; print_indent(); std::cout << "├─" << prev->first << '\n'; prev->second.printRecursively(next_indent); } - std::cout << &prev->second.writable() << " "; + std::cout << &prev->second.writable() << " [" << prev->second.dirty() << "," + << prev->second.dirtyRecursive() << "] "; print_indent(); std::cout << "└─" << prev->first << '\n'; prev->second.printRecursively(indent + " "); diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 46b9f91bde..b82bffa0ed 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -370,6 +370,9 @@ void Iteration::flush(internal::FlushParams const &flushParams) { Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); + + customHierarchyFlush(flushParams, /* unset_dirty = */ false); + if (access::readOnly(IOHandler()->m_frontendAccess)) { for (auto &m : meshes) @@ -438,8 +441,6 @@ void Iteration::flush(internal::FlushParams const &flushParams) flushAttributes(flushParams); } - customHierarchyFlush(flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { determineUnsetDirty(flushParams.flushLevel); diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 39fe128ec5..ed0cb29000 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -192,8 +192,9 @@ void Attributable::iterationFlush(std::string backendConfig) } void Attributable::customHierarchyFlush( - internal::FlushParams const &flushParams) + internal::FlushParams const &flushParams, bool unset_dirty) { + // customHierarchies().printRecursively(); if (!dirtyRecursive()) { return; @@ -230,14 +231,15 @@ void Attributable::customHierarchyFlush( pCreate.path = name; IOHandler()->enqueue(IOTask(&subpath, pCreate)); } - subpath.flush(name, flushParams); + subpath.customHierarchyFlush(flushParams, true); } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + if (unset_dirty && flushParams.flushLevel != FlushLevel::SkeletonOnly && flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) { setDirty(false); } + // customHierarchies().printRecursively(); } Series Attributable::retrieveSeries() const From c39ce242ec0cc269b1b4104f933a0048cfa1d84e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 18 May 2026 14:30:52 +0200 Subject: [PATCH 50/58] Cleanup --- src/CustomHierarchy.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp index c6c84b0c0f..3f4a7634ef 100644 --- a/src/CustomHierarchy.cpp +++ b/src/CustomHierarchy.cpp @@ -31,8 +31,14 @@ namespace traits void DeferredInitPolicy>::call( Container_const_or_not &container) { + // Need to sync backend objects into the CustomHierarchy instance + // Need to be a bit sneaky, we must modify my_container&, but this + // method might be called as const. shared_ptr<>s implement interior + // mutability, so use that here. + + // auto &container_front = container.container_front(); auto &container_front = container.m_containerData->m_container; - auto &container_back = container.Attributable::get().m_children; + auto &container_back = container.container_back(/* verify = */ false); auto size_front = container_front.size(); auto size_back = container_back.size(); @@ -70,10 +76,6 @@ namespace traits print(container_back) << '\n'; throw error::Internal(error.str()); } - // Need to sync backend objects into the CustomHierarchy instance - // Need to be a bit sneaky, we must modify my_container&, but this - // method might be called as const. shared_ptr<>s implement interior - // mutability, so use that here. GenerationPolicy gen; From f1f7305811892be83ffe8aa274097bdc3fbf8327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 18 May 2026 15:45:16 +0200 Subject: [PATCH 51/58] Rename --- include/openPMD/CustomHierarchy.hpp | 3 ++- include/openPMD/backend/Attributable.hpp | 2 +- src/backend/Attributable.cpp | 17 ++++++++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp index 2d0a22d5ca..f29ffa5738 100644 --- a/include/openPMD/CustomHierarchy.hpp +++ b/include/openPMD/CustomHierarchy.hpp @@ -138,7 +138,8 @@ namespace traits auto emplaced_pointer = it->second.m_attri.get(); if (backpointer == emplaced_pointer) { - (**cont.m_attri).m_children_object_storage[it->first] = + (**cont.m_attri) + .m_children_managed_as_custom_hierarchy[it->first] = it->second; } } diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 054627dbfa..2a0ec6fd81 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -107,7 +107,7 @@ namespace internal // Store these objects in the parent to avoid reference cycles. using children_object_storage_t = std::map; - children_object_storage_t m_children_object_storage; + children_object_storage_t m_children_managed_as_custom_hierarchy; }; /* diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index ed0cb29000..ee1294fda2 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -215,7 +215,7 @@ void Attributable::customHierarchyFlush( } Parameter pCreate; - for (auto &[name, subpath] : data.m_children_object_storage) + for (auto &[name, subpath] : data.m_children_managed_as_custom_hierarchy) { auto backpointer = subpath.writable().attributable; auto casted_backpointer = @@ -223,7 +223,8 @@ void Attributable::customHierarchyFlush( if (!casted_backpointer) { throw error::Internal( - "SharedAttributableData::m_children_object_storage contained " + "SharedAttributableData::m_children_managed_as_custom_" + "hierarchy contained " "an object that should be flushed conventionally."); } if (!subpath.written()) @@ -409,10 +410,11 @@ OpenpmdStandard Attributable::openPMDStandard() const auto Attributable::customHierarchies() -> CustomHierarchy { // No need to emplace this in - // SharedAttributableData::m_children_object_storage. Only those instances - // of CustomHierarchy need to be emplaced that do not have a counter-object - // inside the openPMD hierarchy keeping it alive, e.g. children created or - // read by the returned instance outside the openPMD hierarchy. + // SharedAttributableData::m_children_managed_as_custom_hierarchy. Only + // those instances of CustomHierarchy need to be emplaced that do not have a + // counter-object inside the openPMD hierarchy keeping it alive, e.g. + // children created or read by the returned instance outside the openPMD + // hierarchy. return CustomHierarchy{*this}; } @@ -684,7 +686,8 @@ void Attributable::preferCurrentBackpointer() const } auto count_of_erased_elements = (*w.parent->attributable) - ->m_children_object_storage.erase(w.ownKeyWithinParent); + ->m_children_managed_as_custom_hierarchy.erase( + w.ownKeyWithinParent); if (count_of_erased_elements != 1) { throw error::Internal( From 220dddcaaeca0ec668c90ab8de92545398c50784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 18 May 2026 16:27:30 +0200 Subject: [PATCH 52/58] Add custom hierarchy flush at series level --- src/Series.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/Series.cpp b/src/Series.cpp index 7e22d8b90e..14529fca2b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1492,6 +1492,12 @@ void Series::flushFileBased( case IO::HasBeenOpened: // continue below it->second.flush(flushParams); + + if (it == begin) + { + customHierarchyFlush( + flushParams, /* unset_dirty = */ false); + } break; } @@ -1551,6 +1557,12 @@ void Series::flushFileBased( it->second.flushFileBased(filename, it->first, flushParams); + if (it == begin) + { + customHierarchyFlush( + flushParams, /* unset_dirty = */ false); + } + series.iterations.flush( auxiliary::replace_first(basePath(), "%T/", ""), flushParams); @@ -1635,7 +1647,14 @@ void Series::flushGorVBased( series.m_snapshotToStep.at(it->first)}; IOHandler()->enqueue(IOTask(this, std::move(param))); } + it->second.flush(flushParams); + + if (it == begin) + { + customHierarchyFlush( + flushParams, /* unset_dirty = */ false); + } break; } @@ -1717,6 +1736,13 @@ void Series::flushGorVBased( throw std::runtime_error( "[Series] Internal control flow error"); } + + if (it == begin) + { + customHierarchyFlush( + flushParams, /* unset_dirty = */ false); + } + break; case IO::RemainsClosed: break; From dea0fa824432e722d792db67fbb2672dc3cbccf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 18 May 2026 17:20:06 +0200 Subject: [PATCH 53/58] Rudimentary test, to be extended --- CMakeLists.txt | 1 + test/CoreTest.cpp | 5 ++ test/Files_Core/CoreTests.hpp | 5 ++ test/Files_Core/custom_hierarchy.cpp | 86 ++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+) create mode 100644 test/Files_Core/custom_hierarchy.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a24803011..d26b6b6ac0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -867,6 +867,7 @@ if(openPMD_BUILD_TESTING) list(APPEND ${out_list} test/Files_Core/automatic_variable_encoding.cpp test/Files_Core/read_nonexistent_attribute.cpp + test/Files_Core/custom_hierarchy.cpp ) endif() endmacro() diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 821907b38e..d737f4eebd 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1769,6 +1769,11 @@ TEST_CASE("read_nonexistent_attribute", "[core]") read_nonexistent_attribute::read_nonexistent_attribute(); } +TEST_CASE("custom_hierarchy", "[core]") +{ + custom_hierarchy::custom_hierarchy(); +} + TEST_CASE("unique_ptr", "[core]") { auto stdptr = std::make_unique(5); diff --git a/test/Files_Core/CoreTests.hpp b/test/Files_Core/CoreTests.hpp index f769beeb99..1f2d7072d9 100644 --- a/test/Files_Core/CoreTests.hpp +++ b/test/Files_Core/CoreTests.hpp @@ -29,3 +29,8 @@ namespace read_nonexistent_attribute { auto read_nonexistent_attribute() -> void; } + +namespace custom_hierarchy +{ +auto custom_hierarchy() -> void; +} diff --git a/test/Files_Core/custom_hierarchy.cpp b/test/Files_Core/custom_hierarchy.cpp new file mode 100644 index 0000000000..b762cb1770 --- /dev/null +++ b/test/Files_Core/custom_hierarchy.cpp @@ -0,0 +1,86 @@ + +/* Copyright 2026 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IterationEncoding.hpp" +#define OPENPMD_private public: +#define OPENPMD_protected public: + +#include "CoreTests.hpp" + +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/IOTask.hpp" + +#include +namespace custom_hierarchy +{ +using namespace openPMD; + +void write( + char const *filename, std::string const &json_params, IterationEncoding ie) +{ + std::cout << json_params << std::endl; + Series series(filename, Access::CREATE_LINEAR, json_params); + auto add_custom_hierarchy = [](Attributable &attr) { + attr.customHierarchies()["rabimmel"].setAttribute("rabammel", "rabumm"); + }; + auto iteration = series.snapshots()[0]; + + add_custom_hierarchy(series); + add_custom_hierarchy(iteration); + iteration.close(); +} + +struct test_config +{ + char const *filename; + char const *json_params; + IterationEncoding encoding; +}; + +void custom_hierarchy() +{ + test_config configs[] = { + {"../samples/custom_hierarchy/groupbased.%E", + R"({"iteration_encoding": "group_based"})", + IterationEncoding::groupBased}, + {"../samples/custom_hierarchy/filebased_%T.%E", + R"({"iteration_encoding": "file_based"})", + IterationEncoding::fileBased}, + {"../samples/custom_hierarchy/variablebased.%E", + R"({"iteration_encoding": "variable_based"})", + IterationEncoding::variableBased}}; + + for (auto const &backend : {"adios2", "hdf5", "json", "toml"}) + { + for (auto const &[filename, json_params, encoding] : configs) + { + auto json_params_ = json::merge( + json_params, + std::string( + R"({"adios2": {"engine": {"type": "file"}}, "backend": ")") + + backend + R"("})"); + write(filename, json_params_, encoding); + } + } +} +} // namespace custom_hierarchy From 9f4e6149256bdd1d2b9d724a3173993b16f24634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 18 May 2026 17:42:59 +0200 Subject: [PATCH 54/58] Fix ADIOS2 path creation relative to / --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 036b4f6098..6f8c36a9eb 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -803,7 +803,11 @@ void ADIOS2IOHandlerImpl::createPath( /* Sanitize path */ if (!auxiliary::starts_with(parameters.path, '/')) { - path = filePositionToString(setAndGetFilePosition(writable)) + "/" + + auto file_position = + filePositionToString(setAndGetFilePosition(writable)); + path = + (auxiliary::ends_with(file_position, '/') ? file_position + : file_position + "/") + auxiliary::removeSlashes(parameters.path); } else From a7b62bd3bf5f34cf730718422e187ff951fb2651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 18 May 2026 18:13:25 +0200 Subject: [PATCH 55/58] Reading test --- src/IO/ADIOS/ADIOS2File.cpp | 7 ++++- test/Files_Core/custom_hierarchy.cpp | 46 +++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 0260836136..1d8a325964 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -1356,7 +1356,12 @@ static std::vector availableAttributesOrVariablesPrefixed( { if (auxiliary::starts_with(it->first, var)) { - ret.emplace_back(auxiliary::replace_first(it->first, var, "")); + if (it->first.size() > var.size()) + { + ret.emplace_back(auxiliary::replace_first(it->first, var, "")); + } + // else we have just found the prefix itself. + // might happen if prefix == "/" } else { diff --git a/test/Files_Core/custom_hierarchy.cpp b/test/Files_Core/custom_hierarchy.cpp index b762cb1770..4bc54301c2 100644 --- a/test/Files_Core/custom_hierarchy.cpp +++ b/test/Files_Core/custom_hierarchy.cpp @@ -36,9 +36,10 @@ namespace custom_hierarchy using namespace openPMD; void write( - char const *filename, std::string const &json_params, IterationEncoding ie) + std::string const &filename, + std::string const &json_params, + IterationEncoding) { - std::cout << json_params << std::endl; Series series(filename, Access::CREATE_LINEAR, json_params); auto add_custom_hierarchy = [](Attributable &attr) { attr.customHierarchies()["rabimmel"].setAttribute("rabammel", "rabumm"); @@ -50,6 +51,27 @@ void write( iteration.close(); } +void read( + std::string const &filename, + std::string const &json_params, + IterationEncoding) +{ + Series series(filename, Access::READ_LINEAR, json_params); + auto require_custom_hierarchy = [](Attributable &attr) { + auto ch = attr.customHierarchies(); + REQUIRE(ch.find("rabimmel") == ch.end()); + ch.read(0); + REQUIRE( + ch["rabimmel"].getAttribute("rabammel").get() == + "rabumm"); + }; + auto iteration = series.snapshots()[0]; + + require_custom_hierarchy(series); + require_custom_hierarchy(iteration); + iteration.close(); +} + struct test_config { char const *filename; @@ -60,13 +82,13 @@ struct test_config void custom_hierarchy() { test_config configs[] = { - {"../samples/custom_hierarchy/groupbased.%E", + {"groupbased.%E", R"({"iteration_encoding": "group_based"})", IterationEncoding::groupBased}, - {"../samples/custom_hierarchy/filebased_%T.%E", + {"filebased_%T.%E", R"({"iteration_encoding": "file_based"})", IterationEncoding::fileBased}, - {"../samples/custom_hierarchy/variablebased.%E", + {"variablebased.%E", R"({"iteration_encoding": "variable_based"})", IterationEncoding::variableBased}}; @@ -77,9 +99,19 @@ void custom_hierarchy() auto json_params_ = json::merge( json_params, std::string( - R"({"adios2": {"engine": {"type": "file"}}, "backend": ")") + + R"( + { + "adios2": { + "engine": { + "type": "file" + } + }, + "backend": ")") + backend + R"("})"); - write(filename, json_params_, encoding); + auto filename_ = std::string("../samples/custom_hierarchy/") + + backend + "/" + filename; + write(filename_, json_params_, encoding); + read(filename_, json_params_, encoding); } } } From 52ff8dc859d879392e0c317fdd72e2cc7a81d64b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 18 May 2026 19:04:55 +0200 Subject: [PATCH 56/58] Add more CustomHierarchyFlush operations --- include/openPMD/backend/ContainerImpl.tpp | 6 +++++- src/Mesh.cpp | 5 +++++ src/ParticleSpecies.cpp | 1 + src/Record.cpp | 6 +++++- src/RecordComponent.cpp | 7 +++++++ src/Series.cpp | 10 ++++++++++ src/backend/Attributable.cpp | 10 ++++++++++ src/backend/PatchRecord.cpp | 8 ++++---- 8 files changed, 47 insertions(+), 6 deletions(-) diff --git a/include/openPMD/backend/ContainerImpl.tpp b/include/openPMD/backend/ContainerImpl.tpp index 6144b050a5..144f9f373d 100644 --- a/include/openPMD/backend/ContainerImpl.tpp +++ b/include/openPMD/backend/ContainerImpl.tpp @@ -359,7 +359,11 @@ auto Container::flush( IOHandler()->enqueue(IOTask(this, pCreate)); } - flushAttributes(flushParams); + customHierarchyFlush(flushParams, /* unset_dirty = */ false); + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } } template diff --git a/src/Mesh.cpp b/src/Mesh.cpp index b4f0c7efab..6294b99bbc 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -451,9 +451,11 @@ void Mesh::flush_impl( } else { + customHierarchyFlush(flushParams, /* unset_dirty = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } + // TODO why is there no unsetDirty operation here? } else { @@ -469,6 +471,8 @@ void Mesh::flush_impl( Parameter pCreate; pCreate.path = name; IOHandler()->enqueue(IOTask(this, pCreate)); + + customHierarchyFlush(flushParams, /* unset_dirty = */ false); for (auto &comp : *this) { comp.second.parent() = &this->writable(); @@ -484,6 +488,7 @@ void Mesh::flush_impl( } else { + customHierarchyFlush(flushParams, /* unset_dirty = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 21cb43ab2e..aa4d295a56 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -182,6 +182,7 @@ void ParticleSpecies::flush( } if (access::readOnly(IOHandler()->m_frontendAccess)) { + customHierarchyFlush(flushParams, /* unset_dirty = */ false); for (auto &record : *this) record.second.flush(record.first, flushParams); for (auto &patch : particlePatches) diff --git a/src/Record.cpp b/src/Record.cpp index 8f7a7736a1..0d646c3504 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -71,9 +71,11 @@ void Record::flush_impl( } else { + customHierarchyFlush(flushParams, /* unset_dirty = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } + // TODO why is there no unsetDirty operation here? } else { @@ -89,6 +91,8 @@ void Record::flush_impl( Parameter pCreate; pCreate.path = name; IOHandler()->enqueue(IOTask(this, pCreate)); + + customHierarchyFlush(flushParams, /* unset_dirty = */ false); for (auto &comp : *this) { comp.second.parent() = getWritable(this); @@ -98,13 +102,13 @@ void Record::flush_impl( } else { - if (scalar()) { T_RecordComponent::flush(name, flushParams); } else { + customHierarchyFlush(flushParams, /* unset_dirty = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index bce3da6aeb..092b8ef3fb 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -433,6 +433,9 @@ void RecordComponent::flush( } if (access::readOnly(IOHandler()->m_frontendAccess)) { + // TODO maybe guard against custom hierarchies on datasets? + // they can be created upon constant components however.. + customHierarchyFlush(flushParams, /* unset_dirty = */ false); while (!rc.m_chunks.empty()) { IOHandler()->enqueue(rc.m_chunks.front()); @@ -552,6 +555,10 @@ void RecordComponent::flush( rc.m_chunks.pop(); } + // TODO maybe guard against custom hierarchies on datasets? + // they can be created upon constant components however.. + customHierarchyFlush(flushParams, /* unset_dirty = */ false); + flushAttributes(flushParams); } determineUnsetDirty(flushParams.flushLevel); diff --git a/src/Series.cpp b/src/Series.cpp index 14529fca2b..0c89101ba6 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1667,6 +1667,11 @@ void Series::flushGorVBased( } } + if (begin == end) + { + customHierarchyFlush(flushParams, /* unset_dirty = */ false); + } + // Phase 3 Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); @@ -1757,6 +1762,11 @@ void Series::flushGorVBased( } } + if (begin == end) + { + customHierarchyFlush(flushParams, /* unset_dirty = */ false); + } + flushAttributes(flushParams); Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index ee1294fda2..78c823beb0 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -22,6 +22,7 @@ #include "openPMD/CustomHierarchy.hpp" #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/RecordComponent.hpp" @@ -197,6 +198,11 @@ void Attributable::customHierarchyFlush( // customHierarchies().printRecursively(); if (!dirtyRecursive()) { + if (!unset_dirty) + { + throw std::runtime_error( + "Control flow error: Should be called upon a dirty object."); + } return; } @@ -431,6 +437,10 @@ template void Attributable::seriesFlush_impl( void Attributable::flushAttributes(internal::FlushParams const &flushParams) { + if (access::readOnly(IOHandler()->m_frontendAccess)) + { + throw std::runtime_error("Control flow error"); + } if (!flush_level::write_attributes(flushParams.flushLevel)) { return; diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 8fe89380e4..dbbdcd402c 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -61,10 +61,10 @@ void PatchRecord::flush_impl( } if (!this->scalar()) { - if (IOHandler()->m_frontendAccess != Access::READ_ONLY) - Container::flush( - path, flushParams); // warning (clang-tidy-10): - // bugprone-parent-virtual-call + Container::flush( + path, flushParams); // warning (clang-tidy-10): + // bugprone-parent-virtual-call + for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } From 3e27cece7dea978f82c8f702d85bb56075eb2b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 18 May 2026 19:19:32 +0200 Subject: [PATCH 57/58] Add series.iterations to test --- src/Series.cpp | 12 +++++++++++- src/backend/Attributable.cpp | 5 ----- test/Files_Core/custom_hierarchy.cpp | 8 +++++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index 0c89101ba6..85f86a6c11 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2360,7 +2360,17 @@ creating new iterations. readableIterations.reserve(pList.paths->size()); for (auto const &it : *pList.paths) { - IterationIndex_t index = std::stoull(it); + IterationIndex_t index; + try + { + index = std::stoull(it); + } + catch (std::exception const &e) + { + std::cerr << "[Warning] Could not parse '" << it + << "' as an Iteration index. Will skip." << std::endl; + continue; + } if (read_only_this_single_iteration.has_value() && index != *read_only_this_single_iteration) { diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 78c823beb0..06e4b7db93 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -198,11 +198,6 @@ void Attributable::customHierarchyFlush( // customHierarchies().printRecursively(); if (!dirtyRecursive()) { - if (!unset_dirty) - { - throw std::runtime_error( - "Control flow error: Should be called upon a dirty object."); - } return; } diff --git a/test/Files_Core/custom_hierarchy.cpp b/test/Files_Core/custom_hierarchy.cpp index 4bc54301c2..250631aaa6 100644 --- a/test/Files_Core/custom_hierarchy.cpp +++ b/test/Files_Core/custom_hierarchy.cpp @@ -41,12 +41,13 @@ void write( IterationEncoding) { Series series(filename, Access::CREATE_LINEAR, json_params); - auto add_custom_hierarchy = [](Attributable &attr) { + auto add_custom_hierarchy = [](auto &&attr) { attr.customHierarchies()["rabimmel"].setAttribute("rabammel", "rabumm"); }; auto iteration = series.snapshots()[0]; add_custom_hierarchy(series); + add_custom_hierarchy(series.snapshots()); add_custom_hierarchy(iteration); iteration.close(); } @@ -57,8 +58,8 @@ void read( IterationEncoding) { Series series(filename, Access::READ_LINEAR, json_params); - auto require_custom_hierarchy = [](Attributable &attr) { - auto ch = attr.customHierarchies(); + auto require_custom_hierarchy = [](auto &&attr) { + CustomHierarchy ch = attr.customHierarchies(); REQUIRE(ch.find("rabimmel") == ch.end()); ch.read(0); REQUIRE( @@ -68,6 +69,7 @@ void read( auto iteration = series.snapshots()[0]; require_custom_hierarchy(series); + require_custom_hierarchy(series.snapshots()); require_custom_hierarchy(iteration); iteration.close(); } From 3922027a2312099745beb13c252659d34e3b8c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 20 May 2026 15:12:16 +0200 Subject: [PATCH 58/58] explain that nonsense lmao --- src/backend/Attributable.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 06e4b7db93..1bf3e3f71e 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -666,6 +666,12 @@ void Attributable::readAttributes(ReadMode mode) void Attributable::preferCurrentBackpointer() const { + /* + * This is called when reopening some object as a specific type (e.g. + * RecordComponent) that had originally been opened generically as + * CustomHierarchy already. In this case, the specific type's pointer should + * be preferred for Writable::attributable as it has more information. + */ auto this_as_custom_hierarchy = dynamic_cast(this); if (this_as_custom_hierarchy) { @@ -682,13 +688,21 @@ void Attributable::preferCurrentBackpointer() const return; } + // Now: + // !this_as_custom_hierarchy && backpointer_as_custom_hierarchy + w.attributable = m_attri.get(); + // Now: + // !this_as_custom_hierarchy && !backpointer_as_custom_hierarchy + if (!w.parent) { throw error::Internal( "CustomHierarchy object was created without parent. Why?"); } + + // dont manage this as a customhierarchy instance auto count_of_erased_elements = (*w.parent->attributable) ->m_children_managed_as_custom_hierarchy.erase(