diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 94edc99565..f2d7c42218 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -303,6 +303,7 @@ jobs: find ../../../build/samples/ \ ! -path '*thetaMode*' \ ! -path '*/many_iterations/*' \ + ! -path '*/custom_hierarchy/*' \ ! -name 'profiling.json' \ ! -name '*config.json' \ ! -path '*/file_state_lifetime/*' \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 42ac7e8723..cb542d5e64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -402,6 +402,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) set(CORE_SOURCE src/config.cpp src/ChunkInfo.cpp + src/CustomHierarchy.cpp src/Dataset.cpp src/Datatype.cpp src/Error.cpp @@ -429,7 +430,10 @@ set(CORE_SOURCE src/backend/Attribute.cpp src/backend/BaseRecord.cpp src/backend/BaseRecordComponent.cpp + src/backend/ChildrenCache.cpp + src/backend/ChildView.cpp src/backend/Container.cpp + src/backend/HierarchyVisitor.cpp src/backend/MeshRecordComponent.cpp src/backend/PatchRecord.cpp src/backend/PatchRecordComponent.cpp @@ -869,9 +873,11 @@ if(openPMD_BUILD_TESTING) list(APPEND ${out_list} test/Files_Core/automatic_variable_encoding.cpp test/Files_Core/read_nonexistent_attribute.cpp + test/Files_Core/custom_hierarchy.cpp ) elseif(${test_name} STREQUAL "Auxiliary") list(APPEND ${out_list} + test/Files_Auxiliary/children_cache.cpp test/Files_Auxiliary/drain_set.cpp ) endif() diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp new file mode 100644 index 0000000000..ccb852c2b0 --- /dev/null +++ b/include/openPMD/CustomHierarchy.hpp @@ -0,0 +1,257 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/auxiliary/Export.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Container.hpp" +#include "openPMD/backend/Writable.hpp" + +#include +#include +#include +#include + +namespace openPMD +{ +class CustomHierarchy; +namespace internal +{ + struct CustomHierarchyData : public AttributableData + { + bool has_been_read = false; + }; +} // namespace internal + +class CustomDataset : public RecordComponent +{ + template + friend class Container; + template + friend class ConvertibleContainer; + friend class CustomHierarchy; + friend class Attributable; + +private: + CustomDataset() = default; +}; + +/* + * This is its own class, so the return value of asContainerOf() is also + * convsertible again. + */ +template +class ConvertibleContainer : public Container +{ + template + friend class ConvertibleContainer; + friend class CustomHierarchy; + +protected: + using Container_t = Container; + + using Container_t::Container_t; + +private: + explicit ConvertibleContainer() = default; + ConvertibleContainer(Attributable::NoInit) + : Container_t(Attributable::NoInit{}) + {} + +public: + [[nodiscard]] auto isDataset() const -> bool; + auto asDataset() -> CustomDataset; + + auto datasets() -> ConvertibleContainer; + auto subgroups() -> ConvertibleContainer; +}; + +namespace traits +{ + template <> + struct ElementAccessPolicy + { + static void call(CustomHierarchy &); + static void call(CustomHierarchy const &); + }; + + namespace detail + { + template + void emplace_object_as_customly_managed(Container &, Iterator &it) + { + auto &writable = it->second.writable(); + + // If this frontend instance is the first one for its backend + // object, the reference graph (Writable::attributable -> frontend + // data -> SharedAttributableData) is rooted at it. The typed cache + // now owns that instance, so only record that the object was + // created through the custom hierarchy. + auto backpointer = writable.attributable; + auto emplaced_pointer = it->second.m_attri.get(); + if (backpointer == emplaced_pointer) + { + (**it->second.m_attri).managed_as_custom_hierarchy = true; + } + } + } // namespace detail + + template <> + struct GenerationPolicy + { + constexpr static bool is_noop = false; + template + void operator()(Container &cont, Iterator &it) + { + detail::emplace_object_as_customly_managed( + cont, it); + } + }; + + template <> + struct GenerationPolicy + { + constexpr static bool is_noop = false; + template + void operator()(Container &cont, Iterator &it) + { + detail::emplace_object_as_customly_managed( + cont, it); + } + }; +} // namespace traits + +class CustomHierarchy : public ConvertibleContainer +{ + friend class Iteration; + friend class Container; + friend class Attributable; + friend struct traits::ElementAccessPolicy; + template + friend class Container; + template + friend class ConvertibleContainer; + template + friend struct internal::ChildMaterializer; + +private: + using Parent_t = ConvertibleContainer; + using Container_t = typename Parent_t::Container_t; + using Data_t = internal::CustomHierarchyData; + + std::shared_ptr m_customHierarchyData; + + inline void setData(std::shared_ptr customHierarchyData) + { + m_customHierarchyData = std::move(customHierarchyData); + ConvertibleContainer::setData(m_customHierarchyData); + } + + inline auto get() -> Data_t & + { + return *m_customHierarchyData; + } + + [[nodiscard]] inline auto get() const -> Data_t const & + { + return *m_customHierarchyData; + } + +protected: + CustomHierarchy(); + CustomHierarchy(NoInit); + CustomHierarchy(std::shared_ptr other); + CustomHierarchy(Attributable const &other); + + void flush_internal( + internal::FlushParams const &, std::vector currentPath); + void flush(std::string const &path, internal::FlushParams const &); + + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w) override; + +public: + CustomHierarchy(CustomHierarchy const &other) = default; + CustomHierarchy(CustomHierarchy &&other) = default; + + CustomHierarchy &operator=(CustomHierarchy const &) = default; + CustomHierarchy &operator=(CustomHierarchy &&) = default; + + // TODO should we automatically read upon returning / instantiating a + // CustomHierarchy object? i.e. upon Attributable::customHierarchy() and + // CustomHierarchy::operator[](). + // + // set max_recursion_depth = 0 for infinite cycling + // recursion depth includes the current object + // recursion will not continue expanding into regions that are already known + // (hence not transitively expand into unknown subregions of known regions) + auto read(size_t max_recursion_depth = 1) -> CustomHierarchy &; + + void printRecursively(); + +private: + void printRecursively(std::string indent); + +protected: + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; +}; +namespace internal +{ + template <> + struct ChildMaterializer + { + static auto materialize( + std::string const &, + std::shared_ptr const &child) + -> std::optional + { + return CustomHierarchy(child); + } + }; + + template <> + struct ChildMaterializer + { + static auto materialize( + std::string const &, + std::shared_ptr const &child) + -> std::optional + { + CustomHierarchy custom{child}; + if (!custom.isDataset()) + { + return std::nullopt; + } + return custom.asDataset(); + } + }; +} // namespace internal +} // namespace openPMD diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index f1a4fd436b..e959e6db66 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -153,14 +153,14 @@ namespace internal class Meshes : public Container { -public: - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; +protected: + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; class Particles : public Container { -public: - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; +protected: + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; /** @brief Logical compilation of data from one snapshot (e.g. a single @@ -301,8 +301,6 @@ class Iteration [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool closedByWriter() const; - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - Meshes meshes{}; Particles particles{}; @@ -478,6 +476,8 @@ class Iteration protected: void scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) override; + + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; // Iteration namespace traits @@ -486,8 +486,8 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = false; - template - void operator()(Iterator &it) + template + void operator()(Container &, Iterator &it) { it->second.get().m_iterationIndex = it->first; } @@ -544,7 +544,7 @@ class IndexedIteration : public Iteration class Iterations : public Container { -public: - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; +protected: + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; } // namespace openPMD diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index bf8ad794fb..3cbcb3e53b 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -327,8 +327,6 @@ class Mesh : public BaseRecord typename = std::enable_if_t::value>> Mesh &setTimeOffset(T timeOffset); - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - private: Mesh(); @@ -340,6 +338,8 @@ class Mesh : public BaseRecord protected: void scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) override; + + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; // Mesh template diff --git a/include/openPMD/ParticlePatches.hpp b/include/openPMD/ParticlePatches.hpp index ffe539d5f1..32c7f29b4b 100644 --- a/include/openPMD/ParticlePatches.hpp +++ b/include/openPMD/ParticlePatches.hpp @@ -38,11 +38,12 @@ class ParticlePatches : public Container size_t numPatches() const; ~ParticlePatches() override = default; - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - private: ParticlePatches() = default; void read(); + +protected: + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; // ParticlePatches } // namespace openPMD diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index df6024db31..f4b604b8f1 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -48,24 +48,24 @@ class ParticleSpecies public: ParticlePatches particlePatches; - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - private: ParticleSpecies(); void read(); - void flush(std::string const &, internal::FlushParams const &) override; + void flush(std::string const &, internal::FlushParams const &); - using Data_t = Container::ContainerData; + using Data_t = internal::AttributableData; inline std::shared_ptr getShared() { - return m_containerData; + return m_attri; } protected: void scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) override; + + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; namespace traits @@ -74,8 +74,8 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = false; - template - void operator()(T &it) + template + void operator()(Container &, T &it) { it->second.particlePatches.linkHierarchy(it->second.writable()); } diff --git a/include/openPMD/Record.hpp b/include/openPMD/Record.hpp index af397925dc..f9bf79f974 100644 --- a/include/openPMD/Record.hpp +++ b/include/openPMD/Record.hpp @@ -50,8 +50,6 @@ class Record : public BaseRecord template Record &setTimeOffset(T); - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - private: Record(); @@ -63,6 +61,8 @@ class Record : public BaseRecord protected: void scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) override; + + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; // Record template diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index e8052d2f3e..b274d6f94b 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -72,38 +72,41 @@ namespace internal RecordComponentData &operator=(RecordComponentData const &) = delete; RecordComponentData &operator=(RecordComponentData &&) = delete; - /** - * Chunk reading/writing requests on the contained dataset. - */ - std::queue m_chunks; + [[nodiscard]] auto constantValue() const -> auto const & + { + return dataset_meta()->m_constantValue; + } + auto constantValue() -> auto & + { + return dataset_meta()->m_constantValue; + } + [[nodiscard]] auto isEmpty() const -> auto const & + { + return dataset_meta()->m_isEmpty; + } + auto isEmpty() -> auto & + { + return dataset_meta()->m_isEmpty; + } + [[nodiscard]] auto hasBeenExtended() const -> auto const & + { + return dataset_meta()->m_hasBeenExtended; + } + auto hasBeenExtended() -> auto & + { + return dataset_meta()->m_hasBeenExtended; + } - void push_chunk(IOTask &&task); - /** - * Stores the value for constant record components. - * Ignored otherwise. - */ - Attribute m_constantValue{-1}; - /** - * True if this component is an empty dataset, i.e. its extent is zero - * in at least one dimension. - * Treated by the openPMD-api as a special case of constant record - * components. - */ - bool m_isEmpty = false; - /** - * User has extended the dataset, but the EXTEND task must yet be - * flushed to the backend - */ - bool m_hasBeenExtended = false; - - void reset() override + [[nodiscard]] auto chunks() const -> auto const & + { + return dataset_meta()->m_chunks; + } + auto chunks() -> auto & { - BaseRecordComponentData::reset(); - m_chunks = std::queue(); - m_constantValue = -1; - m_isEmpty = false; - m_hasBeenExtended = false; + return dataset_meta()->m_chunks; } + + void push_chunk(IOTask &&task); }; template class BaseRecordData; @@ -134,6 +137,9 @@ class RecordComponent friend T &internal::makeOwning(T &self, Series_type); friend class internal::ScientificDefaults; friend class Attributable; + template + friend class ConvertibleContainer; + friend class CustomHierarchy; public: enum class Allocation @@ -488,14 +494,14 @@ class RecordComponent auto visit(Args &&...args) -> decltype(Visitor::template call( std::declval(), std::forward(args)...)); - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - static constexpr char const *const SCALAR = "\vScalar"; protected: void flush(std::string const &, internal::FlushParams const &); void read(); + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; + private: /** * Internal method to be called by all methods that create an empty dataset. diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 951a520cfb..0e8c157751 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -96,7 +96,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) * so we might need to do it now. */ auto &rc = get(); - if (!rc.m_dataset.has_value()) + if (!rc.dataset().has_value()) { throw error::WrongAPIUsage( "[RecordComponent] Must specify dataset type and extent before " @@ -134,7 +134,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) } if (!this->written()) { - Parameter dCreate(rc.m_dataset.value()); + Parameter dCreate(rc.dataset().value()); dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 33ee1e9a88..eb1a9e8550 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -27,6 +27,7 @@ #include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Streaming.hpp" +#include "openPMD/auxiliary/Export.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" @@ -453,6 +454,11 @@ class Series : public Attributable * basePath. */ std::string meshesPath() const; + /** + * @return String representing the path to mesh records, relative(!) to + * basePath. + */ + std::optional meshesPathOptional() const; /** Set the path to mesh * records, relative(!) to basePath. @@ -508,6 +514,11 @@ class Series : public Attributable * basePath. */ std::string particlesPath() const; + /** + * @return String representing the path to particle species, relative(!) to + * basePath. + */ + std::optional particlesPathOptional() const; /** Set the path to groups for each particle * species, relative(!) to basePath. @@ -786,8 +797,6 @@ class Series : public Attributable [[nodiscard]] bool closed() const; - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - /** * This overrides Attributable::iterationFlush() which will fail on Series. */ @@ -1010,6 +1019,9 @@ OPENPMD_private Series &setIterationEncoding_internal( IterationEncoding iterationEncoding, internal::default_or_explicit); + Series &setParticlesPath_internal(std::string const &particlesPath); + Series &setMeshesPath_internal(std::string const &meshesPath); + /* * Returns the current content of the /data/snapshot attribute. * (We could also add this to the public API some time) @@ -1038,6 +1050,9 @@ OPENPMD_private [[nodiscard]] bool randomAccessSteps() const; std::vector availableDatasets(); + +protected: + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; // Series namespace debug diff --git a/include/openPMD/auxiliary/Export.hpp b/include/openPMD/auxiliary/Export.hpp index 394397a092..1ab25a077c 100644 --- a/include/openPMD/auxiliary/Export.hpp +++ b/include/openPMD/auxiliary/Export.hpp @@ -22,7 +22,15 @@ #ifndef OPENPMDAPI_EXPORT #ifdef _MSC_VER +// openPMD_EXPORTS is defined by CMake while building the library. Consumers +// (including the tests) must not see dllexport on these symbols: MSVC would +// otherwise expect the annotated non-inline members to be defined in the +// consuming translation unit. +#ifdef openPMD_EXPORTS #define OPENPMDAPI_EXPORT __declspec(dllexport) +#else +#define OPENPMDAPI_EXPORT +#endif #elif defined(__NVCC__) #define OPENPMDAPI_EXPORT #else diff --git a/include/openPMD/auxiliary/MonadicOperations.hpp b/include/openPMD/auxiliary/MonadicOperations.hpp new file mode 100644 index 0000000000..c96f682bc8 --- /dev/null +++ b/include/openPMD/auxiliary/MonadicOperations.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace openPMD::auxiliary +{ +template +auto optional_and_then(Optional &&arg, F &&then) + -> std::invoke_result_t())> +{ + if (!arg.has_value()) + { + return std::nullopt; + } + return std::forward(then)(*std::forward(arg)); +} +} // namespace openPMD::auxiliary diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index 8d617aad3f..b395c199e1 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -163,6 +163,9 @@ using ScalarType_t = typename detail::ScalarType::type; template using VectorType_t = std::vector>; +template +using dependent_const = std::conditional_t; + /** Emulate in the C++ concept ContiguousContainer * * Users can implement this trait for a type to signal it can be used as diff --git a/include/openPMD/auxiliary/Variant.hpp b/include/openPMD/auxiliary/Variant.hpp index 988076b94c..50bc23a0e2 100644 --- a/include/openPMD/auxiliary/Variant.hpp +++ b/include/openPMD/auxiliary/Variant.hpp @@ -97,6 +97,8 @@ namespace auxiliary */ [[nodiscard]] size_t index() const; + void print() const; + T_DTYPES dtype; private: diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index b69e4e7a46..4032d640c9 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -23,6 +23,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/ThrowError.hpp" +#include "openPMD/auxiliary/Export.hpp" #include "openPMD/auxiliary/OutOfRangeMsg.hpp" #include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" @@ -47,6 +48,13 @@ namespace traits { template struct GenerationPolicy; + template + struct ElementAccessPolicy; + namespace detail + { + template + void emplace_object_as_customly_managed(Container &cont, Iterator &it); + } } // namespace traits class AbstractFilePosition; class Attributable; @@ -57,9 +65,12 @@ namespace internal { class IterationData; class SeriesData; + template + class ChildView; struct HomogenizeExtents; struct ConfigAttribute; class ScientificDefaults; + class AttributableData; class SharedAttributableData { @@ -87,6 +98,18 @@ namespace internal * The attributes defined by this Attributable. */ A_MAP m_attributes; + + /** + * True if the frontend instance owning Writable::attributable was + * created through a custom-hierarchy view and has not yet been + * replaced by a regular typed view. + * + * This cannot be inferred from the typed caches: a cache can contain + * either a custom-created handle or an alias materialized from the + * regular hierarchy. The flag distinguishes those cases for + * custom-hierarchy flush/visit and for preferCurrentBackpointer(). + */ + bool managed_as_custom_hierarchy = false; }; /* @@ -112,6 +135,8 @@ namespace internal using SharedData_t = std::shared_ptr; using A_MAP = SharedData_t::element_type::A_MAP; + using parent_t = std::shared_ptr; + friend class openPMD::CustomHierarchy; public: AttributableData(); @@ -120,12 +145,24 @@ namespace internal AttributableData(AttributableData &&) = delete; virtual ~AttributableData() = default; + inline auto asSharedPtrOfAttributable() + -> std::shared_ptr & + { + return *this; + } + + [[nodiscard]] inline auto asSharedPtrOfAttributable() const + -> std::shared_ptr const & + { + return *this; + } + AttributableData &operator=(AttributableData const &) = delete; AttributableData &operator=(AttributableData &&) = delete; // Make copies explicit, only to be used under the conditions described // above - void cloneFrom(AttributableData const &other); + void cloneFrom(parent_t const &other); template T asInternalCopyOf() @@ -244,11 +281,19 @@ class Attributable template friend T &internal::makeOwning(T &self, Series_type); friend class StatefulSnapshotsContainer; + template + friend class internal::ChildView; friend class internal::AttributableData; friend class Snapshots; friend struct internal::HomogenizeExtents; friend struct internal::ConfigAttribute; friend class internal::ScientificDefaults; + friend class CustomHierarchy; + template + friend struct traits::ElementAccessPolicy; + template + friend void traits::detail::emplace_object_as_customly_managed( + Container &cont, Iterator &it); protected: // tag for internal constructor @@ -294,6 +339,14 @@ class Attributable */ Attribute getAttribute(std::string const &key) const; + /** Retrieve value of Attribute stored with provided key. + * + * @param key Key (i.e. name) of the Attribute to retrieve value for. + * @return If found, the stored Attribute in Variant form. Otherwise, an + * empty std::optional. + */ + std::optional getAttributeOptional(std::string const &key) const; + /** Remove Attribute of provided value both logically and physically. * * @param key Key (i.e. name) of the Attribute to remove. @@ -418,7 +471,7 @@ class Attributable * @param visitor Operations to run for each object. * @param recursive Extend the operation recursively to children. */ - virtual void visitHierarchy(HierarchyVisitor &visitor, bool recursive); + void visitHierarchy(HierarchyVisitor &visitor, bool recursive); /** * Visitor pattern for the openPMD object hierarchy in postfix traversal, @@ -464,6 +517,18 @@ class Attributable */ [[nodiscard]] uintptr_t memoryID() const; + /** Return a generic view of this object's hierarchy. + * + * The returned view is lazy: children are read from the backend and + * represented by frontend handles only when they are accessed. + * + * Children created or erased through the returned view affect the regular + * openPMD hierarchy. Handles created through this view are kept alive by + * the containing backend group, independently of the lifetime of the + * returned handle. + */ + auto customHierarchies() -> CustomHierarchy; + // clang-format off OPENPMD_protected // clang-format on @@ -489,6 +554,9 @@ OPENPMD_protected void flushAttributes(internal::FlushParams const &); + void customHierarchyFlush( + internal::FlushParams const &, bool managed_as_custom_object); + enum ReadMode { /** @@ -589,6 +657,8 @@ OPENPMD_protected return (*m_attri)->m_writable; } + void preferCurrentBackpointer() const; + inline void setData(std::shared_ptr attri) { m_attri = std::move(attri); @@ -699,6 +769,9 @@ OPENPMD_protected */ void setWritten(bool val, EnqueueAsynchronously); +protected: + virtual void visitHierarchyImpl(HierarchyVisitor &visitor, bool recursive); + private: /** * @brief Link with parent. @@ -765,6 +838,6 @@ Attributable::readVectorFloatingpoint(std::string const &key) const std::is_floating_point::value, "Type of attribute must be floating point"); - return getAttribute(key).get >(); + return getAttribute(key).get>(); } } // namespace openPMD diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 3acdb04b86..fd5c6ebbb0 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -49,7 +49,7 @@ namespace internal */ typename T_RecordComponentData = typename T_elem::Data_t> class BaseRecordData final - : public ContainerData + : virtual public AttributableData , public T_RecordComponentData { using T_RecordComponent = T_elem; @@ -248,22 +248,18 @@ class BaseRecord using pointer = typename Container::pointer; using const_pointer = typename Container::const_pointer; - using iterator = internal::ScalarIterator< - T_Self, - Data_t, - typename T_Container::InternalContainer::iterator>; + using iterator = internal:: + ScalarIterator; using const_iterator = internal::ScalarIterator< T_Self const, Data_t const, - typename T_Container::InternalContainer::const_iterator>; - using reverse_iterator = internal::ScalarIterator< - T_Self, - Data_t, - typename T_Container::InternalContainer::reverse_iterator>; + typename T_Container::const_iterator>; + using reverse_iterator = internal:: + ScalarIterator; using const_reverse_iterator = internal::ScalarIterator< T_Self const, Data_t const, - typename T_Container::InternalContainer::const_reverse_iterator>; + typename T_Container::const_reverse_iterator>; private: template @@ -322,7 +318,6 @@ class BaseRecord template void insert(InputIt first, InputIt last); void insert(std::initializer_list ilist); - void swap(BaseRecord &other) noexcept; bool contains(key_type const &key) const; template auto emplace(Args &&...args) -> std::pair; @@ -352,7 +347,7 @@ class BaseRecord void setDatasetDefined(BaseRecordComponent::Data_t &data) override { - if (!T_Container::empty()) + if (this->writable().objectType.isGroup() && !T_Container::empty()) { throw error::WrongAPIUsage( "A scalar component can not be contained at the same time as " @@ -368,7 +363,7 @@ class BaseRecord bool scalar() const; private: - void flush(std::string const &, internal::FlushParams const &) final; + void flush(std::string const &, internal::FlushParams const &); virtual void flush_impl(std::string const &, internal::FlushParams const &) = 0; @@ -382,7 +377,7 @@ class BaseRecord namespace detail { constexpr char const *const NO_SCALAR_INSERT = - "[BaseRecord] emplace()/insert()/swap() API invalid for scalar " + "[BaseRecord] emplace()/insert() API invalid for scalar " "records. Use the Record directly as a RecordComponent."; template @@ -404,15 +399,28 @@ namespace detail constexpr bool IsBaseRecord_v = IsBaseRecord::value; } // namespace detail +template +template +auto BaseRecord::insert(InputIt first, InputIt last) -> void +{ + detail::verifyNonscalar(this); + T_Container::insert(first, last); + /* + * We skip checking for the SCALAR key as that would change the runtime of + * this call from O(last-first) to O(container().size()). + */ +} + template template auto BaseRecord::emplace(Args &&...args) -> std::pair { detail::verifyNonscalar(this); - auto res = this->container().emplace(std::forward(args)...); + auto res = T_Container::emplace(std::forward(args)...); if (res.first->first == RecordComponent::SCALAR) { - this->container().erase(res.first); + auto const key = res.first->first; + this->erase_unchecked(key); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; diff --git a/include/openPMD/backend/BaseRecordComponent.hpp b/include/openPMD/backend/BaseRecordComponent.hpp index af23e08468..9b34d12fc6 100644 --- a/include/openPMD/backend/BaseRecordComponent.hpp +++ b/include/openPMD/backend/BaseRecordComponent.hpp @@ -24,6 +24,7 @@ #include "openPMD/Dataset.hpp" #include "openPMD/Error.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Writable.hpp" #include @@ -39,38 +40,76 @@ namespace internal class BaseRecordComponentData : virtual public AttributableData { public: - /** - * The type and extent of the dataset defined by this component. - */ - std::optional m_dataset; - /** - * True if this is defined as a constant record component as specified - * in the openPMD standard. - * If yes, then no heavy-weight dataset is created and the dataset is - * instead defined via light-weight attributes. - */ - bool m_isConstant = false; - /** - * Tracks if there was any write access to the record component. - * Necessary in BaseRecord to track if the scalar component has been - * used and is used by BaseRecord to determine the return value of - * the BaseRecord::scalar() method. - */ - bool m_datasetDefined = false; - BaseRecordComponentData(BaseRecordComponentData const &) = delete; BaseRecordComponentData(BaseRecordComponentData &&) = delete; BaseRecordComponentData & operator=(BaseRecordComponentData const &) = delete; BaseRecordComponentData &operator=(BaseRecordComponentData &&) = delete; + internal::object_type::DatasetMetaData *m_dataset_meta_ptr = nullptr; + + void setDatasetDefined() + { + m_dataset_meta_ptr = (**this).m_writable.objectType.initDataset(); + } + BaseRecordComponentData() = default; + auto dataset_meta() const + -> internal::object_type::DatasetMetaData const * + { + if (!m_dataset_meta_ptr) + { + throw error::WrongAPIUsage( + "Requesting dataset properties on an object that has no " + "dataset defined."); + } + return m_dataset_meta_ptr; + } + auto dataset_meta() -> internal::object_type::DatasetMetaData * + { + return const_cast( + static_cast(this) + ->dataset_meta()); + } + + [[nodiscard]] inline auto dataset() const -> auto const & + { + return dataset_meta()->m_dataset; + } + inline auto dataset() -> auto & + { + return dataset_meta()->m_dataset; + } + + [[nodiscard]] inline auto isConstant() const -> auto const & + { + return dataset_meta()->m_isConstant; + } + inline auto isConstant() -> auto & + { + return dataset_meta()->m_isConstant; + } + + [[nodiscard]] inline auto datasetDefined() const -> bool + { + return (**this).m_writable.objectType.isDataset(); + } + virtual void reset() { - m_dataset = std::nullopt; - m_isConstant = false; - m_datasetDefined = false; + (**this).m_writable.objectType.initGroup(); + m_dataset_meta_ptr = nullptr; + } + + template + void cloneFrom(Arg &&arg) + { + AttributableData::cloneFrom(std::forward(arg)); + if (datasetDefined()) + { + setDatasetDefined(); + } } }; } // namespace internal @@ -172,11 +211,14 @@ class BaseRecordComponent : virtual public Attributable inline Data_t const &get() const { + // cannot call this in the const overload + // setDatasetDefined(*m_baseRecordComponentData); return *m_baseRecordComponentData; } inline Data_t &get() { + setDatasetDefined(*m_baseRecordComponentData); return *m_baseRecordComponentData; } diff --git a/include/openPMD/backend/ChildView.hpp b/include/openPMD/backend/ChildView.hpp new file mode 100644 index 0000000000..d32be63309 --- /dev/null +++ b/include/openPMD/backend/ChildView.hpp @@ -0,0 +1,344 @@ +/* Copyright 2026 The openPMD Community + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/Error.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/ChildrenCache.hpp" +#include "openPMD/backend/Writable.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace openPMD::internal +{ +class ChildViewIteratorTest; + +/** Traversal filter shared by all projection views of a group. */ +struct ChildFilter +{ + enum class Kind + { + All, + Groups, + Datasets + }; + + Kind kind = Kind::All; + // Non-owning; the referenced string must outlive the view and its + // iterators. The only current use is a string literal. + std::string_view excludedKey; + + auto + accepts(std::string const &key, SharedAttributableData const &child) const + -> bool + { + if (!excludedKey.empty() && key == excludedKey) + { + return false; + } + switch (kind) + { + case Kind::All: + return true; + case Kind::Groups: + return child.m_writable.objectType.isGroup(); + case Kind::Datasets: + return child.m_writable.objectType.isDataset(); + } + return false; + } +}; + +// Numeric keys are the materialized special case (Series::iterations). +template +inline constexpr bool non_string_key_v = !std::is_same_v; + +inline auto default_child_filter() -> ChildFilter +{ + return {}; +} + +/** Materialize a backend child as a typed frontend handle. + * + * The primary template has no defined conversion and throws. View types with a + * defined conversion specialize this. + */ +template +struct ChildMaterializer +{ + static auto materialize( + std::string const &, std::shared_ptr const &) + -> std::optional + { + throw error::WrongAPIUsage( + "Cannot materialize a backend child as this view type. Only " + "CustomHierarchy and CustomDataset have defined backend-to-" + "frontend materializations."); + } +}; + +template +auto materialize_child_at( + ChildrenCache const &cache, + std::string const &canonicalKey, + std::shared_ptr const &child, + bool access_policy = true) -> typename std::map::iterator; + +/** Const-independent state shared by mutable and const projected iterators. */ +template +struct ProjectedChildIteratorMembers +{ + using children_iterator = + typename ChildrenCache::children_map_t::const_iterator; + + ChildrenCache const *cache = nullptr; + children_iterator iterator; + children_iterator end; + ChildFilter filter; + bool access_policy = true; +}; + +template +class ProjectedChildIterator +{ +public: + using children_map_t = ChildrenCache::children_map_t; + // Const-independent: both mutable and const projected iterators traverse + // the authoritative children map read-only. + using children_iterator = typename children_map_t::const_iterator; + using typed_map = std::map; + using typed_iterator = std::conditional_t< + Const, + typename typed_map::const_iterator, + typename typed_map::iterator>; + + using value_type = typename typed_map::value_type; + using reference = + std::conditional_t; + using pointer = std::conditional_t; + using difference_type = std::ptrdiff_t; + using iterator_category = std::bidirectional_iterator_tag; + +private: + using Members = ProjectedChildIteratorMembers; + + template + friend class ProjectedChildIterator; + +public: + ProjectedChildIterator() = default; + + // Convert a mutable iterator to a const iterator. This is safe because + // both only reference the authoritative children map and the shared, + // logically mutable cache; the constness is enforced on the returned + // reference. + template + ProjectedChildIterator( + ProjectedChildIterator const &other, + std::enable_if_t = 0) + : m_members(other.m_members) + {} + + ProjectedChildIterator( + ChildrenCache const *cache, + children_iterator iterator, + children_iterator end, + ChildFilter filter, + bool access_policy); + + auto key() const -> std::string const &; + + auto operator*() const -> reference; + + auto operator->() const -> pointer; + + auto operator++() -> ProjectedChildIterator &; + auto operator++(int) -> ProjectedChildIterator; + + auto operator--() -> ProjectedChildIterator &; + auto operator--(int) -> ProjectedChildIterator; + + auto operator==(ProjectedChildIterator const &other) const -> bool; + auto operator!=(ProjectedChildIterator const &other) const -> bool; + +private: + void skip_forward(); + void skip_backward(); + + Members m_members; +}; + +/** Stateless, filtered projection of one group's authoritative children. */ +template +class ChildView +{ +public: + using materialized_t = std::true_type; + using projected_t = std::false_type; + static constexpr bool non_string = non_string_key_v; + static constexpr bool is_const = Const; + + using children_map_t = ChildrenCache::children_map_t; + using cache_t = + std::conditional_t; + using InternalContainer = std::map; + using key_type = Key; + using mapped_type = T; + using value_type = typename InternalContainer::value_type; + using size_type = typename InternalContainer::size_type; + using difference_type = typename InternalContainer::difference_type; + using allocator_type = typename InternalContainer::allocator_type; + using reference = + std::conditional_t; + using const_reference = value_type const &; + using pointer = std::conditional_t; + using const_pointer = value_type const *; + + using mut_iterator = std::conditional_t< + non_string_key_v, + typename InternalContainer::iterator, + ProjectedChildIterator>; + using const_iterator = std::conditional_t< + non_string_key_v, + typename InternalContainer::const_iterator, + ProjectedChildIterator>; + using iterator = std::conditional_t; + using mut_reverse_iterator = std::conditional_t< + non_string_key_v, + typename InternalContainer::reverse_iterator, + std::reverse_iterator>; + using const_reverse_iterator = std::conditional_t< + non_string_key_v, + typename InternalContainer::const_reverse_iterator, + std::reverse_iterator>; + using reverse_iterator = + std::conditional_t; + + using CreateChild = T (*)(Key const &, Writable &); + + ChildView( + cache_t *cache, + Writable *self, + ChildFilter const &filter, + CreateChild create_child); + + auto begin() -> iterator; + auto begin() const -> const_iterator; + auto cbegin() const -> const_iterator; + + auto end() -> iterator; + auto end() const -> const_iterator; + auto cend() const -> const_iterator; + + auto rbegin() -> reverse_iterator; + auto rbegin() const -> const_reverse_iterator; + auto crbegin() const -> const_reverse_iterator; + + auto rend() -> reverse_iterator; + auto rend() const -> const_reverse_iterator; + auto crend() const -> const_reverse_iterator; + + auto empty() const -> bool; + auto size() const -> size_type; + + auto find(Key const &key) -> iterator; + auto find(Key const &key) const -> const_iterator; + + auto contains(Key const &key) const -> bool; + auto count(Key const &key) const -> size_type; + + auto at(Key const &key) -> std::conditional_t; + auto at(Key const &key) const -> T const &; + + auto operator[](Key const &key) -> T &; + auto get_or_create(Key const &key, bool access_policy) -> T &; + + auto insert(value_type const &value) -> std::pair; + auto insert(value_type &&value) -> std::pair; + auto insert(const_iterator hint, value_type const &value) -> iterator; + auto insert(const_iterator hint, value_type &&value) -> iterator; + + template + void insert(InputIt first, InputIt last) + { + static_assert(!Const, "Cannot modify a const ChildView."); + for (auto it = first; it != last; ++it) + { + insert(*it); + } + } + + template + auto emplace(Args &&...args) -> std::pair + { + static_assert(!Const, "Cannot modify a const ChildView."); + return insert_impl(value_type(std::forward(args)...)); + } + + auto erase(Key const &key) -> size_type; + auto erase(iterator it) -> iterator; + void clear(); + + auto filter() const -> ChildFilter const &; + void set_access_policy(bool access_policy) noexcept; + +private: + template + auto make_begin() const -> Iterator; + + template + auto make_end() const -> Iterator; + + auto typed() const -> ChildrenCacheMap; + void sync() const; + + auto matches( + std::string const &canonicalKey, + SharedAttributableData const &child) const -> bool; + + void check_writable() const; + + auto insert_impl(value_type const &value) -> std::pair; + auto insert_impl(value_type &&value) -> std::pair; + + cache_t *m_cache; + Writable *m_self; + // References the owning Container's filter and is therefore valid for the + // lifetime of that Container. Iterators copy the (trivial) filter. + std::optional> m_filter; + bool m_access_policy = !non_string; + // Runtime pointer because ChildView is not templated on the container. + // It is empty for const views; get_or_create() asserts it is set. + std::optional m_create_child; +}; +} // namespace openPMD::internal diff --git a/include/openPMD/backend/ChildViewImpl.tpp b/include/openPMD/backend/ChildViewImpl.tpp new file mode 100644 index 0000000000..e48115fe47 --- /dev/null +++ b/include/openPMD/backend/ChildViewImpl.tpp @@ -0,0 +1,693 @@ +/* Copyright 2026 The openPMD Community + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#define OPENPMD_GUARD_HEADER_AGAINST_PUBLIC_INCLUSION + +#include "openPMD/backend/ChildView.hpp" + +namespace openPMD::internal +{ +template +auto materialize_child_at( + ChildrenCache const &cache, + std::string const &canonicalKey, + std::shared_ptr const &child, + bool access_policy) -> typename std::map::iterator +{ + auto typed_cache = cache.template get_or_create_cache(); + auto const &key = + children_cache_detail::key_from_string_or_throw(canonicalKey); + + auto it = typed_cache.find(key); + if (it == typed_cache.end()) + { + auto materialized = + ChildMaterializer::materialize(canonicalKey, child); + assert(materialized.has_value()); + if (!materialized) + { + // A materializer returned "no value" for a child that passed the + // view filter. That is a broken materializer contract, not a user + // error. + throw error::Internal( + "No frontend handle was produced while materializing child '" + + canonicalKey + "'."); + } + auto inserted = cache.template try_emplace( + key, child, std::move(*materialized)); + it = inserted.first; + } + + if (access_policy) + { + // The cache is mutable even for const views; only the returned + // reference is const-qualified. + traits::ElementAccessPolicy::call(it->second); + } + return it; +} + +template +ProjectedChildIterator::ProjectedChildIterator( + ChildrenCache const *cache, + children_iterator iterator, + children_iterator end, + ChildFilter filter, + bool access_policy) + : m_members{cache, iterator, end, std::move(filter), access_policy} +{ + if (m_members.iterator != m_members.end) + { + skip_forward(); + } +} + +template +auto ProjectedChildIterator::key() const -> std::string const & +{ + return m_members.iterator->first; +} + +template +auto ProjectedChildIterator::operator*() const -> reference +{ + auto it = materialize_child_at( + *m_members.cache, + m_members.iterator->first, + m_members.iterator->second, + m_members.access_policy); + return *typed_iterator(it); +} + +template +auto ProjectedChildIterator::operator->() const -> pointer +{ + return &operator*(); +} + +template +auto ProjectedChildIterator::operator++() + -> ProjectedChildIterator & +{ + ++m_members.iterator; + skip_forward(); + return *this; +} + +template +auto ProjectedChildIterator::operator++(int) + -> ProjectedChildIterator +{ + auto res = *this; + ++(*this); + return res; +} + +template +auto ProjectedChildIterator::operator--() + -> ProjectedChildIterator & +{ + --m_members.iterator; + skip_backward(); + return *this; +} + +template +auto ProjectedChildIterator::operator--(int) + -> ProjectedChildIterator +{ + auto res = *this; + --(*this); + return res; +} + +template +auto ProjectedChildIterator::operator==( + ProjectedChildIterator const &other) const -> bool +{ + return m_members.iterator == other.m_members.iterator; +} + +template +auto ProjectedChildIterator::operator!=( + ProjectedChildIterator const &other) const -> bool +{ + return !(*this == other); +} + +template +void ProjectedChildIterator::skip_forward() +{ + while (m_members.iterator != m_members.end && + !m_members.filter.accepts( + m_members.iterator->first, *m_members.iterator->second)) + { + ++m_members.iterator; + } +} + +template +void ProjectedChildIterator::skip_backward() +{ + while (!m_members.filter.accepts( + m_members.iterator->first, *m_members.iterator->second)) + { + --m_members.iterator; + } +} + +template +ChildView::ChildView( + cache_t *cache, + Writable *self, + ChildFilter const &filter, + CreateChild create_child) + : m_cache(cache) + , m_self(self) + , m_filter(std::cref(filter)) + , m_create_child( + create_child ? std::optional(create_child) + : std::nullopt) +{} + +template +auto ChildView::begin() -> iterator +{ + return make_begin(); +} +template +auto ChildView::begin() const -> const_iterator +{ + return make_begin(); +} +template +auto ChildView::cbegin() const -> const_iterator +{ + return begin(); +} +template +auto ChildView::end() -> iterator +{ + return make_end(); +} +template +auto ChildView::end() const -> const_iterator +{ + return make_end(); +} +template +auto ChildView::cend() const -> const_iterator +{ + return end(); +} +template +auto ChildView::rbegin() -> reverse_iterator +{ + if constexpr (non_string) + { + sync(); + if constexpr (Const) + { + auto const typed_view = typed(); + return typed_view.rbegin(); + } + else + { + return typed().rbegin(); + } + } + else + { + return reverse_iterator(end()); + } +} +template +auto ChildView::rbegin() const -> const_reverse_iterator +{ + if constexpr (non_string) + { + sync(); + auto const typed_view = typed(); + return typed_view.rbegin(); + } + else + { + return const_reverse_iterator(end()); + } +} +template +auto ChildView::crbegin() const -> const_reverse_iterator +{ + return rbegin(); +} +template +auto ChildView::rend() -> reverse_iterator +{ + if constexpr (non_string) + { + if constexpr (Const) + { + auto const typed_view = typed(); + return typed_view.rend(); + } + else + { + return typed().rend(); + } + } + else + { + return reverse_iterator(begin()); + } +} +template +auto ChildView::rend() const -> const_reverse_iterator +{ + if constexpr (non_string) + { + auto const typed_view = typed(); + return typed_view.rend(); + } + else + { + return const_reverse_iterator(begin()); + } +} +template +auto ChildView::crend() const -> const_reverse_iterator +{ + return rend(); +} + +template +auto ChildView::empty() const -> bool +{ + auto const &children = m_cache->children(); + for (auto const &[key, child] : children) + { + if (matches(key, *child)) + { + return false; + } + } + return true; +} + +template +auto ChildView::size() const -> size_type +{ + size_type res = 0; + for (auto const &[key, child] : m_cache->children()) + { + (void)child; + if (matches(key, *child)) + { + ++res; + } + } + return res; +} + +template +auto ChildView::find(Key const &key) -> iterator +{ + auto const &canonical = children_cache_detail::key_to_string(key); + auto it = m_cache->children().find(canonical); + if (it == m_cache->children().end() || + !filter().accepts(canonical, *it->second)) + { + return end(); + } + + if constexpr (non_string) + { + auto typed_view = typed(); + auto typed_it = typed_view.find(key); + if (typed_it == typed_view.end()) + { + typed_it = materialize_child_at( + *m_cache, canonical, it->second, m_access_policy); + } + return typed_it; + } + else + { + return iterator( + m_cache, it, m_cache->children().end(), filter(), m_access_policy); + } +} +template +auto ChildView::find(Key const &key) const -> const_iterator +{ + auto const &canonical = children_cache_detail::key_to_string(key); + auto it = m_cache->children().find(canonical); + if (it == m_cache->children().end() || + !filter().accepts(canonical, *it->second)) + { + return end(); + } + + if constexpr (non_string) + { + auto typed_view = typed(); + auto typed_it = typed_view.find(key); + if (typed_it == typed_view.end()) + { + typed_it = materialize_child_at( + *m_cache, canonical, it->second, m_access_policy); + } + return typed_it; + } + else + { + return const_iterator( + m_cache, it, m_cache->children().end(), filter(), m_access_policy); + } +} + +template +auto ChildView::contains(Key const &key) const -> bool +{ + auto const &canonical = children_cache_detail::key_to_string(key); + auto it = m_cache->children().find(canonical); + return it != m_cache->children().end() && + filter().accepts(canonical, *it->second); +} + +template +auto ChildView::count(Key const &key) const -> size_type +{ + return contains(key) ? 1 : 0; +} + +template +auto ChildView::at(Key const &key) + -> std::conditional_t +{ + auto it = find(key); + if (it == end()) + { + throw std::out_of_range( + "ChildView::at(): key not found in the child hierarchy."); + } + return it->second; +} +template +auto ChildView::at(Key const &key) const -> T const & +{ + auto it = find(key); + if (it == end()) + { + throw std::out_of_range( + "ChildView::at(): key not found in the child hierarchy."); + } + return it->second; +} + +template +auto ChildView::operator[](Key const &key) -> T & +{ + static_assert(!Const, "Cannot modify a const ChildView."); + return get_or_create(key, /* access_policy = */ true); +} + +template +auto ChildView::get_or_create(Key const &key, bool access_policy) + -> T & +{ + static_assert(!Const, "Cannot modify a const ChildView."); + auto const &canonical = children_cache_detail::key_to_string(key); + auto &children = m_cache->children(); + auto it = children.find(canonical); + + if (it != children.end()) + { + if (!filter().accepts(canonical, *it->second)) + { + throw error::WrongAPIUsage( + "Cannot access child '" + canonical + + "' through a filtered view."); + } + auto typed_it = materialize_child_at( + *m_cache, canonical, it->second, access_policy); + return typed_it->second; + } + + check_writable(); + assert(m_self != nullptr); + assert(m_create_child.has_value()); + T child = (*m_create_child)(key, *m_self); + auto child_ptr = child.m_attri->asSharedPtrOfAttributable(); + auto inserted = + m_cache->template try_emplace(key, child_ptr, std::move(child)); + assert(inserted.second); + traits::GenerationPolicy generation; + generation(*this, inserted.first); + return inserted.first->second; +} + +template +auto ChildView::insert(value_type const &value) + -> std::pair +{ + static_assert(!Const, "Cannot modify a const ChildView."); + return insert_impl(value); +} +template +auto ChildView::insert(value_type &&value) + -> std::pair +{ + static_assert(!Const, "Cannot modify a const ChildView."); + return insert_impl(std::move(value)); +} +template +auto ChildView::insert( + const_iterator hint, value_type const &value) -> iterator +{ + static_assert(!Const, "Cannot modify a const ChildView."); + (void)hint; + return insert_impl(value).first; +} +template +auto ChildView::insert(const_iterator hint, value_type &&value) + -> iterator +{ + static_assert(!Const, "Cannot modify a const ChildView."); + (void)hint; + return insert_impl(std::move(value)).first; +} + +template +auto ChildView::erase(Key const &key) -> size_type +{ + static_assert(!Const, "Cannot modify a const ChildView."); + auto const &canonical = children_cache_detail::key_to_string(key); + auto it = m_cache->children().find(canonical); + if (it == m_cache->children().end() || + !filter().accepts(canonical, *it->second)) + { + return 0; + } + m_cache->erase(canonical); + return 1; +} + +template +auto ChildView::erase(iterator it) -> iterator +{ + static_assert(!Const, "Cannot modify a const ChildView."); + auto next = it; + ++next; + if constexpr (non_string) + { + m_cache->erase(children_cache_detail::key_to_string(it->first)); + } + else + { + m_cache->erase(it.key()); + } + return next; +} + +template +void ChildView::clear() +{ + static_assert(!Const, "Cannot modify a const ChildView."); + m_cache->clear(); +} + +template +auto ChildView::filter() const -> ChildFilter const & +{ + assert(m_filter.has_value()); + return m_filter->get(); +} + +template +void ChildView::set_access_policy(bool access_policy) noexcept +{ + static_assert(!Const, "Cannot modify a const ChildView."); + assert(!non_string); + m_access_policy = access_policy; +} + +template +template +auto ChildView::make_begin() const -> Iterator +{ + if constexpr (non_string) + { + sync(); + return typed().begin(); + } + else + { + auto it = m_cache->children().begin(); + Iterator res( + m_cache, it, m_cache->children().end(), filter(), m_access_policy); + return res; + } +} + +template +template +auto ChildView::make_end() const -> Iterator +{ + if constexpr (non_string) + { + return typed().end(); + } + else + { + return Iterator( + m_cache, + m_cache->children().end(), + m_cache->children().end(), + filter(), + m_access_policy); + } +} + +template +auto ChildView::typed() const -> ChildrenCacheMap +{ + return m_cache->template get_or_create_cache(); +} + +template +void ChildView::sync() const +{ + if constexpr (non_string) + { + auto typed_view = typed(); + for (auto const &[canonical, child] : m_cache->children()) + { + auto const key = + children_cache_detail::key_from_string(canonical); + if (!key) + { + continue; + } + if (typed_view.find(*key) != typed_view.end()) + { + continue; + } + auto materialized = + ChildMaterializer::materialize(canonical, child); + if (materialized) + { + m_cache->template try_emplace( + *key, child, std::move(*materialized)); + } + } + } +} + +template +auto ChildView::matches( + std::string const &canonicalKey, SharedAttributableData const &child) const + -> bool +{ + if (!filter().accepts(canonicalKey, child)) + { + return false; + } + if constexpr (std::is_same_v) + { + return true; + } + else + { + return children_cache_detail::key_from_string(canonicalKey) + .has_value(); + } +} + +template +void ChildView::check_writable() const +{ + if (m_self == nullptr || !m_self->IOHandler || !*m_self->IOHandler) + { + return; + } + auto const &handler = **m_self->IOHandler; + if (handler && access::readOnly(handler->m_frontendAccess) && + handler->m_seriesStatus != internal::SeriesStatus::Parsing) + { + throw std::out_of_range( + "Cannot create a new child in a read-only Series."); + } +} + +template +auto ChildView::insert_impl(value_type const &value) + -> std::pair +{ + return insert_impl(value_type(value)); +} + +template +auto ChildView::insert_impl(value_type &&value) + -> std::pair +{ + auto const &canonical = children_cache_detail::key_to_string(value.first); + auto const &children = m_cache->children(); + auto child_it = children.find(canonical); + auto child_ptr = value.second.m_attri->asSharedPtrOfAttributable(); + if (child_it != children.end()) + { + child_ptr = child_it->second; + value.second.m_attri->asSharedPtrOfAttributable() = child_ptr; + value.second.preferCurrentBackpointer(); + } + + auto inserted = m_cache->template try_emplace( + value.first, child_ptr, std::move(value).second); + if (!inserted.second) + { + return {find(value.first), false}; + } + traits::GenerationPolicy generation; + generation(*this, inserted.first); + return {find(value.first), true}; +} +} // namespace openPMD::internal diff --git a/include/openPMD/backend/ChildrenCache.hpp b/include/openPMD/backend/ChildrenCache.hpp new file mode 100644 index 0000000000..12e7508f32 --- /dev/null +++ b/include/openPMD/backend/ChildrenCache.hpp @@ -0,0 +1,300 @@ +/* Copyright 2026 The openPMD Community + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/auxiliary/TypeTraits.hpp" +#include "openPMD/backend/ChildrenCacheMap.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace openPMD::internal +{ +class SharedAttributableData; + +/** Stable identifier for a container element type. + * + * Defined in ChildrenCache.cpp for all explicitly instantiated container + * types. Unlike std::type_index, the returned value does not depend on RTTI + * identity across shared-library boundaries. + */ +template +auto container_type_id() -> std::uint8_t; + +namespace children_cache_detail +{ + template + auto key_from_string(std::string const &key) -> std::optional + { + if constexpr (std::is_same_v) + { + static_assert( + !std::is_same_v, + "key_from_string() would copy the key. Use " + "key_from_string_or_throw() instead."); + } + else if constexpr (std::is_integral_v) + { + if (key.empty()) + { + return std::nullopt; + } + std::size_t pos = 0; + try + { + if constexpr (std::is_signed_v) + { + auto const value = std::stoll(key, &pos); + if (pos != key.size()) + { + return std::nullopt; + } + if (value < static_cast( + std::numeric_limits::min()) || + value > static_cast( + std::numeric_limits::max())) + { + return std::nullopt; + } + auto const casted = static_cast(value); + if (std::to_string(casted) != key) + { + return std::nullopt; + } + return casted; + } + else + { + auto const value = std::stoull(key, &pos); + if (pos != key.size()) + { + return std::nullopt; + } + if (value > static_cast( + std::numeric_limits::max())) + { + return std::nullopt; + } + auto const casted = static_cast(value); + if (std::to_string(casted) != key) + { + return std::nullopt; + } + return casted; + } + } + catch (...) + { + return std::nullopt; + } + } + else + { + static_assert( + auxiliary::dependent_false_v, + "ChildrenCache does not know how to convert this key type " + "from its canonical string representation."); + } + } + + /** Parse a canonical key, throwing if it cannot be represented. */ + template + decltype(auto) key_from_string_or_throw(std::string const &key) + { + if constexpr (std::is_same_v) + { + return key; + } + else if constexpr (std::is_integral_v) + { + auto const parsed = key_from_string(key); + if (!parsed) + { + throw std::invalid_argument( + "Cannot convert '" + key + "' to the typed key."); + } + // Return by value: returning *parsed with decltype(auto) would + // deduce Key& and refer to the local optional. + return static_cast(*parsed); + } + else + { + static_assert( + auxiliary::dependent_false_v, + "ChildrenCache does not know how to convert this key type " + "from its canonical string representation."); + } + } + + /** Convert a typed key to the canonical string representation. */ + template + decltype(auto) key_to_string(Key const &key) + { + if constexpr (std::is_same_v) + { + return key; + } + else if constexpr (std::is_integral_v) + { + return std::to_string(key); + } + else + { + static_assert( + auxiliary::dependent_false_v, + "ChildrenCache does not know how to convert this key type " + "to its canonical string representation."); + } + } +} // namespace children_cache_detail + +/** Collection of lazily materialized, type-erased child maps. + * + * This class owns the authoritative child map and the lazily materialized + * typed maps derived from it. The typed maps are projections, not independent + * containers, and structural removal is applied to all of them together. + * + * Invariants: + * - at most one map exists per (T, Key) pair; + * - every materialized map has erase, clear, contains, size, and iteration + * callbacks registered; + * - erase(key) removes key from every materialized map and leaves no stale + * entry behind; + * - clear() empties every materialized map. + * + * ChildrenCache enforces these invariants on insertion, erase(), and clear(), + * and validates them in assert_consistent(). Typed insertion goes through + * try_emplace(), which also maintains the authoritative child map. + * + * Assertions validate these invariants. In particular, assert_consistent() + * verifies that every materialized key can be reconstructed from its canonical + * string representation, since a failed reconstruction would make erase() + * silently leave an owning handle behind. + */ +class ChildrenCache +{ + struct EntryI + { + std::shared_ptr storage; + virtual void erase(std::string const &) = 0; + virtual void clear() = 0; + virtual auto contains(std::string const &) -> bool = 0; + virtual void + for_each_key(std::function const &) = 0; + virtual auto size() -> std::size_t = 0; + virtual ~EntryI() = default; + }; + + template + struct TypedEntry; + + using Entry = std::shared_ptr; + +public: + using children_map_t = + std::map>; + + ChildrenCache() = default; + ChildrenCache(ChildrenCache const &) = delete; + ChildrenCache(ChildrenCache &&) noexcept = default; + ChildrenCache &operator=(ChildrenCache const &) = delete; + ChildrenCache &operator=(ChildrenCache &&) noexcept = default; + + /** Get or create the cache for (T, Key). + * + * Materialization is a cache operation and therefore logically const. + * + * Defined in ChildrenCacheImpl.tpp and explicitly instantiated for all + * container types in ChildrenCache.cpp. Type-specific materialization is + * handled by ChildMaterializer instead. + */ + template + auto get_or_create_cache() const -> ChildrenCacheMap; + + template + [[nodiscard]] auto find_cache() const + -> std::optional>; + + template + [[nodiscard]] auto contains_cache() const noexcept -> bool; + + /** Insert a typed child handle. + * + * The non-const overload also inserts the supplied backend child into the + * authoritative children map if it is not present yet. The const overload + * is used for lazy materialization and requires the child to exist. + */ + template + auto try_emplace( + Key const &key, + std::shared_ptr const &child, + T &&value) -> std::pair::iterator, bool>; + + template + auto try_emplace( + Key const &key, + std::shared_ptr const &child, + T &&value) const + -> std::pair::iterator, bool>; + + children_map_t &children(); + children_map_t const &children() const; + + /** Remove canonicalKey from the authoritative map and every cache. */ + void erase(std::string const &canonicalKey); + + /** Clear the authoritative map and every cache. */ + void clear(); + + [[nodiscard]] bool empty() const noexcept; + [[nodiscard]] std::size_t num_maps() const noexcept; + + /** Validate internal cache invariants. */ + void assert_consistent() const; + +private: + template + auto get_or_create_typed_entry() const -> TypedEntry &; + + /** Authoritative children of this group. */ + children_map_t m_children; + + /** + * Type-erased typed caches, stored once per ChildrenCache instance (i.e. + * once per group node) and shared by all views of that node. The + * ChildrenCacheMap instances handed out by get_or_create_cache() are + * non-owning views and store no state. Keys are container_type_id() + * values, not RTTI-dependent type indices. + */ + mutable std::map m_entries; +}; +} // namespace openPMD::internal diff --git a/include/openPMD/backend/ChildrenCacheImpl.tpp b/include/openPMD/backend/ChildrenCacheImpl.tpp new file mode 100644 index 0000000000..d5ca438314 --- /dev/null +++ b/include/openPMD/backend/ChildrenCacheImpl.tpp @@ -0,0 +1,178 @@ +/* Copyright 2026 The openPMD Community + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#define OPENPMD_GUARD_HEADER_AGAINST_PUBLIC_INCLUSION + +#include "openPMD/backend/ChildrenCache.hpp" + +namespace openPMD::internal +{ +template +struct ChildrenCache::TypedEntry : ChildrenCache::EntryI +{ + using map_type = std::map; + map_type map; + void erase(std::string const &key) override + { + if constexpr (std::is_same_v) + { + map.erase(key); + } + else + { + auto const parsed = + children_cache_detail::key_from_string(key); + // A key that cannot be represented by this map cannot be + // contained in it. assert_consistent() validates the round-trip + // for entries that actually exist. + if (parsed) + { + map.erase(*parsed); + } + } + } + void clear() override + { + map.clear(); + } + auto contains(std::string const &key) -> bool override + { + if constexpr (std::is_same_v) + { + return map.find(key) != map.end(); + } + else + { + auto const parsed = + children_cache_detail::key_from_string(key); + return parsed.has_value() && map.find(*parsed) != map.end(); + } + } + void for_each_key( + std::function const &functor) override + { + for (auto const &[key, value] : map) + { + (void)value; + functor(children_cache_detail::key_to_string(key)); + } + } + auto size() -> std::size_t override + { + return map.size(); + } +}; + +template +auto ChildrenCache::get_or_create_typed_entry() const -> TypedEntry & +{ + using entry_type = TypedEntry; + auto const type = container_type_id(); + auto it = m_entries.find(type); + if (it == m_entries.end()) + { + auto typed_entry = std::make_shared(); + it = m_entries.emplace(type, std::move(typed_entry)).first; + assert_consistent(); + } + + auto *typed_entry = static_cast(it->second.get()); + assert(typed_entry != nullptr); + return *typed_entry; +} + +template +auto ChildrenCache::get_or_create_cache() const -> ChildrenCacheMap +{ + return ChildrenCacheMap(get_or_create_typed_entry().map); +} + +template +auto ChildrenCache::try_emplace( + Key const &key, + std::shared_ptr const &child, + T &&value) -> std::pair::iterator, bool> +{ + auto const &canonicalKey = children_cache_detail::key_to_string(key); + auto &typed_entry = get_or_create_typed_entry(); + + auto typed_it = typed_entry.map.find(key); + if (typed_it != typed_entry.map.end()) + { + assert(m_children.find(canonicalKey) != m_children.end()); + return {typed_it, false}; + } + + auto child_it = m_children.find(canonicalKey); + if (child_it == m_children.end()) + { + child_it = m_children.emplace(canonicalKey, child).first; + } + else + { + assert(child_it->second == child); + } + + auto inserted = typed_entry.map.try_emplace(key, std::move(value)); + assert(inserted.second); + assert_consistent(); + return inserted; +} + +template +auto ChildrenCache::try_emplace( + Key const &key, + std::shared_ptr const &child, + T &&value) const -> std::pair::iterator, bool> +{ + auto const &canonicalKey = children_cache_detail::key_to_string(key); + auto const child_it = m_children.find(canonicalKey); + assert(child_it != m_children.end()); + assert(child_it->second == child); + + auto &typed_entry = get_or_create_typed_entry(); + auto inserted = typed_entry.map.try_emplace(key, std::move(value)); + assert_consistent(); + return inserted; +} + +template +auto ChildrenCache::find_cache() const + -> std::optional> +{ + using entry_type = TypedEntry; + auto const it = m_entries.find(container_type_id()); + if (it == m_entries.end()) + { + return std::nullopt; + } + auto *typed_entry = static_cast(it->second.get()); + assert(typed_entry != nullptr); + return ChildrenCacheMap(typed_entry->map); +} + +template +auto ChildrenCache::contains_cache() const noexcept -> bool +{ + return m_entries.find(container_type_id()) != m_entries.end(); +} +} // namespace openPMD::internal diff --git a/include/openPMD/backend/ChildrenCacheMap.hpp b/include/openPMD/backend/ChildrenCacheMap.hpp new file mode 100644 index 0000000000..b311f1af61 --- /dev/null +++ b/include/openPMD/backend/ChildrenCacheMap.hpp @@ -0,0 +1,127 @@ +/* Copyright 2026 The openPMD Community + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include +#include +#include +#include + +namespace openPMD::internal +{ +class ChildrenCache; + +/** Non-owning, restricted view of one typed map inside a ChildrenCache. + * + * The only purpose of this class is to ban access to modifying calls and + * thereby enforce code discipline: structural removal and typed insertion must + * go through ChildrenCache, which keeps the authoritative child map and every + * materialized typed map in sync. Apart from that restriction, all accessors + * forward to the underlying std::map. + */ +template +class ChildrenCacheMap +{ + friend class ChildrenCache; + +public: + using map_type = std::map; + using key_type = typename map_type::key_type; + using mapped_type = typename map_type::mapped_type; + using value_type = typename map_type::value_type; + using size_type = typename map_type::size_type; + using iterator = typename map_type::iterator; + using const_iterator = typename map_type::const_iterator; + using reverse_iterator = typename map_type::reverse_iterator; + using const_reverse_iterator = typename map_type::const_reverse_iterator; + + auto begin() noexcept -> iterator + { + return m_map->begin(); + } + auto begin() const noexcept -> const_iterator + { + return m_map->begin(); + } + auto end() noexcept -> iterator + { + return m_map->end(); + } + auto end() const noexcept -> const_iterator + { + return m_map->end(); + } + auto rbegin() noexcept -> reverse_iterator + { + return m_map->rbegin(); + } + auto rbegin() const noexcept -> const_reverse_iterator + { + return m_map->rbegin(); + } + auto rend() noexcept -> reverse_iterator + { + return m_map->rend(); + } + auto rend() const noexcept -> const_reverse_iterator + { + return m_map->rend(); + } + + [[nodiscard]] auto empty() const noexcept -> bool + { + return m_map->empty(); + } + [[nodiscard]] auto size() const noexcept -> size_type + { + return m_map->size(); + } + + auto find(Key const &key) noexcept -> iterator + { + return m_map->find(key); + } + auto find(Key const &key) const noexcept -> const_iterator + { + return m_map->find(key); + } + [[nodiscard]] auto contains(Key const &key) const noexcept -> bool + { + return m_map->find(key) != m_map->end(); + } + + auto at(Key const &key) -> T & + { + return m_map->at(key); + } + auto at(Key const &key) const -> T const & + { + return m_map->at(key); + } + +private: + explicit ChildrenCacheMap(map_type &map) noexcept : m_map(&map) + {} + + map_type *m_map; +}; + +} // namespace openPMD::internal diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 839f6452c9..a57f816190 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -24,15 +24,20 @@ #include "openPMD/IO/Access.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/ChildView.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/Writable.hpp" +#include #include #include +#include #include #include #include #include #include +#include // expose private and protected members for invasive testing #ifndef OPENPMD_protected @@ -53,40 +58,51 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = true; - template - void operator()(T &) + template + void operator()(Container &, T &) {} }; + + template + struct ElementAccessPolicy + { + static void call(Element_t &) + {} + static void call(Element_t const &) + {} + }; + + template + struct ElementAccessPolicy + { + static void call(Element_t const &el) + { + ElementAccessPolicy::call(el); + } + }; } // namespace traits +class CustomHierarchy; +class CustomDataset; + namespace internal { class SeriesData; template class EraseStaleEntries; - template < - typename T, - typename T_key = std::string, - typename T_container = std::map> - class ContainerData : virtual public AttributableData - { - public: - using InternalContainer = T_container; - - /** - * The wrapped container holding all the actual data, e.g. std::map. - */ - InternalContainer m_container; - - ContainerData() = default; + template + constexpr inline bool isDerivedFromAttributable = + std::is_base_of_v; - ContainerData(ContainerData const &) = delete; - ContainerData(ContainerData &&) = delete; + /* + * Opt out from this check due to the recursive definition of + * class CustomHierarchy : public Container{ ... }; + * Cannot check this while CustomHierarchy is still an incomplete type. + */ + template <> + constexpr inline bool isDerivedFromAttributable = true; - ContainerData &operator=(ContainerData const &) = delete; - ContainerData &operator=(ContainerData &&) = delete; - }; } // namespace internal /** @brief Map-like container that enforces openPMD requirements and handles IO. @@ -105,8 +121,8 @@ template < class Container : virtual public Attributable { static_assert( - std::is_base_of::value, - "Type of container element must be derived from Writable"); + internal::isDerivedFromAttributable, + "Type of container element must be derived from Attributable"); friend class Iteration; friend class ParticleSpecies; @@ -116,65 +132,112 @@ class Container : virtual public Attributable template friend class internal::EraseStaleEntries; friend class StatefulIterator; + friend struct traits::ElementAccessPolicy; + + using Self_t = Container; protected: - using ContainerData = internal::ContainerData; using InternalContainer = T_container; + using View = internal::ChildView; + using ConstView = internal::ChildView; + + internal::ChildFilter m_child_filter = internal::default_child_filter(); - std::shared_ptr m_containerData; + inline void setData(std::shared_ptr data) + { + Attributable::setData(std::move(data)); + } + + inline auto group_data() const + -> internal::object_type::GroupMetaData const & + { + auto const *res = (**m_attri).m_writable.objectType.requireGroup(); + assert(res != nullptr); + return *res; + } + + inline auto group_data() -> internal::object_type::GroupMetaData & + { + return *this->writable().objectType.requireGroup(); + } + + // Static, so it receives the container's own Writable explicitly. This is + // the same object as this->writable() for non-static callers. + static auto create_child(T_key const &key, Writable &self) -> T + { + T child; + child.linkHierarchy(self); + child.writable().ownKeyWithinParent = + internal::children_cache_detail::key_to_string(key); + return child; + } - inline void setData(std::shared_ptr containerData) + inline auto container() -> View { - m_containerData = std::move(containerData); - Attributable::setData(m_containerData); + return View( + &group_data().m_children_cache, + &this->writable(), + m_child_filter, + &Container::create_child); } - inline InternalContainer const &container() const + inline auto container() const -> ConstView { - return m_containerData->m_container; + return ConstView( + &group_data().m_children_cache, nullptr, m_child_filter, nullptr); } - inline InternalContainer &container() + OPENPMD_protected + /** Remove a key from the authoritative children and every typed cache. + * + * Performs no IO checks or DELETE_PATH enqueuing. + */ + void erase_unchecked(typename InternalContainer::key_type const &key) + { + container().erase(key); + } + + /** Clear all children and every typed cache without IO checks. */ + void clear_children_unchecked() { - return m_containerData->m_container; + container().clear(); } public: - using key_type = typename InternalContainer::key_type; - using mapped_type = typename InternalContainer::mapped_type; - using value_type = typename InternalContainer::value_type; - using size_type = typename InternalContainer::size_type; - using difference_type = typename InternalContainer::difference_type; - using allocator_type = typename InternalContainer::allocator_type; - using reference = typename InternalContainer::reference; - using const_reference = typename InternalContainer::const_reference; - using pointer = typename InternalContainer::pointer; - using const_pointer = typename InternalContainer::const_pointer; - using iterator = typename InternalContainer::iterator; - using const_iterator = typename InternalContainer::const_iterator; - using reverse_iterator = typename InternalContainer::reverse_iterator; - using const_reverse_iterator = - typename InternalContainer::const_reverse_iterator; - - iterator begin() noexcept; - const_iterator begin() const noexcept; - const_iterator cbegin() const noexcept; - - iterator end() noexcept; - const_iterator end() const noexcept; - const_iterator cend() const noexcept; - - reverse_iterator rbegin() noexcept; - const_reverse_iterator rbegin() const noexcept; - const_reverse_iterator crbegin() const noexcept; - - reverse_iterator rend() noexcept; - const_reverse_iterator rend() const noexcept; - const_reverse_iterator crend() const noexcept; - - bool empty() const noexcept; - - size_type size() const noexcept; + using key_type = typename View::key_type; + using mapped_type = typename View::mapped_type; + using value_type = typename View::value_type; + using size_type = typename View::size_type; + using difference_type = typename View::difference_type; + using allocator_type = typename View::allocator_type; + using reference = typename View::reference; + using const_reference = typename View::const_reference; + using pointer = typename View::pointer; + using const_pointer = typename View::const_pointer; + using iterator = typename View::iterator; + using const_iterator = typename View::const_iterator; + using reverse_iterator = typename View::reverse_iterator; + using const_reverse_iterator = typename View::const_reverse_iterator; + + iterator begin(); + const_iterator begin() const; + const_iterator cbegin() const; + + iterator end(); + const_iterator end() const; + const_iterator cend() const; + + reverse_iterator rbegin(); + const_reverse_iterator rbegin() const; + const_reverse_iterator crbegin() const; + + reverse_iterator rend(); + const_reverse_iterator rend() const; + const_reverse_iterator crend() const; + + bool empty() const; + + size_type size() const; /** Remove all objects from the container and (if written) from disk. * @@ -188,15 +251,23 @@ class Container : virtual public Attributable std::pair insert(value_type &&value); iterator insert(const_iterator hint, value_type const &value); iterator insert(const_iterator hint, value_type &&value); + /** + * Insert all elements in the range [first, last) in a single batch, + * keeping the front and backend containers synchronized. + * + * The initializer_list overload below is implemented in terms of this + * one. + */ template void insert(InputIt first, InputIt last) { - container().insert(first, last); + for (; first != last; ++first) + { + insert(*first); + } } void insert(std::initializer_list ilist); - void swap(Container &other); - mapped_type &at(key_type const &key); mapped_type const &at(key_type const &key) const; @@ -257,16 +328,15 @@ class Container : virtual public Attributable // virtual iterator erase(const_iterator first, const_iterator last) template - auto emplace(Args &&...args) - -> decltype(InternalContainer().emplace(std::forward(args)...)) + auto emplace(Args &&...args) -> std::pair { return container().emplace(std::forward(args)...); } template - void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) + void visitHierarchyContainer(HierarchyVisitor &v, bool recursive) { - if (recursive) + if (recursive && this->writable().objectType.isGroup()) { for (auto &p : *this) { @@ -282,7 +352,7 @@ OPENPMD_protected void clear_unchecked(); - virtual void + void flush(std::string const &path, internal::FlushParams const &flushParams); Container(); diff --git a/include/openPMD/backend/ContainerImpl.tpp b/include/openPMD/backend/ContainerImpl.tpp index de11c91f14..d01871c432 100644 --- a/include/openPMD/backend/ContainerImpl.tpp +++ b/include/openPMD/backend/ContainerImpl.tpp @@ -19,7 +19,9 @@ * If not, see . */ +#include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/Writable.hpp" /* * Instantiations in src/backend/Container.cpp @@ -31,81 +33,82 @@ namespace openPMD { template -auto Container::begin() noexcept -> iterator +auto Container::begin() -> iterator { return container().begin(); } template -auto Container::begin() const noexcept -> const_iterator +auto Container::begin() const -> const_iterator { - return container().begin(); + auto const view = container(); + return view.begin(); } template -auto Container::cbegin() const noexcept -> const_iterator +auto Container::cbegin() const -> const_iterator { - return container().cbegin(); + auto const view = container(); + return view.cbegin(); } - template -auto Container::end() noexcept -> iterator +auto Container::end() -> iterator { return container().end(); } template -auto Container::end() const noexcept -> const_iterator +auto Container::end() const -> const_iterator { - return container().end(); + auto const view = container(); + return view.end(); } template -auto Container::cend() const noexcept -> const_iterator +auto Container::cend() const -> const_iterator { - return container().cend(); + auto const view = container(); + return view.cend(); } - template -auto Container::rbegin() noexcept -> reverse_iterator +auto Container::rbegin() -> reverse_iterator { return container().rbegin(); } template -auto Container::rbegin() const noexcept - -> const_reverse_iterator +auto Container::rbegin() const -> const_reverse_iterator { - return container().rbegin(); + auto const view = container(); + return view.rbegin(); } template -auto Container::crbegin() const noexcept - -> const_reverse_iterator +auto Container::crbegin() const -> const_reverse_iterator { - return container().crbegin(); + auto const view = container(); + return view.crbegin(); } - template -auto Container::rend() noexcept -> reverse_iterator +auto Container::rend() -> reverse_iterator { return container().rend(); } template -auto Container::rend() const noexcept - -> const_reverse_iterator +auto Container::rend() const -> const_reverse_iterator { - return container().rend(); + auto const view = container(); + return view.rend(); } template -auto Container::crend() const noexcept - -> const_reverse_iterator +auto Container::crend() const -> const_reverse_iterator { - return container().crend(); + auto const view = container(); + return view.crend(); } template -auto Container::empty() const noexcept -> bool +auto Container::empty() const -> bool { return container().empty(); } template -auto Container::size() const noexcept -> size_type +auto Container::size() const -> size_type { return container().size(); } @@ -119,74 +122,21 @@ template auto Container::at(key_type const &key) const -> mapped_type const & { - return container().at(key); + auto const view = container(); + return view.at(key); } template auto Container::operator[](key_type const &key) -> mapped_type & { - auto it = container().find(key); - if (it != container().end()) - return it->second; - else - { - if (IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing && - access::readOnly(IOHandler()->m_frontendAccess)) - { - auxiliary::OutOfRangeMsg const out_of_range_msg; - throw std::out_of_range(out_of_range_msg(key)); - } - - T t = T(); - t.linkHierarchy(writable()); - auto inserted_iterator = container().insert({key, std::move(t)}).first; - auto &ret = inserted_iterator->second; - if constexpr (std::is_same_v) - { - ret.writable().ownKeyWithinParent = key; - } - else - { - ret.writable().ownKeyWithinParent = std::to_string(key); - } - traits::GenerationPolicy gen; - gen(inserted_iterator); - return ret; - } + return container()[key]; } template auto Container::operator[](key_type &&key) -> mapped_type & { - auto it = container().find(key); - if (it != container().end()) - return it->second; - else - { - if (IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing && - access::readOnly(IOHandler()->m_frontendAccess)) - { - auxiliary::OutOfRangeMsg out_of_range_msg; - throw std::out_of_range(out_of_range_msg(key)); - } - - T t = T(); - t.linkHierarchy(writable()); - auto inserted_iterator = container().insert({key, std::move(t)}).first; - auto &ret = inserted_iterator->second; - if constexpr (std::is_same_v) - { - ret.writable().ownKeyWithinParent = std::move(key); - } - else - { - ret.writable().ownKeyWithinParent = std::to_string(std::move(key)); - } - traits::GenerationPolicy gen; - gen(inserted_iterator); - return ret; - } + return container()[key]; } template @@ -209,7 +159,7 @@ template auto Container::insert(value_type &&value) -> std::pair { - return container().insert(value); + return container().insert(std::move(value)); } template auto Container::insert( @@ -221,19 +171,14 @@ template auto Container::insert( const_iterator hint, value_type &&value) -> iterator { - return container().insert(hint, value); + return container().insert(hint, std::move(value)); } + template auto Container::insert( std::initializer_list ilist) -> void { - container().insert(ilist); -} - -template -auto Container::swap(Container &other) -> void -{ - container().swap(other.container()); + insert(ilist.begin(), ilist.end()); } template @@ -245,14 +190,10 @@ template auto Container::find(key_type const &key) const -> const_iterator { - return container().find(key); + auto const view = container(); + return view.find(key); } -/** This returns either 1 if the key is found in the container of 0 if not. - * - * @param key key value of the element to count - * @return since keys are unique in this container, returns 0 or 1 - */ template auto Container::count(key_type const &key) const -> size_type @@ -260,17 +201,11 @@ auto Container::count(key_type const &key) const return container().count(key); } -/** Checks if there is an element with a key equivalent to an exiting key in - * the container. - * - * @param key key value of the element to search for - * @return true of key is found, else false - */ template auto Container::contains(key_type const &key) const -> bool { - return container().find(key) != container().end(); + return container().contains(key); } template @@ -280,15 +215,16 @@ auto Container::erase(key_type const &key) -> size_type throw std::runtime_error( "Can not erase from a container in a read-only Series."); - auto res = container().find(key); - if (res != container().end() && res->second.written()) + auto view = container(); + auto res = view.find(key); + if (res != view.end() && res->second.written()) { Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); IOHandler()->flush(internal::defaultFlushParams); } - return container().erase(key); + return view.erase(key); } template @@ -298,7 +234,7 @@ auto Container::erase(iterator res) -> iterator throw std::runtime_error( "Can not erase from a container in a read-only Series."); - if (res != container().end() && res->second.written()) + if (res->second.written()) { Parameter pDelete; pDelete.path = "."; @@ -315,7 +251,7 @@ auto Container::clear_unchecked() -> void throw std::runtime_error( "Clearing a written container not (yet) implemented."); - container().clear(); + clear_children_unchecked(); } template @@ -329,13 +265,17 @@ auto Container::flush( IOHandler()->enqueue(IOTask(this, pCreate)); } - flushAttributes(flushParams); + customHierarchyFlush(flushParams, /* managed_as_custom_object = */ false); + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } } template Container::Container() : Attributable(NoInit()) { - setData(std::make_shared()); + setData(std::make_shared()); } template @@ -347,7 +287,7 @@ Container::Container(Container const &other) : Attributable(NoInit()) { m_attri = other.m_attri; - m_containerData = other.m_containerData; + m_child_filter = other.m_child_filter; } template @@ -358,7 +298,7 @@ Container::Container(Container &&other) noexcept { m_attri = std::move(other.m_attri); } - m_containerData = std::move(other.m_containerData); + m_child_filter = std::move(other.m_child_filter); } template @@ -366,7 +306,7 @@ auto Container::operator=(Container const &other) -> Container & { m_attri = other.m_attri; - m_containerData = other.m_containerData; + m_child_filter = other.m_child_filter; return *this; } @@ -378,7 +318,7 @@ auto Container::operator=(Container &&other) noexcept { m_attri = std::move(other.m_attri); } - m_containerData = std::move(other.m_containerData); + m_child_filter = std::move(other.m_child_filter); return *this; } @@ -421,22 +361,20 @@ namespace internal template EraseStaleEntries::~EraseStaleEntries() { - auto &map = m_originalContainer.container(); - using iterator_t = - typename Container_t::InternalContainer::const_iterator; - std::vector deleteMe; - deleteMe.reserve(map.size() - m_accessedKeys.size()); - for (iterator_t it = map.begin(); it != map.end(); ++it) + std::vector deleteMe; + for (auto it = m_originalContainer.begin(); + it != m_originalContainer.end(); + ++it) { auto lookup = m_accessedKeys.find(it->first); if (lookup == m_accessedKeys.end()) { - deleteMe.push_back(it); + deleteMe.push_back(it->first); } } - for (auto &it : deleteMe) + for (auto const &key : deleteMe) { - map.erase(it); + m_originalContainer.erase_unchecked(key); } } } // namespace internal diff --git a/include/openPMD/backend/HierarchyVisitor.hpp b/include/openPMD/backend/HierarchyVisitor.hpp index 389974bb5b..66325142df 100644 --- a/include/openPMD/backend/HierarchyVisitor.hpp +++ b/include/openPMD/backend/HierarchyVisitor.hpp @@ -15,6 +15,7 @@ class PatchRecordComponent; class Record; class RecordComponent; class Series; +class CustomHierarchy; class HierarchyVisitor { @@ -32,5 +33,6 @@ class HierarchyVisitor virtual void operator()(Record &) = 0; virtual void operator()(RecordComponent &) = 0; virtual void operator()(Series &) = 0; + virtual void operator()(CustomHierarchy &); }; } // namespace openPMD diff --git a/include/openPMD/backend/HierarchyVisitorImpl.hpp b/include/openPMD/backend/HierarchyVisitorImpl.hpp index 91ba076570..702a294c09 100644 --- a/include/openPMD/backend/HierarchyVisitorImpl.hpp +++ b/include/openPMD/backend/HierarchyVisitorImpl.hpp @@ -75,6 +75,10 @@ class HierarchyVisitorFromLambda : public HierarchyVisitor { lambda(obj); } + void operator()(CustomHierarchy &obj) override + { + lambda(obj); + } }; template diff --git a/include/openPMD/backend/MeshRecordComponent.hpp b/include/openPMD/backend/MeshRecordComponent.hpp index f359d5904c..c5dc6cdfa5 100644 --- a/include/openPMD/backend/MeshRecordComponent.hpp +++ b/include/openPMD/backend/MeshRecordComponent.hpp @@ -95,11 +95,11 @@ class MeshRecordComponent : public RecordComponent template MeshRecordComponent &makeConstant(T); - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - protected: void scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) override; + + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; template diff --git a/include/openPMD/backend/PatchRecord.hpp b/include/openPMD/backend/PatchRecord.hpp index 49a6f1448d..120486e067 100644 --- a/include/openPMD/backend/PatchRecord.hpp +++ b/include/openPMD/backend/PatchRecord.hpp @@ -40,13 +40,13 @@ class PatchRecord : public BaseRecord PatchRecord &setUnitDimension(unit_representations::AsArray const &udim); ~PatchRecord() override = default; - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - private: PatchRecord() = default; void flush_impl(std::string const &, internal::FlushParams const &) override; void read(); + + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; // PatchRecord } // namespace openPMD diff --git a/include/openPMD/backend/PatchRecordComponent.hpp b/include/openPMD/backend/PatchRecordComponent.hpp index b99de6d266..9be0256e9e 100644 --- a/include/openPMD/backend/PatchRecordComponent.hpp +++ b/include/openPMD/backend/PatchRecordComponent.hpp @@ -89,8 +89,6 @@ class PatchRecordComponent : public RecordComponent template void store(T); - void visitHierarchy(HierarchyVisitor &v, bool recursive) override; - // clang-format off OPENPMD_private // clang-format on @@ -111,6 +109,8 @@ OPENPMD_protected protected: void scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) override; + + void visitHierarchyImpl(HierarchyVisitor &v, bool recursive) override; }; // PatchRecordComponent template diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index accd1f9843..8ab405cf69 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -20,11 +20,15 @@ */ #pragma once +#include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/InvalidatableFile.hpp" +#include "openPMD/backend/ChildrenCache.hpp" #include +#include #include +#include #include // expose private and protected members for invasive testing @@ -47,12 +51,205 @@ template class Span; class Series; +namespace traits +{ + template + struct GenerationPolicy; + template + struct ElementAccessPolicy; + namespace detail + { + template + void emplace_object_as_customly_managed(Container &cont, Iterator &it); + } +} // namespace traits + namespace internal { class SharedAttributableData; class AttributableData; class SeriesData; class ScientificDefaults; + class BaseRecordComponentData; + class RecordComponentData; + template + class ChildView; + struct ChildFilter; +} // namespace internal + +namespace internal::object_type +{ + struct DatasetMetaData + { + /** + * Chunk reading/writing requests on the contained dataset. + */ + std::queue m_chunks; + /** + * The type and extent of the dataset defined by this component. + */ + std::optional m_dataset; + /** + * Stores the value for constant record components. + * Ignored otherwise. + */ + Attribute m_constantValue{-1}; + /** + * True if this is defined as a constant record component as specified + * in the openPMD standard. + * If yes, then no heavy-weight dataset is created and the dataset is + * instead defined via light-weight attributes. + */ + bool m_isConstant = false; + + /** + * True if this component is an empty dataset, i.e. its extent is zero + * in at least one dimension. + * Treated by the openPMD-api as a special case of constant record + * components. + */ + bool m_isEmpty = false; + /** + * User has extended the dataset, but the EXTEND task must yet be + * flushed to the backend + */ + bool m_hasBeenExtended = false; + }; + + struct GroupMetaData + { + // Using shared_ptr in here because + // AttributableData is non-copyable and non-movable, but we need + // movability for map handling + // Disallowing move in AttributableData is only a measure for code + // discipline anyway. + using children_map_t = ChildrenCache::children_map_t; + + /** + * Authoritative child hierarchy plus lazily materialized typed views. + * This is the only persistent storage for children; all views of this + * group share it. + */ + mutable ChildrenCache m_children_cache; + + inline auto children() -> children_map_t & + { + return m_children_cache.children(); + } + inline auto children() const -> children_map_t const & + { + return m_children_cache.children(); + } + + /** Remove a child from the authoritative map and every cache. */ + void erase_child(std::string const &key) + { + m_children_cache.erase(key); + } + + /** Clear the authoritative map and every cache. */ + void clear_children() + { + m_children_cache.clear(); + } + + bool phantom = false; + }; +} // namespace internal::object_type + +namespace internal +{ + struct ObjectType + : std::variant + { + using variant_t = std:: + variant; + using variant_t::variant; + + [[nodiscard]] auto as_base() const -> variant_t const & + { + return *this; + } + auto as_base() -> variant_t & + { + return *this; + } + [[nodiscard]] auto isDataset() const -> bool + { + return std::holds_alternative( + as_base()); + } + [[nodiscard]] auto isGroup() const -> bool + { + return std::holds_alternative( + as_base()); + } + auto initDataset() -> object_type::DatasetMetaData * + { + if (auto res = + std::get_if(&as_base()); + res) + { + return res; + } + return &as_base().emplace(); + } + auto initGroup() -> object_type::GroupMetaData * + { + if (auto res = std::get_if(&as_base()); + res) + { + return res; + } + return &as_base().emplace(); + } + + auto requireGroup() -> object_type::GroupMetaData * + { + if (auto res = std::get_if(&as_base()); + res) + { + return res; + } + throw error::Internal( + "Internal object status: Group type required."); + } + + auto requireGroup() const -> object_type::GroupMetaData const * + { + if (auto res = std::get_if(&as_base()); + res) + { + return res; + } + throw error::Internal( + "Internal object status: Group type required."); + } + + template + void ifGroup(Functor &&f) + { + std::visit( + auxiliary::overloaded{ + [&f](object_type::GroupMetaData &object_metadata) { + std::forward(f)(object_metadata); + }, + [](object_type::DatasetMetaData &) {}}, + as_base()); + } + + template + void ifGroup(Functor &&f) const + { + std::visit( + auxiliary::overloaded{ + [&f](object_type::GroupMetaData const &object_metadata) { + std::forward(f)(object_metadata); + }, + [](object_type::DatasetMetaData const &) {}}, + as_base()); + } + }; } // namespace internal namespace detail { @@ -111,6 +308,21 @@ class Writable final friend struct Parameter; friend struct Parameter; friend class internal::ScientificDefaults; + template + friend class ConvertibleContainer; + friend class CustomHierarchy; + template + friend class internal::ChildView; + friend struct internal::ChildFilter; + template + friend struct traits::GenerationPolicy; + friend class internal::BaseRecordComponentData; + friend class internal::RecordComponentData; + template + friend struct traits::ElementAccessPolicy; + template + friend void traits::detail::emplace_object_as_customly_managed( + Container &cont, Iterator &it); private: Writable(internal::AttributableData *); @@ -159,6 +371,7 @@ OPENPMD_private * If multiple Attributables share the same Writable, then the creating one. * (See SharedAttributableData) */ + // TODO turn this into a weak pointer internal::AttributableData *attributable = nullptr; Writable *parent = nullptr; @@ -201,5 +414,7 @@ OPENPMD_private * */ bool written = false; + + internal::ObjectType objectType{internal::object_type::GroupMetaData{}}; }; } // namespace openPMD diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index 91e5f79195..e5da821699 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -26,6 +26,7 @@ namespace openPMD {} // IWYU pragma: begin_exports +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/Error.hpp" diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp new file mode 100644 index 0000000000..5bfcede848 --- /dev/null +++ b/src/CustomHierarchy.cpp @@ -0,0 +1,422 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/CustomHierarchy.hpp" +#include "openPMD/auxiliary/Defer.hpp" +#include "openPMD/backend/Container.hpp" + +namespace openPMD +{ +namespace traits +{ + + void ElementAccessPolicy::call(CustomHierarchy &cont) + { + if (!cont.get().has_been_read && + access::read(cont.IOHandler()->m_backendAccess)) + { + bool skip = false; + cont.writable().objectType.ifGroup([&](auto &group_data) { + if (group_data.phantom) + { + skip = true; + } + }); + if (!skip) + { + cont.read(1); + } + } + } + + void ElementAccessPolicy::call(CustomHierarchy const &) + {} +} // namespace traits + +// TODO: Add visitor over real type, using HierarchyVisitor class +// Then, maybe even implement HierarchyVisitor in terms of CustomHierarchy + +template +auto ConvertibleContainer::isDataset() const -> bool +{ + return this->writable().objectType.isDataset(); +} + +template +auto ConvertibleContainer::asDataset() -> CustomDataset +{ + auto castableToConstantDataset = [&]() { + return this->containsAttribute("value") && + this->containsAttribute("shape") && this->empty(); + }; + auto initDataset = [&]() { this->writable().objectType.initDataset(); }; + auto makeResult = [&]() { + CustomDataset res; + res.get().cloneFrom(*this->m_attri); + return res; + }; + if (this->written()) + { + if (this->writable().objectType.isGroup()) + { + if (!castableToConstantDataset()) + { + throw error::WrongAPIUsage( + "Can't cast a group object into a dataset."); + } + else + { // Also, what about shape elision in SoA structures? + // Probably does not make sense outside of Meshes / Particles + initDataset(); + auto res = makeResult(); + res.get().isConstant() = true; + res.read(); + return res; + } + } + else + { + return makeResult(); + } + } + else + { + if (this->writable().objectType.isGroup() && !this->empty()) + { + throw error::WrongAPIUsage( + "Can't cast a group that has contained objects into a " + "dataset."); + } + if (this->writable().objectType.isDataset() || + access::write(this->IOHandler()->m_frontendAccess) || + this->IOHandler()->m_seriesStatus == + internal::SeriesStatus::Parsing) + { + // this is now a dataset. + initDataset(); + return makeResult(); + } + else + { + // TODO check if this has valid uses + throw error::WrongAPIUsage( + "Read only: Trying to access a non-written object as a " + "RecordComponent."); + } + } +} + +template <> +auto ConvertibleContainer::datasets() + -> ConvertibleContainer +{ + ConvertibleContainer res; + res.m_attri->cloneFrom(*this->m_attri); + res.m_child_filter.kind = internal::ChildFilter::Kind::Datasets; + return res; +} + +template <> +auto ConvertibleContainer::datasets() + -> ConvertibleContainer +{ + return *this; +} + +template +auto ConvertibleContainer::subgroups() + -> ConvertibleContainer +{ + ConvertibleContainer res; + res.m_attri->cloneFrom(*this->m_attri); + res.m_child_filter.kind = internal::ChildFilter::Kind::Groups; + return res; +} + +template class ConvertibleContainer; +template class ConvertibleContainer; + +CustomHierarchy::CustomHierarchy() : ConvertibleContainer(NoInit{}) +{ + setData(std::make_shared()); +} + +CustomHierarchy::CustomHierarchy(NoInit) : ConvertibleContainer(NoInit{}) +{} + +CustomHierarchy::CustomHierarchy( + std::shared_ptr other) + : ConvertibleContainer(NoInit{}) +{ + auto data = std::make_shared(); + data->asSharedPtrOfAttributable() = std::move(other); + setData(std::move(data)); +} + +CustomHierarchy::CustomHierarchy(Attributable const &other) + : CustomHierarchy(other.m_attri->asSharedPtrOfAttributable()) +{} + +auto CustomHierarchy::read(size_t const max_recursion_depth) + -> CustomHierarchy & +{ + auxiliary::opaque_defer_type reset_parsing_status; + if (IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) + { + IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; + reset_parsing_status = auxiliary::defer([&]() { + IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; + }); + } + auto was_written = writable().written; + if (!was_written) + { + auto do_throw = [&]() { + throw error::WrongAPIUsage( + "Cannot read contents of CustomHierarchy path '" + + myPath().openPMDPath() + + "', since the backend does not yet / no longer know about this " + "object. Ensure that the object is open (e.g. by opening its " + "containing Iteration or by reading parent paths first)"); + }; + auto parent = writable().parent; + if (!parent) + { + do_throw(); + } + Attributable parent_attributable(NoInit{}); + parent_attributable.setData( + std::shared_ptr{ + parent->attributable, [](auto const *) {}}); + CustomHierarchy parent_(parent_attributable); + parent_.read(1); + if (!writable() /* still not */.written) + { + do_throw(); + } + } + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Attributable::readAttributes(ReadMode::FullyReread); + + auto &data = get(); + auto defer_bookkeeping = auxiliary::defer([&]() { + data.has_been_read = true; + setDirty(false); + }); + + if (writable().objectType.isDataset()) + { + { + asDataset().read(); + } + return *this; + } + + auto do_recurse = [this, max_recursion_depth](CustomHierarchy &child) { + switch (max_recursion_depth) + { + case 0: + IOHandler()->flush(internal::defaultFlushParams); + child.read(0); + break; + case 1: + break; + default: + IOHandler()->flush(internal::defaultFlushParams); + child.read(max_recursion_depth - 1); + break; + } + }; + + if (!data.has_been_read) + { + Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); + + Parameter dList; + IOHandler()->enqueue(IOTask(this, dList)); + + IOHandler()->flush(internal::defaultFlushParams); + + for (auto const &path : *pList.paths) + { + auto &subpath = this->container().get_or_create( + path, /* access_policy = */ false); + subpath.linkHierarchy(this->writable()); + } + + for (auto const &path : *dList.datasets) + { + auto &subpath = this->container().get_or_create( + path, /* access_policy = */ false); + subpath.linkHierarchy(this->writable()); + + subpath.asDataset(); + do_recurse(subpath); + } + } + + auto view = this->container(); + view.set_access_policy(false); + for (auto &pair : view) + { + auto &path = pair.first; + auto &subpath = pair.second; + if (subpath.get().has_been_read) + { + continue; + } + + std::visit( + auxiliary::overloaded{ + [&](internal::object_type::GroupMetaData &group_data) { + if (group_data.phantom) + { + return; + } + if (!subpath.written()) + { + Parameter pOpen; + pOpen.path = path; + subpath.linkHierarchy(this->writable()); + IOHandler()->enqueue(IOTask(&subpath, pOpen)); + } + do_recurse(subpath); + }, + [&](internal::object_type::DatasetMetaData &) { + if (!subpath.written()) + { + Parameter dOpen; + dOpen.name = path; + + IOHandler()->enqueue(IOTask(&subpath, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + + subpath.setWritten( + false, Attributable::EnqueueAsynchronously::No); + subpath.asDataset().resetDataset( + Dataset(*dOpen.dtype, *dOpen.extent)); + subpath.setWritten( + true, Attributable::EnqueueAsynchronously::No); + } + do_recurse(subpath); + }}, + subpath.writable().objectType.as_base()); + } + + return *this; +} + +void CustomHierarchy::flush( + std::string const & /* path */, internal::FlushParams const &) +{ + throw std::runtime_error("Use Attributable::customHierarchyFlush instead!"); +} + +void CustomHierarchy::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); +} + +void CustomHierarchy::printRecursively() +{ + std::cout << &writable() << " [" << dirty() << "," << dirtyRecursive() + << "] "; + if (empty()) + { + std::cout << "\n"; + return; + } + else + { + std::cout << writable().ownKeyWithinParent << '\n'; + printRecursively(""); + std::cout.flush(); + } +} + +void CustomHierarchy::printRecursively(std::string indent) +{ + if (!writable().objectType.isGroup()) + { + return; + } + auto print_indent = [&indent]() { std::cout << indent; }; + auto it = begin(); + auto end_ = end(); + if (it == end_) + { + return; + } + auto prev = it; + ++it; + auto next_indent = indent + "│ "; + for (; it != end_; prev = it, ++it) + { + std::cout << &prev->second.writable() << " [" << prev->second.dirty() + << "," << prev->second.dirtyRecursive() << "] "; + print_indent(); + std::cout << "├─" << prev->first << '\n'; + prev->second.printRecursively(next_indent); + } + std::cout << &prev->second.writable() << " [" << prev->second.dirty() << "," + << prev->second.dirtyRecursive() << "] "; + print_indent(); + std::cout << "└─" << prev->first << '\n'; + prev->second.printRecursively(indent + " "); +} + +void CustomHierarchy::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) +{ + /* + * The frontend container of a CustomHierarchy mirrors the backend group's + * children, which is a superset of the objects flagged as + * managed_as_custom_hierarchy: The flag marks only those objects that were + * *created* by a CustomHierarchy view, while the frontend container + * additionally aliases objects managed by the regular openPMD hierarchy + * (e.g. Meshes, Particles and their descendants). + * + * Attributable::visitHierarchy() recurses into the custom-managed children + * after calling this method. To visit every object exactly once, only + * traverse those frontend children that are *not* marked as custom-managed + * here, and leave the rest to the base class. + */ + if (recursive) + { + writable().objectType.ifGroup([&](auto &) { + for (auto &pair : *this) + { + if (!pair.second.Attributable::get() + .managed_as_custom_hierarchy) + { + pair.second.visitHierarchy(v, recursive); + } + } + }); + } + v(*this); +} +} // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index d56cea7eb2..35f09a986e 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -1350,7 +1350,12 @@ static std::vector availableAttributesOrVariablesPrefixed( { if (auxiliary::starts_with(it->first, var)) { - ret.emplace_back(auxiliary::replace_first(it->first, var, "")); + if (it->first.size() > var.size()) + { + ret.emplace_back(auxiliary::replace_first(it->first, var, "")); + } + // else we have just found the prefix itself. + // might happen if prefix == "/" } else { diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index f50b0d1a21..6d9e0dad76 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -857,7 +857,11 @@ void ADIOS2IOHandlerImpl::createPath( /* Sanitize path */ if (!auxiliary::starts_with(parameters.path, '/')) { - path = filePositionToString(*setAndGetFilePosition(writable)) + "/" + + auto file_position = + filePositionToString(*setAndGetFilePosition(writable)); + path = + (auxiliary::ends_with(file_position, '/') ? file_position + : file_position + "/") + auxiliary::removeSlashes(parameters.path); } else diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 153601677c..1e3e4d9a5a 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -217,6 +217,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) ", extent=", [¶meter]() { return vec_as_string(parameter.extent); }); createDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::EXTEND_DATASET: { @@ -230,6 +231,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) i.writable, "] EXTEND_DATASET"); extendDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::OPEN_FILE: { @@ -286,6 +288,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) "] OPEN_DATASET: ", parameter.name); openDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::DELETE_FILE: { @@ -345,6 +348,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) ", extent=", [¶meter]() { return vec_as_string(parameter.extent); }); writeDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::WRITE_ATT: { @@ -376,6 +380,7 @@ std::future AbstractIOHandlerImpl::flush(FlushLevel l) ", extent=", [¶meter]() { return vec_as_string(parameter.extent); }); readDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::GET_BUFFER_VIEW: { diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 7e708ca4e4..038517fe51 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -54,14 +54,14 @@ namespace openPMD using internal::CloseStatus; using internal::DeferredParseAccess; -void Meshes::visitHierarchy(HierarchyVisitor &v, bool recursive) +void Meshes::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { - visitHierarchyImpl(v, recursive); + visitHierarchyContainer(v, recursive); } -void Particles::visitHierarchy(HierarchyVisitor &v, bool recursive) +void Particles::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { - visitHierarchyImpl(v, recursive); + visitHierarchyContainer(v, recursive); } Iteration::Iteration() : Attributable(NoInit()) @@ -254,7 +254,7 @@ bool Iteration::closedByWriter() const } } -void Iteration::visitHierarchy(HierarchyVisitor &v, bool recursive) +void Iteration::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { if (recursive) { @@ -370,6 +370,9 @@ void Iteration::flush(internal::FlushParams const &flushParams) { Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); + + customHierarchyFlush(flushParams, /* managed_as_custom_object = */ false); + if (access::readOnly(IOHandler()->m_frontendAccess)) { for (auto &m : meshes) @@ -403,6 +406,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) { auto meshesPath = set_and_get_mp_path( "meshesPath", "meshes/", &Series::setMeshesPath); + meshes.writable().objectType.requireGroup()->phantom = false; if (meshes.dirtyRecursive()) { meshes.flush(meshesPath, flushParams); @@ -421,6 +425,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) { auto particlesPath = set_and_get_mp_path( "particlesPath", "particles/", &Series::setParticlesPath); + particles.writable().objectType.requireGroup()->phantom = false; if (particles.dirtyRecursive()) { particles.flush(particlesPath, flushParams); @@ -437,6 +442,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) flushAttributes(flushParams); } + if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { determineUnsetDirty(flushParams.flushLevel); @@ -561,13 +567,14 @@ void Iteration::read_impl(std::string const &groupPath) try { readMeshes(s.meshesPath()); + meshes.writable().objectType.requireGroup()->phantom = false; } catch (error::ReadError const &err) { std::cerr << "Cannot read meshes in iteration " << groupPath << " and will skip them due to read error:\n" << err.what() << std::endl; - meshes.container().clear(); + meshes.clear_children_unchecked(); } } meshes.setDirty(false); @@ -577,13 +584,14 @@ void Iteration::read_impl(std::string const &groupPath) try { readParticles(s.particlesPath()); + particles.writable().objectType.requireGroup()->phantom = false; } catch (error::ReadError const &err) { std::cerr << "Cannot read particles in iteration " << groupPath << " and will skip them due to read error:\n" << err.what() << std::endl; - particles.container().clear(); + particles.clear_children_unchecked(); } } particles.setDirty(false); @@ -645,7 +653,7 @@ void Iteration::readMeshes(std::string const &meshesPath) MeshRecordComponent &mrc = m; IOHandler()->enqueue(IOTask(&mrc, pOpen)); IOHandler()->flush(internal::defaultFlushParams); - mrc.get().m_isConstant = true; + mrc.get().isConstant() = true; } try { @@ -911,6 +919,27 @@ void Iteration::linkHierarchy(Writable &w) meshes.linkHierarchy(this->writable()); particles.linkHierarchy(this->writable()); get().m_perIterationData.m_rankTableAttributable.linkHierarchy(*w.parent); + + auto &container_back = + this->writable().objectType.requireGroup()->children(); + auto s = retrieveSeries(); + auto link_mp = [&](auto &meshes_or_particles, + std::optional const &mp_path, + char const *default_) { + meshes_or_particles.writable().objectType.requireGroup()->phantom = + true; + if (mp_path) + { + container_back[auxiliary::replace_all_nonrecursively( + *mp_path, "/", "")] = *meshes_or_particles.m_attri; + } + else + { + container_back[default_] = *meshes_or_particles.m_attri; + } + }; + link_mp(meshes, s.meshesPathOptional(), "meshes"); + link_mp(particles, s.particlesPathOptional(), "particles"); } void Iteration::runDeferredParseAccess() @@ -980,9 +1009,9 @@ void Iteration::scientificDefaults_impl( .withReader(int_types, require_type())(wor); } -void Iterations::visitHierarchy(HierarchyVisitor &v, bool recursive) +void Iterations::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { - visitHierarchyImpl(v, recursive); + visitHierarchyContainer(v, recursive); } template float Iteration::time() const; diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 8d8227f23d..8883fec77a 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -37,9 +37,9 @@ namespace openPMD { -void Mesh::visitHierarchy(HierarchyVisitor &v, bool recursive) +void Mesh::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { - visitHierarchyImpl(v, recursive); + visitHierarchyContainer(v, recursive); } Mesh::Mesh() = default; @@ -446,15 +446,19 @@ void Mesh::flush_impl( if (access::readOnly(IOHandler()->m_frontendAccess)) { auto &m = get(); - if (m.m_datasetDefined) + if (m.datasetDefined()) { T_RecordComponent::flush(SCALAR, flushParams); } else { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } + // Dirty flags are unset centrally by BaseRecord::flush() after + // flush_impl() returns, so no unsetDirty operation is needed here. } else { @@ -470,6 +474,9 @@ void Mesh::flush_impl( Parameter pCreate; pCreate.path = name; IOHandler()->enqueue(IOTask(this, pCreate)); + + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) { comp.second.parent() = &this->writable(); @@ -485,6 +492,8 @@ void Mesh::flush_impl( } else { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } @@ -497,7 +506,6 @@ void Mesh::read() { internal::HomogenizeExtents homogenizeExtents( IOHandler()->m_verify_homogeneous_extents); - internal::EraseStaleEntries map{*this}; if (scalar()) { @@ -506,6 +514,7 @@ void Mesh::read() } else { + internal::EraseStaleEntries map{*this}; Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); IOHandler()->flush(internal::defaultFlushParams); @@ -516,7 +525,7 @@ void Mesh::read() MeshRecordComponent &rc = map[component]; pOpen.path = component; IOHandler()->enqueue(IOTask(&rc, pOpen)); - rc.get().m_isConstant = true; + rc.get().isConstant() = true; try { rc.read(); diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index e27208e596..f8c157f1a3 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -35,9 +35,9 @@ size_t ParticlePatches::numPatches() const return this->at("numParticles").getExtent()[0]; } -void ParticlePatches::visitHierarchy(HierarchyVisitor &v, bool recursive) +void ParticlePatches::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { - visitHierarchyImpl(v, recursive); + visitHierarchyContainer(v, recursive); } void ParticlePatches::read() @@ -61,7 +61,7 @@ void ParticlePatches::read() std::cerr << "Cannot read patch record '" << record_name << "' due to read error and will skip it:" << err.what() << std::endl; - this->container().erase(record_name); + this->erase_unchecked(record_name); } } @@ -113,7 +113,7 @@ void ParticlePatches::read() << "Cannot read record component '" << component_name << "' in particle patch and will skip it due to read error:\n" << err.what() << std::endl; - Container::container().erase(component_name); + this->erase_unchecked(component_name); } } setDirty(false); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 718d3e847a..47398de3e9 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -27,18 +27,21 @@ namespace openPMD { -void ParticleSpecies::visitHierarchy(HierarchyVisitor &v, bool recursive) +void ParticleSpecies::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { if (recursive) { particlePatches.visitHierarchy(v, recursive); } - visitHierarchyImpl(v, recursive); + visitHierarchyContainer(v, recursive); } ParticleSpecies::ParticleSpecies() { + m_child_filter.excludedKey = "particlePatches"; particlePatches.writable().ownKeyWithinParent = "particlePatches"; + writable().objectType.requireGroup()->children()["particlePatches"] = + *particlePatches.m_attri; } void ParticleSpecies::read() @@ -91,7 +94,7 @@ void ParticleSpecies::read() RecordComponent &rc = r; IOHandler()->enqueue(IOTask(&rc, pOpen)); IOHandler()->flush(internal::defaultFlushParams); - rc.get().m_isConstant = true; + rc.get().isConstant() = true; } internal::HomogenizeExtents recordExtents; try @@ -113,9 +116,8 @@ void ParticleSpecies::read() if (!hasParticlePatches) { - auto &container = particlePatches.container(); - container.erase("numParticles"); - container.erase("numParticlesOffset"); + particlePatches.erase_unchecked("numParticles"); + particlePatches.erase_unchecked("numParticlesOffset"); particlePatches.setDirty(false); } @@ -178,6 +180,8 @@ void ParticleSpecies::flush( } if (access::readOnly(IOHandler()->m_frontendAccess)) { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &record : *this) record.second.flush(record.first, flushParams); for (auto &patch : particlePatches) diff --git a/src/Record.cpp b/src/Record.cpp index f99f26f015..4943cb4cc0 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -31,9 +31,9 @@ namespace openPMD { -void Record::visitHierarchy(HierarchyVisitor &v, bool recursive) +void Record::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { - visitHierarchyImpl(v, recursive); + visitHierarchyContainer(v, recursive); } Record::Record() = default; @@ -71,9 +71,13 @@ void Record::flush_impl( } else { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } + // Dirty flags are unset centrally by BaseRecord::flush() after + // flush_impl() returns, so no unsetDirty operation is needed here. } else { @@ -89,6 +93,9 @@ void Record::flush_impl( Parameter pCreate; pCreate.path = name; IOHandler()->enqueue(IOTask(this, pCreate)); + + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) { comp.second.parent() = getWritable(this); @@ -98,13 +105,14 @@ void Record::flush_impl( } else { - if (scalar()) { T_RecordComponent::flush(name, flushParams); } else { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } @@ -151,7 +159,7 @@ auto Record::read() -> internal::HomogenizeExtents RecordComponent &rc = (*this)[component]; pOpen.path = component; IOHandler()->enqueue(IOTask(&rc, pOpen)); - rc.get().m_isConstant = true; + rc.get().isConstant() = true; try { rc.read(); @@ -161,7 +169,7 @@ auto Record::read() -> internal::HomogenizeExtents std::cerr << "Cannot read record component '" << component << "' and will skip it due to read error:\n" << err.what() << std::endl; - this->container().erase(component); + this->erase_unchecked(component); continue; } check_extent(rc); @@ -190,7 +198,7 @@ auto Record::read() -> internal::HomogenizeExtents std::cerr << "Cannot read record component '" << component << "' and will skip it due to read error:\n" << err.what() << std::endl; - this->container().erase(component); + this->erase_unchecked(component); continue; } check_extent(rc); diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 42738e1222..246b603c2c 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -72,7 +72,7 @@ namespace internal } #endif a.setDirtyRecursive(true); - m_chunks.push(std::move(task)); + chunks().push(std::move(task)); } static constexpr char const *note_on_deactivating_this_check = R"( @@ -195,6 +195,7 @@ auto resource(T &t) -> attribute_types & RecordComponent::RecordComponent() : BaseRecordComponent(NoInit()) { setData(std::make_shared()); + setDatasetDefined(*m_recordComponentData); } RecordComponent::RecordComponent(NoInit) : BaseRecordComponent(NoInit()) @@ -219,6 +220,7 @@ RecordComponent &RecordComponent::setUnitSI(double usi) RecordComponent &RecordComponent::resetDataset(Dataset d) { auto &rc = get(); + auto &dataset = rc.dataset(); /* * For backwards-compatibility reasons, we do not actually run the below @@ -239,7 +241,7 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) if (written()) { - if (!rc.m_dataset.has_value()) + if (!dataset.has_value()) { throw error::Internal( "Internal control flow error: Written record component must " @@ -247,14 +249,14 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) } if (d.dtype == Datatype::UNDEFINED) { - d.dtype = rc.m_dataset.value().dtype; + d.dtype = dataset.value().dtype; } - else if (d.dtype != rc.m_dataset.value().dtype) + else if (d.dtype != dataset.value().dtype) { throw std::runtime_error( "Cannot change the datatype of a dataset."); } - rc.m_hasBeenExtended = true; + rc.hasBeenExtended() = true; } if (d.extent.empty()) @@ -275,26 +277,26 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) } else { - rc.m_dataset = std::move(d); + dataset = std::move(d); setDirty(true); return *this; } } - rc.m_isEmpty = false; + rc.isEmpty() = false; if (written()) { - if (!rc.m_dataset.has_value()) + if (!dataset.has_value()) { throw error::Internal( "Internal control flow error: Written record component must " "have defined datatype and extent."); } - rc.m_dataset.value().extend(std::move(d.extent)); + dataset.value().extend(std::move(d.extent)); } else { - rc.m_dataset = std::move(d); + dataset = std::move(d); } setDirty(true); @@ -304,9 +306,10 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) uint8_t RecordComponent::getDimensionality() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + auto &dataset = rc.dataset(); + if (dataset.has_value()) { - return rc.m_dataset.value().rank; + return dataset.value().rank; } else { @@ -317,9 +320,10 @@ uint8_t RecordComponent::getDimensionality() const Extent RecordComponent::getExtent() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + auto &dataset = rc.dataset(); + if (dataset.has_value()) { - return rc.m_dataset.value().extent; + return dataset.value().extent; } else { @@ -354,9 +358,10 @@ RecordComponent &RecordComponent::makeEmpty(Datatype dt, uint8_t dimensions) RecordComponent &RecordComponent::makeEmpty(Dataset d) { auto &rc = get(); + auto &dataset = rc.dataset(); if (written()) { - if (!rc.m_dataset.has_value()) + if (!dataset.has_value()) { throw error::Internal( "Internal control flow error: Written record component must " @@ -371,40 +376,40 @@ RecordComponent &RecordComponent::makeEmpty(Dataset d) } if (d.dtype == Datatype::UNDEFINED) { - d.dtype = rc.m_dataset.value().dtype; + d.dtype = dataset.value().dtype; } - else if (d.dtype != rc.m_dataset.value().dtype) + else if (d.dtype != dataset.value().dtype) { throw std::runtime_error( "Cannot change the datatype of a dataset."); } - rc.m_dataset.value().extend(std::move(d.extent)); - rc.m_hasBeenExtended = true; + dataset.value().extend(std::move(d.extent)); + rc.hasBeenExtended() = true; } else { - rc.m_dataset = std::move(d); + dataset = std::move(d); } - if (rc.m_dataset.value().extent.size() == 0) + if (dataset.value().extent.size() == 0) throw std::runtime_error("Dataset extent must be at least 1D."); - rc.m_isEmpty = true; + rc.isEmpty() = true; setDirty(true); if (!written()) { switchType>( - rc.m_dataset.value().dtype, *this); + dataset.value().dtype, *this); } return *this; } bool RecordComponent::empty() const { - return get().m_isEmpty; + return get().isEmpty(); } -void RecordComponent::visitHierarchy(HierarchyVisitor &v, bool) +void RecordComponent::visitHierarchyImpl(HierarchyVisitor &v, bool) { v(*this); } @@ -423,10 +428,12 @@ void RecordComponent::flush( } if (access::readOnly(IOHandler()->m_frontendAccess)) { - while (!rc.m_chunks.empty()) + // TODO: if we are going to allow custom hierarchies on constant + // components (technically possible), the flush would go here + while (!rc.chunks().empty()) { - IOHandler()->enqueue(rc.m_chunks.front()); - rc.m_chunks.pop(); + IOHandler()->enqueue(rc.chunks().front()); + rc.chunks().pop(); } } else @@ -434,11 +441,11 @@ void RecordComponent::flush( /* * This catches when a user forgets to use resetDataset. */ - if (!rc.m_dataset.has_value()) + if (!rc.dataset().has_value()) { // The check for !written() is technically not needed, just // defensive programming against internal bugs that go on us. - if (!written() && rc.m_chunks.empty() && !rc.m_isConstant) + if (!written() && rc.chunks().empty() && !rc.isConstant()) { // No data written yet, just accessed the object so far without // doing anything @@ -471,8 +478,8 @@ void RecordComponent::flush( IOHandler()->enqueue(IOTask(this, pCreate)); Parameter aWrite; aWrite.name = "value"; - aWrite.dtype = rc.m_constantValue.dtype; - aWrite.m_resource = rc.m_constantValue.getAny(); + aWrite.dtype = rc.constantValue().dtype; + aWrite.m_resource = rc.constantValue().getAny(); if (isVBased) { aWrite.changesOverSteps = Parameter< @@ -496,13 +503,13 @@ void RecordComponent::flush( else { Parameter dCreate( - rc.m_dataset.value()); + rc.dataset().value()); dCreate.name = name; IOHandler()->enqueue(IOTask(this, dCreate)); } } - if (rc.m_hasBeenExtended) + if (rc.hasBeenExtended()) { if (constant()) { @@ -530,18 +537,23 @@ void RecordComponent::flush( else { Parameter pExtend( - rc.m_dataset.value().extent); + rc.dataset().value().extent); IOHandler()->enqueue(IOTask(this, std::move(pExtend))); - rc.m_hasBeenExtended = false; + rc.hasBeenExtended() = false; } } - while (!rc.m_chunks.empty()) + while (!rc.chunks().empty()) { - IOHandler()->enqueue(rc.m_chunks.front()); - rc.m_chunks.pop(); + IOHandler()->enqueue(rc.chunks().front()); + rc.chunks().pop(); } + // TODO maybe guard against custom hierarchies on datasets? + // they can be created upon constant components however.. + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + flushAttributes(flushParams); } determineUnsetDirty(flushParams.flushLevel); @@ -623,6 +635,7 @@ void RecordComponent::readBase() setWritten(true, Attributable::EnqueueAsynchronously::No); }; + // why !empty() ?? if (constant() && !empty()) { read_constant(); @@ -755,8 +768,8 @@ RecordComponent &RecordComponent::makeConstant(T value) auto &rc = get(); - rc.m_constantValue = Attribute(value); - rc.m_isConstant = true; + rc.constantValue() = Attribute(value); + rc.isConstant() = true; return *this; } @@ -902,7 +915,7 @@ void RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) std::optional val = switchNonVectorType>( - /* dt = */ getDatatype(), rc.m_constantValue); + /* dt = */ getDatatype(), rc.constantValue()); if (val.has_value()) { diff --git a/src/Series.cpp b/src/Series.cpp index b08996c2c4..dcbe3e100f 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -36,6 +36,7 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/auxiliary/MonadicOperations.hpp" #include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" @@ -272,6 +273,13 @@ std::string Series::meshesPath() const return getAttribute("meshesPath").get(); } +std::optional Series::meshesPathOptional() const +{ + return auxiliary::optional_and_then( + getAttributeOptional("meshesPath"), + [](Attribute const &attr) { return attr.getOptional(); }); +} + Series &Series::setMeshesPath(std::string const &mp) { auto &series = get(); @@ -285,12 +293,7 @@ Series &Series::setMeshesPath(std::string const &mp) "A files meshesPath can not (yet) be changed after it has been " "written."); - if (auxiliary::ends_with(mp, '/')) - setAttribute("meshesPath", mp); - else - setAttribute("meshesPath", mp + "/"); - setDirty(true); - return *this; + return setMeshesPath_internal(mp); } std::vector Series::availableDatasets() @@ -610,6 +613,13 @@ std::string Series::particlesPath() const return getAttribute("particlesPath").get(); } +std::optional Series::particlesPathOptional() const +{ + return auxiliary::optional_and_then( + getAttributeOptional("particlesPath"), + [](Attribute const &attr) { return attr.getOptional(); }); +} + Series &Series::setParticlesPath(std::string const &pp) { auto &series = get(); @@ -623,12 +633,7 @@ Series &Series::setParticlesPath(std::string const &pp) "A files particlesPath can not (yet) be changed after it has been " "written."); - if (auxiliary::ends_with(pp, '/')) - setAttribute("particlesPath", pp); - else - setAttribute("particlesPath", pp + "/"); - setDirty(true); - return *this; + return setParticlesPath_internal(pp); } std::string Series::author() const @@ -1222,6 +1227,10 @@ void Series::initSeries( series.iterations.linkHierarchy(writable); series.iterations.writable().ownKeyWithinParent = "data"; + series->m_writable.ownKeyWithinParent = "ROOT"; + auto &container_back = + series->m_writable.objectType.requireGroup()->children(); + container_back["data"] = *series.iterations.m_attri; series.m_perIterationData.m_rankTableAttributable.linkHierarchy(writable); series.m_name = input->name; @@ -1484,6 +1493,12 @@ void Series::flushFileBased( case IO::HasBeenOpened: // continue below it->second.flush(flushParams); + + if (it == begin) + { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + } break; } @@ -1544,6 +1559,12 @@ void Series::flushFileBased( it->second.flushFileBased(filename, it->first, flushParams); + if (it == begin) + { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + } + series.iterations.flush( auxiliary::replace_first(basePath(), "%T/", ""), flushParams); @@ -1625,7 +1646,14 @@ void Series::flushGorVBased( series.m_snapshotToStep.at(it->first)}; IOHandler()->enqueue(IOTask(this, std::move(param))); } + it->second.flush(flushParams); + + if (it == begin) + { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + } break; } @@ -1638,6 +1666,12 @@ void Series::flushGorVBased( } } + if (begin == end) + { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + } + // Phase 3 Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); @@ -1707,6 +1741,13 @@ void Series::flushGorVBased( throw std::runtime_error( "[Series] Internal control flow error"); } + + if (it == begin) + { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + } + break; case IO::RemainsClosed: break; @@ -1721,6 +1762,12 @@ void Series::flushGorVBased( } } + if (begin == end) + { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + } + flushAttributes(flushParams); Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); @@ -1941,7 +1988,7 @@ void Series::readFileBased( for (auto index : unparseableIterations) { - series.iterations.container().erase(index); + series.iterations.erase_unchecked(index); } if (padding > 0) @@ -2277,7 +2324,7 @@ creating new iterations. std::cerr << "Cannot read iteration '" << index << "' and will skip it due to read error:\n" << err.what() << std::endl; - series.iterations.container().erase(index); + series.iterations.erase_unchecked(index); return {err}; } i.get().m_closed = internal::CloseStatus::Open; @@ -2308,7 +2355,17 @@ creating new iterations. readableIterations.reserve(pList.paths->size()); for (auto const &it : *pList.paths) { - IterationIndex_t index = std::stoull(it); + IterationIndex_t index; + try + { + index = std::stoull(it); + } + catch (std::exception const &e) + { + std::cerr << "[Warning] Could not parse '" << it + << "' as an Iteration index. Will skip." << std::endl; + continue; + } if (read_only_this_single_iteration.has_value() && index != *read_only_this_single_iteration) { @@ -2439,7 +2496,6 @@ creating new iterations. void Series::readBase() { - auto &series = get(); Parameter aRead; aRead.name = "openPMD"; @@ -2525,16 +2581,9 @@ void Series::readBase() .getOptional(); val.has_value()) { - /* allow setting the meshes path after completed IO */ - for (auto &it : series.iterations) - it.second.meshes.setWritten( - false, Attributable::EnqueueAsynchronously::No); - - setMeshesPath(val.value()); - - for (auto &it : series.iterations) - it.second.meshes.setWritten( - true, Attributable::EnqueueAsynchronously::No); + /* use internal api to allow setting the meshes path after completed + * IO */ + setMeshesPath_internal(val.value()); } else throw error::ReadError( @@ -2567,16 +2616,9 @@ void Series::readBase() .getOptional(); val.has_value()) { - /* allow setting the meshes path after completed IO */ - for (auto &it : series.iterations) - it.second.particles.setWritten( - false, Attributable::EnqueueAsynchronously::No); - - setParticlesPath(val.value()); - - for (auto &it : series.iterations) - it.second.particles.setWritten( - true, Attributable::EnqueueAsynchronously::No); + /* use internal api to allow setting the meshes path after completed + * IO */ + setParticlesPath_internal(val.value()); } else throw error::ReadError( @@ -2940,6 +2982,25 @@ Series &Series::setIterationEncoding_internal( return *this; } +Series &Series::setParticlesPath_internal(std::string const &pp) +{ + if (auxiliary::ends_with(pp, '/')) + setAttribute("particlesPath", pp); + else + setAttribute("particlesPath", pp + "/"); + setDirty(true); + return *this; +} + +Series &Series::setMeshesPath_internal(std::string const &mp) +{ + if (auxiliary::ends_with(mp, '/')) + setAttribute("meshesPath", mp); + else + setAttribute("meshesPath", mp + "/"); + setDirty(true); + return *this; +} auto Series::openIterationIfDirty(IterationIndex_t index, Iteration &iteration) -> IterationOpened { @@ -3366,7 +3427,7 @@ namespace internal } // Not strictly necessary, but clear the map of iterations // This releases the openPMD hierarchy - iterations.container().clear(); + iterations.clear_children_unchecked(); // Release the IO Handler if (IOHandler) { @@ -3605,7 +3666,7 @@ bool Series::closed() const return !w.IOHandler->has_value(); } -void Series::visitHierarchy(HierarchyVisitor &v, bool recursive) +void Series::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { if (recursive) { diff --git a/src/auxiliary/Variant.cpp b/src/auxiliary/Variant.cpp index 15d629e9a2..29e4598566 100644 --- a/src/auxiliary/Variant.cpp +++ b/src/auxiliary/Variant.cpp @@ -21,6 +21,8 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/DatatypeMacros.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" #include #include @@ -68,6 +70,24 @@ size_t Variant::index() const return getVariant>().index(); } +template +void Variant::print() const +{ + std::visit( + [](auto const &val) { + using T = std::remove_cv_t>; + if constexpr (auxiliary::IsVector_v || auxiliary::IsArray_v) + { + auxiliary::write_vec_to_stream(std::cout, val) << std::endl; + } + else + { + std::cout << val << std::endl; + } + }, + getVariant>()); +} + #define OPENPMD_ENUMERATE_TYPES(type) , type using VariantInstantiated = Variant; diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 2587bc76c1..3846251ab7 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -19,8 +19,10 @@ * If not, see . */ #include "openPMD/backend/Attributable.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/RecordComponent.hpp" @@ -29,6 +31,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/HierarchyVisitorImpl.hpp" +#include "openPMD/backend/Writable.hpp" #include #include @@ -53,12 +56,10 @@ namespace internal : SharedData_t({raw_ptr, [](auto const *) {}}) {} - void AttributableData::cloneFrom(AttributableData const &other) + void AttributableData::cloneFrom(parent_t const &other) { - using parent_t = std::shared_ptr; static_cast(*this) = static_cast(other); } - } // namespace internal Attributable::Attributable() @@ -105,13 +106,25 @@ bool Attributable::setAttribute(std::string const &key, Attribute attribute) } Attribute Attributable::getAttribute(std::string const &key) const +{ + auto attribute = getAttributeOptional(key); + if (attribute.has_value()) + { + return *attribute; + } + + throw no_such_attribute_error(key); +} + +std::optional +Attributable::getAttributeOptional(std::string const &key) const { auto &attri = get(); auto it = attri.m_attributes.find(key); if (it != attri.m_attributes.cend()) return it->second; - throw no_such_attribute_error(key); + return std::nullopt; } bool Attributable::deleteAttribute(std::string const &key) @@ -179,6 +192,68 @@ void Attributable::iterationFlush(std::string backendConfig) std::move(backendConfig)); } +void Attributable::customHierarchyFlush( + internal::FlushParams const &flushParams, bool managed_as_custom_object) +{ + // customHierarchies().printRecursively(); + if (!dirtyRecursive()) + { + return; + } + + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + // No need to do anything in access::readOnly since meshes and particles + // are initialized as aliases for subgroups at parsing time + if (managed_as_custom_object && + access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } + + auto &data = get(); + Parameter pCreate; + + data.m_writable.objectType.ifGroup([&](auto &group_metadata) { + for (auto const &[name, child] : group_metadata.children()) + { + if (!child->managed_as_custom_hierarchy) + { + continue; + } + CustomHierarchy subpath{child}; + std::visit( + auxiliary::overloaded{ + [&, &name_lambda = name]( + internal::object_type::GroupMetaData const &) { + if (!subpath.written()) + { + pCreate.path = name_lambda; + IOHandler()->enqueue(IOTask(&subpath, pCreate)); + } + subpath.customHierarchyFlush(flushParams, true); + }, + [&, name_lambda = name]( + internal::object_type::DatasetMetaData const &) { + subpath.asDataset().flush(name_lambda, flushParams); + }}, + child->m_writable.objectType); + } + }); + + if (managed_as_custom_object && + flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + setDirty(false); + } + // customHierarchies().printRecursively(); +} + Series Attributable::retrieveSeries() const { Writable const *findSeries = &writable(); @@ -297,10 +372,31 @@ void Attributable::touch() setDirtyRecursive(true); } -void Attributable::visitHierarchy(HierarchyVisitor &, bool) +void Attributable::visitHierarchy(HierarchyVisitor &visitor, bool recursive) { - throw error::Internal( - "[Attributable::visitHierarchy] Cannot call this on base class."); + this->visitHierarchyImpl(visitor, recursive); + if (recursive) + { + writable().objectType.ifGroup([&, recursive](auto &group_info) { + for (auto const &[name, child] : group_info.children()) + { + (void)name; + if (!child->managed_as_custom_hierarchy) + { + continue; + } + CustomHierarchy obj{child}; + if (obj.isDataset()) + { + obj.asDataset().visitHierarchy(visitor, recursive); + } + else + { + obj.visitHierarchy(visitor, recursive); + } + } + }); + } } void Attributable::populateMissingMetadata(bool recursive) @@ -348,6 +444,16 @@ uintptr_t Attributable::memoryID() const return reinterpret_cast(&retrieveSeries().Attributable::get()); } +auto Attributable::customHierarchies() -> CustomHierarchy +{ + // The returned instance does not need to be registered anywhere: typed + // handles are owned by the ChildrenCache, so children created through this + // view stay alive in the backend hierarchy. + auto res = CustomHierarchy{*this}; + traits::ElementAccessPolicy::call(res); + return res; +} + template void Attributable::seriesFlush_impl( internal::FlushParams const &flushParams, bool flush_io_handler) @@ -361,6 +467,10 @@ template void Attributable::seriesFlush_impl( void Attributable::flushAttributes(internal::FlushParams const &flushParams) { + if (access::readOnly(IOHandler()->m_frontendAccess)) + { + throw std::runtime_error("Control flow error"); + } if (!flush_level::write_attributes(flushParams.flushLevel)) { return; @@ -589,6 +699,39 @@ void Attributable::readAttributes(ReadMode mode) setDirty(false); } +void Attributable::preferCurrentBackpointer() const +{ + /* + * This is called when reopening some object as a specific type (e.g. + * RecordComponent) that had originally been opened generically as + * CustomHierarchy already. In this case, the specific type's pointer should + * be preferred for Writable::attributable as it has more information. + */ + auto this_as_custom_hierarchy = dynamic_cast(this); + if (this_as_custom_hierarchy) + { + return; + } + + auto &shareddata = **m_attri; + auto &w = shareddata.m_writable; + + auto backpointer_as_custom_hierarchy = + dynamic_cast(w.attributable); + if (!backpointer_as_custom_hierarchy) + { + return; + } + + // Now: + // !this_as_custom_hierarchy && backpointer_as_custom_hierarchy + + w.attributable = m_attri.get(); + shareddata.managed_as_custom_hierarchy = false; + + // std::cout << "REWIRED '" << myPath().openPMDPath() << "'." << std::endl; +} + void Attributable::setWritten(bool val, EnqueueAsynchronously ea) { switch (ea) @@ -612,6 +755,12 @@ void Attributable::setWritten(bool val, EnqueueAsynchronously ea) writable().written = val; } +void Attributable::visitHierarchyImpl(HierarchyVisitor &, bool) +{ + throw error::Internal( + "[Attributable::visitHierarchy] Cannot call this on base class."); +} + void Attributable::linkHierarchy(Writable &w) { auto handler = w.IOHandler; diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index 89777af8d6..fb33318423 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -172,20 +172,19 @@ namespace internal template class internal::ScalarIterator< \ baserecordtype, \ baserecordtype::Data_t, \ - baserecordtype::T_Container::InternalContainer::iterator>; \ + baserecordtype::T_Container::iterator>; \ template class internal::ScalarIterator< \ baserecordtype const, \ baserecordtype::Data_t const, \ - baserecordtype::T_Container::InternalContainer::const_iterator>; \ + baserecordtype::T_Container::const_iterator>; \ template class internal::ScalarIterator< \ baserecordtype, \ baserecordtype::Data_t, \ - baserecordtype::T_Container::InternalContainer::reverse_iterator>; \ + baserecordtype::T_Container::reverse_iterator>; \ template class internal::ScalarIterator< \ baserecordtype const, \ baserecordtype::Data_t const, \ - baserecordtype::T_Container::InternalContainer:: \ - const_reverse_iterator>; + baserecordtype::T_Container::const_reverse_iterator>; #define OPENPMD_INSTANTIATE(recordcomponenttype) \ INSTANTIATE_ITERATORS_FOR_BASERECORD(BaseRecord) @@ -198,7 +197,7 @@ namespace internal template auto BaseRecord::begin() -> iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ false); } @@ -211,7 +210,7 @@ auto BaseRecord::begin() -> iterator template auto BaseRecord::begin() const -> const_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ false); } @@ -224,7 +223,7 @@ auto BaseRecord::begin() const -> const_iterator template auto BaseRecord::cbegin() const -> const_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ false); } @@ -237,7 +236,7 @@ auto BaseRecord::cbegin() const -> const_iterator template auto BaseRecord::end() -> iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ true); } @@ -250,7 +249,7 @@ auto BaseRecord::end() -> iterator template auto BaseRecord::end() const -> const_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ true); } @@ -263,7 +262,7 @@ auto BaseRecord::end() const -> const_iterator template auto BaseRecord::cend() const -> const_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ true); } @@ -276,78 +275,78 @@ auto BaseRecord::cend() const -> const_iterator template auto BaseRecord::rbegin() -> reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ false); } else { - return makeReverseIterator(this->container().rbegin()); + return makeReverseIterator(T_Container::rbegin()); } } template auto BaseRecord::rbegin() const -> const_reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ false); } else { - return makeReverseIterator(this->container().rbegin()); + return makeReverseIterator(T_Container::rbegin()); } } template auto BaseRecord::crbegin() const -> const_reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ false); } else { - return makeReverseIterator(this->container().crbegin()); + return makeReverseIterator(T_Container::crbegin()); } } template auto BaseRecord::rend() -> reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ true); } else { - return makeReverseIterator(this->container().rend()); + return makeReverseIterator(T_Container::rend()); } } template auto BaseRecord::rend() const -> const_reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ true); } else { - return makeReverseIterator(this->container().rend()); + return makeReverseIterator(T_Container::rend()); } } template auto BaseRecord::crend() const -> const_reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ true); } else { - return makeReverseIterator(this->container().crend()); + return makeReverseIterator(T_Container::crend()); } } @@ -456,13 +455,18 @@ auto BaseRecord::at(key_type const &key) const -> mapped_type const & bool const keyScalar = (key == RecordComponent::SCALAR); if (keyScalar) { - if (!get().m_datasetDefined) + if (!get().datasetDefined()) { throw std::out_of_range( "[at()] Requested scalar entry from non-scalar record."); } return static_cast(*this); } + else if (scalar()) + { + throw std::out_of_range( + "[at()] Requested non-scalar entry from a scalar record."); + } else { return T_Container::at(key); @@ -474,7 +478,11 @@ auto BaseRecord::erase(key_type const &key) -> size_type { bool const keyScalar = (key == RecordComponent::SCALAR); size_type res; - if (!keyScalar) + if (!keyScalar && scalar()) + { + res = 0; + } + else if (!keyScalar) res = Container::erase(key); else { @@ -510,7 +518,7 @@ template auto BaseRecord::find(key_type const &key) -> iterator { auto &r = get(); - if (r.m_datasetDefined) + if (r.datasetDefined()) { if (key == RecordComponent::SCALAR) { @@ -527,7 +535,7 @@ auto BaseRecord::find(key_type const &key) -> iterator } else { - return makeIterator(r.m_container.find(key)); + return makeIterator(T_Container::find(key)); } } @@ -535,7 +543,7 @@ template auto BaseRecord::find(key_type const &key) const -> const_iterator { auto &r = get(); - if (r.m_datasetDefined) + if (r.datasetDefined()) { if (key == RecordComponent::SCALAR) { @@ -552,7 +560,7 @@ auto BaseRecord::find(key_type const &key) const -> const_iterator } else { - return makeIterator(r.m_container.find(key)); + return makeIterator(T_Container::find(key)); } } @@ -561,7 +569,11 @@ auto BaseRecord::count(key_type const &key) const -> size_type { if (key == RecordComponent::SCALAR) { - return get().m_datasetDefined ? 1 : 0; + return get().datasetDefined() ? 1 : 0; + } + else if (scalar()) + { + return 0; } else { @@ -626,10 +638,11 @@ auto BaseRecord::insert(value_type const &value) -> std::pair { detail::verifyNonscalar(this); - auto res = this->container().insert(value); + auto res = T_Container::insert(value); if (res.first->first == RecordComponent::SCALAR) { - this->container().erase(res.first); + auto const key = res.first->first; + this->erase_unchecked(key); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; @@ -639,10 +652,11 @@ template auto BaseRecord::insert(value_type &&value) -> std::pair { detail::verifyNonscalar(this); - auto res = this->container().insert(std::move(value)); + auto res = T_Container::insert(std::move(value)); if (res.first->first == RecordComponent::SCALAR) { - this->container().erase(res.first); + auto const key = res.first->first; + this->erase_unchecked(key); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; @@ -657,15 +671,14 @@ auto BaseRecord::insert(const_iterator hint, value_type const &value) auxiliary::overloaded{ [](typename const_iterator::Left left) { return left; }, [this](typename const_iterator::Right) { - return static_cast const *>(this) - ->container() - .begin(); + return T_Container::cbegin(); }}, hint.m_iterator); - auto res = this->container().insert(base_hint, value); + auto res = T_Container::insert(base_hint, value); if (res->first == RecordComponent::SCALAR) { - this->container().erase(res); + auto const key = res->first; + this->erase_unchecked(key); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return makeIterator(res); @@ -680,65 +693,24 @@ auto BaseRecord::insert(const_iterator hint, value_type &&value) auxiliary::overloaded{ [](typename const_iterator::Left left) { return left; }, [this](typename const_iterator::Right) { - return static_cast const *>(this) - ->container() - .begin(); + return T_Container::cbegin(); }}, hint.m_iterator); - auto res = this->container().insert(base_hint, std::move(value)); + auto res = T_Container::insert(base_hint, std::move(value)); if (res->first == RecordComponent::SCALAR) { - this->container().erase(res); + auto const key = res->first; + this->erase_unchecked(key); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return makeIterator(res); } -template -template -auto BaseRecord::insert(InputIt first, InputIt last) -> void -{ - detail::verifyNonscalar(this); - this->container().insert(first, last); - /* - * We skip this check as it changes the runtime of this call from - * O(last-first) to O(container().size()). - */ - // for (auto it = this->container().begin(); it != end; ++it) - // { - // if (it->first == RecordComponent::SCALAR) - // { - // this->container().erase(it); - // throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); - // } - // } -} - template auto BaseRecord::insert(std::initializer_list ilist) -> void { detail::verifyNonscalar(this); - this->container().insert(std::move(ilist)); - /* - * We skip this check as it changes the runtime of this call from - * O(last-first) to O(container().size()). - */ - // for (auto it = this->container().begin(); it != end; ++it) - // { - // if (it->first == RecordComponent::SCALAR) - // { - // this->container().erase(it); - // throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); - // } - // } -} - -template -auto BaseRecord::swap(BaseRecord &other) noexcept -> void -{ - detail::verifyNonscalar(this); - detail::verifyNonscalar(&other); - this->container().swap(other.container()); + this->insert(ilist.begin(), ilist.end()); } template @@ -791,7 +763,8 @@ inline void BaseRecord::flush( * Defensive programming. Normally, this error should yield as soon as * possible. */ - if (scalar() && !T_Container::empty()) + if (scalar() && this->writable().objectType.isGroup() && + !T_Container::empty()) { throw error::WrongAPIUsage( "A scalar component can not be contained at the same time as " @@ -799,9 +772,12 @@ inline void BaseRecord::flush( } this->flush_impl(name, flushParams); + /* + * The dirty flag is unset centrally here, after flush_impl() has updated + * the middle-end state. The flush_impl() overrides (Mesh, Record, + * PatchRecord) must therefore not unset dirty flags themselves. + */ this->determineUnsetDirty(flushParams.flushLevel); - // flush_impl must take care to correctly set the dirty() flag so this - // method doesn't do it } template diff --git a/src/backend/BaseRecordComponent.cpp b/src/backend/BaseRecordComponent.cpp index 21a7352850..ed30f88601 100644 --- a/src/backend/BaseRecordComponent.cpp +++ b/src/backend/BaseRecordComponent.cpp @@ -32,19 +32,20 @@ double BaseRecordComponent::unitSI() const BaseRecordComponent &BaseRecordComponent::resetDatatype(Datatype d) { + auto &rc = get(); + auto &dataset = rc.dataset(); if (written()) throw std::runtime_error( "A Records Datatype can not (yet) be changed after it has been " "written."); - auto &rc = get(); - if (rc.m_dataset.has_value()) + if (dataset.has_value()) { - rc.m_dataset.value().dtype = d; + dataset.value().dtype = d; } else { - rc.m_dataset = Dataset{d, {1}}; + dataset = Dataset{d, {1}}; } setDirty(true); return *this; @@ -53,9 +54,10 @@ BaseRecordComponent &BaseRecordComponent::resetDatatype(Datatype d) Datatype BaseRecordComponent::getDatatype() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + auto &dataset = rc.dataset(); + if (dataset.has_value()) { - return rc.m_dataset.value().dtype; + return dataset.value().dtype; } else { @@ -65,15 +67,17 @@ Datatype BaseRecordComponent::getDatatype() const bool BaseRecordComponent::constant() const { - return get().m_isConstant; + auto &data = get(); + return data.datasetDefined() && data.isConstant(); } std::optional BaseRecordComponent::joinedDimension() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + auto &dataset = rc.dataset(); + if (dataset.has_value()) { - return rc.m_dataset.value().joinedDimension(); + return dataset.value().joinedDimension(); } else { @@ -84,14 +88,15 @@ std::optional BaseRecordComponent::joinedDimension() const ChunkTable BaseRecordComponent::availableChunks() { auto &rc = get(); - if (rc.m_isConstant) + auto &dataset = rc.dataset(); + if (rc.isConstant()) { - if (!rc.m_dataset.has_value()) + if (!dataset.has_value()) { return ChunkTable{}; } - Offset offset(rc.m_dataset.value().extent.size(), 0); - return ChunkTable{{std::move(offset), rc.m_dataset.value().extent}}; + Offset offset(dataset.value().extent.size(), 0); + return ChunkTable{{std::move(offset), dataset.value().extent}}; } if (auto iteration_data = containingIteration().first; iteration_data.has_value()) @@ -122,12 +127,12 @@ BaseRecordComponent::BaseRecordComponent(NoInit) : Attributable(NoInit()) void BaseRecordComponent::setDatasetDefined( internal::BaseRecordComponentData &data) { - data.m_datasetDefined = true; + data.setDatasetDefined(); } bool BaseRecordComponent::datasetDefined() const { auto &data = get(); - return data.m_datasetDefined; + return data.datasetDefined(); } } // namespace openPMD diff --git a/src/backend/ChildView.cpp b/src/backend/ChildView.cpp new file mode 100644 index 0000000000..2614a06ea6 --- /dev/null +++ b/src/backend/ChildView.cpp @@ -0,0 +1,99 @@ +/* Copyright 2026 The openPMD Community + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#include "openPMD/backend/ChildView.hpp" + +#include "ContainerTypeMacros.hpp" + +#include "openPMD/CustomHierarchy.hpp" +#include "openPMD/Iteration.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticlePatches.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/Record.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/ChildViewImpl.tpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecord.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" + +namespace openPMD::internal +{ +#define OPENPMD_INSTANTIATE_CONTAINER_VIEW(type, key) \ + template class ChildView; \ + template class ProjectedChildIterator; \ + template class ProjectedChildIterator; + +OPENPMD_FOREACH_CONTAINER_TYPE(OPENPMD_INSTANTIATE_CONTAINER_VIEW) +#undef OPENPMD_INSTANTIATE_CONTAINER_VIEW + +// Explicit instantiation of a const ChildView would also instantiate its +// intentionally ill-formed mutating members. Instantiate only the read-only +// API of the const specializations. +#define OPENPMD_INSTANTIATE_CONST_CONTAINER_VIEW(type, key) \ + template ChildView::ChildView( \ + typename ChildView::cache_t *, \ + Writable *, \ + ChildFilter const &, \ + typename ChildView::CreateChild); \ + template auto ChildView::begin() -> \ + typename ChildView::const_iterator; \ + template auto ChildView::begin() const -> \ + typename ChildView::const_iterator; \ + template auto ChildView::cbegin() const -> \ + typename ChildView::const_iterator; \ + template auto ChildView::end() -> \ + typename ChildView::const_iterator; \ + template auto ChildView::end() const -> \ + typename ChildView::const_iterator; \ + template auto ChildView::cend() const -> \ + typename ChildView::const_iterator; \ + template auto ChildView::rbegin() -> \ + typename ChildView::const_reverse_iterator; \ + template auto ChildView::rbegin() const -> \ + typename ChildView::const_reverse_iterator; \ + template auto ChildView::crbegin() const -> \ + typename ChildView::const_reverse_iterator; \ + template auto ChildView::rend() -> \ + typename ChildView::const_reverse_iterator; \ + template auto ChildView::rend() const -> \ + typename ChildView::const_reverse_iterator; \ + template auto ChildView::crend() const -> \ + typename ChildView::const_reverse_iterator; \ + template auto ChildView::empty() const -> bool; \ + template auto ChildView::size() const -> \ + typename ChildView::size_type; \ + template auto ChildView::find(key const &) -> \ + typename ChildView::const_iterator; \ + template auto ChildView::find(key const &) const -> \ + typename ChildView::const_iterator; \ + template auto ChildView::contains(key const &) const \ + -> bool; \ + template auto ChildView::count(key const &) const -> \ + typename ChildView::size_type; \ + template auto ChildView::at(key const &) -> type const &; \ + template auto ChildView::at(key const &) const \ + -> type const &; \ + template auto ChildView::filter() const \ + -> ChildFilter const &; + +OPENPMD_FOREACH_CONTAINER_TYPE(OPENPMD_INSTANTIATE_CONST_CONTAINER_VIEW) +#undef OPENPMD_INSTANTIATE_CONST_CONTAINER_VIEW +} // namespace openPMD::internal diff --git a/src/backend/ChildrenCache.cpp b/src/backend/ChildrenCache.cpp new file mode 100644 index 0000000000..95e85ff274 --- /dev/null +++ b/src/backend/ChildrenCache.cpp @@ -0,0 +1,159 @@ +/* Copyright 2026 The openPMD Community + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#include "openPMD/backend/ChildrenCache.hpp" + +#include "ContainerTypeMacros.hpp" + +#include "openPMD/CustomHierarchy.hpp" +#include "openPMD/Iteration.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticlePatches.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/Record.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/ChildrenCacheImpl.tpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecord.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" + +#include + +namespace openPMD::internal +{ +namespace +{ + enum class ContainerTypeId : std::uint8_t + { +#define OPENPMD_ENUMERATE_CONTAINER_TYPE(type, key) type, + OPENPMD_FOREACH_CONTAINER_TYPE(OPENPMD_ENUMERATE_CONTAINER_TYPE) +#undef OPENPMD_ENUMERATE_CONTAINER_TYPE + }; +} // namespace + +#define OPENPMD_DEFINE_CONTAINER_TYPE_ID(type, key) \ + template <> \ + auto container_type_id() -> std::uint8_t \ + { \ + return static_cast(ContainerTypeId::type); \ + } +OPENPMD_FOREACH_CONTAINER_TYPE(OPENPMD_DEFINE_CONTAINER_TYPE_ID) +#undef OPENPMD_DEFINE_CONTAINER_TYPE_ID + +#define OPENPMD_INSTANTIATE_CONTAINER_CACHE(type, key) \ + template ChildrenCacheMap \ + ChildrenCache::get_or_create_cache() const; \ + template std::optional> \ + ChildrenCache::find_cache() const; \ + template bool ChildrenCache::contains_cache() const noexcept; \ + template std::pair::iterator, bool> \ + ChildrenCache::try_emplace( \ + key const &, \ + std::shared_ptr const &, \ + type &&); \ + template std::pair::iterator, bool> \ + ChildrenCache::try_emplace( \ + key const &, std::shared_ptr const &, type &&) \ + const; + +OPENPMD_FOREACH_CONTAINER_TYPE(OPENPMD_INSTANTIATE_CONTAINER_CACHE) +#undef OPENPMD_INSTANTIATE_CONTAINER_CACHE + +ChildrenCache::children_map_t &ChildrenCache::children() +{ + return m_children; +} + +ChildrenCache::children_map_t const &ChildrenCache::children() const +{ + return m_children; +} + +void ChildrenCache::erase(std::string const &canonicalKey) +{ + for (auto &[type, entry] : m_entries) + { + (void)type; + entry->erase(canonicalKey); + } + + for (auto const &[type, entry] : m_entries) + { + (void)type; + assert(!entry->contains(canonicalKey)); + } + m_children.erase(canonicalKey); +} + +void ChildrenCache::clear() +{ + for (auto &[type, entry] : m_entries) + { + (void)type; + entry->clear(); + } + + for (auto const &[type, entry] : m_entries) + { + (void)type; + assert(entry->size() == 0); + } + m_children.clear(); + + assert_consistent(); +} + +bool ChildrenCache::empty() const noexcept +{ + for (auto const &[type, entry] : m_entries) + { + (void)type; + if (entry->size() != 0) + { + return false; + } + } + return true; +} + +std::size_t ChildrenCache::num_maps() const noexcept +{ + return m_entries.size(); +} + +void ChildrenCache::assert_consistent() const +{ +#ifndef NDEBUG + for (auto const &pair : m_entries) + { + auto &entry = pair.second; + + std::size_t const registered_size = entry->size(); + std::size_t counted_size = 0; + entry->for_each_key([&](std::string const &key) { + ++counted_size; + assert(entry->contains(key)); + assert(m_children.find(key) != m_children.end()); + }); + assert(counted_size == registered_size); + } +#endif +} +} // namespace openPMD::internal diff --git a/src/backend/Container.cpp b/src/backend/Container.cpp index fc0785d2c1..2412110c7c 100644 --- a/src/backend/Container.cpp +++ b/src/backend/Container.cpp @@ -21,30 +21,27 @@ #include "openPMD/backend/ContainerImpl.tpp" +#include "ContainerTypeMacros.hpp" + +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" #include "openPMD/ParticleSpecies.hpp" +#include "openPMD/Record.hpp" +#include "openPMD/RecordComponent.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/HierarchyVisitorImpl.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecord.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" namespace openPMD { -#define OPENPMD_COMMA , -#define OPENPMD_INSTANTIATE(type) template class Container; +#define OPENPMD_INSTANTIATE(type, key) template class Container; -OPENPMD_INSTANTIATE(Mesh) -OPENPMD_INSTANTIATE(MeshRecordComponent) -OPENPMD_INSTANTIATE(ParticlePatches) -OPENPMD_INSTANTIATE(ParticleSpecies) -OPENPMD_INSTANTIATE(PatchRecord) -OPENPMD_INSTANTIATE(PatchRecordComponent) -OPENPMD_INSTANTIATE(Record) -OPENPMD_INSTANTIATE(RecordComponent) -OPENPMD_INSTANTIATE(Iteration OPENPMD_COMMA Iteration::IterationIndex_t) +OPENPMD_FOREACH_CONTAINER_TYPE(OPENPMD_INSTANTIATE) #undef OPENPMD_INSTANTIATE -#undef OPENPMD_COMMA namespace internal { diff --git a/src/backend/ContainerTypeMacros.hpp b/src/backend/ContainerTypeMacros.hpp new file mode 100644 index 0000000000..bfd8bcf73a --- /dev/null +++ b/src/backend/ContainerTypeMacros.hpp @@ -0,0 +1,39 @@ +/* Copyright 2026 The openPMD Community + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +/** Enumerate all explicitly instantiated Container element types. + * + * The macro is invoked as MACRO(type, key_type). All container types use + * std::string as key except Iteration, which is indexed by IterationIndex_t. + */ +#define OPENPMD_FOREACH_CONTAINER_TYPE(MACRO) \ + MACRO(Mesh, std::string) \ + MACRO(MeshRecordComponent, std::string) \ + MACRO(ParticlePatches, std::string) \ + MACRO(ParticleSpecies, std::string) \ + MACRO(PatchRecord, std::string) \ + MACRO(PatchRecordComponent, std::string) \ + MACRO(Record, std::string) \ + MACRO(RecordComponent, std::string) \ + MACRO(CustomHierarchy, std::string) \ + MACRO(CustomDataset, std::string) \ + MACRO(Iteration, Iteration::IterationIndex_t) diff --git a/src/backend/HierarchyVisitor.cpp b/src/backend/HierarchyVisitor.cpp new file mode 100644 index 0000000000..2b9b136c22 --- /dev/null +++ b/src/backend/HierarchyVisitor.cpp @@ -0,0 +1,8 @@ +#include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/CustomHierarchy.hpp" + +namespace openPMD +{ +void HierarchyVisitor::operator()(CustomHierarchy &) +{} +} // namespace openPMD diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index 2b297f2731..bedf952d5b 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -66,7 +66,7 @@ MeshRecordComponent &MeshRecordComponent::setPosition(std::vector pos) return *this; } -void MeshRecordComponent::visitHierarchy(HierarchyVisitor &v, bool) +void MeshRecordComponent::visitHierarchyImpl(HierarchyVisitor &v, bool) { v(*this); } diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 7d68b16035..fd4dbd0d3a 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -47,9 +47,9 @@ PatchRecord::setUnitDimension(unit_representations::AsArray const &udim) return *this; } -void PatchRecord::visitHierarchy(HierarchyVisitor &v, bool recursive) +void PatchRecord::visitHierarchyImpl(HierarchyVisitor &v, bool recursive) { - visitHierarchyImpl(v, recursive); + visitHierarchyContainer(v, recursive); } void PatchRecord::flush_impl( @@ -61,16 +61,17 @@ void PatchRecord::flush_impl( } if (!this->scalar()) { - if (IOHandler()->m_frontendAccess != Access::READ_ONLY) - Container::flush( - path, flushParams); // warning (clang-tidy-10): - // bugprone-parent-virtual-call + Container::flush( + path, flushParams); // warning (clang-tidy-10): + // bugprone-parent-virtual-call + for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } else T_RecordComponent::flush(path, flushParams); - determineUnsetDirty(flushParams.flushLevel); + // Dirty flags are unset centrally by BaseRecord::flush() after + // flush_impl() returns, so no unsetDirty operation is needed here. } void PatchRecord::read() @@ -100,7 +101,7 @@ void PatchRecord::read() << component_name << "' and will skip it due to read error:" << err.what() << std::endl; - this->container().erase(component_name); + this->erase_unchecked(component_name); } } diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index 5c35d918fe..c2fbebb7aa 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -44,9 +44,10 @@ uint8_t PatchRecordComponent::getDimensionality() const Extent PatchRecordComponent::getExtent() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + auto &dataset = rc.dataset(); + if (dataset.has_value()) { - return rc.m_dataset.value().extent; + return dataset.value().extent; } else { @@ -54,7 +55,7 @@ Extent PatchRecordComponent::getExtent() const } } -void PatchRecordComponent::visitHierarchy(HierarchyVisitor &v, bool) +void PatchRecordComponent::visitHierarchyImpl(HierarchyVisitor &v, bool) { v(*this); } @@ -72,10 +73,8 @@ void PatchRecordComponent::read() internal::ScientificDefaults::readDefaults(*this, IOHandler()->m_standard); } -PatchRecordComponent::PatchRecordComponent() : RecordComponent(NoInit()) -{ - setData(std::make_shared()); -} +PatchRecordComponent::PatchRecordComponent() : RecordComponent() +{} PatchRecordComponent::PatchRecordComponent(NoInit) : RecordComponent(NoInit()) {} diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 2a7f994873..adc1c2ed7f 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -657,7 +657,7 @@ std::optional StatefulIterator::loopBody(Seek const &seek) else if ( series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) { - data.series.iterations.container().erase( + data.series.iterations.erase_unchecked( *maybe_current_iteration); } } @@ -853,7 +853,7 @@ void StatefulIterator::deactivateDeadIteration(iteration_index_t index) } break; } - data.series.iterations.container().erase(index); + data.series.iterations.erase_unchecked(index); } StatefulIterator &StatefulIterator::operator++() diff --git a/test/AuxiliaryTest.cpp b/test/AuxiliaryTest.cpp index d7b07b2969..229c6a461c 100644 --- a/test/AuxiliaryTest.cpp +++ b/test/AuxiliaryTest.cpp @@ -31,6 +31,8 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/ChildViewImpl.tpp" +#include "openPMD/backend/ChildrenCacheImpl.tpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/ContainerImpl.tpp" #include "openPMD/backend/Variant_internal.hpp" @@ -169,6 +171,15 @@ struct S : public TestHelper }; } // namespace openPMD::test +namespace openPMD::internal +{ +template <> +auto container_type_id() -> std::uint8_t +{ + return 246; +} +} // namespace openPMD::internal + TEST_CASE("container_default_test", "[auxiliary]") { #if openPMD_USE_INVASIVE_TESTS @@ -209,6 +220,15 @@ struct structure : public TestHelper }; } // namespace openPMD::test +namespace openPMD::internal +{ +template <> +auto container_type_id() -> std::uint8_t +{ + return 247; +} +} // namespace openPMD::internal + TEST_CASE("container_retrieve_test", "[auxiliary]") { #if openPMD_USE_INVASIVE_TESTS @@ -284,6 +304,15 @@ struct Widget : public TestHelper }; } // namespace openPMD::test +namespace openPMD::internal +{ +template <> +auto container_type_id() -> std::uint8_t +{ + return 248; +} +} // namespace openPMD::internal + TEST_CASE("container_access_test", "[auxiliary]") { #if openPMD_USE_INVASIVE_TESTS @@ -319,6 +348,32 @@ TEST_CASE("container_access_test", "[auxiliary]") REQUIRE(c.erase(c.find("4fourthWidget")) == c.end()); REQUIRE(c.size() == 0); REQUIRE(c.empty()); + + // Range insert, both the iterator-pair and the initializer_list overloads. + std::vector::value_type> widgets; + widgets.emplace_back("6sixthWidget", Widget(6)); + widgets.emplace_back("7seventhWidget", Widget(7)); + c.insert(widgets.begin(), widgets.end()); + REQUIRE(c.size() == 2); + REQUIRE(c.contains("6sixthWidget")); + REQUIRE(c.contains("7seventhWidget")); + c.insert( + {Container::value_type{"8eighthWidget", Widget(8)}, + Container::value_type{"9ninthWidget", Widget(9)}}); + REQUIRE(c.size() == 4); + REQUIRE(c.contains("8eighthWidget")); + REQUIRE(c.contains("9ninthWidget")); + + // Post-increment/decrement must yield the *previous* iterator position. + { + auto it = c.begin(); + auto old = it++; + REQUIRE(old->first == "6sixthWidget"); + REQUIRE(it != c.begin()); + auto back = it--; + REQUIRE(it == c.begin()); + REQUIRE(back->first != "6sixthWidget"); + } #else std::cerr << "Invasive tests not enabled. Hierarchy is not visible.\n"; #endif @@ -545,3 +600,8 @@ TEST_CASE("drain_set_test", "[auxiliary]") { drain_set::drain_set(); } + +TEST_CASE("children_cache_test", "[auxiliary]") +{ + children_cache::children_cache(); +} diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 676b2b0765..39aff1ca91 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -19,6 +19,7 @@ * If not, see . */ // expose private and protected members for invasive testing +#include "openPMD/Error.hpp" #if openPMD_USE_INVASIVE_TESTS #define OPENPMD_private public: #define OPENPMD_protected public: @@ -278,6 +279,17 @@ TEST_CASE("versions_test", "[core]") REQUIRE(featureVariants.at("json") == true); } +TEST_CASE("dataset_initialization", "[core]") +{ + Series s("../samples/f.json", Access::CREATE); + auto rc = s.iterations[0].meshes["E"]["x"]; + rc.getDatatype(); + auto scalar_rc = s.iterations[0].meshes["theta"]; + REQUIRE_THROWS_AS(scalar_rc.getDatatype(), error::WrongAPIUsage); + scalar_rc.resetDataset(Dataset{Datatype::INT, {5}}); + scalar_rc.getDatatype(); +} + TEST_CASE("attribute_dtype_test", "[core]") { Attribute a = Attribute(static_cast(' ')); @@ -586,6 +598,25 @@ TEST_CASE("output_constructor_test", "[core]") o.setParticlesPath("particles"); } +TEST_CASE("optional_attribute_test", "[core]") +{ + Series o = Series("../samples/new_openpmd_output.json", Access::CREATE); + + // Missing attributes yield an empty optional instead of throwing. + REQUIRE(!o.getAttributeOptional("meshesPath").has_value()); + REQUIRE(!o.meshesPathOptional().has_value()); + REQUIRE(!o.particlesPathOptional().has_value()); + REQUIRE_THROWS_AS(o.getAttribute("meshesPath"), no_such_attribute_error); + + o.setMeshesPath("customMeshes").setParticlesPath("customParticles"); + REQUIRE(o.getAttributeOptional("meshesPath").has_value()); + REQUIRE(o.meshesPathOptional() == "customMeshes/"); + REQUIRE(o.particlesPathOptional() == "customParticles/"); + // getAttribute still throws for a missing key, returns for a present one. + REQUIRE(o.getAttribute("meshesPath").get() == "customMeshes/"); + o.iterations[0]; +} + TEST_CASE("output_modification_test", "[core]") { Series o = Series("../samples/MyOutput_%T.json", Access::CREATE); @@ -1151,8 +1182,8 @@ TEST_CASE("wrapper_test", "[core]") o.flush(); REQUIRE(all_data.get()[0] == value); #if openPMD_USE_INVASIVE_TESTS - REQUIRE(o.iterations[4].meshes["E"]["y"].get().m_chunks.empty()); - REQUIRE(mrc2.get().m_chunks.empty()); + REQUIRE(o.iterations[4].meshes["E"]["y"].get().chunks().empty()); + REQUIRE(mrc2.get().chunks().empty()); #endif MeshRecordComponent mrc3 = o.iterations[5].meshes["E"]["y"]; @@ -1167,13 +1198,13 @@ TEST_CASE("wrapper_test", "[core]") std::shared_ptr storeData = std::make_shared(44); o.iterations[5].meshes["E"]["y"].storeChunk(storeData, {0}, {1}); #if openPMD_USE_INVASIVE_TESTS - REQUIRE(o.iterations[5].meshes["E"]["y"].get().m_chunks.size() == 1); - REQUIRE(mrc3.get().m_chunks.size() == 1); + REQUIRE(o.iterations[5].meshes["E"]["y"].get().chunks().size() == 1); + REQUIRE(mrc3.get().chunks().size() == 1); #endif o.flush(); #if openPMD_USE_INVASIVE_TESTS - REQUIRE(o.iterations[5].meshes["E"]["y"].get().m_chunks.empty()); - REQUIRE(mrc3.get().m_chunks.empty()); + REQUIRE(o.iterations[5].meshes["E"]["y"].get().chunks().empty()); + REQUIRE(mrc3.get().chunks().empty()); #endif o.iterations[6] @@ -1211,8 +1242,9 @@ TEST_CASE("wrapper_test", "[core]") .particles["electrons"] .particlePatches["numParticles"][RecordComponent::SCALAR] .get() - .m_chunks.empty()); - REQUIRE(pp["numParticles"][RecordComponent::SCALAR].get().m_chunks.empty()); + .chunks() + .empty()); + REQUIRE(pp["numParticles"][RecordComponent::SCALAR].get().chunks().empty()); #endif pp["numParticles"][RecordComponent::SCALAR].store(idx, val); #if openPMD_USE_INVASIVE_TESTS @@ -1221,9 +1253,10 @@ TEST_CASE("wrapper_test", "[core]") .particles["electrons"] .particlePatches["numParticles"][RecordComponent::SCALAR] .get() - .m_chunks.size() == 1); + .chunks() + .size() == 1); REQUIRE( - pp["numParticles"][RecordComponent::SCALAR].get().m_chunks.size() == 1); + pp["numParticles"][RecordComponent::SCALAR].get().chunks().size() == 1); #endif std::stringstream u64str; u64str << determineDatatype(); @@ -1245,9 +1278,10 @@ TEST_CASE("wrapper_test", "[core]") .particles["electrons"] .particlePatches["numParticles"][RecordComponent::SCALAR] .get() - .m_chunks.size() == 2); + .chunks() + .size() == 2); REQUIRE( - pp["numParticles"][RecordComponent::SCALAR].get().m_chunks.size() == 2); + pp["numParticles"][RecordComponent::SCALAR].get().chunks().size() == 2); #endif /* * Fill the remaining standard particle patch components. The openPMD @@ -1279,8 +1313,9 @@ TEST_CASE("wrapper_test", "[core]") .particles["electrons"] .particlePatches["numParticles"][RecordComponent::SCALAR] .get() - .m_chunks.empty()); - REQUIRE(pp["numParticles"][RecordComponent::SCALAR].get().m_chunks.empty()); + .chunks() + .empty()); + REQUIRE(pp["numParticles"][RecordComponent::SCALAR].get().chunks().empty()); #endif } @@ -1303,6 +1338,15 @@ TEST_CASE("baserecord_test", "[core]") auto inserted = B.insert(B.find("x"), std::pair{"y", E["y"]}); REQUIRE(inserted->first == "y"); B.erase(inserted); + + // Range insert overload of BaseRecord, in a single batch. + { + std::vector components; + components.emplace_back("y", E["y"]); + B.insert(components.begin(), components.end()); + REQUIRE(B.contains("y")); + B.erase("y"); + } } TEST_CASE("use_count_test", "[core]") @@ -1335,7 +1379,7 @@ TEST_CASE("use_count_test", "[core]") REQUIRE( std::get>( static_cast *>( - pprc.get().m_chunks.front().parameter.get()) + pprc.get().chunks().front().parameter.get()) ->data.as_variant()) .use_count() == 1); /* @@ -1841,6 +1885,21 @@ TEST_CASE("read_nonexistent_attribute", "[core]") read_nonexistent_attribute::read_nonexistent_attribute(); } +TEST_CASE("custom_hierarchy", "[core]") +{ + custom_hierarchy::custom_hierarchy(); +} + +TEST_CASE("custom_hierarchy_materialization_error", "[core]") +{ + custom_hierarchy::materialization_error(); +} + +TEST_CASE("custom_hierarchy_iteration_filtering", "[core]") +{ + custom_hierarchy::iteration_filtering(); +} + TEST_CASE("unique_ptr", "[core]") { auto stdptr = std::make_unique(5); diff --git a/test/Files_Auxiliary/AuxiliaryTests.hpp b/test/Files_Auxiliary/AuxiliaryTests.hpp index c791ee472d..fc3e7cd0f1 100644 --- a/test/Files_Auxiliary/AuxiliaryTests.hpp +++ b/test/Files_Auxiliary/AuxiliaryTests.hpp @@ -18,6 +18,11 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ +namespace children_cache +{ +auto children_cache() -> void; +} // namespace children_cache + namespace drain_set { auto drain_set() -> void; diff --git a/test/Files_Auxiliary/children_cache.cpp b/test/Files_Auxiliary/children_cache.cpp new file mode 100644 index 0000000000..6da3083eed --- /dev/null +++ b/test/Files_Auxiliary/children_cache.cpp @@ -0,0 +1,231 @@ +/* Copyright 2026 The openPMD Community + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#include "AuxiliaryTests.hpp" + +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/ChildrenCache.hpp" +#include "openPMD/backend/ChildrenCacheImpl.tpp" +#include "openPMD/backend/Writable.hpp" + +#include + +#include +#include +#include + +namespace openPMD::internal +{ +template <> +auto container_type_id() -> std::uint8_t +{ + return 240; +} +template <> +auto container_type_id() -> std::uint8_t +{ + return 241; +} +template <> +auto container_type_id() -> std::uint8_t +{ + return 242; +} +} // namespace openPMD::internal + +namespace children_cache +{ +using namespace openPMD; + +namespace +{ + auto make_child() -> std::shared_ptr + { + return std::make_shared(nullptr); + } +} // namespace + +auto children_cache() -> void +{ + SECTION("get_or_create_returns_same_map") + { + internal::ChildrenCache cache; + cache.try_emplace("a", make_child(), 1); + + auto second = cache.get_or_create_cache(); + REQUIRE(second.contains("a")); + REQUIRE(second.at("a") == 1); + REQUIRE(cache.num_maps() == 1); + + cache.assert_consistent(); + } + + SECTION("different_types_are_independent") + { + internal::ChildrenCache cache; + auto child = make_child(); + cache.try_emplace("a", child, 1); + cache.try_emplace("a", child, 2.5); + auto ints = cache.get_or_create_cache(); + auto doubles = cache.get_or_create_cache(); + + REQUIRE(cache.num_maps() == 2); + REQUIRE(ints.at("a") == 1); + REQUIRE(doubles.at("a") == 2.5); + + cache.assert_consistent(); + } + + SECTION("erase_removes_from_every_map") + { + internal::ChildrenCache cache; + cache.try_emplace("a", make_child(), 1); + cache.try_emplace("b", make_child(), 2); + cache.try_emplace(42, make_child(), 3); + auto strings = cache.get_or_create_cache(); + auto numbers = cache.get_or_create_cache(); + + cache.erase("a"); + REQUIRE(!strings.contains("a")); + REQUIRE(strings.contains("b")); + + cache.erase("42"); + REQUIRE(!numbers.contains(42)); + REQUIRE(numbers.empty()); + + cache.assert_consistent(); + } + + SECTION("clear_empties_every_map") + { + internal::ChildrenCache cache; + cache.try_emplace("a", make_child(), 1); + cache.try_emplace(42, make_child(), 3); + auto strings = cache.get_or_create_cache(); + auto numbers = cache.get_or_create_cache(); + REQUIRE(!cache.empty()); + + cache.clear(); + REQUIRE(cache.empty()); + REQUIRE(strings.empty()); + REQUIRE(numbers.empty()); + + cache.assert_consistent(); + } + + SECTION("erased_keys_do_not_resurrect") + { + internal::ChildrenCache cache; + cache.try_emplace("a", make_child(), 1); + cache.try_emplace(42, make_child(), 1); + auto strings = cache.get_or_create_cache(); + auto numbers = cache.get_or_create_cache(); + + cache.erase("a"); + cache.erase("42"); + + // Reinserting the same keys must create fresh entries, not revive + // stale handles from the caches. + cache.try_emplace("a", make_child(), 2); + cache.try_emplace(42, make_child(), 2); + + REQUIRE(strings.at("a") == 2); + REQUIRE(numbers.at(42) == 2); + cache.assert_consistent(); + } + + SECTION("find_and_contains") + { + internal::ChildrenCache cache; + REQUIRE(!cache.contains_cache()); + REQUIRE(!cache.find_cache().has_value()); + + cache.try_emplace("a", make_child(), 1); + + REQUIRE(cache.contains_cache()); + auto found = cache.find_cache(); + REQUIRE(found.has_value()); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + REQUIRE(found->at("a") == 1); + } + + SECTION("typed_insert_registers_authoritative_child") + { + internal::ChildrenCache cache; + auto child = make_child(); + + auto inserted = cache.try_emplace("a", child, 1); + REQUIRE(inserted.second); + REQUIRE(cache.children().size() == 1); + REQUIRE(cache.children().at("a") == child); + + auto duplicate = cache.try_emplace("a", child, 2); + REQUIRE(!duplicate.second); + REQUIRE(cache.children().size() == 1); + REQUIRE(cache.get_or_create_cache().at("a") == 1); + cache.assert_consistent(); + } + + SECTION("integral_keys_must_be_canonical") + { + using namespace internal::children_cache_detail; + using U = std::uint64_t; + using I = int; + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + REQUIRE(key_from_string("42").value() == 42); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + REQUIRE(key_from_string("-42").value() == -42); + REQUIRE(!key_from_string("042")); + REQUIRE(!key_from_string("+42")); + REQUIRE(!key_from_string(" 42")); + REQUIRE(!key_from_string("-1")); + REQUIRE(!key_from_string("042")); + REQUIRE(!key_from_string("+42")); + REQUIRE(!key_from_string(" 42")); + REQUIRE(!key_from_string("-0")); + } + + SECTION("group_metadata_erase_clears_every_cache") + { + internal::object_type::GroupMetaData group; + auto child = make_child(); + group.m_children_cache.try_emplace("x", child, 1); + group.m_children_cache.try_emplace( + "x", child, 2.0); + auto ints = + group.m_children_cache.get_or_create_cache(); + auto doubles = + group.m_children_cache.get_or_create_cache(); + + group.erase_child("x"); + REQUIRE(!ints.contains("x")); + REQUIRE(!doubles.contains("x")); + + auto child_y = make_child(); + group.m_children_cache.try_emplace("y", child_y, 3); + group.m_children_cache.try_emplace( + "y", child_y, 4.0); + group.clear_children(); + REQUIRE(ints.empty()); + REQUIRE(doubles.empty()); + } +} +} // namespace children_cache diff --git a/test/Files_Core/CoreTests.hpp b/test/Files_Core/CoreTests.hpp index f769beeb99..650acb00ee 100644 --- a/test/Files_Core/CoreTests.hpp +++ b/test/Files_Core/CoreTests.hpp @@ -29,3 +29,10 @@ namespace read_nonexistent_attribute { auto read_nonexistent_attribute() -> void; } + +namespace custom_hierarchy +{ +auto custom_hierarchy() -> void; +auto materialization_error() -> void; +auto iteration_filtering() -> void; +} // namespace custom_hierarchy diff --git a/test/Files_Core/custom_hierarchy.cpp b/test/Files_Core/custom_hierarchy.cpp new file mode 100644 index 0000000000..0db9ac0c9c --- /dev/null +++ b/test/Files_Core/custom_hierarchy.cpp @@ -0,0 +1,315 @@ + +/* Copyright 2026 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IterationEncoding.hpp" +#include "openPMD/version.hpp" +#define OPENPMD_private public: +#define OPENPMD_protected public: + +#include "CoreTests.hpp" + +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/IOTask.hpp" +#include "openPMD/backend/HierarchyVisitorImpl.hpp" + +#include + +#include +namespace custom_hierarchy +{ +using namespace openPMD; + +void write( + std::string const &filename, + std::string const &json_params, + IterationEncoding) +{ + Series series(filename, Access::CREATE_LINEAR, json_params); + auto add_custom_hierarchy = [](auto &&attr) { + attr.customHierarchies()["rabimmel"].setAttribute("rabammel", "rabumm"); + }; + auto iteration = series.snapshots()[0]; + + add_custom_hierarchy(series); + add_custom_hierarchy(series.snapshots()); + add_custom_hierarchy(iteration); + + auto Ex = iteration.meshes["E"]["x"]; + Ex.resetDataset({Datatype::INT, {3}, R"( + resizable = true + hdf5.dataset.chunks = "auto")"}); + std::vector Exdata{1, 2, 3}; + Ex.storeChunk(Exdata, {0}, {3}); + + auto Ex_ = iteration.customHierarchies()["meshes"]["E"]["x"].asDataset(); + REQUIRE(Ex_.getDatatype() == Ex.getDatatype()); + REQUIRE(Ex_.getExtent() == Ex.getExtent()); + iteration.seriesFlush(); + + // Test resizing + Ex_.resetDataset({{6}}); + Ex.storeChunk(Exdata, {3}, {3}); + + auto fully_custom_dataset = + iteration.customHierarchies()["fully"]["custom"]["dataset"].asDataset(); + fully_custom_dataset.resetDataset({Datatype::INT, {3}}); + // fully_custom_dataset.setComment( + // "ALS PRIMA BALLERINA ALS WEIB ALS FEMME FATALE"); + fully_custom_dataset.storeChunk(Exdata, {0}, {3}); + + auto custom_groups = iteration.customHierarchies()["fully"]["custom"]; + auto custom_datasets = custom_groups.datasets(); + auto fully_custom_dataset_2 = custom_datasets["dataset2"]; + REQUIRE(custom_groups.contains("dataset2")); + fully_custom_dataset_2.resetDataset({Datatype::INT, {3}}); + fully_custom_dataset.setComment( + "ALS PRIMA BALLERINA ALS WEIB ALS FEMME FATALE"); + fully_custom_dataset_2.storeChunk(Exdata, {0}, {3}); + + auto fully_custom_constant_dataset = + iteration.customHierarchies()["fully"]["custom"]["constant_dataset"] + .asDataset(); + fully_custom_constant_dataset.makeConstant(67); + fully_custom_constant_dataset.resetDataset({{10, 10}}); + + /* + * A group that has contained objects cannot be reinterpreted as a dataset. + */ + { + CustomHierarchy custom_group = + iteration.customHierarchies()["fully"]["custom"]; + REQUIRE(custom_group.contains("dataset")); + REQUIRE_THROWS_AS(custom_group.asDataset(), error::WrongAPIUsage); + } + + /* + * Erasing an object that is managed as custom hierarchy must remove it from + * the backend as well, otherwise it would be re-created upon flushing. + */ + { + CustomHierarchy custom_objects = iteration.customHierarchies(); + custom_objects["to_erase"]["nested"].setAttribute("a", "b"); + REQUIRE(custom_objects.contains("to_erase")); + REQUIRE(custom_objects.erase("to_erase") == 1); + REQUIRE(!custom_objects.contains("to_erase")); + } + iteration.close(); +} + +void read( + std::string const &filename, + std::string const &json_params, + IterationEncoding) +{ + Series series(filename, Access::READ_LINEAR, json_params); + auto require_custom_hierarchy = [](auto &&attr) { + CustomHierarchy ch = attr.customHierarchies(); + REQUIRE( + ch["rabimmel"].getAttribute("rabammel").get() == + "rabumm"); + }; + auto iteration = series.snapshots()[0]; + + require_custom_hierarchy(series); + require_custom_hierarchy(series.snapshots()); + require_custom_hierarchy(iteration); + + auto Ex_data = iteration.customHierarchies()["meshes"]["E"]["x"] + .asDataset() + .loadChunk(); + iteration.seriesFlush(); + for (size_t i = 0; i < 3; ++i) + { + REQUIRE(Ex_data.get()[i] == (int)i + 1); + REQUIRE(Ex_data.get()[i + 3] == (int)i + 1); + } + + auto custom_data = iteration.customHierarchies()["fully"]["custom"] + .datasets()["dataset"] + .loadChunk(); + iteration.seriesFlush(); + for (size_t i = 0; i < 3; ++i) + { + REQUIRE(custom_data.get()[i] == (int)i + 1); + } + + auto custom_data_2 = iteration.customHierarchies()["fully"] + .datasets() + .subgroups()["custom"]["dataset2"] + .asDataset() + .loadChunk(); + iteration.seriesFlush(); + for (size_t i = 0; i < 3; ++i) + { + REQUIRE(custom_data_2.get()[i] == (int)i + 1); + } + + { + auto custom = iteration.customHierarchies()["fully"]["custom"]; + auto subgroups = custom.subgroups(); + REQUIRE(!subgroups.contains("dataset")); + REQUIRE(subgroups.contains("constant_dataset")); + + auto datasets = custom.datasets(); + REQUIRE(datasets.contains("dataset")); + REQUIRE(datasets.contains("dataset2")); + REQUIRE(!datasets.contains("constant_dataset")); + } + + iteration.customHierarchies().read(0).printRecursively(); + auto constant_dataset = + iteration.customHierarchies()["fully"]["custom"]["constant_dataset"] + .asDataset(); + auto custom_constant_data = constant_dataset.loadChunk(); + iteration.seriesFlush(); + for (size_t i = 0; i < 100; ++i) + { + REQUIRE(custom_constant_data.get()[i] == 67); + } + + /* + * HierarchyVisitor must traverse every object of the hierarchy exactly + * once, including objects managed as custom hierarchy. Regression test for + * a bug where custom-managed objects were visited once per traversal path, + * i.e. at least twice. + */ + { + std::map visit_counts; + iteration.visitHierarchyFromLambda( + [&visit_counts](auto &obj) { + ++visit_counts[obj.myPath().openPMDPath()]; + }, + /* recursive = */ true); + for (auto const &[path, count] : visit_counts) + { + INFO("visited path: " << path); + REQUIRE(count == 1); + } + // The custom dataset must be part of the traversal at all. + REQUIRE(visit_counts.count("/data/0/fully/custom/dataset") == 1); + } + + // The object that was erased in write() must not reappear here. + { + CustomHierarchy reopened = iteration.customHierarchies(); + REQUIRE(!reopened.contains("to_erase")); + } + + iteration.close(); +} + +struct test_config +{ + char const *filename; + char const *json_params; + IterationEncoding encoding; +}; + +void iteration_filtering() +{ + Series series( + "../samples/custom_hierarchy/iteration_filtering.json", Access::CREATE); + series.iterations[0]; + auto custom = series.iterations.customHierarchies(); + custom["not_an_index"].setAttribute("x", "y"); + + REQUIRE(series.iterations.size() == 1); + REQUIRE(series.iterations.begin()->first == 0); + + custom["42"].setAttribute("x", "y"); + REQUIRE(series.iterations.size() == 2); + REQUIRE(series.iterations.contains(42)); + REQUIRE_THROWS_AS(series.iterations.begin(), error::WrongAPIUsage); + + custom.erase("42"); + custom.erase("not_an_index"); +} + +void materialization_error() +{ + Series series( + "../samples/custom_hierarchy/materialization_error.json", + Access::CREATE); + auto iteration = series.iterations[0]; + iteration.meshes["E"]["x"].resetDataset({Datatype::INT, {3}}); + iteration.customHierarchies()["meshes"]["custom_group"].setAttribute( + "a", "b"); + + // Already materialized children and their iterators remain valid. + REQUIRE(iteration.meshes.contains("E")); + auto it = iteration.meshes.begin(); + REQUIRE(it != iteration.meshes.end()); + REQUIRE(it->first == "E"); + REQUIRE(&it->second == &iteration.meshes.at("E")); + + // The custom-created group has no Mesh materialization. Dereferencing it + // throws, but the iterator remains valid and can still be advanced. + ++it; + REQUIRE(it != iteration.meshes.end()); + REQUIRE_THROWS_AS(*it, error::WrongAPIUsage); + ++it; + REQUIRE(it == iteration.meshes.end()); +} + +void custom_hierarchy() +{ + test_config configs[] = { + {"groupbased.%E", + R"({"iteration_encoding": "group_based"})", + IterationEncoding::groupBased}, + {"filebased_%T.%E", + R"({"iteration_encoding": "file_based"})", + IterationEncoding::fileBased}, + {"variablebased.%E", + R"({"iteration_encoding": "variable_based"})", + IterationEncoding::variableBased}}; + + auto const &variants = getVariants(); + for (auto const &backend : {"adios2", "hdf5", "json", "toml"}) + { + if (!variants.at(backend)) + { + continue; + } + for (auto const &[filename, json_params, encoding] : configs) + { + auto json_params_ = json::merge( + json_params, + std::string( + R"( + { + "adios2": { + "engine": { + "type": "file" + } + }, + "backend": ")") + + backend + R"("})"); + auto filename_ = std::string("../samples/custom_hierarchy/") + + backend + "/" + filename; + write(filename_, json_params_, encoding); + read(filename_, json_params_, encoding); + } + } +} +} // namespace custom_hierarchy