From 0723076231e4ead001a9d794de70529bcb081aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 4 Jul 2022 16:28:49 +0200 Subject: [PATCH 1/8] Initialize Series attributes and datasets from template --- CMakeLists.txt | 1 + include/openPMD/Iteration.hpp | 12 ++ include/openPMD/auxiliary/TemplateFile.hpp | 10 ++ src/Iteration.cpp | 54 ++++++- src/auxiliary/TemplateFile.cpp | 173 +++++++++++++++++++++ test/SerialIOTest.cpp | 7 + 6 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 include/openPMD/auxiliary/TemplateFile.hpp create mode 100644 src/auxiliary/TemplateFile.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b3a0cc30c2..b7789bbf34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,7 @@ set(CORE_SOURCE src/auxiliary/Filesystem.cpp src/auxiliary/JSON.cpp src/auxiliary/Mpi.cpp + src/auxiliary/TemplateFile.cpp src/backend/Attributable.cpp src/backend/BaseRecordComponent.cpp src/backend/MeshRecordComponent.cpp diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 43ee1084bb..d97b55b32c 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -134,6 +134,15 @@ namespace internal * Otherwise empty. */ std::optional m_deferredParseAccess{}; + + enum TernaryBool + { + Undefined, + True, + False + }; + TernaryBool hasMeshes = TernaryBool::Undefined; + TernaryBool hasParticles = TernaryBool::Undefined; }; } // namespace internal /** @brief Logical compilation of data from one snapshot (e.g. a single @@ -271,6 +280,9 @@ class Iteration : public Attributable Container meshes{}; Container particles{}; // particleSpecies? + bool hasMeshes() const; + bool hasParticles() const; + virtual ~Iteration() = default; private: diff --git a/include/openPMD/auxiliary/TemplateFile.hpp b/include/openPMD/auxiliary/TemplateFile.hpp new file mode 100644 index 0000000000..95ea7e9cf3 --- /dev/null +++ b/include/openPMD/auxiliary/TemplateFile.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include "openPMD/Series.hpp" + +namespace openPMD::auxiliary +{ +// @todo replace uint64_t with proper type after merging #1285 +Series &initializeFromTemplate( + Series &initializeMe, Series const &fromTemplate, uint64_t iteration); +} // namespace openPMD::auxiliary diff --git a/src/Iteration.cpp b/src/Iteration.cpp index c991c2b79b..83a9ab204d 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -47,6 +47,50 @@ namespace openPMD using internal::CloseStatus; using internal::DeferredParseAccess; +bool Iteration::hasMeshes() const +{ + /* + * Currently defined at the Series level, but might be defined at the + * Iteration level in next standard iterations. + * Hence an Iteration:: method. + */ + + switch (get().hasMeshes) + { + case internal::IterationData::TernaryBool::True: + return true; + case internal::IterationData::TernaryBool::False: + return false; + case internal::IterationData::TernaryBool::Undefined: { + Series s = retrieveSeries(); + return !meshes.empty() || s.containsAttribute("meshesPath"); + }; + } + throw std::runtime_error("Unreachable!"); +} + +bool Iteration::hasParticles() const +{ + /* + * Currently defined at the Series level, but might be defined at the + * Iteration level in next standard iterations. + * Hence an Iteration:: method. + */ + + switch (get().hasParticles) + { + case internal::IterationData::TernaryBool::True: + return true; + case internal::IterationData::TernaryBool::False: + return false; + case internal::IterationData::TernaryBool::Undefined: { + Series s = retrieveSeries(); + return !particles.empty() || s.containsAttribute("particlesPath"); + }; + } + throw std::runtime_error("Unreachable!"); +} + Iteration::Iteration() : Attributable(NoInit()) { setData(std::make_shared()); @@ -354,7 +398,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); - if (!meshes.empty() || s.containsAttribute("meshesPath")) + if (hasMeshes()) { if (!s.containsAttribute("meshesPath")) { @@ -370,7 +414,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) meshes.setDirty(false); } - if (!particles.empty() || s.containsAttribute("particlesPath")) + if (hasParticles()) { if (!s.containsAttribute("particlesPath")) { @@ -562,6 +606,12 @@ void Iteration::read_impl(std::string const &groupPath) hasMeshes = s.containsAttribute("meshesPath"); hasParticles = s.containsAttribute("particlesPath"); } + { + using TB = internal::IterationData::TernaryBool; + auto &data = get(); + data.hasMeshes = hasMeshes ? TB::True : TB::False; + data.hasParticles = hasParticles ? TB::True : TB::False; + } if (hasMeshes) { diff --git a/src/auxiliary/TemplateFile.cpp b/src/auxiliary/TemplateFile.cpp new file mode 100644 index 0000000000..8eab0727dc --- /dev/null +++ b/src/auxiliary/TemplateFile.cpp @@ -0,0 +1,173 @@ +#include "openPMD/auxiliary/TemplateFile.hpp" +#include "openPMD/DatatypeHelpers.hpp" + +#include + +namespace openPMD::auxiliary +{ +namespace +{ + // Some forward declarations + template + void initializeFromTemplate( + Container &initializeMe, Container const &fromTemplate); + + struct SetAttribute + { + template + static void + call(Attributable &object, std::string const &name, Attribute attr) + { + object.setAttribute(name, attr.get()); + } + + template + static void call(Attributable &, std::string const &name, Attribute) + { + std::cerr << "Unknown datatype for template attribute '" << name + << "'. Will skip it." << std::endl; + } + }; + + void copyAttributes( + Attributable &target, + Attributable const &source, + std::vector ignore = {}) + { + auto shouldBeIgnored = [&ignore](std::string const &attrName) { + // `ignore` is empty by default and normally has only a handful of + // entries otherwise. + // So just use linear search. + for (auto const &ignored : ignore) + { + if (attrName == ignored) + { + return true; + } + } + return false; + }; + + for (auto const &attrName : source.attributes()) + { + if (shouldBeIgnored(attrName)) + { + continue; + } + auto attr = source.getAttribute(attrName); + auto dtype = attr.dtype; + switchType(dtype, target, attrName, std::move(attr)); + } + } + + void initializeFromTemplate( + BaseRecordComponent &initializeMe, + BaseRecordComponent const &fromTemplate) + { + copyAttributes(initializeMe, fromTemplate); + } + + void initializeFromTemplate( + RecordComponent &initializeMe, RecordComponent const &fromTemplate) + { + if (fromTemplate.getDatatype() != Datatype::UNDEFINED) + { + initializeMe.resetDataset( + Dataset{fromTemplate.getDatatype(), fromTemplate.getExtent()}); + } + initializeFromTemplate( + static_cast(initializeMe), + static_cast(fromTemplate)); + } + + void initializeFromTemplate( + PatchRecordComponent &initializeMe, + PatchRecordComponent const &fromTemplate) + { + if (fromTemplate.getDatatype() != Datatype::UNDEFINED) + { + initializeMe.resetDataset( + Dataset{fromTemplate.getDatatype(), fromTemplate.getExtent()}); + } + initializeFromTemplate( + static_cast(initializeMe), + static_cast(fromTemplate)); + } + + void initializeFromTemplate( + ParticleSpecies &initializeMe, ParticleSpecies const &fromTemplate) + { + if (!fromTemplate.particlePatches.empty()) + { + initializeFromTemplate( + static_cast &>( + initializeMe.particlePatches), + static_cast const &>( + fromTemplate.particlePatches)); + } + initializeFromTemplate( + static_cast &>(initializeMe), + static_cast const &>(fromTemplate)); + } + + template + void initializeFromTemplate( + Container &initializeMe, Container const &fromTemplate) + { + copyAttributes(initializeMe, fromTemplate); + for (auto const &pair : fromTemplate) + { + initializeFromTemplate(initializeMe[pair.first], pair.second); + } + } + + void initializeFromTemplate( + Iteration &initializeMe, Iteration const &fromTemplate) + { + copyAttributes(initializeMe, fromTemplate, {"snapshot"}); + if (fromTemplate.hasMeshes()) + { + initializeFromTemplate(initializeMe.meshes, fromTemplate.meshes); + } + if (fromTemplate.hasParticles()) + { + initializeFromTemplate( + initializeMe.particles, fromTemplate.particles); + } + } +} // namespace + +Series &initializeFromTemplate( + Series &initializeMe, Series const &fromTemplate, uint64_t iteration) +{ + if (!initializeMe.containsAttribute("from_template")) + { + copyAttributes( + initializeMe, + fromTemplate, + {"basePath", "iterationEncoding", "iterationFormat", "openPMD"}); + initializeMe.setAttribute("from_template", fromTemplate.name()); + } + + uint64_t sourceIteration = iteration; + if (!fromTemplate.iterations.contains(sourceIteration)) + { + if (fromTemplate.iterations.empty()) + { + std::cerr << "Template file has no iterations, will only fill in " + "global attributes." + << std::endl; + return initializeMe; + } + else + { + sourceIteration = fromTemplate.iterations.begin()->first; + } + } + + initializeFromTemplate( + initializeMe.iterations[iteration], + fromTemplate.iterations.at(sourceIteration)); + return initializeMe; +} +} // namespace openPMD::auxiliary diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 0b050384fb..13168fae70 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -14,6 +14,7 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/auxiliary/TemplateFile.hpp" #include "openPMD/openPMD.hpp" #include @@ -1485,6 +1486,12 @@ inline void dtype_test( if (activateTemplateMode.has_value()) { + Series out( + "../samples/dtype_test_from_template." + backend, + Access::CREATE, + activateTemplateMode.value()); + auxiliary::initializeFromTemplate(out, s, 1000); + out.flush(); return; } // same implementation types (not necessary aliases) detection From 17cc79b65662c98429706f98d6cb4a7ff43dab30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 10 Mar 2023 15:43:53 +0100 Subject: [PATCH 2/8] Further testing (@todo: cleanup) --- test/SerialIOTest.cpp | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 13168fae70..782c8ded99 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -3035,21 +3035,21 @@ TEST_CASE("git_hdf5_legacy_picongpu", "[serial][hdf5]") TEST_CASE("git_hdf5_sample_attribute_test", "[serial][hdf5]") { - try - { - Series o = Series("../samples/git-sample/data%T.h5", Access::READ_ONLY); - + auto verifySeries = [](Series o, bool this_is_the_original_file) { REQUIRE(o.openPMD() == "1.1.0"); REQUIRE(o.openPMDextension() == 1); REQUIRE(o.basePath() == "/data/%T/"); REQUIRE(o.meshesPath() == "fields/"); REQUIRE(o.particlesPath() == "particles/"); - REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); - REQUIRE(o.iterationFormat() == "data%T.h5"); - REQUIRE(o.name() == "data%T"); + if (this_is_the_original_file) + { + REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); + REQUIRE(o.iterationFormat() == "data%T.h5"); + REQUIRE(o.name() == "data%T"); - REQUIRE(o.iterations.size() == 5); - REQUIRE(o.iterations.count(100) == 1); + REQUIRE(o.iterations.size() == 5); + REQUIRE(o.iterations.count(100) == 1); + } Iteration &iteration_100 = o.iterations[100]; REQUIRE(iteration_100.time() == 3.2847121452090077e-14); @@ -3279,6 +3279,30 @@ TEST_CASE("git_hdf5_sample_attribute_test", "[serial][hdf5]") REQUIRE(weighting_scalar.getDatatype() == Datatype::DOUBLE); REQUIRE(weighting_scalar.getDimensionality() == 1); REQUIRE(weighting_scalar.getExtent() == e); + }; + + try + { + { + Series o = + Series("../samples/git-sample/data%T.h5", Access::READ_ONLY); + verifySeries(o, true); + + Series fromTemplate( + "../samples/initialized_from_git_sample.json", + Access::CREATE, + R"(json.mode = "template")"); + auxiliary::initializeFromTemplate(fromTemplate, o, 100); + fromTemplate.flush(); + } + + { + Series o( + "../samples/initialized_from_git_sample.json", + Access::READ_ONLY, + R"(json.mode = "template")"); + verifySeries(o, false); + } } catch (error::ReadError &e) { From 04e6cff240d5ace208c96f3a50da63000f6f0891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 19 May 2022 17:29:24 +0200 Subject: [PATCH 3/8] Extend example --- examples/14_toml_template.cpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp index 29d19fb53a..9f0b3662db 100644 --- a/examples/14_toml_template.cpp +++ b/examples/14_toml_template.cpp @@ -1,3 +1,5 @@ +#include "openPMD/Dataset.hpp" +#include #include std::string backendEnding() @@ -101,7 +103,23 @@ void read() "../samples/tomlTemplate." + backendEnding(), openPMD::Access::READ_LINEAR); read.parseBase(); - openPMD::helper::listSeries(read); + + std::string jsonConfig = R"( +{ + "iteration_encoding": "variable_based", + "json": { + "mode": "template" + } +} +)"; + openPMD::Series cloned( + "../samples/jsonTemplate.json", openPMD::Access::CREATE, jsonConfig); + openPMD::auxiliary::initializeFromTemplate(cloned, read, 0); + // Have to define the dataset for E/z as it is not defined in the template + // @todo check that the dataset is defined only upon destruction, not at + // flushing already + cloned.writeIterations()[0].meshes["E"]["z"].resetDataset( + {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}}); } int main() From 0282f84136bcd7eff46545b7ba566542c246e1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 20 Mar 2023 12:19:54 +0100 Subject: [PATCH 4/8] Only opt-into tests for TOML Has the same implementation as JSON anyway, and it makes tests run into timeouts otherwise. --- examples/14_toml_template.cpp | 3 ++- test/SerialIOTest.cpp | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp index 9f0b3662db..e22fafc05d 100644 --- a/examples/14_toml_template.cpp +++ b/examples/14_toml_template.cpp @@ -108,7 +108,8 @@ void read() { "iteration_encoding": "variable_based", "json": { - "mode": "template" + "dataset": {"mode": "template"}, + "attribute": {"mode": "short"} } } )"; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 782c8ded99..0f2a04675a 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1564,7 +1564,15 @@ TEST_CASE("dtype_test", "[serial]") { for (auto const &t : testedFileExtensions()) { - dtype_test(t); + if (t == "json") + { + dtype_test(t); + dtype_test(t, R"(json.mode = "template")"); + } + else + { + dtype_test(t); + } } dtype_test("json", R"( { From f3eb94d528adc69fe7301037341c2bf48ea810bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 22 Dec 2023 19:17:51 +0100 Subject: [PATCH 5/8] Adapt this to changed SCALAR API --- src/auxiliary/TemplateFile.cpp | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/auxiliary/TemplateFile.cpp b/src/auxiliary/TemplateFile.cpp index 8eab0727dc..59d71ec613 100644 --- a/src/auxiliary/TemplateFile.cpp +++ b/src/auxiliary/TemplateFile.cpp @@ -34,6 +34,25 @@ namespace Attributable const &source, std::vector ignore = {}) { +#if 0 // leave this in for potential future debugging + std::cout << "COPYING ATTRIBUTES FROM '" << [&source]() -> std::string { + auto vec = source.myPath().group; + if (vec.empty()) + { + return "[]"; + } + std::stringstream sstream; + auto it = vec.begin(); + sstream << "[" << *it++; + for (; it != vec.end(); ++it) + { + sstream << ", " << *it; + } + sstream << "]"; + return sstream.str(); + }() << "'" + << std::endl; +#endif auto shouldBeIgnored = [&ignore](std::string const &attrName) { // `ignore` is empty by default and normally has only a handful of // entries otherwise. @@ -94,6 +113,25 @@ namespace static_cast(fromTemplate)); } + template + void initializeFromTemplate( + BaseRecord &initializeMe, BaseRecord const &fromTemplate) + { + if (fromTemplate.scalar()) + { + initializeMe[RecordComponent::SCALAR]; + initializeFromTemplate( + static_cast(initializeMe), + static_cast(fromTemplate)); + } + else + { + initializeFromTemplate( + static_cast &>(initializeMe), + static_cast const &>(fromTemplate)); + } + } + void initializeFromTemplate( ParticleSpecies &initializeMe, ParticleSpecies const &fromTemplate) { From 09fefe08cddf29cea3f31c31e7b5dd264774a3a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 4 Jan 2024 13:14:12 +0100 Subject: [PATCH 6/8] Avoid CI warnings --- src/auxiliary/TemplateFile.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/auxiliary/TemplateFile.cpp b/src/auxiliary/TemplateFile.cpp index 59d71ec613..bd88c7756a 100644 --- a/src/auxiliary/TemplateFile.cpp +++ b/src/auxiliary/TemplateFile.cpp @@ -16,13 +16,14 @@ namespace { template static void - call(Attributable &object, std::string const &name, Attribute attr) + call(Attributable &object, std::string const &name, Attribute &attr) { object.setAttribute(name, attr.get()); } template - static void call(Attributable &, std::string const &name, Attribute) + static void + call(Attributable &, std::string const &name, Attribute const &) { std::cerr << "Unknown datatype for template attribute '" << name << "'. Will skip it." << std::endl; From 47b9967c2ba215bb21483d98178e4836d1c172a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 23 Jan 2024 17:34:09 +0100 Subject: [PATCH 7/8] Fix after rebasing --- src/auxiliary/TemplateFile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auxiliary/TemplateFile.cpp b/src/auxiliary/TemplateFile.cpp index bd88c7756a..1ac672a524 100644 --- a/src/auxiliary/TemplateFile.cpp +++ b/src/auxiliary/TemplateFile.cpp @@ -76,7 +76,7 @@ namespace } auto attr = source.getAttribute(attrName); auto dtype = attr.dtype; - switchType(dtype, target, attrName, std::move(attr)); + switchType(dtype, target, attrName, attr); } } From 77c023bdb491d70267e865f5b417ea3b9b596149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 26 Mar 2024 12:49:18 +0100 Subject: [PATCH 8/8] Ensure that E.z is there --- examples/14_toml_template.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp index e22fafc05d..10d775bf6f 100644 --- a/examples/14_toml_template.cpp +++ b/examples/14_toml_template.cpp @@ -119,7 +119,7 @@ void read() // Have to define the dataset for E/z as it is not defined in the template // @todo check that the dataset is defined only upon destruction, not at // flushing already - cloned.writeIterations()[0].meshes["E"]["z"].resetDataset( + cloned.writeIterations()[0].meshes["E"].at("z").resetDataset( {openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}}); }