From 1e4976f5518383bde387ef554f99161d0687e2ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Dec 2024 14:49:57 +0100 Subject: [PATCH 01/33] Add preparseSnapshots --- include/openPMD/IO/AbstractIOHandlerImpl.hpp | 4 + include/openPMD/IO/IOTask.hpp | 32 +++++ include/openPMD/Series.hpp | 10 +- include/openPMD/backend/Attribute.hpp | 38 +++--- src/IO/ADIOS/ADIOS2File.cpp | 1 + src/IO/ADIOS/ADIOS2IOHandler.cpp | 1 + src/IO/AbstractIOHandlerImpl.cpp | 31 +++++ src/Series.cpp | 127 ++++++++++++++++--- test/SerialIOTest.cpp | 4 +- 9 files changed, 211 insertions(+), 37 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 10b0fd0c97..cb866c7ffe 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -360,6 +360,10 @@ class AbstractIOHandlerImpl */ virtual void readAttribute(Writable *, Parameter &) = 0; + /** COLLECTIVE! + */ + virtual void readAttributeAllsteps( + Writable *, Parameter &); /** List all paths/sub-groups inside a group, non-recursively. * * The operation should fail if the Writable was not marked written. diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 5589db6923..316722e99a 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -26,6 +26,7 @@ #include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Export.hpp" #include "openPMD/auxiliary/Memory.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/ParsePreference.hpp" @@ -72,6 +73,7 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ DELETE_ATT, WRITE_ATT, READ_ATT, + READ_ATT_ALLSTEPS, LIST_ATTS, ADVANCE, @@ -602,6 +604,36 @@ struct OPENPMDAPI_EXPORT Parameter std::make_shared(); }; +template <> +struct OPENPMDAPI_EXPORT Parameter + : public AbstractParameter +{ + Parameter() = default; + Parameter(Parameter &&) = default; + Parameter(Parameter const &) = default; + Parameter &operator=(Parameter &&) = default; + Parameter &operator=(Parameter const &) = default; + + std::unique_ptr to_heap() && override + { + return std::unique_ptr( + new Parameter(std::move(*this))); + } + + std::string name = ""; + std::shared_ptr dtype = std::make_shared(); + + struct to_vector_type + { + template + using type = std::vector; + }; + using result_type = typename auxiliary::detail:: + map_variant::type; + + std::shared_ptr resource = std::make_shared(); +}; + template <> struct OPENPMDAPI_EXPORT Parameter : public AbstractParameter diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index c387c803f7..7cbac21f2d 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -118,6 +118,10 @@ namespace internal * snapshot attribute. */ std::set m_currentlyActiveIterations; + /** + * For reading: In which IO step do I need to look for an Iteration? + */ + std::unordered_map m_snapshotToStep; /** * This map contains the filenames of those Iterations which were found * on the file system upon opening the Series for reading in file-based @@ -969,12 +973,16 @@ OPENPMD_private * Returns the current content of the /data/snapshot attribute. * (We could also add this to the public API some time) */ - std::optional> currentSnapshot() const; + std::optional> currentSnapshot(); AbstractIOHandler *runDeferredInitialization(); AbstractIOHandler *IOHandler(); AbstractIOHandler const *IOHandler() const; + + std::optional>> + preparseSnapshots(); + [[nodiscard]] bool randomAccessSteps() const; }; // Series using SnapshotWorkflow = Series::SnapshotWorkflow; diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index a183b7818a..e2ba7e1dd5 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -310,6 +310,25 @@ namespace detail #else } #endif + + template + auto doConvertOptional(T const *pv) -> std::optional + { + auto eitherValueOrError = doConvert(pv); + return std::visit( + [](auto &containedValue) -> std::optional { + using Res = std::decay_t; + if constexpr (std::is_same_v) + { + return std::nullopt; + } + else + { + return {std::move(containedValue)}; + } + }, + eitherValueOrError); + } } // namespace detail template @@ -339,25 +358,12 @@ U Attribute::get() const template std::optional Attribute::getOptional() const { - auto eitherValueOrError = std::visit( - [](auto &&containedValue) -> std::variant { - using containedType = std::decay_t; - return detail::doConvert(&containedValue); - }, - Variant::getResource()); return std::visit( [](auto &&containedValue) -> std::optional { - using T = std::decay_t; - if constexpr (std::is_same_v) - { - return std::nullopt; - } - else - { - return {std::move(containedValue)}; - } + using containedType = std::decay_t; + return detail::doConvertOptional(&containedValue); }, - std::move(eitherValueOrError)); + Variant::getResource()); } } // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index c6f4747aff..8006b39e82 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -62,6 +62,7 @@ void DatasetReader::call( { adios2::Variable var = impl->verifyDataset(bp.param.offset, bp.param.extent, IO, bp.name); + // var.SetStepSelection({}); if (!var) { throw std::runtime_error( diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 5bd3803446..f5a7ee5332 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -2010,6 +2010,7 @@ namespace detail file, ADIOS2IOHandlerImpl::IfFileNotOpen::ThrowError); auto &IO = fileData.m_IO; adios2::Variable var = IO.InquireVariable(varName); + // var.SetStepSelection(); if (!var) { throw std::runtime_error( diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 7675cc3e07..ad093dad1d 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -21,7 +21,9 @@ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/IOTask.hpp" #include "openPMD/auxiliary/Environment.hpp" +#include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -344,6 +346,20 @@ std::future AbstractIOHandlerImpl::flush() readAttribute(i.writable, parameter); break; } + case O::READ_ATT_ALLSTEPS: { + auto ¶meter = + deref_dynamic_cast>( + i.parameter.get()); + writeToStderr( + "[", + i.writable->parent, + "->", + i.writable, + "] READ_ATT_ALLSTEPS: ", + parameter.name); + readAttributeAllsteps(i.writable, parameter); + break; + } case O::LIST_PATHS: { auto ¶meter = deref_dynamic_cast>( i.parameter.get()); @@ -485,6 +501,21 @@ std::future AbstractIOHandlerImpl::flush() return std::future(); } +void AbstractIOHandlerImpl::readAttributeAllsteps( + Writable *w, Parameter ¶m) +{ + using result_type = Parameter::result_type; + Parameter param_internal; + param_internal.name = param.name; + param_internal.dtype = param.dtype; + readAttribute(w, param_internal); + *param.resource = std::visit( + [](auto &val) -> result_type { + return result_type{std::vector{std::move(val)}}; + }, + *param_internal.resource); +} + void AbstractIOHandlerImpl::setWritten( Writable *w, Parameter const ¶m) { diff --git a/src/Series.cpp b/src/Series.cpp index 792be0555f..ab33237122 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -38,6 +38,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Attribute.hpp" #include "openPMD/snapshots/ContainerImpls.hpp" #include "openPMD/snapshots/IteratorTraits.hpp" #include "openPMD/snapshots/RandomAccessIterator.hpp" @@ -57,6 +58,7 @@ #include #include #include +#include #include #include @@ -1949,18 +1951,9 @@ auto Series::readGorVBased( else if (encoding == "variableBased") { series.m_iterationEncoding = IterationEncoding::variableBased; - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) - { - std::cerr << R"( -The opened Series uses variable-based encoding, but is being accessed by -READ_ONLY mode which operates in random-access manner. -Random-access is (currently) unsupported by variable-based encoding -and some iterations may not be found by this access mode. -Consider using Access::READ_LINEAR and Series::readIterations().)" - << std::endl; - } - else if (IOHandler()->m_frontendAccess == Access::READ_WRITE) + if (IOHandler()->m_frontendAccess == Access::READ_WRITE) { + // @todo hmm see what happens throw error::WrongAPIUsage(R"( The opened Series uses variable-based encoding, but is being accessed by READ_WRITE mode which does not (yet) support variable-based encoding. @@ -3223,8 +3216,7 @@ void Series::close() m_attri.reset(); } -auto Series::currentSnapshot() const - -> std::optional> +auto Series::currentSnapshot() -> std::optional> { using vec_t = std::vector; auto &series = get(); @@ -3235,7 +3227,31 @@ auto Series::currentSnapshot() const * `/data/snapshot`. This makes it possible to retrieve it from * `series.iterations`. */ - if (series.iterations.containsAttribute("snapshot")) + if (!series.iterations.containsAttribute("snapshot")) + { + return std::nullopt; + } + else if (randomAccessSteps()) + { + auto preparsed = preparseSnapshots(); + if (!preparsed.has_value()) + { + return std::nullopt; + } + std::set res; + for (size_t step = 0; step < preparsed->size(); ++step) + { + for (IterationIndex_t iteration : (*preparsed)[step]) + { + res.emplace(iteration); + // If an Iteration is found in multiple steps, this logic will + // prefer the Iteration from the later step. + series.m_snapshotToStep[iteration] = step; + } + } + return vec_t{res.begin(), res.end()}; + } + else { auto const &attribute = series.iterations.getAttribute("snapshot"); auto res = attribute.getOptional(); @@ -3257,10 +3273,6 @@ auto Series::currentSnapshot() const s.str()); } } - else - { - return std::optional>{}; - } } AbstractIOHandler *Series::runDeferredInitialization() @@ -3294,6 +3306,85 @@ AbstractIOHandler const *Series::IOHandler() const return res; } +auto Series::preparseSnapshots() + -> std::optional>> +{ + using vec_t = std::vector; + auto &series = get(); + if (!series.m_snapshotToStep.empty()) + { + throw error::Internal( + "Control flow error: This is an expensive operation and should be " + "called only once upon initialization."); + } + auto io_handler = IOHandler(); + if (!series.iterations.containsAttribute("snapshot")) + { + return std::nullopt; + } + Parameter readAttr; + readAttr.name = "snapshot"; + io_handler->enqueue(IOTask(&series.iterations, readAttr)); + io_handler->flush(internal::defaultFlushParams); + auto wrong_datatype = [&]() { + std::stringstream s; + s << "Unexpected datatype for '/data/snapshot': " << *readAttr.dtype + << " (expected a vector of integer)."; + return error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + s.str()); + }; + return std::visit( + [&](auto &vec) -> std::vector { + using containedType = + typename std::decay_t::value_type; + if constexpr (std::is_same_v) + { + // need to keep the vector overload away from the + // below implementation + throw wrong_datatype(); + } + else + { + std::vector res; + res.reserve(vec.size()); + for (auto const &val : vec) + { + auto converted = + detail::doConvertOptional(&val); + if (!converted.has_value()) + { + throw wrong_datatype(); + } + res.emplace_back(*converted); + } + return res; + } + }, + *readAttr.resource); +} + +bool Series::randomAccessSteps() const +{ + auto randomAccess = [](Access access) { + switch (access) + { + case Access::READ_RANDOM_ACCESS: + case Access::READ_WRITE: + return true; + case Access::READ_LINEAR: + case Access::CREATE: + case Access::APPEND: + return false; + } + return false; + }; + return iterationEncoding() == IterationEncoding::variableBased && + randomAccess(IOHandler()->m_backendAccess); +} + namespace { CleanedFilename cleanFilename( diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index ab8c2ed15c..b9313efe33 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5293,8 +5293,7 @@ void variableBasedSingleIteration(std::string const &file) } { - Series readSeries(file, Access::READ_LINEAR); - readSeries.parseBase(); + Series readSeries(file, Access::READ_RANDOM_ACCESS); auto E_x = readSeries.iterations[0].meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); @@ -5789,6 +5788,7 @@ void variableBasedSeries(std::string const &file) { if (i > 0 && is_not_adios2) { + return; REQUIRE_THROWS_AS( iterations[i], error::OperationUnsupportedInBackend); return; From 559a97e530eea2e9ef6f23ac22dcd0014d17554e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Dec 2024 15:51:27 +0100 Subject: [PATCH 02/33] Serial ADIOS2 implementation for readAttributeAllsteps --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 10 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 291 ++++++++++++------- 2 files changed, 197 insertions(+), 104 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index fa300da472..327ccabbc5 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -190,6 +190,9 @@ class ADIOS2IOHandlerImpl void readAttribute(Writable *, Parameter &) override; + void readAttributeAllsteps( + Writable *, Parameter &) override; + void listPaths(Writable *, Parameter &) override; void @@ -533,11 +536,8 @@ namespace detail struct AttributeReader { template - static Datatype call( - ADIOS2IOHandlerImpl &, - adios2::IO &IO, - std::string name, - Attribute::resource &resource); + static Datatype + call(adios2::IO &IO, std::string name, Attribute::resource &resource); template static Datatype call(Params &&...); diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index f5a7ee5332..6bd10a2513 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -28,6 +28,7 @@ #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/IterationEncoding.hpp" +#include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" @@ -35,6 +36,8 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" +#include +#include #include #include // std::tolower #include @@ -1216,10 +1219,191 @@ void ADIOS2IOHandlerImpl::readAttribute( } Datatype ret = switchType( - type, *this, ba.m_IO, name, *parameters.resource); + type, ba.m_IO, name, *parameters.resource); *parameters.dtype = ret; } +namespace +{ + template + Datatype + genericReadAttribute(Functor &&fun, adios2::IO &IO, std::string const &name) + { + /* + * If we store an attribute of boolean type, we store an additional + * attribute prefixed with '__is_boolean__' to indicate this information + * that would otherwise be lost. Check whether this has been done. + */ + using rep = detail::AttributeTypes::rep; + + if constexpr (std::is_same::value) + { + auto attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Internal error: Failed reading attribute '" + + name + "'."); + } + + std::string metaAttr; + metaAttr = adios_defaults::str_isBoolean + name; + /* + * In verbose mode, attributeInfo will yield a warning if not + * finding the requested attribute. Since we expect the attribute + * not to be present in many cases (i.e. when it is actually not + * a boolean), let's tell attributeInfo to be quiet. + */ + auto type = detail::attributeInfo( + IO, + metaAttr, + /* verbose = */ false); + + if (type == determineDatatype()) + { + auto meta = IO.InquireAttribute(metaAttr); + if (meta.Data().size() == 1 && meta.Data()[0] == 1) + { + std::forward(fun)( + detail::bool_repr::fromRep(attr.Data()[0])); + return determineDatatype(); + } + } + std::forward(fun)(attr.Data()[0]); + } + else if constexpr (detail::IsUnsupportedComplex_v) + { + throw std::runtime_error( + "[ADIOS2] Internal error: no support for long double complex " + "attribute types"); + } + else if constexpr (auxiliary::IsVector_v) + { + auto attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Internal error: Failed reading attribute '" + + name + "'."); + } + std::forward(fun)(attr.Data()); + } + else if constexpr (auxiliary::IsArray_v) + { + auto attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Internal error: Failed reading attribute '" + + name + "'."); + } + auto data = attr.Data(); + T res; + for (size_t i = 0; i < data.size(); i++) + { + res[i] = data[i]; + } + std::forward(fun)(res); + } + else if constexpr (std::is_same_v) + { + throw std::runtime_error( + "Observed boolean attribute. ADIOS2 does not have these?"); + } + else + { + auto attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Internal error: Failed reading attribute '" + + name + "'."); + } + std::forward(fun)(attr.Data()[0]); + } + + return determineDatatype(); + } + + struct ReadAttributeAllsteps + { + template + static void call( + adios2::IO &IO, + adios2::Engine &engine, + std::string const &name, + adios2::StepStatus status, + Parameter::result_type + &put_result_here) + { + std::vector res; + res.reserve(engine.Steps()); + while (status == adios2::StepStatus::OK) + { + genericReadAttribute( + [&res](auto &&val) { + using type = std::remove_reference_t; + if constexpr (std::is_same_v) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "ADIOS2", + "[ReadAttributeAllsteps] No support for " + "Boolean attributes."); + } + else + { + res.emplace_back(static_cast(val)); + } + }, + IO, + name); + engine.EndStep(); + status = engine.BeginStep(); + } + switch (status) + { + case adios2::StepStatus::OK: + throw error::Internal("Control flow error."); + case adios2::StepStatus::NotReady: + case adios2::StepStatus::OtherError: + throw error::ReadError( + error::AffectedObject::File, + error::Reason::CannotRead, + "ADIOS2", + "Unexpected step status while preparsing snapshots."); + case adios2::StepStatus::EndOfStream: + break; + } + put_result_here = std::move(res); + } + + static constexpr char const *errorMsg = "ReadAttributeAllsteps"; + }; +} // namespace + +void ADIOS2IOHandlerImpl::readAttributeAllsteps( + Writable *writable, Parameter ¶m) +{ + std::cout << "ADIOS2: ReadAttributeAllSteps" << '\n'; + auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); + auto pos = setAndGetFilePosition(writable); + auto name = nameOfAttribute(writable, param.name); + + adios2::ADIOS adios; + auto IO = adios.DeclareIO("PreparseSnapshots"); + // @todo check engine type + // @todo MPI implementation + IO.SetEngine(realEngineType()); + auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); + auto status = engine.BeginStep(); + auto type = detail::attributeInfo(IO, name, /* verbose = */ true); + switchAdios2AttributeType( + type, IO, engine, name, status, *param.resource); + engine.Close(); +} + void ADIOS2IOHandlerImpl::listPaths( Writable *writable, Parameter ¶meters) { @@ -1745,105 +1929,14 @@ namespace detail { template Datatype AttributeReader::call( - ADIOS2IOHandlerImpl &impl, - adios2::IO &IO, - std::string name, - Attribute::resource &resource) + adios2::IO &IO, std::string name, Attribute::resource &resource) { - (void)impl; - /* - * If we store an attribute of boolean type, we store an additional - * attribute prefixed with '__is_boolean__' to indicate this information - * that would otherwise be lost. Check whether this has been done. - */ - using rep = AttributeTypes::rep; - - if constexpr (std::is_same::value) - { - auto attr = IO.InquireAttribute(name); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed reading attribute '" + - name + "'."); - } - - std::string metaAttr; - metaAttr = adios_defaults::str_isBoolean + name; - /* - * In verbose mode, attributeInfo will yield a warning if not - * finding the requested attribute. Since we expect the attribute - * not to be present in many cases (i.e. when it is actually not - * a boolean), let's tell attributeInfo to be quiet. - */ - auto type = attributeInfo( - IO, - metaAttr, - /* verbose = */ false); - - if (type == determineDatatype()) - { - auto meta = IO.InquireAttribute(metaAttr); - if (meta.Data().size() == 1 && meta.Data()[0] == 1) - { - resource = bool_repr::fromRep(attr.Data()[0]); - return determineDatatype(); - } - } - resource = attr.Data()[0]; - } - else if constexpr (IsUnsupportedComplex_v) - { - throw std::runtime_error( - "[ADIOS2] Internal error: no support for long double complex " - "attribute types"); - } - else if constexpr (auxiliary::IsVector_v) - { - auto attr = IO.InquireAttribute(name); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed reading attribute '" + - name + "'."); - } - resource = attr.Data(); - } - else if constexpr (auxiliary::IsArray_v) - { - auto attr = IO.InquireAttribute(name); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed reading attribute '" + - name + "'."); - } - auto data = attr.Data(); - T res; - for (size_t i = 0; i < data.size(); i++) - { - res[i] = data[i]; - } - resource = res; - } - else if constexpr (std::is_same_v) - { - throw std::runtime_error( - "Observed boolean attribute. ADIOS2 does not have these?"); - } - else - { - auto attr = IO.InquireAttribute(name); - if (!attr) - { - throw std::runtime_error( - "[ADIOS2] Internal error: Failed reading attribute '" + - name + "'."); - } - resource = attr.Data()[0]; - } - - return determineDatatype(); + return genericReadAttribute( + [&resource](auto &&value) { + resource = static_cast(value); + }, + IO, + name); } template From d5616627237692494c919d4c1d2072cfb6ea18b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Dec 2024 17:04:39 +0100 Subject: [PATCH 03/33] Feature-complete, untested --- include/openPMD/IO/ADIOS/ADIOS2File.hpp | 6 ++- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 3 +- include/openPMD/IO/IOTask.hpp | 7 +++- include/openPMD/Iteration.hpp | 22 ++++++++++- src/IO/ADIOS/ADIOS2File.cpp | 41 ++++++++++++++++++-- src/IO/ADIOS/ADIOS2IOHandler.cpp | 27 +++++++++++-- src/IO/AbstractIOHandlerImpl.cpp | 30 ++++++++++---- src/Iteration.cpp | 34 +++++++++++----- src/Series.cpp | 39 +++++++++++++++---- 9 files changed, 172 insertions(+), 37 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 5d519a85ad..2f53ba6686 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -79,7 +79,8 @@ struct DatasetReader BufferedGet &bp, adios2::IO &IO, adios2::Engine &engine, - std::string const &fileName); + std::string const &fileName, + std::optional stepSelection); static constexpr char const *errorMsg = "ADIOS2: readDataset()"; }; @@ -412,6 +413,8 @@ class ADIOS2File StreamStatus streamStatus = StreamStatus::OutsideOfStep; size_t currentStep(); + void setStepSelection(size_t); + [[nodiscard]] std::optional stepSelection() const; private: ADIOS2IOHandlerImpl *m_impl; @@ -422,6 +425,7 @@ class ADIOS2File * implement this manually. */ size_t m_currentStep = 0; + bool useStepSelection = false; /* * ADIOS2 does not give direct access to its internal attribute and diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 327ccabbc5..78fc156ada 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -562,7 +562,8 @@ namespace detail ADIOS2IOHandlerImpl *impl, InvalidatableFile const &, std::string const &varName, - Parameter ¶meters); + Parameter ¶meters, + std::optional stepSelection); static constexpr char const *errorMsg = "ADIOS2: openDataset()"; }; diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 316722e99a..d65f0dda72 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -670,8 +670,13 @@ struct OPENPMDAPI_EXPORT Parameter new Parameter(std::move(*this))); } + struct StepSelection + { + size_t step; + }; + //! input parameter - AdvanceMode mode; + std::variant mode; bool isThisStepMandatory = false; //! output parameter std::shared_ptr status = diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 8c39c0a518..b084dcd193 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -52,6 +52,23 @@ namespace internal propagated to the backend */ }; + namespace BeginStepTypes + { + struct DontBeginStep + {}; + struct BeginStepSynchronously + {}; + struct BeginStepRandomAccess + { + size_t step; + }; + } // namespace BeginStepTypes + + using BeginStep = std::variant< + BeginStepTypes::DontBeginStep, + BeginStepTypes::BeginStepSynchronously, + BeginStepTypes::BeginStepRandomAccess>; + struct DeferredParseAccess { /** @@ -69,7 +86,7 @@ namespace internal * (Group- and variable-based parsing shares the same code logic.) */ bool fileBased = false; - bool beginStep = false; + BeginStep beginStep = BeginStepTypes::DontBeginStep{}; }; class IterationData : public AttributableData @@ -305,7 +322,8 @@ class Iteration : public Attributable std::string const &filePath, std::string const &groupPath, bool beginStep); - void readGorVBased(std::string const &groupPath, bool beginStep); + void readGorVBased( + std::string const &groupPath, internal::BeginStep const &beginStep); void read_impl(std::string const &groupPath); void readMeshes(std::string const &meshesPath); void readParticles(std::string const &particlesPath); diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 8006b39e82..20df672f68 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -27,6 +27,7 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include #include #if openPMD_USE_VERIFY @@ -58,11 +59,15 @@ void DatasetReader::call( detail::BufferedGet &bp, adios2::IO &IO, adios2::Engine &engine, - std::string const &fileName) + std::string const &fileName, + std::optional stepSelection) { adios2::Variable var = impl->verifyDataset(bp.param.offset, bp.param.extent, IO, bp.name); - // var.SetStepSelection({}); + if (stepSelection.has_value()) + { + var.SetStepSelection({*stepSelection, 1}); + } if (!var) { throw std::runtime_error( @@ -134,7 +139,13 @@ void WriteDataset::call(Params &&...) void BufferedGet::run(ADIOS2File &ba) { switchAdios2VariableType( - param.dtype, ba.m_impl, *this, ba.m_IO, ba.getEngine(), ba.m_file); + param.dtype, + ba.m_impl, + *this, + ba.m_IO, + ba.getEngine(), + ba.m_file, + ba.stepSelection()); } void BufferedPut::run(ADIOS2File &ba) @@ -338,6 +349,30 @@ size_t ADIOS2File::currentStep() } } +void ADIOS2File::setStepSelection(size_t step) +{ + if (streamStatus != StreamStatus::ReadWithoutStream) + { + throw error::Internal( + "ADIOS2 backend: Cannot only use random-access step selections " + "when reading without streaming mode."); + } + m_currentStep = step; + useStepSelection = true; +} + +std::optional ADIOS2File::stepSelection() const +{ + if (useStepSelection) + { + return {m_currentStep}; + } + else + { + return std::nullopt; + } +} + void ADIOS2File::configure_IO_Read() { bool upfrontParsing = supportsUpfrontParsing( diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 6bd10a2513..04943494c5 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -27,7 +27,9 @@ #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" +#include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" +#include "openPMD/Streaming.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" @@ -35,6 +37,7 @@ #include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include #include @@ -998,7 +1001,12 @@ void ADIOS2IOHandlerImpl::openDataset( *parameters.dtype = detail::fromADIOS2Type(fileData.m_IO.VariableType(varName)); switchAdios2VariableType( - *parameters.dtype, this, file, varName, parameters); + *parameters.dtype, + this, + file, + varName, + parameters, + fileData.stepSelection()); writable->written = true; } @@ -1609,7 +1617,14 @@ void ADIOS2IOHandlerImpl::advance( { auto file = m_files.at(writable); auto &ba = getFileData(file, IfFileNotOpen::ThrowError); - *parameters.status = ba.advance(parameters.mode); + std::visit( + auxiliary::overloaded{ + [&](AdvanceMode mode) { *parameters.status = ba.advance(mode); }, + [&](Parameter::StepSelection step) { + ba.setStepSelection(step.step); + *parameters.status = AdvanceStatus::RANDOMACCESS; + }}, + parameters.mode); } void ADIOS2IOHandlerImpl::closePath( @@ -2097,13 +2112,17 @@ namespace detail ADIOS2IOHandlerImpl *impl, InvalidatableFile const &file, const std::string &varName, - Parameter ¶meters) + Parameter ¶meters, + std::optional stepSelection) { auto &fileData = impl->getFileData( file, ADIOS2IOHandlerImpl::IfFileNotOpen::ThrowError); auto &IO = fileData.m_IO; adios2::Variable var = IO.InquireVariable(varName); - // var.SetStepSelection(); + if (stepSelection.has_value()) + { + var.SetStepSelection({*stepSelection, 1}); + } if (!var) { throw std::runtime_error( diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index ad093dad1d..dbce88ccac 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -22,7 +22,9 @@ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/IO/IOTask.hpp" +#include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Environment.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/Writable.hpp" @@ -30,6 +32,7 @@ #include #include #include +#include namespace openPMD { @@ -399,15 +402,26 @@ std::future AbstractIOHandlerImpl::flush() i.writable, "] ADVANCE ", [&]() { - switch (parameter.mode) - { + return std::visit( + auxiliary::overloaded{ + [](AdvanceMode mode) -> std::string { + switch (mode) + { - case AdvanceMode::BEGINSTEP: - return "BEGINSTEP"; - case AdvanceMode::ENDSTEP: - return "ENDSTEP"; - } - throw std::runtime_error("Unreachable!"); + case AdvanceMode::BEGINSTEP: + return "BEGINSTEP"; + case AdvanceMode::ENDSTEP: + return "ENDSTEP"; + } + throw std::runtime_error("Unreachable!"); + }, + [](Parameter::StepSelection + step) { + std::stringstream s; + s << "RANDOMACCESS '" << step.step << "'"; + return s.str(); + }}, + parameter.mode); }()); advance(i.writable, parameter); break; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 931a1f9e3d..3d2615e630 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -29,6 +29,7 @@ #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Writable.hpp" @@ -39,6 +40,7 @@ #include #include #include +#include namespace openPMD { @@ -440,15 +442,27 @@ void Iteration::readFileBased( read_impl(groupPath); } -void Iteration::readGorVBased(std::string const &groupPath, bool doBeginStep) +void Iteration::readGorVBased( + std::string const &groupPath, internal::BeginStep const &doBeginStep) { - if (doBeginStep) - { - /* - * beginStep() must take care to open files - */ - beginStep(/* reread = */ false); - } + std::visit( + auxiliary::overloaded{ + [](internal::BeginStepTypes::DontBeginStep const &) {}, + [&](internal::BeginStepTypes::BeginStepSynchronously const &) { + /* + * beginStep() must take care to open files + */ + beginStep(/* reread = */ false); + }, + [&](internal::BeginStepTypes::BeginStepRandomAccess const + &randomAccess) { + Parameter param; + param.mode = Parameter::StepSelection{ + randomAccess.step}; + IOHandler()->enqueue(IOTask(this, std::move(param))); + }, + }, + doBeginStep); read_impl(groupPath); } @@ -920,7 +934,9 @@ void Iteration::runDeferredParseAccess() deferred.iteration, filename, deferred.path, - deferred.beginStep); + std::holds_alternative< + internal::BeginStepTypes::BeginStepSynchronously>( + deferred.beginStep)); } else { diff --git a/src/Series.cpp b/src/Series.cpp index ab33237122..1b77ce55b2 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1448,6 +1448,13 @@ void Series::flushGorVBased( break; case IO::HasBeenOpened: // continue below + if (randomAccessSteps() && !series.m_snapshotToStep.empty()) + { + Parameter param; + param.mode = Parameter::StepSelection{ + series.m_snapshotToStep.at(it->first)}; + IOHandler()->enqueue(IOTask(&it->second, std::move(param))); + } it->second.flush(flushParams); break; } @@ -2050,11 +2057,11 @@ creating new iterations. /* * Return error if one is caught. */ - auto readSingleIteration = - [&series, &pOpen, this]( - IterationIndex_t index, - std::string const &path, - bool beginStep) -> std::optional { + auto readSingleIteration = [&series, &pOpen, this]( + IterationIndex_t index, + std::string const &path, + internal::BeginStep const &beginStep) + -> std::optional { if (series.iterations.contains(index)) { // maybe re-read @@ -2126,7 +2133,12 @@ creating new iterations. } if (auto err = internal::withRWAccess( IOHandler()->m_seriesStatus, - [&]() { return readSingleIteration(index, it, false); }); + [&]() { + return readSingleIteration( + index, + it, + internal::BeginStepTypes::DontBeginStep{}); + }); err) { std::cerr << "Cannot read iteration " << index @@ -2183,10 +2195,16 @@ creating new iterations. * Variable-based iteration encoding relies on steps, so parsing * must happen after opening the first step. */ + internal::BeginStep beginStep = randomAccessSteps() + ? internal:: + BeginStep{internal::BeginStepTypes::BeginStepRandomAccess{ + series.m_snapshotToStep.at(it)}} + : internal::BeginStep{ + internal::BeginStepTypes::BeginStepSynchronously{}}; if (auto err = internal::withRWAccess( IOHandler()->m_seriesStatus, - [&readSingleIteration, it]() { - return readSingleIteration(it, "", true); + [&readSingleIteration, it, &beginStep]() { + return readSingleIteration(it, "", beginStep); }); err) { @@ -3249,6 +3267,11 @@ auto Series::currentSnapshot() -> std::optional> series.m_snapshotToStep[iteration] = step; } } + for (auto const &[iteration, step] : series.m_snapshotToStep) + { + std::cout << '\t' << iteration << "\t-> " << step << '\n'; + } + std::cout.flush(); return vec_t{res.begin(), res.end()}; } else From 9e8a590aa0642f356e9d4aa34a080ef84f9aa7e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Dec 2024 17:50:16 +0100 Subject: [PATCH 04/33] Fixes --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 7 ++++++- src/IO/ADIOS/ADIOS2File.cpp | 20 ++++++++++++-------- src/IO/ADIOS/ADIOS2IOHandler.cpp | 8 ++------ src/IO/AbstractIOHandlerImpl.cpp | 5 ++++- src/IO/IOTask.cpp | 17 +++++++++++++++-- src/Iteration.cpp | 3 ++- src/Series.cpp | 12 ++++++++---- test/SerialIOTest.cpp | 1 - 8 files changed, 49 insertions(+), 24 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 78fc156ada..84aabc5076 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -434,7 +434,8 @@ class ADIOS2IOHandlerImpl Offset const &offset, Extent const &extent, adios2::IO &IO, - std::string const &varName) + std::string const &varName, + std::optional stepSelection) { { auto requiredType = adios2::GetType(); @@ -461,6 +462,10 @@ class ADIOS2IOHandlerImpl throw std::runtime_error( "[ADIOS2] Internal error: Failed opening ADIOS2 variable."); } + if (stepSelection.has_value()) + { + var.SetStepSelection({*stepSelection, 1}); + } // TODO leave this check to ADIOS? adios2::Dims shape = var.Shape(); auto actualDim = shape.size(); diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 20df672f68..a1bd21f381 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -62,12 +62,8 @@ void DatasetReader::call( std::string const &fileName, std::optional stepSelection) { - adios2::Variable var = - impl->verifyDataset(bp.param.offset, bp.param.extent, IO, bp.name); - if (stepSelection.has_value()) - { - var.SetStepSelection({*stepSelection, 1}); - } + adios2::Variable var = impl->verifyDataset( + bp.param.offset, bp.param.extent, IO, bp.name, stepSelection); if (!var) { throw std::runtime_error( @@ -96,7 +92,11 @@ void WriteDataset::call(ADIOS2File &ba, detail::BufferedPut &bp) auto ptr = static_cast(arg.get()); adios2::Variable var = ba.m_impl->verifyDataset( - bp.param.offset, bp.param.extent, ba.m_IO, bp.name); + bp.param.offset, + bp.param.extent, + ba.m_IO, + bp.name, + std::nullopt); ba.getEngine().Put(var, ptr); } @@ -160,7 +160,11 @@ struct RunUniquePtrPut { auto ptr = static_cast(bufferedPut.data.get()); adios2::Variable var = ba.m_impl->verifyDataset( - bufferedPut.offset, bufferedPut.extent, ba.m_IO, bufferedPut.name); + bufferedPut.offset, + bufferedPut.extent, + ba.m_IO, + bufferedPut.name, + std::nullopt); ba.getEngine().Put(var, ptr); } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 04943494c5..b0ec06aaa3 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -34,15 +34,11 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" -#include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/auxiliary/Variant.hpp" -#include -#include #include -#include // std::tolower #include #include #include @@ -1091,7 +1087,7 @@ namespace detail auto &IO = ba.m_IO; auto &engine = ba.getEngine(); adios2::Variable variable = impl->verifyDataset( - params.offset, params.extent, IO, varName); + params.offset, params.extent, IO, varName, std::nullopt); adios2::Dims offset(params.offset.begin(), params.offset.end()); adios2::Dims extent(params.extent.begin(), params.extent.end()); variable.SetSelection({std::move(offset), std::move(extent)}); @@ -1407,7 +1403,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); auto status = engine.BeginStep(); auto type = detail::attributeInfo(IO, name, /* verbose = */ true); - switchAdios2AttributeType( + switchType( type, IO, engine, name, status, *param.resource); engine.Close(); } diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index dbce88ccac..d362c91b2e 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -319,7 +319,10 @@ std::future AbstractIOHandlerImpl::flush() i.writable->parent, "->", i.writable, - "] READ_DATASET"); + "] READ_DATASET, offset=", + [¶meter]() { return vec_as_string(parameter.offset); }, + ", extent=", + [¶meter]() { return vec_as_string(parameter.extent); }); readDataset(i.writable, parameter); break; } diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 47b0bea4ca..0692547745 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -149,10 +149,23 @@ namespace internal case Operation::AVAILABLE_CHUNKS: return "AVAILABLE_CHUNKS"; break; - default: - return "unknown"; + case Operation::CHECK_FILE: + return "CHECK_FILE"; + break; + case Operation::READ_ATT_ALLSTEPS: + return "READ_ATT_ALLSTEPS"; + break; + case Operation::DEREGISTER: + return "DEREGISTER"; + break; + case Operation::TOUCH: + return "TOUCH"; + break; + case Operation::SET_WRITTEN: + return "SET_WRITTEN"; break; } + return "unknown"; } } // namespace internal diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 3d2615e630..a4fbf33149 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -459,7 +459,8 @@ void Iteration::readGorVBased( Parameter param; param.mode = Parameter::StepSelection{ randomAccess.step}; - IOHandler()->enqueue(IOTask(this, std::move(param))); + IOHandler()->enqueue( + IOTask(&retrieveSeries().writable(), std::move(param))); }, }, doBeginStep); diff --git a/src/Series.cpp b/src/Series.cpp index 1b77ce55b2..2d631e3221 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1453,7 +1453,7 @@ void Series::flushGorVBased( Parameter param; param.mode = Parameter::StepSelection{ series.m_snapshotToStep.at(it->first)}; - IOHandler()->enqueue(IOTask(&it->second, std::move(param))); + IOHandler()->enqueue(IOTask(this, std::move(param))); } it->second.flush(flushParams); break; @@ -2196,9 +2196,13 @@ creating new iterations. * must happen after opening the first step. */ internal::BeginStep beginStep = randomAccessSteps() - ? internal:: - BeginStep{internal::BeginStepTypes::BeginStepRandomAccess{ - series.m_snapshotToStep.at(it)}} + ? (series.m_snapshotToStep.empty() + ? internal::BeginStep{internal::BeginStepTypes:: + DontBeginStep{}} + : internal::BeginStep{internal::BeginStepTypes:: + BeginStepRandomAccess{ + series.m_snapshotToStep.at( + it)}}) : internal::BeginStep{ internal::BeginStepTypes::BeginStepSynchronously{}}; if (auto err = internal::withRWAccess( diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index b9313efe33..fed2c8c1bd 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5788,7 +5788,6 @@ void variableBasedSeries(std::string const &file) { if (i > 0 && is_not_adios2) { - return; REQUIRE_THROWS_AS( iterations[i], error::OperationUnsupportedInBackend); return; From eee83f626b8d49fc1b73724432e00196319476a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 18 Dec 2024 19:15:47 +0100 Subject: [PATCH 05/33] wip: parallel impl. for readAttributeAllsteps huh this works?? --- include/openPMD/auxiliary/Mpi.hpp | 5 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 197 ++++++++++++++++++++++++++++-- 2 files changed, 189 insertions(+), 13 deletions(-) diff --git a/include/openPMD/auxiliary/Mpi.hpp b/include/openPMD/auxiliary/Mpi.hpp index f8eefe0cc5..a808d6ecbd 100644 --- a/include/openPMD/auxiliary/Mpi.hpp +++ b/include/openPMD/auxiliary/Mpi.hpp @@ -61,8 +61,9 @@ namespace } else { - static_assert( - dependent_false_v, "openPMD_MPI_type: Unsupported type."); + throw std::runtime_error("Unimplemented"); + // static_assert( + // dependent_false_v, "openPMD_MPI_type: Unsupported type."); } } } // namespace diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index b0ec06aaa3..1833df29b4 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -34,6 +34,7 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/auxiliary/Variant.hpp" @@ -43,6 +44,8 @@ #include #include #include +#include +#include #include #include #include @@ -1385,6 +1388,143 @@ namespace static constexpr char const *errorMsg = "ReadAttributeAllsteps"; }; + + template + auto vec_as_string(Vec const &vec) -> std::string + { + if (vec.empty()) + { + return "[]"; + } + else + { + std::stringstream res; + res << '['; + auto it = vec.begin(); + res << *it++; + auto end = vec.end(); + for (; it != end; ++it) + { + res << ", " << *it; + } + res << ']'; + return res.str(); + } + } + +#if openPMD_HAVE_MPI + struct DistributeToAllRanks + { + template + static void call( + Parameter::result_type + &put_result_here_in, + MPI_Comm comm, + int rank) + { + if (rank != 0) + { + put_result_here_in = std::vector{}; + } + std::vector &put_result_here = + std::get>(put_result_here_in); + std::cout << "INIT VECTOR SIZE: " << put_result_here.size() << '\n'; + size_t num_items = put_result_here.size(); + MPI_Bcast( + &num_items, 1, auxiliary::openPMD_MPI_type(), 0, comm); + std::cout << "WILL_COMMUNICATE: " << num_items << '\n'; + if constexpr ( + std::is_same_v || + std::is_same_v> || + std::is_same_v || + std::is_same_v> || auxiliary::IsArray_v) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", + "[readAttributeAllsteps] No support for attributes of type " + "string, bool or std::array in parallel."); + } + else if constexpr ( + // auxiliary::IsArray_v || + auxiliary::IsVector_v) + { + std::vector sizes; + sizes.reserve(num_items); + if (rank == 0) + { + std::transform( + put_result_here.begin(), + put_result_here.end(), + std::back_inserter(sizes), + [](T const &arr) { return arr.size(); }); + } + sizes.resize(num_items); + MPI_Bcast( + sizes.data(), + num_items, + auxiliary::openPMD_MPI_type(), + 0, + comm); + size_t total_flat_size = + std::accumulate(sizes.begin(), sizes.end(), size_t(0)); + using flat_type = typename T::value_type; + std::vector flat_vector; + flat_vector.reserve(total_flat_size); + if (rank == 0) + { + for (auto const &arr : put_result_here) + { + for (auto val : arr) + { + flat_vector.push_back(val); + } + } + } + flat_vector.resize(total_flat_size); + MPI_Bcast( + flat_vector.data(), + total_flat_size, + auxiliary::openPMD_MPI_type(), + 0, + comm); + if (rank != 0) + { + std::cout << "RECEIVED: " << vec_as_string(flat_vector) + << '\n'; + size_t offset = 0; + put_result_here.reserve(num_items); + for (size_t current_extent : sizes) + { + put_result_here.emplace_back( + flat_vector.begin() + offset, + flat_vector.begin() + offset + current_extent); + offset += current_extent; + } + } + } + else + { + std::vector receive; + if (rank != 0) + { + receive.resize(num_items); + } + MPI_Bcast( + rank == 0 ? put_result_here.data() : receive.data(), + num_items, + auxiliary::openPMD_MPI_type(), + 0, + comm); + if (rank != 0) + { + put_result_here = std::move(receive); + std::cout << "RECEIVED: " << vec_as_string(receive) << '\n'; + } + } + } + static constexpr char const *errorMsg = "DistributeToAllRanks"; + }; +#endif } // namespace void ADIOS2IOHandlerImpl::readAttributeAllsteps( @@ -1395,17 +1535,52 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto pos = setAndGetFilePosition(writable); auto name = nameOfAttribute(writable, param.name); - adios2::ADIOS adios; - auto IO = adios.DeclareIO("PreparseSnapshots"); - // @todo check engine type - // @todo MPI implementation - IO.SetEngine(realEngineType()); - auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); - auto status = engine.BeginStep(); - auto type = detail::attributeInfo(IO, name, /* verbose = */ true); - switchType( - type, IO, engine, name, status, *param.resource); - engine.Close(); + auto read_from_file_in_serial = [&]() { + adios2::ADIOS adios; + auto IO = adios.DeclareIO("PreparseSnapshots"); + // @todo check engine type + IO.SetEngine(realEngineType()); + auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); + auto status = engine.BeginStep(); + auto type = detail::attributeInfo(IO, name, /* verbose = */ true); + switchType( + type, IO, engine, name, status, *param.resource); + engine.Close(); + return type; + }; +#if openPMD_HAVE_MPI + if (!m_communicator.has_value()) + { + read_from_file_in_serial(); + return; + } + int rank, size; + MPI_Comm_rank(*m_communicator, &rank); + MPI_Comm_size(*m_communicator, &size); + Datatype type; + constexpr size_t max_string_len = 30; + char distribute_string[max_string_len]{}; + if (rank == 0) + { + type = read_from_file_in_serial(); + auto as_string = datatypeToString(type); + if (as_string.size() + 1 > max_string_len) + { + throw error::Internal( + "String size hardcoded too low for MPI_Bcast"); + } + std::copy(as_string.begin(), as_string.end(), distribute_string); + } + MPI_Bcast(distribute_string, max_string_len, MPI_CHAR, 0, *m_communicator); + if (rank != 0) + { + type = stringToDatatype(std::string(distribute_string)); + } + switchType( + type, *param.resource, *m_communicator, rank); +#else + read_from_file_in_serial(); +#endif } void ADIOS2IOHandlerImpl::listPaths( From 5a780a9513bca3fddf6a37213996ad12ec8b16cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 19 Dec 2024 10:41:09 +0100 Subject: [PATCH 06/33] Simplify, cleanup --- include/openPMD/Datatype.hpp | 4 +-- include/openPMD/auxiliary/Mpi.hpp | 45 ++++++++++++++++++++++++++++--- src/IO/ADIOS/ADIOS2IOHandler.cpp | 21 ++++----------- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index c18b7cd82b..24f77218a0 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -513,7 +513,7 @@ inline bool isFloatingPoint(Datatype d) * @param d Datatype to test * @return true if complex floating point, otherwise false */ -inline bool isComplexFloatingPoint(Datatype d) +constexpr inline bool isComplexFloatingPoint(Datatype d) { using DT = Datatype; @@ -554,7 +554,7 @@ inline bool isFloatingPoint() * @return true if complex floating point, otherwise false */ template -inline bool isComplexFloatingPoint() +constexpr inline bool isComplexFloatingPoint() { Datatype dtype = determineDatatype(); diff --git a/include/openPMD/auxiliary/Mpi.hpp b/include/openPMD/auxiliary/Mpi.hpp index a808d6ecbd..658523f7d9 100644 --- a/include/openPMD/auxiliary/Mpi.hpp +++ b/include/openPMD/auxiliary/Mpi.hpp @@ -47,10 +47,38 @@ namespace { return MPI_CHAR; } + else if constexpr (std::is_same_v) + { + return MPI_SIGNED_CHAR; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED_CHAR; + } + else if constexpr (std::is_same_v) + { + return MPI_SHORT; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED_SHORT; + } else if constexpr (std::is_same_v) { return MPI_UNSIGNED; } + else if constexpr (std::is_same_v) + { + return MPI_INT; + } + else if constexpr (std::is_same_v) + { + return MPI_LONG; + } + else if constexpr (std::is_same_v) + { + return MPI_LONG_LONG; + } else if constexpr (std::is_same_v) { return MPI_UNSIGNED_LONG; @@ -59,11 +87,22 @@ namespace { return MPI_UNSIGNED_LONG_LONG; } + else if constexpr (std::is_same_v) + { + return MPI_FLOAT; + } + else if constexpr (std::is_same_v) + { + return MPI_DOUBLE; + } + else if constexpr (std::is_same_v) + { + return MPI_LONG_DOUBLE; + } else { - throw std::runtime_error("Unimplemented"); - // static_assert( - // dependent_false_v, "openPMD_MPI_type: Unsupported type."); + static_assert( + dependent_false_v, "openPMD_MPI_type: Unsupported type."); } } } // namespace diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 1833df29b4..732b07f697 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1437,12 +1437,14 @@ namespace std::is_same_v || std::is_same_v> || std::is_same_v || - std::is_same_v> || auxiliary::IsArray_v) + std::is_same_v> || + auxiliary::IsArray_v || isComplexFloatingPoint()) { throw error::OperationUnsupportedInBackend( "ADIOS2", "[readAttributeAllsteps] No support for attributes of type " - "string, bool or std::array in parallel."); + "std::string, bool, std::complex or std::array in " + "parallel."); } else if constexpr ( // auxiliary::IsArray_v || @@ -1558,24 +1560,11 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( MPI_Comm_rank(*m_communicator, &rank); MPI_Comm_size(*m_communicator, &size); Datatype type; - constexpr size_t max_string_len = 30; - char distribute_string[max_string_len]{}; if (rank == 0) { type = read_from_file_in_serial(); - auto as_string = datatypeToString(type); - if (as_string.size() + 1 > max_string_len) - { - throw error::Internal( - "String size hardcoded too low for MPI_Bcast"); - } - std::copy(as_string.begin(), as_string.end(), distribute_string); - } - MPI_Bcast(distribute_string, max_string_len, MPI_CHAR, 0, *m_communicator); - if (rank != 0) - { - type = stringToDatatype(std::string(distribute_string)); } + MPI_Bcast(&type, 1, MPI_INT, 0, *m_communicator); switchType( type, *param.resource, *m_communicator, rank); #else From d9d16574eac8760f94525d0d5e24f1003d70e903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 19 Dec 2024 12:08:09 +0100 Subject: [PATCH 07/33] wip testing --- CMakeLists.txt | 4 + test/Files_ParallelIO/ParallelIOTests.hpp | 6 + .../read_variablebased_randomaccess.cpp | 134 ++++++++++++++++++ test/ParallelIOTest.cpp | 7 + 4 files changed, 151 insertions(+) create mode 100644 test/Files_ParallelIO/ParallelIOTests.hpp create mode 100644 test/Files_ParallelIO/read_variablebased_randomaccess.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 03637b1938..6c3feadc35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -787,6 +787,10 @@ if(openPMD_BUILD_TESTING) test/Files_SerialIO/close_and_reopen_test.cpp test/Files_SerialIO/filebased_write_test.cpp ) + elseif(${test_name} STREQUAL "ParallelIO") + list(APPEND ${out_list} + test/Files_ParallelIO/read_variablebased_randomaccess.cpp + ) endif() endmacro() diff --git a/test/Files_ParallelIO/ParallelIOTests.hpp b/test/Files_ParallelIO/ParallelIOTests.hpp new file mode 100644 index 0000000000..c7c879fc8b --- /dev/null +++ b/test/Files_ParallelIO/ParallelIOTests.hpp @@ -0,0 +1,6 @@ +#include + +namespace read_variablebased_randomaccess +{ +auto read_variablebased_randomaccess() -> void; +} diff --git a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp new file mode 100644 index 0000000000..ca5eb66961 --- /dev/null +++ b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp @@ -0,0 +1,134 @@ +#include "ParallelIOTests.hpp" + +#include "openPMD/openPMD.hpp" + +#include + +#include + +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI +#include +#include + +namespace read_variablebased_randomaccess +{ +static void create_file_in_serial() +{ + { + adios2::ADIOS adios; + auto IO = adios.DeclareIO("IO"); + IO.SetEngine("bp5"); + auto engine = + IO.Open("../samples/bp5_no_steps.bp", adios2::Mode::Write); + + auto variable = + IO.DefineVariable("/data/meshes/theta", {10}, {0}, {10}); + + for (size_t step = 0; step < 5; ++step) + { + engine.BeginStep(); + + // write default openPMD attributes + IO.DefineAttribute("/basePath", std::string("/data/%T/")); + IO.DefineAttribute( + "/date", std::string("2021-02-22 11:14:00 +0000")); + IO.DefineAttribute( + "/iterationEncoding", std::string("variableBased")); + IO.DefineAttribute("/iterationFormat", std::string("/data/%T/")); + IO.DefineAttribute("/meshesPath", std::string("meshes/")); + IO.DefineAttribute("/openPMD", std::string("1.1.0")); + IO.DefineAttribute("/openPMDextension", uint32_t(0)); + IO.DefineAttribute("/software", std::string("openPMD-api")); + IO.DefineAttribute("/softwareVersion", std::string("0.17.0-dev")); + + std::vector current_snapshots(10); + std::iota( + current_snapshots.begin(), + current_snapshots.end(), + current_snapshots.size() * step); + IO.DefineAttribute( + "/data/snapshot", + current_snapshots.data(), + current_snapshots.size(), + "", + "/", + /* allowModification = */ true); + + IO.DefineAttribute("/data/dt", double(1)); + IO.DefineAttribute( + "/data/meshes/theta/axisLabels", + std::vector{"x"}.data(), + 1); + IO.DefineAttribute( + "/data/meshes/theta/dataOrder", std::string("C")); + IO.DefineAttribute( + "/data/meshes/theta/geometry", std::string("cartesian")); + IO.DefineAttribute( + "/data/meshes/theta/gridGlobalOffset", double(0)); + IO.DefineAttribute("/data/meshes/theta/gridSpacing", double(1)); + IO.DefineAttribute("/data/meshes/theta/gridUnitSI", double(1)); + IO.DefineAttribute("/data/meshes/theta/position", double(0)); + IO.DefineAttribute("/data/meshes/theta/timeOffset", double(0)); + IO.DefineAttribute( + "/data/meshes/theta/unitDimension", + std::vector(7, 0).data(), + 7); + IO.DefineAttribute("/data/meshes/theta/unitSI", double(1)); + IO.DefineAttribute("/data/time", double(0)); + IO.DefineAttribute("/data/timeUnitSI", double(1)); + + IO.DefineAttribute( + "__openPMD_internal/openPMD2_adios2_schema", 0); + IO.DefineAttribute("__openPMD_internal/useSteps", 0); + + std::vector data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + engine.Put(variable, data.data()); + + engine.EndStep(); + } + + engine.Close(); + } +} + +auto read_variablebased_randomaccess() -> void +{ + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (rank == 0) + { + create_file_in_serial(); + } + MPI_Barrier(MPI_COMM_WORLD); + + std::string const config = R"END( +{ + "adios2": + { + "use_group_table": true + } +})END"; + + { + openPMD::Series read( + "../samples/bp5_no_steps.bp", + openPMD::Access::READ_ONLY, + MPI_COMM_WORLD, + "adios2.engine.type = \"bp5\""); + auto data = + read.iterations[0].meshes["theta"].loadChunk({0}, {10}); + read.flush(); + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(data.get()[i] == int(i)); + } + } +} +} // namespace read_variablebased_randomaccess +#else +namespace read_variablebased_randomaccess +{ +void read_variablebased_randomaccess() +{} +} // namespace read_variablebased_randomaccess +#endif diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 2d4cc6554e..d208cadb53 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1,6 +1,8 @@ /* Running this test in parallel with MPI requires MPI::Init. * To guarantee a correct call to Init, launch the tests manually. */ +#include "Files_ParallelIO/ParallelIOTests.hpp" + #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -2197,4 +2199,9 @@ TEST_CASE("adios2_flush_via_step") } #endif +TEST_CASE("read_variablebased_randomaccess") +{ + read_variablebased_randomaccess::read_variablebased_randomaccess(); +} + #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI From a3cabdd457dda8c2a32b05ea9c32b063f9e85c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 19 Dec 2024 12:42:13 +0100 Subject: [PATCH 08/33] CI fixes --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 4 ++-- test/Files_ParallelIO/read_variablebased_randomaccess.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 732b07f697..9cfea1ffbf 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -44,7 +44,6 @@ #include #include #include -#include #include #include #include @@ -1467,6 +1466,7 @@ namespace auxiliary::openPMD_MPI_type(), 0, comm); + std::cout << "SIZES: " << vec_as_string(sizes) << '\n'; size_t total_flat_size = std::accumulate(sizes.begin(), sizes.end(), size_t(0)); using flat_type = typename T::value_type; @@ -1519,8 +1519,8 @@ namespace comm); if (rank != 0) { - put_result_here = std::move(receive); std::cout << "RECEIVED: " << vec_as_string(receive) << '\n'; + put_result_here = std::move(receive); } } } diff --git a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp index ca5eb66961..e366704871 100644 --- a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp +++ b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp @@ -6,7 +6,7 @@ #include -#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI && openPMD_HAS_ADIOS_2_9 #include #include From f8a9e199f6316ade74a2fe3c132c2660331001f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 19 Dec 2024 14:42:22 +0100 Subject: [PATCH 09/33] Fix test --- test/Files_ParallelIO/read_variablebased_randomaccess.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp index e366704871..3c2640ac43 100644 --- a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp +++ b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp @@ -1,5 +1,6 @@ #include "ParallelIOTests.hpp" +#include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/openPMD.hpp" #include @@ -18,8 +19,9 @@ static void create_file_in_serial() adios2::ADIOS adios; auto IO = adios.DeclareIO("IO"); IO.SetEngine("bp5"); - auto engine = - IO.Open("../samples/bp5_no_steps.bp", adios2::Mode::Write); + auto engine = IO.Open( + "../samples/read_variablebased_randomaccess.bp", + adios2::Mode::Write); auto variable = IO.DefineVariable("/data/meshes/theta", {10}, {0}, {10}); From 73534395c2b9308e12cb7c0dbebc5bcfae05ae6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 19 Dec 2024 14:49:10 +0100 Subject: [PATCH 10/33] Add MPI_CHECK --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 9cfea1ffbf..2da805eef7 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -89,6 +89,17 @@ std::optional joinedDimension(adios2::Dims const &dims) #if openPMD_HAVE_MPI +#define MPI_CHECK(cmd) \ + do \ + { \ + int error = cmd; \ + if (error != MPI_SUCCESS) \ + { \ + std::cerr << "<" << __FILE__ << ">:" << __LINE__; \ + throw std::runtime_error(std::string("[MPI] Error")); \ + } \ + } while (false); + ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler *handler, MPI_Comm communicator, @@ -1429,8 +1440,8 @@ namespace std::get>(put_result_here_in); std::cout << "INIT VECTOR SIZE: " << put_result_here.size() << '\n'; size_t num_items = put_result_here.size(); - MPI_Bcast( - &num_items, 1, auxiliary::openPMD_MPI_type(), 0, comm); + MPI_CHECK(MPI_Bcast( + &num_items, 1, auxiliary::openPMD_MPI_type(), 0, comm)); std::cout << "WILL_COMMUNICATE: " << num_items << '\n'; if constexpr ( std::is_same_v || @@ -1460,12 +1471,12 @@ namespace [](T const &arr) { return arr.size(); }); } sizes.resize(num_items); - MPI_Bcast( + MPI_CHECK(MPI_Bcast( sizes.data(), num_items, auxiliary::openPMD_MPI_type(), 0, - comm); + comm)); std::cout << "SIZES: " << vec_as_string(sizes) << '\n'; size_t total_flat_size = std::accumulate(sizes.begin(), sizes.end(), size_t(0)); @@ -1483,12 +1494,12 @@ namespace } } flat_vector.resize(total_flat_size); - MPI_Bcast( + MPI_CHECK(MPI_Bcast( flat_vector.data(), total_flat_size, auxiliary::openPMD_MPI_type(), 0, - comm); + comm)); if (rank != 0) { std::cout << "RECEIVED: " << vec_as_string(flat_vector) @@ -1511,12 +1522,12 @@ namespace { receive.resize(num_items); } - MPI_Bcast( + MPI_CHECK(MPI_Bcast( rank == 0 ? put_result_here.data() : receive.data(), num_items, auxiliary::openPMD_MPI_type(), 0, - comm); + comm)); if (rank != 0) { std::cout << "RECEIVED: " << vec_as_string(receive) << '\n'; @@ -1564,7 +1575,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( { type = read_from_file_in_serial(); } - MPI_Bcast(&type, 1, MPI_INT, 0, *m_communicator); + MPI_CHECK(MPI_Bcast(&type, 1, MPI_INT, 0, *m_communicator)); switchType( type, *param.resource, *m_communicator, rank); #else From 926581e48cb324fb65fe5c9c11c8cd9076372868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Jan 2025 14:57:35 +0100 Subject: [PATCH 11/33] Stricter warning about group-based encoding in BP5 --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2da805eef7..0c0eea23b6 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -657,13 +657,9 @@ void ADIOS2IOHandlerImpl::createFile( m_writeAttributesFromThisRank && m_handler->m_encoding == IterationEncoding::groupBased) { - // For a peaceful phase-out of group-based encoding in ADIOS2, - // print this warning only in the new layout (with group table) - if (m_useGroupTable.value_or(UseGroupTable::No) == - UseGroupTable::Yes && - (m_engineType == "bp5" || - (m_engineType == "file" || m_engineType == "filestream" || - m_engineType == "bp"))) + if (m_engineType == "bp5" || + (m_engineType == "file" || m_engineType == "filestream" || + m_engineType == "bp")) { std::cerr << warningADIOS2NoGroupbasedEncoding << std::endl; printedWarningsAlready.noGroupBased = true; From 0da9a0168941175de02ceb2a68faa94b99bcde7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Jan 2025 16:13:28 +0100 Subject: [PATCH 12/33] variableBased as default in ADIOS2 --- examples/5_write_parallel.py | 5 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 6 + include/openPMD/IO/AbstractIOHandler.hpp | 1 + include/openPMD/Series.hpp | 10 ++ src/IO/ADIOS/ADIOS2IOHandler.cpp | 1 + src/IO/AbstractIOHandler.cpp | 5 + src/Series.cpp | 137 ++++++++++++------ test/Files_SerialIO/close_and_reopen_test.cpp | 5 +- test/ParallelIOTest.cpp | 5 +- test/SerialIOTest.cpp | 2 +- 10 files changed, 128 insertions(+), 49 deletions(-) diff --git a/examples/5_write_parallel.py b/examples/5_write_parallel.py index 8574c1d66e..64ae652348 100644 --- a/examples/5_write_parallel.py +++ b/examples/5_write_parallel.py @@ -56,8 +56,7 @@ # in streaming setups, e.g. an iteration cannot be opened again once # it has been closed. # `Series.iterations` can be directly accessed in random-access workflows. - series.iterations[1].open() - mymesh = series.iterations[1]. \ + mymesh = series.write_iterations()[1]. \ meshes["mymesh"] # example 1D domain decomposition in first index @@ -92,7 +91,7 @@ # The iteration can be closed in order to help free up resources. # The iteration's content will be flushed automatically. # An iteration once closed cannot (yet) be reopened. - series.iterations[1].close() + series.write_iterations()[1].close() if 0 == comm.rank: print("Dataset content has been fully written to disk") diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 84aabc5076..02920c0207 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -23,6 +23,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" +#include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" @@ -860,6 +861,11 @@ class ADIOS2IOHandler : public AbstractIOHandler return "ADIOS2"; } + bool fullSupportForVariableBasedEncoding() const override + { + return openPMD_HAS_ADIOS_2_9; + } + std::future flush(internal::ParsedFlushParams &) override; }; // ADIOS2IOHandler } // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index e8e55457eb..291bc405a2 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -250,6 +250,7 @@ class AbstractIOHandler /** The currently used backend */ virtual std::string backendName() const = 0; + virtual bool fullSupportForVariableBasedEncoding() const; std::string directory; /* diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 7cbac21f2d..f0487fe238 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -68,6 +68,11 @@ class Series; namespace internal { + enum class default_or_explicit : bool + { + default_, + explicit_ + }; /** * @brief Data members for Series. Pinned at one memory location. * @@ -183,6 +188,8 @@ namespace internal * The iteration encoding used in this series. */ IterationEncoding m_iterationEncoding{}; + default_or_explicit m_iterationEncodingSetExplicitly = + default_or_explicit::default_; /** * Detected IO format (backend). */ @@ -969,6 +976,9 @@ OPENPMD_private */ void flushStep(bool doFlush); + Series &setIterationEncoding_internal( + IterationEncoding iterationEncoding, internal::default_or_explicit); + /* * Returns the current content of the /data/snapshot attribute. * (We could also add this to the public API some time) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 0c0eea23b6..778267e8d7 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1549,6 +1549,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto IO = adios.DeclareIO("PreparseSnapshots"); // @todo check engine type IO.SetEngine(realEngineType()); + IO.SetParameter("StreamReader", "ON"); // this be for BP4 auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); auto status = engine.BeginStep(); auto type = detail::attributeInfo(IO, name, /* verbose = */ true); diff --git a/src/IO/AbstractIOHandler.cpp b/src/IO/AbstractIOHandler.cpp index 8f0bd4524e..c8d3412fe2 100644 --- a/src/IO/AbstractIOHandler.cpp +++ b/src/IO/AbstractIOHandler.cpp @@ -116,4 +116,9 @@ std::future AbstractIOHandler::flush(internal::FlushParams const ¶ms) json::warnGlobalUnusedOptions(parsedParams.backendConfig); return future; } + +bool AbstractIOHandler::fullSupportForVariableBasedEncoding() const +{ + return false; +} } // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index 2d631e3221..b2aa65e121 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -581,45 +581,7 @@ IterationEncoding Series::iterationEncoding() const Series &Series::setIterationEncoding(IterationEncoding ie) { - auto &series = get(); - if (series.m_deferred_initialization) - { - runDeferredInitialization(); - } - if (written()) - throw std::runtime_error( - "A files iterationEncoding can not (yet) be changed after it has " - "been written."); - - series.m_iterationEncoding = ie; - switch (ie) - { - case IterationEncoding::fileBased: - setIterationFormat(series.m_name); - setAttribute("iterationEncoding", std::string("fileBased")); - // This checks that the name contains the expansion pattern - // (e.g. %T) and parses it - if (series.m_filenamePadding < 0) - { - if (!reparseExpansionPattern(series.m_name)) - { - throw error::WrongAPIUsage( - "For fileBased formats the iteration expansion pattern " - "%T must " - "be included in the file name"); - } - } - break; - case IterationEncoding::groupBased: - setIterationFormat(BASEPATH); - setAttribute("iterationEncoding", std::string("groupBased")); - break; - case IterationEncoding::variableBased: - setIterationFormat(auxiliary::replace_first(basePath(), "/%T/", "")); - setAttribute("iterationEncoding", std::string("variableBased")); - break; - } - IOHandler()->setIterationEncoding(ie); + setIterationEncoding_internal(ie, internal::default_or_explicit::default_); return *this; } @@ -1170,7 +1132,9 @@ Given file pattern: ')END" setWritten(false, Attributable::EnqueueAsynchronously::No); initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + setIterationEncoding_internal( + input->iterationEncoding, + series.m_iterationEncodingSetExplicitly); setWritten(true, Attributable::EnqueueAsynchronously::No); } @@ -1186,12 +1150,14 @@ Given file pattern: ')END" } case Access::CREATE: { initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + setIterationEncoding_internal( + input->iterationEncoding, series.m_iterationEncodingSetExplicitly); break; } case Access::APPEND: { initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + setIterationEncoding_internal( + input->iterationEncoding, series.m_iterationEncodingSetExplicitly); if (input->iterationEncoding != IterationEncoding::fileBased) { break; @@ -1857,7 +1823,8 @@ void Series::readOneIterationFileBased(std::string const &filePath) "Unknown iterationEncoding: " + encoding); auto old_written = written(); setWritten(false, Attributable::EnqueueAsynchronously::No); - setIterationEncoding(encoding_out); + setIterationEncoding_internal( + encoding_out, internal::default_or_explicit::explicit_); setWritten(old_written, Attributable::EnqueueAsynchronously::Yes); } else @@ -2645,6 +2612,59 @@ void Series::flushStep(bool doFlush) series.m_wroteAtLeastOneIOStep = true; } +Series &Series::setIterationEncoding_internal( + IterationEncoding ie, internal::default_or_explicit doe) +{ + auto &series = get(); + switch (doe) + { + case internal::default_or_explicit::default_: + case internal::default_or_explicit::explicit_: + // mark this option as set explicitly by the user + series.m_iterationEncodingSetExplicitly = doe; + break; + } + if (series.m_deferred_initialization) + { + runDeferredInitialization(); + } + if (written()) + throw std::runtime_error( + "A files iterationEncoding can not (yet) be changed after it has " + "been written."); + + series.m_iterationEncoding = ie; + switch (ie) + { + case IterationEncoding::fileBased: + setIterationFormat(series.m_name); + setAttribute("iterationEncoding", std::string("fileBased")); + // This checks that the name contains the expansion pattern + // (e.g. %T) and parses it + if (series.m_filenamePadding < 0) + { + if (!reparseExpansionPattern(series.m_name)) + { + throw error::WrongAPIUsage( + "For fileBased formats the iteration expansion pattern " + "%T must " + "be included in the file name"); + } + } + break; + case IterationEncoding::groupBased: + setIterationFormat(BASEPATH); + setAttribute("iterationEncoding", std::string("groupBased")); + break; + case IterationEncoding::variableBased: + setIterationFormat(auxiliary::replace_first(basePath(), "/%T/", "")); + setAttribute("iterationEncoding", std::string("variableBased")); + break; + } + IOHandler()->setIterationEncoding(ie); + return *this; +} + auto Series::openIterationIfDirty(IterationIndex_t index, Iteration &iteration) -> IterationOpened { @@ -2939,6 +2959,8 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) options, "iteration_encoding", iterationEncoding); if (!iterationEncoding.empty()) { + series.m_iterationEncodingSetExplicitly = + internal::default_or_explicit::explicit_; auto it = ieDescriptors.find(iterationEncoding); if (it != ieDescriptors.end()) { @@ -3190,6 +3212,35 @@ Series::snapshots(std::optional const snapshot_workflow) } } + /* + * ADIOS2 should use variable-based encoding as a default when applicable, + * since group-based encoding has severe limitations in ADIOS2. + * The below logic checks if variable-based encoding should be used. + */ + + if ( + // 1. No encoding has been explicitly selected by the user. + // Flag set by Series::setIterationEncoding(). + series.m_iterationEncodingSetExplicitly == + internal::default_or_explicit::default_ && + // 2. Iteration encoding was recognized as groupBased by init() + // procedures (and not file-based). + series.m_iterationEncoding == IterationEncoding::groupBased && + // 3. The IO workflow will be synchronous, necessary for writing + // variable-based data (but not for reading!). + usedSnapshotWorkflow == SnapshotWorkflow::Synchronous && + // 4. The chosen access type is write-only, otherwise the encoding is + // determined by the previous file content. + access::writeOnly(access) && + // 5. The backend is ADIOS2 in a recent enough version to support + // modifiable attributes (v2.9). + IOHandler()->fullSupportForVariableBasedEncoding()) + { + setIterationEncoding_internal( + IterationEncoding::variableBased, + internal::default_or_explicit::default_); + } + switch (usedSnapshotWorkflow) { case SnapshotWorkflow::RandomAccess: { diff --git a/test/Files_SerialIO/close_and_reopen_test.cpp b/test/Files_SerialIO/close_and_reopen_test.cpp index 56be4a39f6..8fad892635 100644 --- a/test/Files_SerialIO/close_and_reopen_test.cpp +++ b/test/Files_SerialIO/close_and_reopen_test.cpp @@ -172,7 +172,10 @@ auto run_test_groupbased( { std::string filename = "../samples/close_iteration_reopen/groupbased." + ext; - Series series(filename, Access::CREATE, write_cfg); + Series series( + filename, + Access::CREATE, + json::merge(write_cfg, R"({"iteration_encoding": "group_based"})")); { auto it = writeIterations(series)[0]; auto E_x = it.meshes["E"]["x"]; diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index d208cadb53..c3f3331bcb 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1213,7 +1213,10 @@ void adios2_streaming(bool variableBasedLayout) Series writeSeries( "../samples/adios2_stream.sst", Access::CREATE, - "adios2.engine.type = \"sst\""); + variableBasedLayout + ? "adios2.engine.type = \"sst\"" + : "adios2.engine.type = \"sst\"\niteration_encoding = " + "\"group_based\""); if (variableBasedLayout) { writeSeries.setIterationEncoding(IterationEncoding::variableBased); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index fed2c8c1bd..02f086112b 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -6993,7 +6993,7 @@ TEST_CASE("unfinished_iteration_test", "[serial]") unfinished_iteration_test( "bp", IterationEncoding::groupBased, - R"({"backend": "adios2"})", + R"({"backend": "adios2", "iteration_encoding": "group_based"})", /* test_linear_access = */ false); unfinished_iteration_test( "bp5", From 432fe2e1b93d097706bf8d0220ba3c840b1c3a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 14 Jan 2025 12:29:02 +0100 Subject: [PATCH 13/33] Add error check for non-homogeneous datasets --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 778267e8d7..c3849b757b 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -2287,6 +2287,29 @@ namespace detail file, ADIOS2IOHandlerImpl::IfFileNotOpen::ThrowError); auto &IO = fileData.m_IO; adios2::Variable var = IO.InquireVariable(varName); + + if (fileData.stepSelection().has_value()) + { + auto file_steps = fileData.getEngine().Steps(); + auto var_steps = var.Steps(); + if (file_steps != var_steps) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", + &R"( +The opened file contains different data per step. +When using variable-based encoding, such files must be opened in linear read +mode, since random-access mode cannot easily associate variable steps +to iterations under these circumstances (yet). +If random-access read mode is required, file-based iteration encoding is more +useful for such data in ADIOS2. You may use the openpmd-pipe command line tool +for converting from variable-based to file-based iteration encoding. +ERROR: Variable ')"[1] + varName + + "' has " + std::to_string(var_steps) + + " step(s), but the file has " + + std::to_string(file_steps) + " step(s)."); + } + } if (stepSelection.has_value()) { var.SetStepSelection({*stepSelection, 1}); From 1c230e1ff4ce62d1d53a84e60e02d64c3047810d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 14 Jan 2025 13:47:49 +0100 Subject: [PATCH 14/33] Reset steps before reading the rankTable --- include/openPMD/IO/ADIOS/ADIOS2File.hpp | 2 +- include/openPMD/IO/IOTask.hpp | 2 +- src/IO/ADIOS/ADIOS2File.cpp | 14 +++++++++++--- src/IO/AbstractIOHandlerImpl.cpp | 6 +++++- src/Series.cpp | 8 ++++++++ test/SerialIOTest.cpp | 9 ++++++++- 6 files changed, 34 insertions(+), 7 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 2f53ba6686..91c5773ba8 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -413,7 +413,7 @@ class ADIOS2File StreamStatus streamStatus = StreamStatus::OutsideOfStep; size_t currentStep(); - void setStepSelection(size_t); + void setStepSelection(std::optional); [[nodiscard]] std::optional stepSelection() const; private: diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index d65f0dda72..5ac4c9f6c5 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -672,7 +672,7 @@ struct OPENPMDAPI_EXPORT Parameter struct StepSelection { - size_t step; + std::optional step; }; //! input parameter diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index a1bd21f381..bb7af51baa 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -353,7 +353,7 @@ size_t ADIOS2File::currentStep() } } -void ADIOS2File::setStepSelection(size_t step) +void ADIOS2File::setStepSelection(std::optional step) { if (streamStatus != StreamStatus::ReadWithoutStream) { @@ -361,8 +361,16 @@ void ADIOS2File::setStepSelection(size_t step) "ADIOS2 backend: Cannot only use random-access step selections " "when reading without streaming mode."); } - m_currentStep = step; - useStepSelection = true; + if (!step.has_value()) + { + m_currentStep = 0; + useStepSelection = false; + } + else + { + m_currentStep = *step; + useStepSelection = true; + } } std::optional ADIOS2File::stepSelection() const diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index d362c91b2e..95fb7dfa09 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -421,7 +421,11 @@ std::future AbstractIOHandlerImpl::flush() [](Parameter::StepSelection step) { std::stringstream s; - s << "RANDOMACCESS '" << step.step << "'"; + s << "RANDOMACCESS '" + << (step.step.has_value() + ? std::to_string(*step.step) + : std::string("RESET")) + << "'"; return s.str(); }}, parameter.mode); diff --git a/src/Series.cpp b/src/Series.cpp index b2aa65e121..9a9d9a2972 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -256,6 +256,14 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) rankTable.m_bufferedRead = chunk_assignment::RankMeta{}; return {}; } + if (iterationEncoding() == IterationEncoding::variableBased && + IOHandler()->m_backendAccess == Access::READ_RANDOM_ACCESS) + { + Parameter advance; + advance.mode = + Parameter::StepSelection{std::nullopt}; + IOHandler()->enqueue(IOTask(this, std::move(advance))); + } Parameter openDataset; openDataset.name = "rankTable"; IOHandler()->enqueue(IOTask(&rankTable.m_attributable, openDataset)); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 02f086112b..45858fd277 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5219,7 +5219,8 @@ void serial_iterator(std::string const &file) { constexpr Extent::value_type extent = 1000; { - Series writeSeries(file, Access::CREATE); + Series writeSeries( + file, Access::CREATE, "rank_table = \"posix_hostname\""); auto iterations = writeSeries.writeIterations(); for (size_t i = 0; i < 10; ++i) { @@ -5250,6 +5251,12 @@ void serial_iterator(std::string const &file) } last_iteration_index = iteration.iterationIndex; } + if (readSeries.iterationEncoding() == IterationEncoding::variableBased) + for (auto const &[rank, host] : readSeries.rankTable(true)) + { + std::cout << "POST Rank '" << rank << "' written from host '" + << host << "'\n"; + } REQUIRE(last_iteration_index == 9); REQUIRE(numberOfIterations == 10); } From 544e765f153ed6a3878802861b2111ee247a0583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 14 Jan 2025 14:29:43 +0100 Subject: [PATCH 15/33] Use writeIterations() in MPI Benchmark --- .../openPMD/benchmark/mpi/MPIBenchmark.hpp | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/include/openPMD/benchmark/mpi/MPIBenchmark.hpp b/include/openPMD/benchmark/mpi/MPIBenchmark.hpp index 3d6d78c7e4..b88e80adc3 100644 --- a/include/openPMD/benchmark/mpi/MPIBenchmark.hpp +++ b/include/openPMD/benchmark/mpi/MPIBenchmark.hpp @@ -325,12 +325,17 @@ MPIBenchmark::BenchmarkExecution::writeBenchmark( m_benchmark->communicator, jsonConfig); + typename Clock::duration ignore{}; + for (Series::IterationIndex_t i = 0; i < iterations; i++) { + auto ignore_start = Clock::now(); auto writeData = datasetFiller->produceData(); + auto ignore_end = Clock::now(); + ignore += ignore_end - ignore_start; - MeshRecordComponent id = - series.iterations[i].meshes["id"][MeshRecordComponent::SCALAR]; + MeshRecordComponent id = series.writeIterations()[i] + .meshes["id"][MeshRecordComponent::SCALAR]; Datatype datatype = determineDatatype(writeData); Dataset dataset = Dataset(datatype, m_benchmark->totalExtent); @@ -342,18 +347,11 @@ MPIBenchmark::BenchmarkExecution::writeBenchmark( id.storeChunk(writeData, offset, extent); series.flush(); } - + series.close(); MPI_Barrier(m_benchmark->communicator); auto end = Clock::now(); - // deduct the time needed for data generation - for (Series::IterationIndex_t i = 0; i < iterations; i++) - { - datasetFiller->produceData(); - } - auto deduct = Clock::now(); - - return end - start - (deduct - end); + return (end - start) - ignore; } template @@ -378,7 +376,7 @@ MPIBenchmark::BenchmarkExecution::readBenchmark( for (Series::IterationIndex_t i = 0; i < iterations; i++) { MeshRecordComponent id = - series.iterations[i].meshes["id"][MeshRecordComponent::SCALAR]; + series.snapshots()[i].meshes["id"][MeshRecordComponent::SCALAR]; auto chunk_data = id.loadChunk(offset, extent); series.flush(); From 547546c7e9f01cfdd098a411a494a815eba2ec18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 14 Jan 2025 14:36:22 +0100 Subject: [PATCH 16/33] Use group tables by default --- src/IO/ADIOS/ADIOS2File.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index bb7af51baa..0fc61700dc 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -480,14 +480,10 @@ void ADIOS2File::configure_IO() { switch (m_impl->m_handler->m_encoding) { - /* - * For variable-based encoding, this does not matter as it is - * new and requires >= v2.9 features anyway. - */ case IterationEncoding::variableBased: + case IterationEncoding::groupBased: m_impl->m_useGroupTable = UseGroupTable::Yes; break; - case IterationEncoding::groupBased: case IterationEncoding::fileBased: m_impl->m_useGroupTable = UseGroupTable::No; break; From 313b9a094171f16bb8784cbd3f7086fc14e2c533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 14 Jan 2025 15:33:43 +0100 Subject: [PATCH 17/33] Add iterate_nonstreaming_series test to parallel tests This covers the use case that the snapshot attribute has more than just one entry. --- CMakeLists.txt | 3 +- test/Files_ParallelIO/ParallelIOTests.hpp | 72 +++++++ .../iterate_nonstreaming_series.cpp | 186 ++++++++++++++++++ test/ParallelIOTest.cpp | 48 ++--- test/SerialIOTest.cpp | 12 +- 5 files changed, 278 insertions(+), 43 deletions(-) create mode 100644 test/Files_ParallelIO/iterate_nonstreaming_series.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c3feadc35..c4af3df142 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -787,9 +787,10 @@ if(openPMD_BUILD_TESTING) test/Files_SerialIO/close_and_reopen_test.cpp test/Files_SerialIO/filebased_write_test.cpp ) - elseif(${test_name} STREQUAL "ParallelIO") + elseif(${test_name} STREQUAL "ParallelIO" AND openPMD_HAVE_MPI) list(APPEND ${out_list} test/Files_ParallelIO/read_variablebased_randomaccess.cpp + test/Files_ParallelIO/iterate_nonstreaming_series.cpp ) endif() endmacro() diff --git a/test/Files_ParallelIO/ParallelIOTests.hpp b/test/Files_ParallelIO/ParallelIOTests.hpp index c7c879fc8b..0cb247700e 100644 --- a/test/Files_ParallelIO/ParallelIOTests.hpp +++ b/test/Files_ParallelIO/ParallelIOTests.hpp @@ -1,6 +1,78 @@ #include +#if openPMD_HAVE_MPI + +using namespace openPMD; + +struct BackendSelection +{ + std::string backendName; + std::string extension; + + [[nodiscard]] inline std::string jsonBaseConfig() const + { + return R"({"backend": ")" + backendName + "\"}"; + } +}; + +inline std::vector testedBackends() +{ + auto variants = getVariants(); + std::map extensions{ + {"adios2", "bp"}, {"hdf5", "h5"}}; + std::vector res; + for (auto const &pair : variants) + { + if (pair.second) + { + auto lookup = extensions.find(pair.first); + if (lookup != extensions.end()) + { + std::string extension = lookup->second; + res.push_back({pair.first, std::move(extension)}); + } + } + } + return res; +} + +inline std::vector getBackends() +{ + // first component: backend file ending + // second component: whether to test 128 bit values + std::vector res; +#if openPMD_HAVE_ADIOS2 + res.emplace_back("bp"); +#endif +#if openPMD_HAVE_HDF5 + res.emplace_back("h5"); +#endif + return res; +} + +inline auto const backends = getBackends(); + +inline std::vector testedFileExtensions() +{ + auto allExtensions = getFileExtensions(); + auto newEnd = std::remove_if( + allExtensions.begin(), allExtensions.end(), [](std::string const &ext) { + // sst and ssc need a receiver for testing + // bp4 is already tested via bp + return ext == "sst" || ext == "ssc" || ext == "bp4" || + ext == "toml" || ext == "json"; + }); + return {allExtensions.begin(), newEnd}; +} + namespace read_variablebased_randomaccess { auto read_variablebased_randomaccess() -> void; } + +namespace iterate_nonstreaming_series +{ +auto iterate_nonstreaming_series() -> void; +} + +#endif diff --git a/test/Files_ParallelIO/iterate_nonstreaming_series.cpp b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp new file mode 100644 index 0000000000..acce61a162 --- /dev/null +++ b/test/Files_ParallelIO/iterate_nonstreaming_series.cpp @@ -0,0 +1,186 @@ +#include "ParallelIOTests.hpp" + +#include "openPMD/IO/ADIOS/macros.hpp" + +#include +#include + +namespace iterate_nonstreaming_series +{ + +static auto run_test( + std::string const &file, + bool variableBasedLayout, + std::string const &jsonConfig) -> void +{ + int mpi_size, mpi_rank; + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + constexpr size_t base_extent = 10; + size_t const extent = base_extent * size_t(mpi_size); + + { + Series writeSeries(file, Access::CREATE, MPI_COMM_WORLD, jsonConfig); + if (variableBasedLayout) + { + writeSeries.setIterationEncoding(IterationEncoding::variableBased); + } + // use conventional API to write iterations + auto iterations = writeSeries.iterations; + for (size_t i = 0; i < 10; ++i) + { + auto iteration = iterations[i]; + auto E_x = iteration.meshes["E"]["x"]; + E_x.resetDataset( + openPMD::Dataset(openPMD::Datatype::INT, {2, extent})); + int value = variableBasedLayout ? 0 : i; + std::vector data(extent, value); + E_x.storeChunk( + data, {0, base_extent * size_t(mpi_rank)}, {1, base_extent}); + bool taskSupportedByBackend = true; + DynamicMemoryView memoryView; + { + auto currentBuffer = memoryView.currentBuffer(); + REQUIRE(currentBuffer.data() == nullptr); + REQUIRE(currentBuffer.size() == 0); + } + memoryView = E_x.storeChunk( + {1, base_extent * size_t(mpi_rank)}, + {1, base_extent}, + /* + * Hijack the functor that is called for buffer creation. + * This allows us to check if the backend has explicit support + * for buffer creation or if the fallback implementation is + * used. + */ + [&taskSupportedByBackend](size_t size) { + taskSupportedByBackend = false; + return std::shared_ptr{ + new int[size], [](auto *ptr) { delete[] ptr; }}; + }); + if (writeSeries.backend() == "ADIOS2") + { + // that backend must support span creation + REQUIRE(taskSupportedByBackend); + } + auto span = memoryView.currentBuffer(); + for (size_t j = 0; j < span.size(); ++j) + { + span[j] = j; + } + + /* + * This is to test whether defaults are correctly written for + * scalar record components since there previously was a bug. + */ + auto scalarMesh = + iteration + .meshes["i_energyDensity"][MeshRecordComponent::SCALAR]; + scalarMesh.resetDataset( + Dataset(Datatype::INT, {5 * size_t(mpi_size)})); + auto scalarSpan = + scalarMesh.storeChunk({5 * size_t(mpi_rank)}, {5}) + .currentBuffer(); + for (size_t j = 0; j < scalarSpan.size(); ++j) + { + scalarSpan[j] = j; + } + // we encourage manually closing iterations, but it should not + // matter so let's do the switcharoo for this test + if (i % 2 == 0) + { + writeSeries.flush(); + } + else + { + iteration.close(); + } + } + } + + for (auto access : {Access::READ_LINEAR, Access::READ_ONLY}) + { + Series readSeries( + file, + access, + MPI_COMM_WORLD, + json::merge(jsonConfig, R"({"defer_iteration_parsing": true})")); + + size_t last_iteration_index = 0; + // conventionally written Series must be readable with streaming-aware + // API! + for (auto iteration : readSeries.readIterations()) + { + // ReadIterations takes care of Iteration::open()ing iterations + auto E_x = iteration.meshes["E"]["x"]; + REQUIRE(E_x.getDimensionality() == 2); + REQUIRE(E_x.getExtent()[0] == 2); + REQUIRE(E_x.getExtent()[1] == extent); + auto chunk = E_x.loadChunk({0, 0}, {1, extent}); + auto chunk2 = E_x.loadChunk({1, 0}, {1, extent}); + // we encourage manually closing iterations, but it should not + // matter so let's do the switcharoo for this test + if (last_iteration_index % 2 == 0) + { + readSeries.flush(); + } + else + { + iteration.close(); + } + + int value = variableBasedLayout ? 0 : iteration.iterationIndex; + for (size_t i = 0; i < extent; ++i) + { + REQUIRE(chunk.get()[i] == value); + REQUIRE(chunk2.get()[i] == int(i % base_extent)); + } + last_iteration_index = iteration.iterationIndex; + } + REQUIRE(last_iteration_index == 9); + } +} + +auto iterate_nonstreaming_series() -> void +{ + for (auto const &backend : testedBackends()) + { + run_test( + "../samples/iterate_nonstreaming_series/parallel_filebased_%T." + + backend.extension, + false, + backend.jsonBaseConfig()); + run_test( + "../samples/iterate_nonstreaming_series/parallel_groupbased." + + backend.extension, + false, + backend.jsonBaseConfig()); +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_ADIOS2_BP5 + if (backend.extension == "bp") + { + run_test( + "../samples/iterate_nonstreaming_series/" + "parallel_filebased_bp5_%T." + + backend.extension, + false, + json::merge( + backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); + run_test( + "../samples/iterate_nonstreaming_series/" + "parallel_groupbased_bp5." + + backend.extension, + false, + json::merge( + backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); + } +#endif + } +#if openPMD_HAVE_ADIOS2 && openPMD_HAS_ADIOS_2_9 + run_test( + "../samples/iterate_nonstreaming_series/parallel_variablebased.bp", + true, + R"({"backend": "adios2"})"); +#endif +} +} // namespace iterate_nonstreaming_series diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index c3f3331bcb..aa98d08b46 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -10,7 +10,12 @@ #include "openPMD/openPMD.hpp" #include -#if openPMD_HAVE_MPI +#if !openPMD_HAVE_MPI +TEST_CASE("none", "[parallel]") +{} + +#else + #include #if openPMD_HAVE_ADIOS2 @@ -34,42 +39,6 @@ using namespace openPMD; -std::vector getBackends() -{ - // first component: backend file ending - // second component: whether to test 128 bit values - std::vector res; -#if openPMD_HAVE_ADIOS2 - res.emplace_back("bp"); -#endif -#if openPMD_HAVE_HDF5 - res.emplace_back("h5"); -#endif - return res; -} - -auto const backends = getBackends(); - -std::vector testedFileExtensions() -{ - auto allExtensions = getFileExtensions(); - auto newEnd = std::remove_if( - allExtensions.begin(), allExtensions.end(), [](std::string const &ext) { - // sst and ssc need a receiver for testing - // bp4 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp4" || - ext == "toml" || ext == "json"; - }); - return {allExtensions.begin(), newEnd}; -} - -#else - -TEST_CASE("none", "[parallel]") -{} -#endif - -#if openPMD_HAVE_MPI TEST_CASE("parallel_multi_series_test", "[parallel]") { std::list allSeries; @@ -2207,4 +2176,9 @@ TEST_CASE("read_variablebased_randomaccess") read_variablebased_randomaccess::read_variablebased_randomaccess(); } +TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") +{ + iterate_nonstreaming_series::iterate_nonstreaming_series(); +} + #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 45858fd277..582a54481e 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -6377,12 +6377,12 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") for (auto const &backend : testedBackends()) { iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_filebased_%T." + + "../samples/iterate_nonstreaming_series/serial_filebased_%T." + backend.extension, false, backend.jsonBaseConfig()); iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_groupbased." + + "../samples/iterate_nonstreaming_series/serial_groupbased." + backend.extension, false, backend.jsonBaseConfig()); @@ -6390,13 +6390,15 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") if (backend.extension == "bp") { iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_filebased_bp5_%T." + + "../samples/iterate_nonstreaming_series/" + "serial_filebased_bp5_%T." + backend.extension, false, json::merge( backend.jsonBaseConfig(), "adios2.engine.type = \"bp5\"")); iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_groupbased_bp5." + + "../samples/iterate_nonstreaming_series/" + "serial_groupbased_bp5." + backend.extension, false, json::merge( @@ -6406,7 +6408,7 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") } #if openPMD_HAVE_ADIOS2 iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_variablebased.bp", + "../samples/iterate_nonstreaming_series/serial_variablebased.bp", true, R"({"backend": "adios2"})"); #endif From 30fe9b353825c7131d379b5ec9b0a1f6099dfc71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 14 Jan 2025 15:43:36 +0100 Subject: [PATCH 18/33] Remove std::cout calls --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 30 ------------------------------ src/Series.cpp | 4 ---- 2 files changed, 34 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index c3849b757b..2fe56eb442 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1395,29 +1395,6 @@ namespace static constexpr char const *errorMsg = "ReadAttributeAllsteps"; }; - template - auto vec_as_string(Vec const &vec) -> std::string - { - if (vec.empty()) - { - return "[]"; - } - else - { - std::stringstream res; - res << '['; - auto it = vec.begin(); - res << *it++; - auto end = vec.end(); - for (; it != end; ++it) - { - res << ", " << *it; - } - res << ']'; - return res.str(); - } - } - #if openPMD_HAVE_MPI struct DistributeToAllRanks { @@ -1434,11 +1411,9 @@ namespace } std::vector &put_result_here = std::get>(put_result_here_in); - std::cout << "INIT VECTOR SIZE: " << put_result_here.size() << '\n'; size_t num_items = put_result_here.size(); MPI_CHECK(MPI_Bcast( &num_items, 1, auxiliary::openPMD_MPI_type(), 0, comm)); - std::cout << "WILL_COMMUNICATE: " << num_items << '\n'; if constexpr ( std::is_same_v || std::is_same_v> || @@ -1473,7 +1448,6 @@ namespace auxiliary::openPMD_MPI_type(), 0, comm)); - std::cout << "SIZES: " << vec_as_string(sizes) << '\n'; size_t total_flat_size = std::accumulate(sizes.begin(), sizes.end(), size_t(0)); using flat_type = typename T::value_type; @@ -1498,8 +1472,6 @@ namespace comm)); if (rank != 0) { - std::cout << "RECEIVED: " << vec_as_string(flat_vector) - << '\n'; size_t offset = 0; put_result_here.reserve(num_items); for (size_t current_extent : sizes) @@ -1526,7 +1498,6 @@ namespace comm)); if (rank != 0) { - std::cout << "RECEIVED: " << vec_as_string(receive) << '\n'; put_result_here = std::move(receive); } } @@ -1539,7 +1510,6 @@ namespace void ADIOS2IOHandlerImpl::readAttributeAllsteps( Writable *writable, Parameter ¶m) { - std::cout << "ADIOS2: ReadAttributeAllSteps" << '\n'; auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); auto pos = setAndGetFilePosition(writable); auto name = nameOfAttribute(writable, param.name); diff --git a/src/Series.cpp b/src/Series.cpp index 9a9d9a2972..c78d7e3c4d 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3330,10 +3330,6 @@ auto Series::currentSnapshot() -> std::optional> series.m_snapshotToStep[iteration] = step; } } - for (auto const &[iteration, step] : series.m_snapshotToStep) - { - std::cout << '\t' << iteration << "\t-> " << step << '\n'; - } std::cout.flush(); return vec_t{res.begin(), res.end()}; } From e53576239acdbd758872173066a6d5f8709415c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 14 Jan 2025 18:20:14 +0100 Subject: [PATCH 19/33] wip: use variable-based encoding in tests --- src/Series.cpp | 32 +++++++++++--- test/ParallelIOTest.cpp | 13 +++--- test/SerialIOTest.cpp | 97 ++++++++++++++++++++++++----------------- 3 files changed, 92 insertions(+), 50 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index c78d7e3c4d..ca3da5fb9b 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2144,10 +2144,29 @@ creating new iterations. case IterationEncoding::variableBased: { if (!currentSteps.has_value() || currentSteps.value().empty()) { - currentSteps = std::vector{ - read_only_this_single_iteration.has_value() - ? *read_only_this_single_iteration - : 0}; + if (!read_only_this_single_iteration.has_value()) + { + Parameter ld; + Parameter lp; + IOHandler()->enqueue(IOTask(&iterations, ld)); + IOHandler()->enqueue(IOTask(&iterations, lp)); + IOHandler()->flush(internal::defaultFlushParams); + if (ld.datasets->empty() && lp.paths->empty()) + { + return {}; // no iterations, just global attributes + } + else + { + // there is data, defaulting to calling this Iteration idx 0 + // when no further info is available + currentSteps = std::vector{0}; + } + } + else + { + currentSteps = std::vector{ + *read_only_this_single_iteration}; + } } else if (read_only_this_single_iteration.has_value()) { @@ -3242,7 +3261,10 @@ Series::snapshots(std::optional const snapshot_workflow) access::writeOnly(access) && // 5. The backend is ADIOS2 in a recent enough version to support // modifiable attributes (v2.9). - IOHandler()->fullSupportForVariableBasedEncoding()) + IOHandler()->fullSupportForVariableBasedEncoding() && + // 6. The Series must not yet be written, otherwise we're too late + // for this + !this->written()) { setIterationEncoding_internal( IterationEncoding::variableBased, diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index aa98d08b46..9ade0c8f44 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -57,7 +57,8 @@ TEST_CASE("parallel_multi_series_test", "[parallel]") .append(".") .append(file_ending), Access::CREATE, - MPI_COMM_WORLD); + MPI_COMM_WORLD, + R"(iteration_encoding = "variable_based")"); allSeries.back().iterations[sn].setAttribute("wululu", sn); allSeries.back().flush(); } @@ -110,7 +111,7 @@ void write_test_zero_extent( for (int step = 0; step <= max_step; step += 20) { - Iteration it = o.iterations[step]; + Iteration it = o.writeIterations()[step]; it.setAttribute("yolo", "yo"); if (rank != 0 || declareFromAll) @@ -467,7 +468,7 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig) // array record component -> array record component // should work - auto E_x = write.iterations[0].meshes["E"]["x"]; + auto E_x = write.writeIterations()[0].meshes["E"]["x"]; E_x.resetDataset(ds1); E_x.storeChunk(data1, {mpi_rank, 0}, {1, 25}); write.flush(); @@ -673,7 +674,7 @@ void write_4D_test(std::string const &file_ending) std::string name = "../samples/parallel_write_4d." + file_ending; Series o = Series(name, Access::CREATE, MPI_COMM_WORLD); - auto it = o.iterations[1]; + auto it = o.writeIterations()[1]; auto E_x = it.meshes["E"]["x"]; // every rank out of mpi_size MPI ranks contributes two writes: @@ -708,7 +709,7 @@ void write_makeconst_some(std::string const &file_ending) std::cout << name << std::endl; Series o = Series(name, Access::CREATE, MPI_COMM_WORLD); - auto it = o.iterations[1]; + auto it = o.writeIterations()[1]; // I would have expected we need this, since the first call that writes // data below (makeConstant) is not executed in MPI collective manner // it.open(); @@ -1138,7 +1139,7 @@ TEST_CASE("independent_write_with_collective_flush", "[parallel]") int size, rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); - auto iteration = write.iterations[0]; + auto iteration = write.writeIterations()[0]; auto E_x = iteration.meshes["E"]["x"]; E_x.resetDataset({Datatype::DOUBLE, {10}}); write.flush(); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 582a54481e..3d72e376ad 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -122,7 +122,7 @@ namespace detail template void writeChar(Series &series, std::string const &component_name) { - auto component = series.iterations[0].meshes["E"][component_name]; + auto component = series.writeIterations()[0].meshes["E"][component_name]; std::vector data(10); component.resetDataset({determineDatatype(), {10}}); component.storeChunk(data, {0}, {10}); @@ -279,7 +279,7 @@ TEST_CASE("multi_series_test", "[serial]") .append(".") .append(file_ending), Access::CREATE); - allSeries.back().iterations[sn].setAttribute("wululu", sn); + allSeries.back().writeIterations()[sn].setAttribute("wululu", sn); allSeries.back().flush(); } } @@ -636,12 +636,13 @@ void close_and_copy_attributable_test(std::string const &file_ending) { if (iteration_ptr) { - *iteration_ptr = series.iterations[i]; + *iteration_ptr = series.writeIterations()[i]; } else { // use copy constructor - iteration_ptr = std::make_unique(series.iterations[i]); + iteration_ptr = + std::make_unique(series.writeIterations()[i]); } Record electronPositions = iteration_ptr->particles["e"]["position"]; // TODO set this automatically to zero if not provided @@ -773,17 +774,20 @@ inline void empty_dataset_test(std::string const &file_ending) "../samples/empty_datasets." + file_ending, Access::CREATE); auto makeEmpty_dim_7_int = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int"]; + series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_int"]; auto makeEmpty_dim_7_long = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool"]; + series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_bool"]; auto makeEmpty_dim_7_int_alt = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int_alt"]; + series.writeIterations()[1] + .meshes["rho"]["makeEmpty_dim_7_int_alt"]; auto makeEmpty_dim_7_long_alt = - series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool_alt"]; + series.writeIterations()[1] + .meshes["rho"]["makeEmpty_dim_7_bool_alt"]; auto makeEmpty_resetDataset_dim3 = - series.iterations[1].meshes["rho"]["makeEmpty_resetDataset_dim3"]; + series.writeIterations()[1] + .meshes["rho"]["makeEmpty_resetDataset_dim3"]; auto makeEmpty_resetDataset_dim3_notallzero = - series.iterations[1] + series.writeIterations()[1] .meshes["rho"]["makeEmpty_resetDataset_dim3_notallzero"]; makeEmpty_dim_7_int.makeEmpty(7); makeEmpty_dim_7_long.makeEmpty(7); @@ -888,17 +892,18 @@ inline void constant_scalar(std::string const &file_ending) Series s = Series("../samples/constant_scalar." + file_ending, Access::CREATE); s.setOpenPMD("2.0.0"); - auto rho = s.iterations[1].meshes["rho"][MeshRecordComponent::SCALAR]; - REQUIRE(s.iterations[1].meshes["rho"].scalar()); + auto rho = + s.writeIterations()[1].meshes["rho"][MeshRecordComponent::SCALAR]; + REQUIRE(s.writeIterations()[1].meshes["rho"].scalar()); rho.resetDataset(Dataset(Datatype::CHAR, {1, 2, 3})); rho.makeConstant(static_cast('a')); REQUIRE(rho.constant()); // mixed constant/non-constant - auto E_x = s.iterations[1].meshes["E"]["x"]; + auto E_x = s.writeIterations()[1].meshes["E"]["x"]; E_x.resetDataset(Dataset(Datatype::FLOAT, {1, 2, 3})); E_x.makeConstant(static_cast(13.37)); - auto E_y = s.iterations[1].meshes["E"]["y"]; + auto E_y = s.writeIterations()[1].meshes["E"]["y"]; E_y.resetDataset(Dataset(Datatype::UINT, {1, 2, 3})); UniquePtrWithLambda E( new unsigned int[6], [](unsigned int const *p) { delete[] p; }); @@ -907,7 +912,7 @@ inline void constant_scalar(std::string const &file_ending) E_y.storeChunk(std::move(E), {0, 0, 0}, {1, 2, 3}); // store a number of predefined attributes in E - Mesh &E_mesh = s.iterations[1].meshes["E"]; + Mesh &E_mesh = s.writeIterations()[1].meshes["E"]; // test that these can be defined successively E_mesh.setGridUnitDimension({{{UnitDimension::L, 1}}, {}, {}}); E_mesh.setGridUnitDimension( @@ -932,21 +937,21 @@ inline void constant_scalar(std::string const &file_ending) std::vector{gridUnitSI, gridUnitSI, gridUnitSI}); // constant scalar - auto pos = - s.iterations[1].particles["e"]["position"][RecordComponent::SCALAR]; + auto pos = s.writeIterations()[1] + .particles["e"]["position"][RecordComponent::SCALAR]; pos.resetDataset(Dataset(Datatype::DOUBLE, {3, 2, 1})); pos.makeConstant(static_cast(42.)); auto posOff = - s.iterations[1] + s.writeIterations()[1] .particles["e"]["positionOffset"][RecordComponent::SCALAR]; posOff.resetDataset(Dataset(Datatype::INT, {3, 2, 1})); posOff.makeConstant(static_cast(-42)); // mixed constant/non-constant - auto vel_x = s.iterations[1].particles["e"]["velocity"]["x"]; + auto vel_x = s.writeIterations()[1].particles["e"]["velocity"]["x"]; vel_x.resetDataset(Dataset(Datatype::SHORT, {3, 2, 1})); vel_x.makeConstant(static_cast(-1)); - auto vel_y = s.iterations[1].particles["e"]["velocity"]["y"]; + auto vel_y = s.writeIterations()[1].particles["e"]["velocity"]["y"]; vel_y.resetDataset(Dataset(Datatype::ULONGLONG, {3, 2, 1})); UniquePtrWithLambda vel( new unsigned long long[6], @@ -1135,7 +1140,7 @@ TEST_CASE("flush_without_position_positionOffset", "[serial]") Series s = Series( "../samples/flush_without_position_positionOffset." + file_ending, Access::CREATE); - ParticleSpecies e = s.iterations[0].particles["e"]; + ParticleSpecies e = s.writeIterations()[0].particles["e"]; RecordComponent weighting = e["weighting"][RecordComponent::SCALAR]; weighting.resetDataset(Dataset(Datatype::FLOAT, Extent{2, 2})); weighting.storeChunk( @@ -1396,7 +1401,7 @@ inline void dtype_test( // should be possible to parse without error upon opening // the series for reading { - auto E = s.iterations[0].meshes["E"]; + auto E = s.writeIterations()[0].meshes["E"]; E.setGridSpacing(std::vector{1.0, 1.0}); auto E_x = E["x"]; E_x.makeEmpty(1); @@ -1611,7 +1616,7 @@ inline void write_test( Series o = Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); - ParticleSpecies &e_1 = o.iterations[1].particles["e"]; + ParticleSpecies &e_1 = o.writeIterations()[1].particles["e"]; std::vector position_global(4); double pos{0.}; @@ -1646,7 +1651,7 @@ inline void write_test( e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); } - ParticleSpecies &e_2 = o.iterations[2].particles["e"]; + ParticleSpecies &e_2 = o.writeIterations()[2].particles["e"]; std::generate(position_global.begin(), position_global.end(), [&pos] { return pos++; @@ -1677,7 +1682,7 @@ inline void write_test( o.flush(); - ParticleSpecies &e_3 = o.iterations[3].particles["e"]; + ParticleSpecies &e_3 = o.writeIterations()[3].particles["e"]; std::generate(position_global.begin(), position_global.end(), [&pos] { return pos++; @@ -1786,7 +1791,8 @@ void test_complex(const std::string &backend) o.setAttribute( "longDoublesYouSay", std::complex(5.5, -4.55)); - auto Cflt = o.iterations[0].meshes["Cflt"][RecordComponent::SCALAR]; + auto Cflt = + o.writeIterations()[0].meshes["Cflt"][RecordComponent::SCALAR]; std::vector> cfloats(3); cfloats.at(0) = {1., 2.}; cfloats.at(1) = {-3., 4.}; @@ -1794,7 +1800,8 @@ void test_complex(const std::string &backend) Cflt.resetDataset(Dataset(Datatype::CFLOAT, {cfloats.size()})); Cflt.storeChunk(cfloats, {0}); - auto Cdbl = o.iterations[0].meshes["Cdbl"][RecordComponent::SCALAR]; + auto Cdbl = + o.writeIterations()[0].meshes["Cdbl"][RecordComponent::SCALAR]; std::vector> cdoubles(3); cdoubles.at(0) = {2., 1.}; cdoubles.at(1) = {-4., 3.}; @@ -1806,7 +1813,7 @@ void test_complex(const std::string &backend) if (o.backend() != "ADIOS2") { auto Cldbl = - o.iterations[0].meshes["Cldbl"][RecordComponent::SCALAR]; + o.writeIterations()[0].meshes["Cldbl"][RecordComponent::SCALAR]; cldoubles.at(0) = {3., 2.}; cldoubles.at(1) = {-5., 4.}; cldoubles.at(2) = {7., -6.}; @@ -2416,7 +2423,10 @@ TEST_CASE("sample_write_thetaMode", "[serial][thetaMode]") inline void bool_test(const std::string &backend) { { - Series o = Series("../samples/serial_bool." + backend, Access::CREATE); + Series o = Series( + "../samples/serial_bool." + backend, + Access::CREATE, + R"({"iteration_encoding": "variable_based"})"); o.setAttribute("Bool attribute true", true); o.setAttribute("Bool attribute false", false); @@ -2452,7 +2462,7 @@ inline void patch_test(const std::string &backend) { Series o = Series("../samples/serial_patch." + backend, Access::CREATE); - auto e = o.iterations[1].particles["e"]; + auto e = o.writeIterations()[1].particles["e"]; uint64_t const num_particles = 1u; auto dset_d = Dataset(Datatype::DOUBLE, {num_particles}); @@ -4636,7 +4646,12 @@ TEST_CASE("adios2_engines_and_file_endings") Series write( name, Access::CREATE, - json::merge("backend = \"adios2\"", jsonCfg)); + json::merge( + R"( + backend = "adios2" + iteration_encoding = "variable_based" + )", + jsonCfg)); } if (directory) { @@ -4701,7 +4716,11 @@ TEST_CASE("adios2_engines_and_file_endings") auto filesystemname = filesystemExt.empty() ? name : basename + filesystemExt; { - Series write(name, Access::CREATE, jsonCfg); + Series write( + name, + Access::CREATE, + json::merge( + R"(iteration_encoding = "variable_based")", jsonCfg)); write.close(); } if (directory) @@ -6087,8 +6106,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]") // automatically (de)activate span-based storeChunking { Series write("../samples/span_based.bp", Access::CREATE); - auto E_uncompressed = write.iterations[0].meshes["E"]["x"]; - auto E_compressed = write.iterations[0].meshes["E"]["y"]; + auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"]; + auto E_compressed = write.writeIterations()[0].meshes["E"]["y"]; Dataset ds{Datatype::INT, {10}}; @@ -6138,8 +6157,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]") } })END"; Series write("../samples/span_based.bp", Access::CREATE, enable); - auto E_uncompressed = write.iterations[0].meshes["E"]["x"]; - auto E_compressed = write.iterations[0].meshes["E"]["y"]; + auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"]; + auto E_compressed = write.writeIterations()[0].meshes["E"]["y"]; Dataset ds{Datatype::INT, {10}}; @@ -6202,8 +6221,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]") } })END"; Series write("../samples/span_based.bp", Access::CREATE, disable); - auto E_uncompressed = write.iterations[0].meshes["E"]["x"]; - auto E_compressed = write.iterations[0].meshes["E"]["y"]; + auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"]; + auto E_compressed = write.writeIterations()[0].meshes["E"]["y"]; Dataset ds{Datatype::INT, {10}}; @@ -6657,7 +6676,7 @@ void deferred_parsing(std::string const &extension) Series series(basename + "%06T." + extension, Access::CREATE); std::vector buffer(20); std::iota(buffer.begin(), buffer.end(), 0.f); - auto dataset = series.iterations[0].meshes["E"]["x"]; + auto dataset = series.writeIterations()[0].meshes["E"]["x"]; dataset.resetDataset({Datatype::FLOAT, {20}}); dataset.storeChunk(buffer, {0}, {20}); series.flush(); From 185d023f903553b5c5cfa14c79be605e0b76a181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 15 Jan 2025 11:56:09 +0100 Subject: [PATCH 20/33] CI fixes --- test/SerialIOTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 3d72e376ad..40d06e8dc1 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5239,7 +5239,7 @@ void serial_iterator(std::string const &file) constexpr Extent::value_type extent = 1000; { Series writeSeries( - file, Access::CREATE, "rank_table = \"posix_hostname\""); + file, Access::CREATE, R"({"rank_table": "posix_hostname"})"); auto iterations = writeSeries.writeIterations(); for (size_t i = 0; i < 10; ++i) { From 02a4fd4b053bb746d4091b1a6a6becdaa2d686c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 12 Feb 2025 18:11:41 +0100 Subject: [PATCH 21/33] Fix Windows testing --- test/SerialIOTest.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 40d06e8dc1..5fb15dd838 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5239,7 +5239,13 @@ void serial_iterator(std::string const &file) constexpr Extent::value_type extent = 1000; { Series writeSeries( - file, Access::CREATE, R"({"rank_table": "posix_hostname"})"); + file, + Access::CREATE +#ifndef _WIN32 + , + R"({"rank_table": "posix_hostname"})" +#endif + ); auto iterations = writeSeries.writeIterations(); for (size_t i = 0; i < 10; ++i) { @@ -5270,12 +5276,18 @@ void serial_iterator(std::string const &file) } last_iteration_index = iteration.iterationIndex; } - if (readSeries.iterationEncoding() == IterationEncoding::variableBased) - for (auto const &[rank, host] : readSeries.rankTable(true)) +#ifndef _WIN32 + if (readSeries.iterationEncoding() != IterationEncoding::fileBased) + { + auto rank_table = readSeries.rankTable(true); + for (auto const &[rank, host] : rank_table) { std::cout << "POST Rank '" << rank << "' written from host '" << host << "'\n"; } + REQUIRE(rank_table.size() == 1); + } +#endif REQUIRE(last_iteration_index == 9); REQUIRE(numberOfIterations == 10); } From 126664c7cb77785b8981909514d1cdfc5cd61def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Feb 2025 13:17:23 +0100 Subject: [PATCH 22/33] Cleanup --- include/openPMD/IO/ADIOS/ADIOS2File.hpp | 2 ++ include/openPMD/IO/AbstractIOHandlerImpl.hpp | 23 ++++++++++++++++- include/openPMD/IO/IOTask.hpp | 2 ++ include/openPMD/Iteration.hpp | 13 ++++++++-- include/openPMD/Series.hpp | 26 ++++++++++++++++++++ src/IO/ADIOS/ADIOS2IOHandler.cpp | 6 ++++- src/Iteration.cpp | 4 +-- src/Series.cpp | 16 ++++++------ 8 files changed, 77 insertions(+), 15 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 91c5773ba8..147e79d2ee 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -423,6 +423,8 @@ class ADIOS2File /* * Not all engines support the CurrentStep() call, so we have to * implement this manually. + * Note: We don't use a std::optional here since the currentStep + * is always being counted. */ size_t m_currentStep = 0; bool useStepSelection = false; diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index cb866c7ffe..a7a5746b19 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -72,6 +72,13 @@ class AbstractIOHandlerImpl * IO actions up to the point of closing a step must be performed now. * * The advance mode is determined by parameters.mode. + * parameters.mode has type std::variant: + * + * 1. AdvanceMode is for processing steps sequentially. In this case, a step + * is either begun or closed. + * 2. StepSelection is for random-accessing steps. A target step number is + * specified. + * * The return status code shall be stored as parameters.status. */ virtual void advance(Writable *, Parameter ¶meters) @@ -360,7 +367,21 @@ class AbstractIOHandlerImpl */ virtual void readAttribute(Writable *, Parameter &) = 0; - /** COLLECTIVE! + /** Collective task to read modifiable attributes over steps. + * + * Has a default implementation for backends that do not support steps; + * here, the task is relayed to normal READ_ATT. + * This task is key for implementing the preparsing logic needed in + * random-access read mode for variable-encoded ADIOS2 files. + * adios2::Mode::ReadRandomAccess does not support modifiable attributes, + * so this task will instead quickly open the file's metadata in + * adios2::Mode::Read, go through all its steps and register the attribute + * values. Expensive and collective operation, run only once at startup. + * Absolutely necessary for reading /data/snapshot. + * Necessary (but not yet used) for having correct values in attributes + * such as /data/time. + * In future: Let this task preparse the entirety of all modifiable + * attributes. */ virtual void readAttributeAllsteps( Writable *, Parameter &); diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 5ac4c9f6c5..a4b2bb9579 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -628,6 +628,8 @@ struct OPENPMDAPI_EXPORT Parameter template using type = std::vector; }; + // std::variant, std::vector, ...> + // for all T_i in openPMD::Datatype. using result_type = typename auxiliary::detail:: map_variant::type; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index b084dcd193..43ee1084bb 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -56,7 +56,7 @@ namespace internal { struct DontBeginStep {}; - struct BeginStepSynchronously + struct BeginStepSequentially {}; struct BeginStepRandomAccess { @@ -66,9 +66,18 @@ namespace internal using BeginStep = std::variant< BeginStepTypes::DontBeginStep, - BeginStepTypes::BeginStepSynchronously, + BeginStepTypes::BeginStepSequentially, BeginStepTypes::BeginStepRandomAccess>; + namespace BeginStepTypes + { + template + constexpr auto make(Args &&...args) -> BeginStep + { + return BeginStep{T{std::forward(args)...}}; + } + } // namespace BeginStepTypes + struct DeferredParseAccess { /** diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index f0487fe238..0a568ed559 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -68,6 +68,8 @@ class Series; namespace internal { + /* Just a more self-documenting boolean used for + * m_iterationEncodingSetExplicitly */ enum class default_or_explicit : bool { default_, @@ -188,6 +190,15 @@ namespace internal * The iteration encoding used in this series. */ IterationEncoding m_iterationEncoding{}; + /* + * ADIOS2 should use variable-based encoding as default rather than + * group-based encoding as much as possible. + * Since this cannot be decided at construction time, groupBased + * encoding is selected first, and re-decided later. + * However, when group-based encoding is selected by the user explcitly, + * that selection should not be changed again. + * Hence, remember that here. + */ default_or_explicit m_iterationEncodingSetExplicitly = default_or_explicit::default_; /** @@ -976,6 +987,12 @@ OPENPMD_private */ void flushStep(bool doFlush); + /* + * setIterationEncoding() should only be called by users of our public API, + * but never internally. We need to distinguish if the iteration encoding + * was selected explicitly or implicitly, see + * m_iterationEncodingSetExplicitly for further details. + */ Series &setIterationEncoding_internal( IterationEncoding iterationEncoding, internal::default_or_explicit); @@ -990,8 +1007,17 @@ OPENPMD_private AbstractIOHandler *IOHandler(); AbstractIOHandler const *IOHandler() const; + /* adios2::Mode::ReadRandomAccess does not support reading modifiable + * attributes. However, we need the values of /data/snapshot as a modifiable + * attribute, so this function quickly opens the file in adios2::Mode::Read + * and retrieves the changings values over time. + * Return std::nullopt if /data/snapshot is not present. + */ std::optional>> preparseSnapshots(); + /* Should adios2::Variable::SetStepSelection() be used for accessing + * steps? + */ [[nodiscard]] bool randomAccessSteps() const; }; // Series diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2fe56eb442..77b84fa180 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1238,6 +1238,10 @@ void ADIOS2IOHandlerImpl::readAttribute( namespace { + /* Used by both readAttribute() and readAttributeAllsteps() tasks. + Functor fun will be called with the value of the retrieved attribute; + both functions use different logic for processing the retrieved values. + */ template Datatype genericReadAttribute(Functor &&fun, adios2::IO &IO, std::string const &name) @@ -1316,7 +1320,7 @@ namespace { res[i] = data[i]; } - std::forward(fun)(res); + std::forward(fun)(std::move(res)); } else if constexpr (std::is_same_v) { diff --git a/src/Iteration.cpp b/src/Iteration.cpp index a4fbf33149..c991c2b79b 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -448,7 +448,7 @@ void Iteration::readGorVBased( std::visit( auxiliary::overloaded{ [](internal::BeginStepTypes::DontBeginStep const &) {}, - [&](internal::BeginStepTypes::BeginStepSynchronously const &) { + [&](internal::BeginStepTypes::BeginStepSequentially const &) { /* * beginStep() must take care to open files */ @@ -936,7 +936,7 @@ void Iteration::runDeferredParseAccess() filename, deferred.path, std::holds_alternative< - internal::BeginStepTypes::BeginStepSynchronously>( + internal::BeginStepTypes::BeginStepSequentially>( deferred.beginStep)); } else diff --git a/src/Series.cpp b/src/Series.cpp index ca3da5fb9b..9bf140501a 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2189,16 +2189,14 @@ creating new iterations. * Variable-based iteration encoding relies on steps, so parsing * must happen after opening the first step. */ - internal::BeginStep beginStep = randomAccessSteps() + using namespace internal; + BeginStep beginStep = randomAccessSteps() ? (series.m_snapshotToStep.empty() - ? internal::BeginStep{internal::BeginStepTypes:: - DontBeginStep{}} - : internal::BeginStep{internal::BeginStepTypes:: - BeginStepRandomAccess{ - series.m_snapshotToStep.at( - it)}}) - : internal::BeginStep{ - internal::BeginStepTypes::BeginStepSynchronously{}}; + ? BeginStepTypes::make() + : BeginStepTypes::make< + BeginStepTypes::BeginStepRandomAccess>( + series.m_snapshotToStep.at(it))) + : BeginStepTypes::make(); if (auto err = internal::withRWAccess( IOHandler()->m_seriesStatus, [&readSingleIteration, it, &beginStep]() { From 182da146e6c23ef30a7c06ec50486ffea27687b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Feb 2025 16:59:37 +0100 Subject: [PATCH 23/33] Warn unimplemented modifiable attributes for now --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 77b84fa180..0f23c2b2c4 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -41,6 +41,7 @@ #include #include +#include #include #include #include @@ -1508,6 +1509,37 @@ namespace } static constexpr char const *errorMsg = "DistributeToAllRanks"; }; + + void warn_ignored_modifiable_attributes( + adios2::IO &IO, std::string const &exempt_this_from_warnings) + { + auto all_attributes = IO.AvailableAttributes(); + std::deque modifiable_attributes; + for (auto const &[identifier, params] : all_attributes) + { + std::cout.flush(); + if (params.at("Modifiable") == "1") + { + modifiable_attributes.emplace_back(identifier); + } + } + if (modifiable_attributes.size() > 1 || + (modifiable_attributes.size() == 1 && + *modifiable_attributes.begin() != exempt_this_from_warnings)) + { + std::cerr << &R"( +Warning: Random-access for variable-encoding in ADIOS2 is currently +experimental. Support for modifiable attributes is currently not implemented +yet, meaning that attributes such as /data/time will show useless values. +Use Access::READ_LINEAR to retrieve those values if needed. +The following modifiable attributes have been found: +)"[1]; + for (auto const &identifier : modifiable_attributes) + { + std::cerr << '\t' << identifier << '\n'; + } + } + } #endif } // namespace @@ -1526,6 +1558,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( IO.SetParameter("StreamReader", "ON"); // this be for BP4 auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); auto status = engine.BeginStep(); + warn_ignored_modifiable_attributes(IO, name); auto type = detail::attributeInfo(IO, name, /* verbose = */ true); switchType( type, IO, engine, name, status, *param.resource); From c2e2b2b7170c977f7b00e912438a4072d625ad27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Feb 2025 16:59:51 +0100 Subject: [PATCH 24/33] Fix test??? --- .../read_variablebased_randomaccess.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp index 3c2640ac43..39f5c88c56 100644 --- a/test/Files_ParallelIO/read_variablebased_randomaccess.cpp +++ b/test/Files_ParallelIO/read_variablebased_randomaccess.cpp @@ -103,17 +103,9 @@ auto read_variablebased_randomaccess() -> void } MPI_Barrier(MPI_COMM_WORLD); - std::string const config = R"END( -{ - "adios2": - { - "use_group_table": true - } -})END"; - { openPMD::Series read( - "../samples/bp5_no_steps.bp", + "../samples/read_variablebased_randomaccess.bp", openPMD::Access::READ_ONLY, MPI_COMM_WORLD, "adios2.engine.type = \"bp5\""); From 9c6f4aaa8e4d0dc8b122339ede40387c39bf7afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Feb 2025 17:27:37 +0100 Subject: [PATCH 25/33] Add this to variableBasedSeries test --- test/SerialIOTest.cpp | 71 +++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 5fb15dd838..0b050384fb 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5883,32 +5883,39 @@ void variableBasedSeries(std::string const &file) auto testRead = [&file, &extent]( std::string const &parseMode, - bool supportsModifiableAttributes) { + bool supportsModifiableAttributes, + Access access = Access::READ_LINEAR) { /* * Need linear read mode to access more than a single iteration in * variable-based iteration encoding. */ - Series readSeries(file, Access::READ_LINEAR, parseMode); + Series readSeries(file, access, parseMode); bool is_adios2 = readSeries.backend() == "ADIOS2"; size_t last_iteration_index = 0; - REQUIRE(!readSeries.containsAttribute("some_global")); + if (access == Access::READ_LINEAR) + { + REQUIRE(!readSeries.containsAttribute("some_global")); + } readSeries.parseBase(); REQUIRE( readSeries.getAttribute("some_global").get() == "attribute"); for (auto iteration : readSeries.readIterations()) { - if (iteration.iterationIndex > 2) - { - REQUIRE( - iteration.getAttribute("iteration_is_larger_than_two") - .get() == "it truly is"); - } - else + if (access == Access::READ_LINEAR) { - REQUIRE_FALSE(iteration.containsAttribute( - "iteration_is_larger_than_two")); + if (iteration.iterationIndex > 2) + { + REQUIRE( + iteration.getAttribute("iteration_is_larger_than_two") + .get() == "it truly is"); + } + else + { + REQUIRE_FALSE(iteration.containsAttribute( + "iteration_is_larger_than_two")); + } } // If modifiable attributes are unsupported, the attribute is @@ -5918,8 +5925,11 @@ void variableBasedSeries(std::string const &file) { REQUIRE( iteration.getAttribute("changing_value").get() == - (supportsModifiableAttributes ? iteration.iterationIndex - : 0)); + (supportsModifiableAttributes + ? (access == Access::READ_LINEAR + ? iteration.iterationIndex + : 9) + : 0)); } auto E_x = iteration.meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); @@ -5937,6 +5947,13 @@ void variableBasedSeries(std::string const &file) Extent changingExtent(dimensionality, len); REQUIRE(E_y.getExtent() == changingExtent); + last_iteration_index = iteration.iterationIndex; + + if (access == Access::READ_RANDOM_ACCESS) + { + continue; + } + // this loop ensures that only the recordcomponent ["E"]["i"] is // present where i == iteration.iterationIndex for (uint64_t otherIteration = 0; otherIteration < 10; @@ -5980,8 +5997,6 @@ void variableBasedSeries(std::string const &file) REQUIRE( constantParticles.getAttribute("value").get() == iteration.iterationIndex); - - last_iteration_index = iteration.iterationIndex; } REQUIRE(last_iteration_index == (is_adios2 ? 9 : 0)); }; @@ -6001,6 +6016,14 @@ void variableBasedSeries(std::string const &file) testRead( "{\"defer_iteration_parsing\": false}", /*supportsModifiableAttributes = */ true); + testRead( + "{\"defer_iteration_parsing\": true}", + /*supportsModifiableAttributes = */ true, + Access::READ_RANDOM_ACCESS); + testRead( + "{\"defer_iteration_parsing\": false}", + /*supportsModifiableAttributes = */ true, + Access::READ_RANDOM_ACCESS); jsonConfig = "{}"; testWrite(jsonConfig); @@ -6010,6 +6033,14 @@ void variableBasedSeries(std::string const &file) testRead( "{\"defer_iteration_parsing\": false}", /*supportsModifiableAttributes = */ true); + testRead( + "{\"defer_iteration_parsing\": true}", + /*supportsModifiableAttributes = */ true, + Access::READ_RANDOM_ACCESS); + testRead( + "{\"defer_iteration_parsing\": false}", + /*supportsModifiableAttributes = */ true, + Access::READ_RANDOM_ACCESS); jsonConfig = R"( { @@ -6024,6 +6055,14 @@ void variableBasedSeries(std::string const &file) testRead( "{\"defer_iteration_parsing\": false}", /*supportsModifiableAttributes = */ false); + testRead( + "{\"defer_iteration_parsing\": true}", + /*supportsModifiableAttributes = */ false, + Access::READ_RANDOM_ACCESS); + testRead( + "{\"defer_iteration_parsing\": false}", + /*supportsModifiableAttributes = */ false, + Access::READ_RANDOM_ACCESS); } TEST_CASE("variableBasedSeries", "[serial][adios2]") From 1794b3a540741a152eff76ed24a3a08c7d651d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Feb 2025 17:41:03 +0100 Subject: [PATCH 26/33] Remove debugging line --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 0f23c2b2c4..73edf0c3eb 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1517,7 +1517,6 @@ namespace std::deque modifiable_attributes; for (auto const &[identifier, params] : all_attributes) { - std::cout.flush(); if (params.at("Modifiable") == "1") { modifiable_attributes.emplace_back(identifier); From 3c95b2c590a325ee82e7f9ca54b3ce46efce24c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 14 Feb 2025 11:39:48 +0100 Subject: [PATCH 27/33] Use own flag for the warning Previously implementation did not work: https://github.com/ornladios/ADIOS2/issues/4466 --- include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 2 + src/IO/ADIOS/ADIOS2File.cpp | 7 ++++ src/IO/ADIOS/ADIOS2IOHandler.cpp | 41 ++++++++------------ 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 2741a91fb0..5f9839769d 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -97,6 +97,8 @@ namespace adios_defaults constexpr const_str str_usesteps = "usesteps"; constexpr const_str str_flushtarget = "preferred_flush_target"; constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps"; + constexpr const_str str_useModifiableAttributes = + "__openPMD_internal/useModifiableAttributes"; constexpr const_str str_adios2Schema = "__openPMD_internal/openPMD2_adios2_schema"; constexpr const_str str_isBoolean = "__is_boolean__"; diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 0fc61700dc..f313869d1f 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -21,6 +21,7 @@ #include "openPMD/IO/ADIOS/ADIOS2File.hpp" #include "openPMD/Error.hpp" +#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IterationEncoding.hpp" @@ -498,6 +499,12 @@ void ADIOS2File::configure_IO() ? ADIOS2IOHandlerImpl::ModifiableAttributes::Yes : ADIOS2IOHandlerImpl::ModifiableAttributes::No; } + m_IO.DefineAttribute( + adios_defaults::str_useModifiableAttributes, + m_impl->m_modifiableAttributes == + ADIOS2IOHandlerImpl::ModifiableAttributes::No + ? 0 + : 1); } // set engine type diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 73edf0c3eb..fd3a650e68 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1509,37 +1509,30 @@ namespace } static constexpr char const *errorMsg = "DistributeToAllRanks"; }; +#endif - void warn_ignored_modifiable_attributes( - adios2::IO &IO, std::string const &exempt_this_from_warnings) + void warn_ignored_modifiable_attributes(adios2::IO &IO) { - auto all_attributes = IO.AvailableAttributes(); - std::deque modifiable_attributes; - for (auto const &[identifier, params] : all_attributes) - { - if (params.at("Modifiable") == "1") - { - modifiable_attributes.emplace_back(identifier); - } - } - if (modifiable_attributes.size() > 1 || - (modifiable_attributes.size() == 1 && - *modifiable_attributes.begin() != exempt_this_from_warnings)) - { - std::cerr << &R"( -Warning: Random-access for variable-encoding in ADIOS2 is currently + auto modifiable_flag = IO.InquireAttribute( + adios_defaults::str_useModifiableAttributes); + auto print_warning = [](std::string const ¬e) { + std::cerr << "Warning: " << note << R"( +Random-access for variable-encoding in ADIOS2 is currently experimental. Support for modifiable attributes is currently not implemented yet, meaning that attributes such as /data/time will show useless values. Use Access::READ_LINEAR to retrieve those values if needed. The following modifiable attributes have been found: -)"[1]; - for (auto const &identifier : modifiable_attributes) - { - std::cerr << '\t' << identifier << '\n'; - } +)"; + }; + if (!modifiable_flag) + { + print_warning("File might be using modifiable attributes."); + } + else if (modifiable_flag.Data().at(0) != 0) + { + print_warning("File uses modifiable attributes."); } } -#endif } // namespace void ADIOS2IOHandlerImpl::readAttributeAllsteps( @@ -1557,7 +1550,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( IO.SetParameter("StreamReader", "ON"); // this be for BP4 auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); auto status = engine.BeginStep(); - warn_ignored_modifiable_attributes(IO, name); + warn_ignored_modifiable_attributes(IO); auto type = detail::attributeInfo(IO, name, /* verbose = */ true); switchType( type, IO, engine, name, status, *param.resource); From dce876e815afec891555f62e9cb993b640458a10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 14 Feb 2025 16:04:20 +0100 Subject: [PATCH 28/33] Documentation --- docs/source/usage/workflow.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index 03fa4ce9d4..c7f64d93a8 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -76,10 +76,13 @@ The openPMD-api distinguishes between a number of different access modes: 3. In streaming backends, random-access is not possible. When using such a backend, the access mode will be coerced automatically to *linear read mode*. Use of Series::readIterations() is mandatory for access. - 4. Reading a variable-based Series is only fully supported with *linear access mode*. - If using *random-access read mode*, the dataset will be considered to only have one single step. - If the dataset only has one single step, this is guaranteed to work as expected. - Otherwise, it is undefined which step's data is returned. + 4. *Random-access read mode* for a variable-based Series is currently experimental. + There is currently only very restricted support for metadata definitions that change across steps: + + 1. Modifiable attributes (except ``/data/snapshot``) can currently not be read. Attributes such as ``/data/time`` that naturally change their value across Iterations will hence not be really well-usable; the last Iteration's value will currently leak into all other Iterations. + 2. There is no support for datasets that do not exist in all Iterations. The internal Iteration layouts should be homogeneous. + If you need this feature, please contact the openPMD-api developers; implementing this is currently not a priority. + 3. Datasets with changing extents are supported. * **Read/Write mode**: Creates a new Series if not existing, otherwise opens an existing Series for reading and writing. New datasets and iterations will be inserted as needed. From 816ae26503201981e8f9d49616cee6c823569186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 14 Feb 2025 16:35:55 +0100 Subject: [PATCH 29/33] Skip inhomogeneous datasets instead of outright failing --- docs/source/usage/workflow.rst | 1 + src/IO/ADIOS/ADIOS2IOHandler.cpp | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index c7f64d93a8..a69a498cb4 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -82,6 +82,7 @@ The openPMD-api distinguishes between a number of different access modes: 1. Modifiable attributes (except ``/data/snapshot``) can currently not be read. Attributes such as ``/data/time`` that naturally change their value across Iterations will hence not be really well-usable; the last Iteration's value will currently leak into all other Iterations. 2. There is no support for datasets that do not exist in all Iterations. The internal Iteration layouts should be homogeneous. If you need this feature, please contact the openPMD-api developers; implementing this is currently not a priority. + Datasets that do not exist in all steps will be skipped at read time (with an error). 3. Datasets with changing extents are supported. * **Read/Write mode**: Creates a new Series if not existing, otherwise opens an existing Series for reading and writing. diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index fd3a650e68..67a18f7c11 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -2293,7 +2293,9 @@ namespace detail auto var_steps = var.Steps(); if (file_steps != var_steps) { - throw error::OperationUnsupportedInBackend( + throw error::ReadError( + error::AffectedObject::Dataset, + error::Reason::UnexpectedContent, "ADIOS2", &R"( The opened file contains different data per step. From 29f64038efb978d9179c2b223bd756304a02bc09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 14 Feb 2025 17:33:04 +0100 Subject: [PATCH 30/33] add test for default iteration encoding --- src/Series.cpp | 6 +- test/CoreTest.cpp | 334 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 336 insertions(+), 4 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index 9bf140501a..2fe870a4a9 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -589,7 +589,7 @@ IterationEncoding Series::iterationEncoding() const Series &Series::setIterationEncoding(IterationEncoding ie) { - setIterationEncoding_internal(ie, internal::default_or_explicit::default_); + setIterationEncoding_internal(ie, internal::default_or_explicit::explicit_); return *this; } @@ -1139,9 +1139,9 @@ Given file pattern: ')END" * allow setting attributes in that case */ setWritten(false, Attributable::EnqueueAsynchronously::No); - initDefaults(input->iterationEncoding); + initDefaults(series.m_iterationEncoding); setIterationEncoding_internal( - input->iterationEncoding, + series.m_iterationEncoding, series.m_iterationEncodingSetExplicitly); setWritten(true, Attributable::EnqueueAsynchronously::No); diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 3f88a18864..f629580399 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1,5 +1,4 @@ // expose private and protected members for invasive testing -#include "openPMD/UnitDimension.hpp" #if openPMD_USE_INVASIVE_TESTS #define OPENPMD_private public: #define OPENPMD_protected public: @@ -1499,6 +1498,339 @@ TEST_CASE("unavailable_backend", "[core]") #endif } +void breakpoint() +{} + +#if openPMD_HAVE_ADIOS2 +TEST_CASE("automatic_variable_encoding", "[adios2]") +{ + size_t filename_counter = 0; + auto filename = [&filename_counter]() { + std::stringstream res; + res << "../samples/automatic_variable_encoding/test_no_" + << filename_counter << ".bp5"; + return res.str(); + }; + auto next_filename = [&filename_counter, &filename]() { + ++filename_counter; + return filename(); + }; + auto require_encoding = [&filename](IterationEncoding ie) { + Series read(filename(), openPMD::Access::READ_RANDOM_ACCESS); + REQUIRE(read.iterationEncoding() == ie); + }; + + // TESTS + + { + Series write(next_filename(), Access::CREATE); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly set variable encoding + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + // explicitly set variable encoding + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly use API call to set variable encoding + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + // explicitly use API call to set group encoding + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly use API call to set group encoding + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + // explicitly use API call to set group encoding a bit late + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.setIterationEncoding(IterationEncoding::groupBased); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::Synchronous); + write.setIterationEncoding(IterationEncoding::groupBased); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly use API call to set variable encoding a bit late + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::Synchronous); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } +} +#endif + TEST_CASE("unique_ptr", "[core]") { auto stdptr = std::make_unique(5); From ffd887d7a411da39c71d2c58c050d6ccdd60533e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 14 Feb 2025 17:56:26 +0100 Subject: [PATCH 31/33] Move test to its own file --- CMakeLists.txt | 4 + test/CoreTest.cpp | 332 +---------------- test/Files_Core/CoreTests.hpp | 6 + .../automatic_variable_encoding.cpp | 339 ++++++++++++++++++ 4 files changed, 352 insertions(+), 329 deletions(-) create mode 100644 test/Files_Core/CoreTests.hpp create mode 100644 test/Files_Core/automatic_variable_encoding.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c4af3df142..b3a0cc30c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -792,6 +792,10 @@ if(openPMD_BUILD_TESTING) test/Files_ParallelIO/read_variablebased_randomaccess.cpp test/Files_ParallelIO/iterate_nonstreaming_series.cpp ) + elseif(${test_name} STREQUAL "Core") + list(APPEND ${out_list} + test/Files_Core/automatic_variable_encoding.cpp + ) endif() endmacro() diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index f629580399..c5d98a73c5 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -5,6 +5,8 @@ #endif #include "openPMD/openPMD.hpp" +#include "Files_Core/CoreTests.hpp" + #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON.hpp" @@ -1498,338 +1500,10 @@ TEST_CASE("unavailable_backend", "[core]") #endif } -void breakpoint() -{} - -#if openPMD_HAVE_ADIOS2 TEST_CASE("automatic_variable_encoding", "[adios2]") { - size_t filename_counter = 0; - auto filename = [&filename_counter]() { - std::stringstream res; - res << "../samples/automatic_variable_encoding/test_no_" - << filename_counter << ".bp5"; - return res.str(); - }; - auto next_filename = [&filename_counter, &filename]() { - ++filename_counter; - return filename(); - }; - auto require_encoding = [&filename](IterationEncoding ie) { - Series read(filename(), openPMD::Access::READ_RANDOM_ACCESS); - REQUIRE(read.iterationEncoding() == ie); - }; - - // TESTS - - { - Series write(next_filename(), Access::CREATE); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.flush(); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.flush(); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - // explicitly set variable encoding - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "variable_based")"); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "variable_based")"); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "variable_based")"); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "variable_based")"); - write.flush(); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "variable_based")"); - write.flush(); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - // explicitly set variable encoding - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "group_based")"); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "group_based")"); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "group_based")"); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "group_based")"); - write.flush(); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write( - next_filename(), - Access::CREATE, - R"(iteration_encoding = "group_based")"); - write.flush(); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - // explicitly use API call to set variable encoding - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.flush(); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.flush(); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - // explicitly use API call to set group encoding - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::groupBased); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::groupBased); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::groupBased); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::groupBased); - write.flush(); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::groupBased); - write.flush(); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - // explicitly use API call to set group encoding - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.flush(); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.setIterationEncoding(IterationEncoding::variableBased); - write.flush(); - write.snapshots(SnapshotWorkflow::Synchronous); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - // explicitly use API call to set group encoding a bit late - - { - Series write(next_filename(), Access::CREATE); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.setIterationEncoding(IterationEncoding::groupBased); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.snapshots(SnapshotWorkflow::Synchronous); - write.setIterationEncoding(IterationEncoding::groupBased); - write.close(); - require_encoding(IterationEncoding::groupBased); - } - - // explicitly use API call to set variable encoding a bit late - - { - Series write(next_filename(), Access::CREATE); - write.snapshots(SnapshotWorkflow::RandomAccess); - write.setIterationEncoding(IterationEncoding::variableBased); - write.close(); - require_encoding(IterationEncoding::variableBased); - } - - { - Series write(next_filename(), Access::CREATE); - write.snapshots(SnapshotWorkflow::Synchronous); - write.setIterationEncoding(IterationEncoding::variableBased); - write.close(); - require_encoding(IterationEncoding::variableBased); - } + automatic_variable_encoding::automatic_variable_encoding(); } -#endif TEST_CASE("unique_ptr", "[core]") { diff --git a/test/Files_Core/CoreTests.hpp b/test/Files_Core/CoreTests.hpp new file mode 100644 index 0000000000..27cb896706 --- /dev/null +++ b/test/Files_Core/CoreTests.hpp @@ -0,0 +1,6 @@ +#include + +namespace automatic_variable_encoding +{ +auto automatic_variable_encoding() -> void; +} diff --git a/test/Files_Core/automatic_variable_encoding.cpp b/test/Files_Core/automatic_variable_encoding.cpp new file mode 100644 index 0000000000..172d7162d1 --- /dev/null +++ b/test/Files_Core/automatic_variable_encoding.cpp @@ -0,0 +1,339 @@ +#include "CoreTests.hpp" +#include + +#include + +namespace automatic_variable_encoding +{ +auto automatic_variable_encoding() -> void +{ +#if openPMD_HAVE_ADIOS2 && openPMD_HAS_ADIOS_2_9 + using namespace openPMD; + + size_t filename_counter = 0; + auto filename = [&filename_counter]() { + std::stringstream res; + res << "../samples/automatic_variable_encoding/test_no_" + << filename_counter << ".bp5"; + return res.str(); + }; + auto next_filename = [&filename_counter, &filename]() { + ++filename_counter; + return filename(); + }; + auto require_encoding = [&filename](IterationEncoding ie) { + Series read(filename(), openPMD::Access::READ_RANDOM_ACCESS); + REQUIRE(read.iterationEncoding() == ie); + }; + + // TESTS + + { + Series write(next_filename(), Access::CREATE); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly set variable encoding + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "variable_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + // explicitly set variable encoding + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write( + next_filename(), + Access::CREATE, + R"(iteration_encoding = "group_based")"); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly use API call to set variable encoding + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + // explicitly use API call to set group encoding + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::groupBased); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly use API call to set group encoding + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.setIterationEncoding(IterationEncoding::variableBased); + write.flush(); + write.snapshots(SnapshotWorkflow::Synchronous); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + // explicitly use API call to set group encoding a bit late + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.setIterationEncoding(IterationEncoding::groupBased); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::Synchronous); + write.setIterationEncoding(IterationEncoding::groupBased); + write.close(); + require_encoding(IterationEncoding::groupBased); + } + + // explicitly use API call to set variable encoding a bit late + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::RandomAccess); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } + + { + Series write(next_filename(), Access::CREATE); + write.snapshots(SnapshotWorkflow::Synchronous); + write.setIterationEncoding(IterationEncoding::variableBased); + write.close(); + require_encoding(IterationEncoding::variableBased); + } +#endif +} +} // namespace automatic_variable_encoding From e15ab8c49b23f4195617f65cb32ee78f90650edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 17 Feb 2025 10:50:44 +0100 Subject: [PATCH 32/33] Don't distinguish ADIOS2 v2.9 any more --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 02920c0207..355d1aa87f 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -863,7 +863,7 @@ class ADIOS2IOHandler : public AbstractIOHandler bool fullSupportForVariableBasedEncoding() const override { - return openPMD_HAS_ADIOS_2_9; + return true; } std::future flush(internal::ParsedFlushParams &) override; From b9158a626bcadab467d6b6af7a080e7f4d2c1c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 19 Feb 2025 19:28:53 +0100 Subject: [PATCH 33/33] Some more documentation --- docs/source/usage/concepts.rst | 3 ++- include/openPMD/IO/IOTask.hpp | 14 +++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/source/usage/concepts.rst b/docs/source/usage/concepts.rst index 91253f8945..e9957e5f4b 100644 --- a/docs/source/usage/concepts.rst +++ b/docs/source/usage/concepts.rst @@ -48,7 +48,7 @@ openPMD-api implements various file-formats (backends) and encoding strategies f **Iteration encoding:** The openPMD-api can encode iterations in different ways. The method ``Series::setIterationEncoding()`` (C++) or ``Series.set_iteration_encoding()`` (Python) may be used in writing for selecting one of the following encodings explicitly: -* **group-based iteration encoding:** This encoding is the default. +* **group-based iteration encoding:** This encoding is the default for HDF5 and JSON. In ADIOS2, variable-based encoding is preferred when possible due to better performance characteristics, see below. It creates a separate group in the hierarchy of the openPMD standard for each iteration. As an example, all data pertaining to iteration 0 may be found in group ``/data/0``, for iteration 100 in ``/data/100``. * **file-based iteration encoding:** A unique file on the filesystem is created for each iteration. @@ -57,6 +57,7 @@ The method ``Series::setIterationEncoding()`` (C++) or ``Series.set_iteration_en A padding may be specified by ``"series_%06T.json"`` to create files ``series_000000.json``, ``series_000100.json`` and ``series_000200.json``. The inner group layout of each file is identical to that of the group-based encoding. * **variable-based iteration encoding:** This experimental encoding uses a feature of some backends (i.e., ADIOS2) to maintain datasets and attributes in several versions (i.e., iterations are stored inside *variables*). + When creating an ADIOS2 Series with steps (e.g. via ``series.writeIterations()`` / ``series.write_iterations()``), this encoding will be picked as a default instead of group-based encoding due to bad performance characteristics of group-based encoding in ADIOS2. No iteration-specific groups are created and the corresponding layer is dropped from the openPMD hierarchy. In backends that do not support this feature, a series created with this encoding can only contain one iteration. diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index a4b2bb9579..fafe9c669b 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -32,8 +32,8 @@ #include "openPMD/backend/ParsePreference.hpp" #include -#include #include +#include #include #include #include @@ -677,10 +677,18 @@ struct OPENPMDAPI_EXPORT Parameter std::optional step; }; - //! input parameter + // input parameters + /** + * AdvanceMode: Is one of BeginStep/EndStep. Used during writing and in + * linear read mode to step sequentially through steps. + * StepSelection: Used in random-access read mode, jump to the specified + * step. Can be nullopt in order to reset the backend to read + * step-agnostically, e.g. for reading global datasets such as + * /rankTable. + */ std::variant mode; bool isThisStepMandatory = false; - //! output parameter + // output parameter std::shared_ptr status = std::make_shared(AdvanceStatus::OK); };