From 3b06c09b0ac0d03a7f4bbbe8f95bd49e5f734b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Aug 2023 11:55:37 +0200 Subject: [PATCH 01/28] Introduce dataset template mode to JSON backend --- CMakeLists.txt | 1 + include/openPMD/IO/JSON/JSONIOHandler.hpp | 1 + include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 23 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 361 ++++++++++++++---- test/SerialIOTest.cpp | 35 +- 5 files changed, 339 insertions(+), 82 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dd3e5a9efd..409a5c771d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -703,6 +703,7 @@ set(openPMD_EXAMPLE_NAMES 10_streaming_read 12_span_write 13_write_dynamic_configuration + 14_toml_template ) set(openPMD_PYTHON_EXAMPLE_NAMES 2_read_serial diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 7cb6870f5b..e22fdb93d1 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -23,6 +23,7 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #if openPMD_HAVE_MPI #include diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index b67ac9138a..875961ea7a 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -265,8 +265,27 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl */ FileFormat m_fileFormat{}; + std::string backendConfigKey() const; + + /* + * First return value: The location of the JSON value (either "json" or + * "toml") Second return value: The value that was maybe found at this place + */ + std::pair> + getBackendConfig(openPMD::json::TracingJSON &) const; + std::string m_originalExtension; + enum class IOMode + { + Dataset, + Template + }; + + IOMode m_mode = IOMode::Dataset; + + IOMode retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + // HELPER FUNCTIONS // will use the IOHandler to retrieve the correct directory. @@ -313,7 +332,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // essentially: m_i = \prod_{j=0}^{i-1} extent_j static Extent getMultiplicators(Extent const &extent); - static Extent getExtent(nlohmann::json &j); + static std::pair getExtent(nlohmann::json &j); // remove single '/' in the beginning and end of a string static std::string removeSlashes(std::string); @@ -371,7 +390,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // check whether the json reference contains a valid dataset template - void verifyDataset(Param const ¶meters, nlohmann::json &); + IOMode verifyDataset(Param const ¶meters, nlohmann::json &); static nlohmann::json platformSpecifics(); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index b1046c4602..72afde50c5 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -126,18 +126,120 @@ namespace } return *accum_ptr; } + + void warnUnusedJson(openPMD::json::TracingJSON const &jsonConfig) + { + auto shadow = jsonConfig.invertShadow(); + if (shadow.size() > 0) + { + switch (jsonConfig.originallySpecifiedAs) + { + case openPMD::json::SupportedLanguages::JSON: + std::cerr << "Warning: parts of the backend configuration for " + "JSON/TOML backend remain unused:\n" + << shadow << std::endl; + break; + case openPMD::json::SupportedLanguages::TOML: { + auto asToml = openPMD::json::jsonToToml(shadow); + std::cerr << "Warning: parts of the backend configuration for " + "JSON/TOML backend remain unused:\n" + << asToml << std::endl; + break; + } + } + } + } } // namespace +auto JSONIOHandlerImpl::retrieveDatasetMode( + openPMD::json::TracingJSON &config) const -> IOMode +{ + IOMode res = m_mode; + if (auto [configLocation, maybeConfig] = getBackendConfig(config); + maybeConfig.has_value()) + { + auto jsonConfig = maybeConfig.value(); + if (jsonConfig.json().contains("dataset")) + { + auto datasetConfig = jsonConfig["dataset"]; + if (datasetConfig.json().contains("mode")) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + datasetConfig["mode"].json()); + if (!modeOption.has_value()) + { + throw error::BackendConfigSchema( + {configLocation, "mode"}, + "Invalid value of non-string type (accepted values are " + "'dataset' and 'template'."); + } + auto mode = modeOption.value(); + if (mode == "dataset") + { + res = IOMode::Dataset; + } + else if (mode == "template") + { + res = IOMode::Template; + } + else + { + throw error::BackendConfigSchema( + {configLocation, "dataset", "mode"}, + "Invalid value: '" + mode + + "' (accepted values are 'dataset' and 'template'."); + } + } + } + } + return res; +} + +std::string JSONIOHandlerImpl::backendConfigKey() const +{ + switch (m_fileFormat) + { + case FileFormat::Json: + return "json"; + case FileFormat::Toml: + return "toml"; + } + throw std::runtime_error("Unreachable!"); +} + +std::pair> +JSONIOHandlerImpl::getBackendConfig(openPMD::json::TracingJSON &config) const +{ + std::string configLocation = backendConfigKey(); + if (config.json().contains(configLocation)) + { + return std::make_pair( + std::move(configLocation), config[configLocation]); + } + else + { + return std::make_pair(std::move(configLocation), std::nullopt); + } +} + JSONIOHandlerImpl::JSONIOHandlerImpl( AbstractIOHandler *handler, - // NOLINTNEXTLINE(performance-unnecessary-value-param) - [[maybe_unused]] openPMD::json::TracingJSON config, + openPMD::json::TracingJSON config, FileFormat format, std::string originalExtension) : AbstractIOHandlerImpl(handler) , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} -{} +{ + m_mode = retrieveDatasetMode(config); + + if (auto [_, backendConfig] = getBackendConfig(config); + backendConfig.has_value()) + { + (void)_; + warnUnusedJson(backendConfig.value()); + } +} #if openPMD_HAVE_MPI JSONIOHandlerImpl::JSONIOHandlerImpl( @@ -292,6 +394,18 @@ void JSONIOHandlerImpl::createDataset( "JSON", "Joined Arrays currently only supported in ADIOS2"); } + openPMD::json::TracingJSON config = openPMD::json::parseOptions( + parameter.options, /* considerFiles = */ false); + // Retrieves mode from dataset-specific configuration, falls back to global + // value if not defined + IOMode localMode = retrieveDatasetMode(config); + + parameter.warnUnusedParameters( + config, + backendConfigKey(), + "Warning: parts of the dataset-specific backend configuration for " + "JSON/TOML backend remain unused"); + if (!writable->written) { /* Sanitize name */ @@ -309,23 +423,41 @@ void JSONIOHandlerImpl::createDataset( setAndGetFilePosition(writable, name); auto &dset = jsonVal[name]; dset["datatype"] = datatypeToString(parameter.dtype); - auto extent = parameter.extent; - switch (parameter.dtype) + + switch (localMode) { - case Datatype::CFLOAT: - case Datatype::CDOUBLE: - case Datatype::CLONG_DOUBLE: { - extent.push_back(2); + case IOMode::Dataset: { + auto extent = parameter.extent; + switch (parameter.dtype) + { + case Datatype::CFLOAT: + case Datatype::CDOUBLE: + case Datatype::CLONG_DOUBLE: { + extent.push_back(2); + break; + } + default: + break; + } + // TOML does not support nulls, so initialize with zero + dset["data"] = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional{} + : parameter.dtype); break; } - default: + case IOMode::Template: + if (parameter.extent != Extent{0}) + { + dset["extent"] = parameter.extent; + } + else + { + // no-op + // If extent is empty, don't bother writing it + } break; } - // TOML does not support nulls, so initialize with zero - dset["data"] = initializeNDArray( - extent, - m_fileFormat == FileFormat::Json ? std::optional() - : parameter.dtype); writable->written = true; m_dirty.emplace(file); } @@ -364,9 +496,11 @@ void JSONIOHandlerImpl::extendDataset( refreshFileFromParent(writable); auto &j = obtainJsonContents(writable); + IOMode localIOMode; try { - auto datasetExtent = getExtent(j); + Extent datasetExtent; + std::tie(datasetExtent, localIOMode) = getExtent(j); VERIFY_ALWAYS( datasetExtent.size() == parameters.extent.size(), "[JSON] Cannot change dimensionality of a dataset") @@ -383,28 +517,40 @@ void JSONIOHandlerImpl::extendDataset( throw std::runtime_error( "[JSON] The specified location contains no valid dataset"); } - auto extent = parameters.extent; - auto datatype = stringToDatatype(j["datatype"].get()); - switch (datatype) + + switch (localIOMode) { - case Datatype::CFLOAT: - case Datatype::CDOUBLE: - case Datatype::CLONG_DOUBLE: { - extent.push_back(2); - break; + case IOMode::Dataset: { + auto extent = parameters.extent; + auto datatype = stringToDatatype(j["datatype"].get()); + switch (datatype) + { + case Datatype::CFLOAT: + case Datatype::CDOUBLE: + case Datatype::CLONG_DOUBLE: { + extent.push_back(2); + break; + } + default: + // nothing to do + break; + } + // TOML does not support nulls, so initialize with zero + nlohmann::json newData = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional{} + : datatype); + nlohmann::json &oldData = j["data"]; + mergeInto(newData, oldData); + j["data"] = newData; } - default: - // nothing to do - break; + break; + case IOMode::Template: { + j["extent"] = parameters.extent; + } + break; } - // TOML does not support nulls, so initialize with zero - nlohmann::json newData = initializeNDArray( - extent, - m_fileFormat == FileFormat::Json ? std::optional() - : datatype); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + writable->written = true; } @@ -700,7 +846,7 @@ void JSONIOHandlerImpl::openDataset( *parameters.dtype = Datatype(stringToDatatype(datasetJson["datatype"].get())); - *parameters.extent = getExtent(datasetJson); + *parameters.extent = getExtent(datasetJson).first; writable->written = true; } @@ -883,7 +1029,16 @@ void JSONIOHandlerImpl::writeDataset( auto file = refreshFileFromParent(writable); auto &j = obtainJsonContents(writable); - verifyDataset(parameters, j); + switch (verifyDataset(parameters, j)) + { + case IOMode::Dataset: + break; + case IOMode::Template: + std::cerr << "[JSON/TOML backend: Warning] Trying to write data to a " + "template dataset. Will skip." + << std::endl; + return; + } switchType(parameters.dtype, j, parameters); @@ -925,22 +1080,55 @@ void JSONIOHandlerImpl::writeAttribute( m_dirty.emplace(file); } +namespace +{ + struct FillWithZeroes + { + template + static void call(void *ptr, Extent const &extent) + { + T *casted = static_cast(ptr); + size_t flattenedExtent = std::accumulate( + extent.begin(), + extent.end(), + size_t(1), + [](size_t left, size_t right) { return left * right; }); + std::fill_n(casted, flattenedExtent, T{}); + } + + static constexpr char const *errorMsg = + "[JSON Backend] Fill with zeroes."; + }; +} // namespace + void JSONIOHandlerImpl::readDataset( Writable *writable, Parameter ¶meters) { refreshFileFromParent(writable); setAndGetFilePosition(writable); auto &j = obtainJsonContents(writable); - verifyDataset(parameters, j); + IOMode localMode = verifyDataset(parameters, j); - try + switch (localMode) { - switchType(parameters.dtype, j["data"], parameters); - } - catch (json::basic_json::type_error &) - { - throw std::runtime_error( - "[JSON] The given path does not contain a valid dataset."); + case IOMode::Template: + std::cerr << "[Warning] Cannot read chunks in Template mode of JSON " + "backend. Will fill with zeroes instead." + << std::endl; + switchNonVectorType( + parameters.dtype, parameters.data.get(), parameters.extent); + return; + case IOMode::Dataset: + try + { + switchType(parameters.dtype, j["data"], parameters); + } + catch (json::basic_json::type_error &) + { + throw std::runtime_error( + "[JSON] The given path does not contain a valid dataset."); + } + break; } } @@ -1196,28 +1384,44 @@ Extent JSONIOHandlerImpl::getMultiplicators(Extent const &extent) return res; } -Extent JSONIOHandlerImpl::getExtent(nlohmann::json &j) +auto JSONIOHandlerImpl::getExtent(nlohmann::json &j) + -> std::pair { Extent res; - nlohmann::json *ptr = &j["data"]; - while (ptr->is_array()) + IOMode ioMode; + if (j.contains("data")) + { + ioMode = IOMode::Dataset; + nlohmann::json *ptr = &j["data"]; + while (ptr->is_array()) + { + res.push_back(ptr->size()); + ptr = &(*ptr)[0]; + } + switch (stringToDatatype(j["datatype"].get())) + { + case Datatype::CFLOAT: + case Datatype::CDOUBLE: + case Datatype::CLONG_DOUBLE: + // the last "dimension" is only the two entries for the complex + // number, so remove that again + res.erase(res.end() - 1); + break; + default: + break; + } + } + else if (j.contains("extent")) { - res.push_back(ptr->size()); - ptr = &(*ptr)[0]; + ioMode = IOMode::Template; + res = j["extent"].get(); } - switch (stringToDatatype(j["datatype"].get())) + else { - case Datatype::CFLOAT: - case Datatype::CDOUBLE: - case Datatype::CLONG_DOUBLE: - // the last "dimension" is only the two entries for the complex - // number, so remove that again - res.erase(res.end() - 1); - break; - default: - break; + ioMode = IOMode::Template; + res = {0}; } - return res; + return std::make_pair(std::move(res), ioMode); } std::string JSONIOHandlerImpl::removeSlashes(std::string s) @@ -1379,7 +1583,14 @@ auto JSONIOHandlerImpl::putJsonContents( return it; } - (*it->second)["platform_byte_widths"] = platformSpecifics(); + switch (m_mode) + { + case IOMode::Dataset: + (*it->second)["platform_byte_widths"] = platformSpecifics(); + break; + case IOMode::Template: + break; + } auto writeSingleFile = [this, &it](std::string const &writeThisFile) { auto [fh, _, fh_with_precision] = @@ -1582,8 +1793,8 @@ bool JSONIOHandlerImpl::isDataset(nlohmann::json const &j) { return false; } - auto i = j.find("data"); - return i != j.end() && i.value().is_array(); + auto i = j.find("datatype"); + return i != j.end() && i.value().is_string(); } bool JSONIOHandlerImpl::isGroup(nlohmann::json::const_iterator const &it) @@ -1594,21 +1805,24 @@ bool JSONIOHandlerImpl::isGroup(nlohmann::json::const_iterator const &it) { return false; } - auto i = j.find("data"); - return i == j.end() || !i.value().is_array(); + + auto i = j.find("datatype"); + return i == j.end() || !i.value().is_string(); } template -void JSONIOHandlerImpl::verifyDataset( - Param const ¶meters, nlohmann::json &j) +auto JSONIOHandlerImpl::verifyDataset( + Param const ¶meters, nlohmann::json &j) -> IOMode { VERIFY_ALWAYS( isDataset(j), "[JSON] Specified dataset does not exist or is not a dataset."); + IOMode res; try { - auto datasetExtent = getExtent(j); + Extent datasetExtent; + std::tie(datasetExtent, res) = getExtent(j); VERIFY_ALWAYS( datasetExtent.size() == parameters.extent.size(), "[JSON] Read/Write request does not fit the dataset's dimension"); @@ -1630,6 +1844,7 @@ void JSONIOHandlerImpl::verifyDataset( throw std::runtime_error( "[JSON] The given path does not contain a valid dataset."); } + return res; } nlohmann::json JSONIOHandlerImpl::platformSpecifics() @@ -1711,7 +1926,7 @@ nlohmann::json JSONIOHandlerImpl::CppToJSON::operator()(const T &val) } template -nlohmann::json JSONIOHandlerImpl::CppToJSON >::operator()( +nlohmann::json JSONIOHandlerImpl::CppToJSON>::operator()( const std::vector &v) { nlohmann::json j; @@ -1724,7 +1939,7 @@ nlohmann::json JSONIOHandlerImpl::CppToJSON >::operator()( } template -nlohmann::json JSONIOHandlerImpl::CppToJSON >::operator()( +nlohmann::json JSONIOHandlerImpl::CppToJSON>::operator()( const std::array &v) { nlohmann::json j; @@ -1743,7 +1958,7 @@ T JSONIOHandlerImpl::JsonToCpp::operator()(nlohmann::json const &json) } template -std::vector JSONIOHandlerImpl::JsonToCpp >::operator()( +std::vector JSONIOHandlerImpl::JsonToCpp>::operator()( nlohmann::json const &json) { std::vector v; @@ -1756,7 +1971,7 @@ std::vector JSONIOHandlerImpl::JsonToCpp >::operator()( } template -std::array JSONIOHandlerImpl::JsonToCpp >::operator()( +std::array JSONIOHandlerImpl::JsonToCpp>::operator()( nlohmann::json const &json) { std::array a; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 41de1bdbcd..96d2e0879b 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1,8 +1,8 @@ // expose private and protected members for invasive testing +#include "openPMD/auxiliary/JSON.hpp" #include "openPMD/ChunkInfo_internal.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" -#include "openPMD/UnitDimension.hpp" #if openPMD_USE_INVASIVE_TESTS #define OPENPMD_private public: #define OPENPMD_protected public: @@ -1276,12 +1276,19 @@ TEST_CASE("particle_patches", "[serial]") } } -inline void dtype_test(const std::string &backend) +inline void dtype_test( + const std::string &backend, + std::optional activateTemplateMode = {}) { bool test_long_double = backend != "json" && backend != "toml"; bool test_long_long = (backend != "json") || sizeof(long long) <= 8; { - Series s = Series("../samples/dtype_test." + backend, Access::CREATE); + Series s = activateTemplateMode.has_value() + ? Series( + "../samples/dtype_test." + backend, + Access::CREATE, + activateTemplateMode.value()) + : Series("../samples/dtype_test." + backend, Access::CREATE); char c = 'c'; s.setAttribute("char", c); @@ -1403,8 +1410,12 @@ inline void dtype_test(const std::string &backend) } } - Series s = Series("../samples/dtype_test." + backend, Access::READ_ONLY); - + Series s = activateTemplateMode.has_value() + ? Series( + "../samples/dtype_test." + backend, + Access::READ_ONLY, + activateTemplateMode.value()) + : Series("../samples/dtype_test." + backend, Access::READ_ONLY); REQUIRE(s.getAttribute("char").get() == 'c'); REQUIRE(s.getAttribute("uchar").get() == 'u'); REQUIRE(s.getAttribute("schar").get() == 's'); @@ -1474,6 +1485,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("bool").get() == true); REQUIRE(s.getAttribute("boolF").get() == false); + if (activateTemplateMode.has_value()) + { + return; + } // same implementation types (not necessary aliases) detection #if !defined(_MSC_VER) REQUIRE(s.getAttribute("short").dtype == Datatype::SHORT); @@ -1546,6 +1561,7 @@ TEST_CASE("dtype_test", "[serial]") { dtype_test(t); } + dtype_test("json", R"({"json":{"dataset":{"mode":"template"}}})"); if (auto extensions = getFileExtensions(); std::find(extensions.begin(), extensions.end(), "toml") != extensions.end()) @@ -1554,6 +1570,7 @@ TEST_CASE("dtype_test", "[serial]") * testing it here. */ dtype_test("toml"); + dtype_test("toml", R"({"toml":{"dataset":{"mode":"template"}}})"); } } @@ -1577,6 +1594,8 @@ inline void write_test(const std::string &backend) host_info::byMethod( host_info::methodFromStringDescription("posix_hostname", false))}}; #endif + jsonCfg = + json::merge(jsonCfg, R"({"json":{"dataset":{"mode":"template"}}})"); Series o = Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); @@ -1604,8 +1623,10 @@ inline void write_test(const std::string &backend) return posOff++; }); std::shared_ptr positionOffset_local_1(new uint64_t); - e_1["positionOffset"]["x"].resetDataset( - Dataset(determineDatatype(positionOffset_local_1), {4})); + e_1["positionOffset"]["x"].resetDataset(Dataset( + determineDatatype(positionOffset_local_1), + {4}, + R"({"json":{"dataset":{"mode":"dataset"}}})")); for (uint64_t i = 0; i < 4; ++i) { From 025800009ee04c2a2f69e5dad7f2484ff7129368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Aug 2023 13:33:24 +0200 Subject: [PATCH 02/28] Write used mode to JSON file --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 10 ++- src/IO/JSON/JSONIOHandlerImpl.cpp | 63 +++++++++++++++++-- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 875961ea7a..541758a4b1 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -284,7 +284,15 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl IOMode m_mode = IOMode::Dataset; - IOMode retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + enum class SpecificationVia + { + DefaultValue, + Manually + }; + SpecificationVia m_IOModeSpecificationVia = SpecificationVia::DefaultValue; + + std::pair + retrieveDatasetMode(openPMD::json::TracingJSON &config) const; // HELPER FUNCTIONS diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 72afde50c5..b164709673 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -64,6 +64,13 @@ namespace openPMD throw std::runtime_error((TEXT)); \ } +namespace JSONDefaults +{ + using const_str = char const *const; + constexpr const_str openpmd_internal = "__openPMD_internal"; + constexpr const_str IOMode = "IO_mode"; +} // namespace JSONDefaults + namespace { struct DefaultValue @@ -151,10 +158,11 @@ namespace } } // namespace -auto JSONIOHandlerImpl::retrieveDatasetMode( - openPMD::json::TracingJSON &config) const -> IOMode +auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) + const -> std::pair { IOMode res = m_mode; + SpecificationVia res_2 = SpecificationVia::DefaultValue; if (auto [configLocation, maybeConfig] = getBackendConfig(config); maybeConfig.has_value()) { @@ -177,10 +185,12 @@ auto JSONIOHandlerImpl::retrieveDatasetMode( if (mode == "dataset") { res = IOMode::Dataset; + res_2 = SpecificationVia::Manually; } else if (mode == "template") { res = IOMode::Template; + res_2 = SpecificationVia::Manually; } else { @@ -192,7 +202,7 @@ auto JSONIOHandlerImpl::retrieveDatasetMode( } } } - return res; + return std::make_pair(res, res_2); } std::string JSONIOHandlerImpl::backendConfigKey() const @@ -231,7 +241,7 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { - m_mode = retrieveDatasetMode(config); + std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); if (auto [_, backendConfig] = getBackendConfig(config); backendConfig.has_value()) @@ -398,7 +408,7 @@ void JSONIOHandlerImpl::createDataset( parameter.options, /* considerFiles = */ false); // Retrieves mode from dataset-specific configuration, falls back to global // value if not defined - IOMode localMode = retrieveDatasetMode(config); + IOMode localMode = retrieveDatasetMode(config).first; parameter.warnUnusedParameters( config, @@ -1558,6 +1568,45 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) auto res = serialImplementation(); #endif + if (res->contains(JSONDefaults::openpmd_internal)) + { + auto const &openpmd_internal = res->at(JSONDefaults::openpmd_internal); + if (openpmd_internal.contains(JSONDefaults::IOMode)) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + openpmd_internal.at(JSONDefaults::IOMode)); + if (!modeOption.has_value()) + { + std::cerr + << "[JSON/TOML backend] Warning: Invalid value of " + "non-string type at internal meta table for entry '" + << JSONDefaults::IOMode << "'. Will ignore and continue." + << std::endl; + } + else if (modeOption.value() == "dataset") + { + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) + { + m_mode = IOMode::Dataset; + } + } + else if (modeOption.value() == "template") + { + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) + { + m_mode = IOMode::Template; + } + } + else + { + std::cerr << "[JSON/TOML backend] Warning: Invalid value '" + << modeOption.value() + << "' at internal meta table for entry '" + << JSONDefaults::IOMode + << "'. Will ignore and continue." << std::endl; + } + } + } m_jsonVals.emplace(file, res); return res; } @@ -1587,8 +1636,12 @@ auto JSONIOHandlerImpl::putJsonContents( { case IOMode::Dataset: (*it->second)["platform_byte_widths"] = platformSpecifics(); + (*it->second)[JSONDefaults::openpmd_internal][JSONDefaults::IOMode] = + "dataset"; break; case IOMode::Template: + (*it->second)[JSONDefaults::openpmd_internal][JSONDefaults::IOMode] = + "template"; break; } From a48e92e61d73a45fa386d3c8e0af05efe6022203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 23 Feb 2023 11:05:39 +0100 Subject: [PATCH 03/28] Use Attribute::getOptional for snapshot attribute --- src/Series.cpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/Series.cpp b/src/Series.cpp index cfd92f84e7..792be0555f 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -3238,19 +3238,13 @@ auto Series::currentSnapshot() const if (series.iterations.containsAttribute("snapshot")) { auto const &attribute = series.iterations.getAttribute("snapshot"); - switch (attribute.dtype) + auto res = attribute.getOptional(); + if (res.has_value()) { - case Datatype::ULONGLONG: - case Datatype::VEC_ULONGLONG: { - auto const &vec = attribute.get>(); - return vec_t{vec.begin(), vec.end()}; + return res.value(); } - case Datatype::ULONG: - case Datatype::VEC_ULONG: { - auto const &vec = attribute.get>(); - return vec_t{vec.begin(), vec.end()}; - } - default: { + else + { std::stringstream s; s << "Unexpected datatype for '/data/snapshot': " << attribute.dtype << " (expected a vector of integer, found " + @@ -3262,7 +3256,6 @@ auto Series::currentSnapshot() const {}, s.str()); } - } } else { From e6e5357b71728bf51684154a14960aa6f36fa0da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Aug 2023 14:49:16 +0200 Subject: [PATCH 04/28] Introduce attribute mode --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 33 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 330 +++++++++++++++++- 2 files changed, 342 insertions(+), 21 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 541758a4b1..0a5a9779fd 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -276,6 +276,16 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl std::string m_originalExtension; + enum class SpecificationVia + { + DefaultValue, + Manually + }; + + ///////////////////// + // Dataset IO mode // + ///////////////////// + enum class IOMode { Dataset, @@ -283,17 +293,28 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl }; IOMode m_mode = IOMode::Dataset; - - enum class SpecificationVia - { - DefaultValue, - Manually - }; SpecificationVia m_IOModeSpecificationVia = SpecificationVia::DefaultValue; std::pair retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + /////////////////////// + // Attribute IO mode // + /////////////////////// + + enum class AttributeMode + { + Short, + Long + }; + + AttributeMode m_attributeMode = AttributeMode::Long; + SpecificationVia m_attributeModeSpecificationVia = + SpecificationVia::DefaultValue; + + std::pair + retrieveAttributeMode(openPMD::json::TracingJSON &config) const; + // HELPER FUNCTIONS // will use the IOHandler to retrieve the correct directory. diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index b164709673..4a3116a62f 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -31,6 +31,7 @@ #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" +#include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -68,7 +69,8 @@ namespace JSONDefaults { using const_str = char const *const; constexpr const_str openpmd_internal = "__openPMD_internal"; - constexpr const_str IOMode = "IO_mode"; + constexpr const_str IOMode = "dataset_mode"; + constexpr const_str AttributeMode = "attribute_mode"; } // namespace JSONDefaults namespace @@ -205,6 +207,54 @@ auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) return std::make_pair(res, res_2); } +auto JSONIOHandlerImpl::retrieveAttributeMode( + openPMD::json::TracingJSON &config) const + -> std::pair +{ + AttributeMode res = m_attributeMode; + SpecificationVia res_2 = SpecificationVia::DefaultValue; + if (auto [configLocation, maybeConfig] = getBackendConfig(config); + maybeConfig.has_value()) + { + auto jsonConfig = maybeConfig.value(); + if (jsonConfig.json().contains("attribute")) + { + auto attributeConfig = jsonConfig["attribute"]; + if (attributeConfig.json().contains("mode")) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + attributeConfig["mode"].json()); + if (!modeOption.has_value()) + { + throw error::BackendConfigSchema( + {configLocation, "mode"}, + "Invalid value of non-string type (accepted values are " + "'dataset' and 'template'."); + } + auto mode = modeOption.value(); + if (mode == "short") + { + res = AttributeMode::Short; + res_2 = SpecificationVia::Manually; + } + else if (mode == "long") + { + res = AttributeMode::Long; + res_2 = SpecificationVia::Manually; + } + else + { + throw error::BackendConfigSchema( + {configLocation, "attribute", "mode"}, + "Invalid value: '" + mode + + "' (accepted values are 'short' and 'long'."); + } + } + } + } + return std::make_pair(res, res_2); +} + std::string JSONIOHandlerImpl::backendConfigKey() const { switch (m_fileFormat) @@ -242,6 +292,8 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_originalExtension{std::move(originalExtension)} { std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); + std::tie(m_attributeMode, m_attributeModeSpecificationVia) = + retrieveAttributeMode(config); if (auto [_, backendConfig] = getBackendConfig(config); backendConfig.has_value()) @@ -1084,8 +1136,17 @@ void JSONIOHandlerImpl::writeAttribute( } nlohmann::json value; switchType(parameter.dtype, value, parameter.resource); - (*jsonVal)[filePosition->id]["attributes"][parameter.name] = { - {"datatype", datatypeToString(parameter.dtype)}, {"value", value}}; + switch (m_attributeMode) + { + case AttributeMode::Long: + (*jsonVal)[filePosition->id]["attributes"][parameter.name] = { + {"datatype", datatypeToString(parameter.dtype)}, {"value", value}}; + break; + case AttributeMode::Short: + // short form + (*jsonVal)[filePosition->id]["attributes"][parameter.name] = value; + break; + } writable->written = true; m_dirty.emplace(file); } @@ -1142,6 +1203,195 @@ void JSONIOHandlerImpl::readDataset( } } +namespace +{ + template + Attribute recoverVectorAttributeFromJson(nlohmann::json const &j) + { + if (!j.is_array()) + { + throw std::runtime_error( + "[JSON backend: recoverVectorAttributeFromJson] Internal " + "control flow error."); + } + + if (j.size() == 7 && + (std::is_same_v || + std::is_same_v || + std::is_same_v)) + { + /* + * The frontend must deal with wrong type reports here. + */ + std::array res; + for (size_t i = 0; i < 7; ++i) + { + res[i] = j[i].get(); + } + return res; + } + else + { + std::vector res; + res.reserve(j.size()); + for (auto const &i : j) + { + res.push_back(i.get()); + } + return res; + } + } + + nlohmann::json::value_t unifyNumericType(nlohmann::json const &j) + { + if (!j.is_array() || j.empty()) + { + throw std::runtime_error( + "[JSON backend: recoverVectorAttributeFromJson] Internal " + "control flow error."); + } + auto dtypeRanking = [](nlohmann::json::value_t dtype) -> unsigned { + switch (dtype) + { + case nlohmann::json::value_t::number_unsigned: + return 0; + case nlohmann::json::value_t::number_integer: + return 1; + case nlohmann::json::value_t::number_float: + return 2; + default: + throw std::runtime_error( + "[JSON backend] Encountered vector with mixed number and " + "non-number datatypes."); + } + }; + auto higherDtype = + [&dtypeRanking]( + nlohmann::json::value_t dt1, + nlohmann::json::value_t dt2) -> nlohmann::json::value_t { + if (dtypeRanking(dt1) > dtypeRanking(dt2)) + { + return dt1; + } + else + { + return dt2; + } + }; + + nlohmann::json::value_t res = j[0].type(); + for (size_t i = 1; i < j.size(); ++i) + { + res = higherDtype(res, j[i].type()); + } + return res; + } + + Attribute recoverAttributeFromJson( + nlohmann::json const &j, std::string const &nameForErrorMessages) + { + // @todo use ReadError once it's mainlined + switch (j.type()) + { + case nlohmann::json::value_t::null: + throw std::runtime_error( + "[JSON backend] Attribute must not be null: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::object: + throw std::runtime_error( + "[JSON backend] Shorthand-style attribute must not be an " + "object: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::array: + if (j.empty()) + { + std::cerr << "Cannot recover datatype of empty vector without " + "explicit type annotation for attribute '" + << nameForErrorMessages + << "'. Will continue with VEC_INT datatype." + << std::endl; + return std::vector{}; + } + else + { + auto valueType = j[0].type(); + /* + * If the vector is of numeric type, it might happen that the + * first entry is an integer, but a later entry is a float. + * We need to pick the most generic datatype in that case. + */ + if (valueType == nlohmann::json::value_t::number_float || + valueType == nlohmann::json::value_t::number_unsigned || + valueType == nlohmann::json::value_t::number_integer) + { + valueType = unifyNumericType(j); + } + switch (valueType) + { + case nlohmann::json::value_t::null: + throw std::runtime_error( + "[JSON backend] Attribute must not be null: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::object: + throw std::runtime_error( + "[JSON backend] Invalid contained datatype (object) " + "inside vector-type attribute: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::array: + throw std::runtime_error( + "[JSON backend] Invalid contained datatype (array) " + "inside vector-type attribute: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::string: + return recoverVectorAttributeFromJson(j); + case nlohmann::json::value_t::boolean: + throw std::runtime_error( + "[JSON backend] Attribute must not be vector of bool: " + "'" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::number_integer: + return recoverVectorAttributeFromJson< + nlohmann::json::number_integer_t>(j); + case nlohmann::json::value_t::number_unsigned: + return recoverVectorAttributeFromJson< + nlohmann::json::number_unsigned_t>(j); + case nlohmann::json::value_t::number_float: + return recoverVectorAttributeFromJson< + nlohmann::json::number_float_t>(j); + case nlohmann::json::value_t::binary: + throw std::runtime_error( + "[JSON backend] Attribute must not have binary type: " + "'" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::discarded: + throw std::runtime_error( + "Internal JSON parser datatype leaked into JSON " + "value."); + } + throw std::runtime_error("Unreachable!"); + } + case nlohmann::json::value_t::string: + return j.get(); + case nlohmann::json::value_t::boolean: + return j.get(); + case nlohmann::json::value_t::number_integer: + return j.get(); + case nlohmann::json::value_t::number_unsigned: + return j.get(); + case nlohmann::json::value_t::number_float: + return j.get(); + case nlohmann::json::value_t::binary: + throw std::runtime_error( + "[JSON backend] Attribute must not have binary type: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::discarded: + throw std::runtime_error( + "Internal JSON parser datatype leaked into JSON value."); + } + throw std::runtime_error("Unreachable!"); + } +} // namespace + void JSONIOHandlerImpl::readAttribute( Writable *writable, Parameter ¶meters) { @@ -1166,9 +1416,19 @@ void JSONIOHandlerImpl::readAttribute( auto &j = jsonLoc[name]; try { - *parameters.dtype = - Datatype(stringToDatatype(j["datatype"].get())); - switchType(*parameters.dtype, j["value"], parameters); + if (j.is_object()) + { + *parameters.dtype = + Datatype(stringToDatatype(j["datatype"].get())); + switchType( + *parameters.dtype, j["value"], parameters); + } + else + { + Attribute attr = recoverAttributeFromJson(j, name); + *parameters.dtype = attr.dtype; + *parameters.resource = attr.getResource(); + } } catch (json::type_error &) { @@ -1571,7 +1831,10 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) if (res->contains(JSONDefaults::openpmd_internal)) { auto const &openpmd_internal = res->at(JSONDefaults::openpmd_internal); - if (openpmd_internal.contains(JSONDefaults::IOMode)) + + // Init dataset mode according to file's default + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue && + openpmd_internal.contains(JSONDefaults::IOMode)) { auto modeOption = openPMD::json::asLowerCaseStringDynamic( openpmd_internal.at(JSONDefaults::IOMode)); @@ -1585,17 +1848,42 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) } else if (modeOption.value() == "dataset") { - if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) - { - m_mode = IOMode::Dataset; - } + m_mode = IOMode::Dataset; } else if (modeOption.value() == "template") { - if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue) - { - m_mode = IOMode::Template; - } + m_mode = IOMode::Template; + } + else + { + std::cerr << "[JSON/TOML backend] Warning: Invalid value '" + << modeOption.value() + << "' at internal meta table for entry '" + << JSONDefaults::IOMode + << "'. Will ignore and continue." << std::endl; + } + } + + if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue && + openpmd_internal.contains(JSONDefaults::AttributeMode)) + { + auto modeOption = openPMD::json::asLowerCaseStringDynamic( + openpmd_internal.at(JSONDefaults::AttributeMode)); + if (!modeOption.has_value()) + { + std::cerr + << "[JSON/TOML backend] Warning: Invalid value of " + "non-string type at internal meta table for entry '" + << JSONDefaults::AttributeMode + << "'. Will ignore and continue." << std::endl; + } + else if (modeOption.value() == "long") + { + m_attributeMode = AttributeMode::Long; + } + else if (modeOption.value() == "short") + { + m_attributeMode = AttributeMode::Short; } else { @@ -1645,6 +1933,18 @@ auto JSONIOHandlerImpl::putJsonContents( break; } + switch (m_attributeMode) + { + case AttributeMode::Short: + (*it->second)[JSONDefaults::openpmd_internal] + [JSONDefaults::AttributeMode] = "short"; + break; + case AttributeMode::Long: + (*it->second)[JSONDefaults::openpmd_internal] + [JSONDefaults::AttributeMode] = "long"; + break; + } + auto writeSingleFile = [this, &it](std::string const &writeThisFile) { auto [fh, _, fh_with_precision] = getFilehandle(File(writeThisFile), Access::CREATE); From 0f2d33f84c7b1bfcac1620f0e8558d8d65ff1322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 4 Aug 2023 14:50:34 +0200 Subject: [PATCH 05/28] Add example 14_toml_template.cpp --- examples/14_toml_template.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 examples/14_toml_template.cpp diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp new file mode 100644 index 0000000000..33b9115129 --- /dev/null +++ b/examples/14_toml_template.cpp @@ -0,0 +1,22 @@ +#include + +int main() +{ + std::string config = R"( +{ + "iteration_encoding": "variable_based", + "toml": { + "dataset": {"mode": "template"}, + "attribute": {"mode": "short"} + } +} +)"; + + openPMD::Series writeTemplate( + "../samples/tomlTemplate.toml", openPMD::Access::CREATE, config); + auto iteration = writeTemplate.writeIterations()[0]; + + auto temperature = + iteration.meshes["temperature"][openPMD::RecordComponent::SCALAR]; + temperature.resetDataset({openPMD::Datatype::FLOAT, {5, 5}}); +} From 52e518d4e5157a6cf3fd2e0ba4b4f7c64ccf11f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 10 Mar 2023 15:42:59 +0100 Subject: [PATCH 06/28] Use Datatype::UNDEFINED to indicate no dataset definition in template --- include/openPMD/Dataset.hpp | 2 +- include/openPMD/RecordComponent.hpp | 2 +- .../openPMD/backend/PatchRecordComponent.hpp | 2 -- src/RecordComponent.cpp | 19 ++++++++++++------- src/backend/PatchRecordComponent.cpp | 17 ----------------- 5 files changed, 14 insertions(+), 28 deletions(-) diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index 0032888541..a610ce6a67 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -44,7 +44,7 @@ class Dataset JOINED_DIMENSION = std::numeric_limits::max() }; - Dataset(Datatype, Extent, std::string options = "{}"); + Dataset(Datatype, Extent = {1}, std::string options = "{}"); /** * @brief Constructor that sets the datatype to undefined. diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 9072b93a32..ee29a6d7fa 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -173,7 +173,7 @@ class RecordComponent : public BaseRecordComponent * * @return RecordComponent& */ - virtual RecordComponent &resetDataset(Dataset); + RecordComponent &resetDataset(Dataset); uint8_t getDimensionality() const; Extent getExtent() const; diff --git a/include/openPMD/backend/PatchRecordComponent.hpp b/include/openPMD/backend/PatchRecordComponent.hpp index 63875b11e2..5c0cf6bfe7 100644 --- a/include/openPMD/backend/PatchRecordComponent.hpp +++ b/include/openPMD/backend/PatchRecordComponent.hpp @@ -66,8 +66,6 @@ class PatchRecordComponent : public RecordComponent PatchRecordComponent &setUnitSI(double); - PatchRecordComponent &resetDataset(Dataset) override; - uint8_t getDimensionality() const; Extent getExtent() const; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 0387268514..fc17909fc6 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -104,15 +104,20 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) rc.m_hasBeenExtended = true; } - if (d.dtype == Datatype::UNDEFINED) + if (d.extent.empty()) + throw std::runtime_error("Dataset extent must be at least 1D."); + if (d.empty()) { - throw error::WrongAPIUsage( - "[RecordComponent] Must set specific datatype."); + if (d.dtype != Datatype::UNDEFINED) + { + return makeEmpty(std::move(d)); + } + else + { + rc.m_dataset = std::move(d); + return *this; + } } - // if( d.extent.empty() ) - // throw std::runtime_error("Dataset extent must be at least 1D."); - if (d.empty()) - return makeEmpty(std::move(d)); rc.m_isEmpty = false; if (written()) diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index af19923fad..2ac202e44a 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -34,23 +34,6 @@ PatchRecordComponent &PatchRecordComponent::setUnitSI(double usi) return *this; } -PatchRecordComponent &PatchRecordComponent::resetDataset(Dataset d) -{ - if (written()) - throw std::runtime_error( - "A Records Dataset can not (yet) be changed after it has been " - "written."); - if (d.extent.empty()) - throw std::runtime_error("Dataset extent must be at least 1D."); - if (d.empty()) - throw std::runtime_error( - "Dataset extent must not be zero in any dimension."); - - get().m_dataset = std::move(d); - setDirty(true); - return *this; -} - uint8_t PatchRecordComponent::getDimensionality() const { return 1; From e03d21d343636360acfbc4b0a298797bf42d3d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 19 May 2022 17:29:24 +0200 Subject: [PATCH 07/28] Extend example --- examples/14_toml_template.cpp | 91 +++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp index 33b9115129..284db9ee84 100644 --- a/examples/14_toml_template.cpp +++ b/examples/14_toml_template.cpp @@ -1,6 +1,21 @@ #include -int main() +std::string backendEnding() +{ + auto extensions = openPMD::getFileExtensions(); + if (auto it = std::find(extensions.begin(), extensions.end(), "toml"); + it != extensions.end()) + { + return *it; + } + else + { + // Fallback for buggy old NVidia compiler + return "json"; + } +} + +void write() { std::string config = R"( { @@ -13,10 +28,80 @@ int main() )"; openPMD::Series writeTemplate( - "../samples/tomlTemplate.toml", openPMD::Access::CREATE, config); + "../samples/tomlTemplate." + backendEnding(), + openPMD::Access::CREATE, + config); auto iteration = writeTemplate.writeIterations()[0]; + openPMD::Dataset ds{openPMD::Datatype::FLOAT, {5, 5}}; + auto temperature = iteration.meshes["temperature"][openPMD::RecordComponent::SCALAR]; - temperature.resetDataset({openPMD::Datatype::FLOAT, {5, 5}}); + temperature.resetDataset(ds); + + auto E = iteration.meshes["E"]; + E["x"].resetDataset(ds); + E["y"].resetDataset(ds); + /* + * Don't specify datatype and extent for this one to indicate that this + * information is not yet known. + */ + E["z"].resetDataset({openPMD::Datatype::UNDEFINED}); + + ds.extent = {10}; + + auto electrons = iteration.particles["e"]; + electrons["position"]["x"].resetDataset(ds); + electrons["position"]["y"].resetDataset(ds); + electrons["position"]["z"].resetDataset(ds); + + electrons["positionOffset"]["x"].resetDataset(ds); + electrons["positionOffset"]["y"].resetDataset(ds); + electrons["positionOffset"]["z"].resetDataset(ds); + electrons["positionOffset"]["x"].makeConstant(3.14); + electrons["positionOffset"]["y"].makeConstant(3.14); + electrons["positionOffset"]["z"].makeConstant(3.14); + + ds.dtype = openPMD::determineDatatype(); + electrons.particlePatches["numParticles"][openPMD::RecordComponent::SCALAR] + .resetDataset(ds); + electrons + .particlePatches["numParticlesOffset"][openPMD::RecordComponent::SCALAR] + .resetDataset(ds); + electrons.particlePatches["offset"]["x"].resetDataset(ds); + electrons.particlePatches["offset"]["y"].resetDataset(ds); + electrons.particlePatches["offset"]["z"].resetDataset(ds); + electrons.particlePatches["extent"]["x"].resetDataset(ds); + electrons.particlePatches["extent"]["y"].resetDataset(ds); + electrons.particlePatches["extent"]["z"].resetDataset(ds); +} + +void read() +{ + /* + * The config is entirely optional, these things are also detected + * automatically when reading + */ + + // std::string config = R"( + // { + // "iteration_encoding": "variable_based", + // "toml": { + // "dataset": {"mode": "template"}, + // "attribute": {"mode": "short"} + // } + // } + // )"; + + openPMD::Series read( + "../samples/tomlTemplate." + backendEnding(), + openPMD::Access::READ_LINEAR); + read.readIterations(); // @todo change to read.parseBase() + openPMD::helper::listSeries(read); +} + +int main() +{ + write(); + read(); } From ce3aab4743cb1a7f72d0e897044d0177595c96ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 7 Aug 2023 11:04:07 +0200 Subject: [PATCH 08/28] Test short attribute mode --- test/SerialIOTest.cpp | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 96d2e0879b..fbbee51c06 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1561,7 +1561,17 @@ TEST_CASE("dtype_test", "[serial]") { dtype_test(t); } - dtype_test("json", R"({"json":{"dataset":{"mode":"template"}}})"); + dtype_test("json", R"( +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})"); if (auto extensions = getFileExtensions(); std::find(extensions.begin(), extensions.end(), "toml") != extensions.end()) @@ -1570,7 +1580,17 @@ TEST_CASE("dtype_test", "[serial]") * testing it here. */ dtype_test("toml"); - dtype_test("toml", R"({"toml":{"dataset":{"mode":"template"}}})"); + dtype_test("toml", R"( +{ + "toml": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})"); } } @@ -1594,8 +1614,17 @@ inline void write_test(const std::string &backend) host_info::byMethod( host_info::methodFromStringDescription("posix_hostname", false))}}; #endif - jsonCfg = - json::merge(jsonCfg, R"({"json":{"dataset":{"mode":"template"}}})"); + jsonCfg = json::merge(jsonCfg, R"( +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})"); Series o = Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); From 06d7ec1caa485ad26e1fd8cbb809ba18df0e8e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 7 Aug 2023 11:26:07 +0200 Subject: [PATCH 09/28] Copy datatypeToString to JSON implementation --- src/IO/JSON/JSONIOHandlerImpl.cpp | 96 ++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 4a3116a62f..aac257df68 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -158,6 +158,95 @@ namespace } } } + + // Does the same as datatypeToString(), but this makes sure that we don't + // accidentally change the JSON schema by modifying datatypeToString() + std::string jsonDatatypeToString(Datatype dt) + { + switch (dt) + { + using DT = Datatype; + case DT::CHAR: + return "CHAR"; + case DT::UCHAR: + return "UCHAR"; + case DT::SCHAR: + return "SCHAR"; + case DT::SHORT: + return "SHORT"; + case DT::INT: + return "INT"; + case DT::LONG: + return "LONG"; + case DT::LONGLONG: + return "LONGLONG"; + case DT::USHORT: + return "USHORT"; + case DT::UINT: + return "UINT"; + case DT::ULONG: + return "ULONG"; + case DT::ULONGLONG: + return "ULONGLONG"; + case DT::FLOAT: + return "FLOAT"; + case DT::DOUBLE: + return "DOUBLE"; + case DT::LONG_DOUBLE: + return "LONG_DOUBLE"; + case DT::CFLOAT: + return "CFLOAT"; + case DT::CDOUBLE: + return "CDOUBLE"; + case DT::CLONG_DOUBLE: + return "CLONG_DOUBLE"; + case DT::STRING: + return "STRING"; + case DT::VEC_CHAR: + return "VEC_CHAR"; + case DT::VEC_SHORT: + return "VEC_SHORT"; + case DT::VEC_INT: + return "VEC_INT"; + case DT::VEC_LONG: + return "VEC_LONG"; + case DT::VEC_LONGLONG: + return "VEC_LONGLONG"; + case DT::VEC_UCHAR: + return "VEC_UCHAR"; + case DT::VEC_USHORT: + return "VEC_USHORT"; + case DT::VEC_UINT: + return "VEC_UINT"; + case DT::VEC_ULONG: + return "VEC_ULONG"; + case DT::VEC_ULONGLONG: + return "VEC_ULONGLONG"; + case DT::VEC_FLOAT: + return "VEC_FLOAT"; + case DT::VEC_DOUBLE: + return "VEC_DOUBLE"; + case DT::VEC_LONG_DOUBLE: + return "VEC_LONG_DOUBLE"; + case DT::VEC_CFLOAT: + return "VEC_CFLOAT"; + case DT::VEC_CDOUBLE: + return "VEC_CDOUBLE"; + case DT::VEC_CLONG_DOUBLE: + return "VEC_CLONG_DOUBLE"; + case DT::VEC_SCHAR: + return "VEC_SCHAR"; + case DT::VEC_STRING: + return "VEC_STRING"; + case DT::ARR_DBL_7: + return "ARR_DBL_7"; + case DT::BOOL: + return "BOOL"; + case DT::UNDEFINED: + return "UNDEFINED"; + } + return "Unreachable!"; + } } // namespace auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) @@ -484,7 +573,7 @@ void JSONIOHandlerImpl::createDataset( } setAndGetFilePosition(writable, name); auto &dset = jsonVal[name]; - dset["datatype"] = datatypeToString(parameter.dtype); + dset["datatype"] = jsonDatatypeToString(parameter.dtype); switch (localMode) { @@ -1140,7 +1229,8 @@ void JSONIOHandlerImpl::writeAttribute( { case AttributeMode::Long: (*jsonVal)[filePosition->id]["attributes"][parameter.name] = { - {"datatype", datatypeToString(parameter.dtype)}, {"value", value}}; + {"datatype", jsonDatatypeToString(parameter.dtype)}, + {"value", value}}; break; case AttributeMode::Short: // short form @@ -2223,7 +2313,7 @@ nlohmann::json JSONIOHandlerImpl::platformSpecifics() Datatype::BOOL}; for (auto it = std::begin(datatypes); it != std::end(datatypes); it++) { - res[datatypeToString(*it)] = toBytes(*it); + res[jsonDatatypeToString(*it)] = toBytes(*it); } return res; } From b29d64d2cbee42e8e6621b2a0af19a98fe06f03c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 22 Sep 2023 17:31:34 +0200 Subject: [PATCH 10/28] Fix after rebase: Init JSON config in parallel mode --- src/IO/JSON/JSONIOHandlerImpl.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index aac257df68..7755f17c23 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -395,16 +395,25 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( #if openPMD_HAVE_MPI JSONIOHandlerImpl::JSONIOHandlerImpl( AbstractIOHandler *handler, - MPI_Comm comm, - // NOLINTNEXTLINE(performance-unnecessary-value-param) - [[maybe_unused]] openPMD::json::TracingJSON config, + MPI_Comm comm,openPMD::json::TracingJSON config, FileFormat format, std::string originalExtension) : AbstractIOHandlerImpl(handler) , m_communicator{comm} , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} -{} +{ + std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); + std::tie(m_attributeMode, m_attributeModeSpecificationVia) = + retrieveAttributeMode(config); + + if (auto [_, backendConfig] = getBackendConfig(config); + backendConfig.has_value()) + { + (void)_; + warnUnusedJson(backendConfig.value()); + } +} #endif JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; From 76f24214788ea337ea0e0310fae93020368de755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 22 Sep 2023 17:32:07 +0200 Subject: [PATCH 11/28] Fix after rebase: Don't erase JSON datasets when writing --- src/IO/JSON/JSONIOHandlerImpl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 7755f17c23..a89d75c41c 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -2159,6 +2159,7 @@ merge the .json files somehow (no tooling provided for this (yet)). #else serialImplementation(); #endif + if (unsetDirty) { m_dirty.erase(filename); From d5534cb759bb4f8eca85b0468e3f7e0004a4fe87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Oct 2023 09:48:23 +0200 Subject: [PATCH 12/28] openpmd-pipe: use short modes for test --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 409a5c771d..87cf2f860e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1328,6 +1328,9 @@ if(openPMD_BUILD_TESTING) ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ --infile ../samples/git-sample/thetaMode/data_%T.bp \ --outfile ../samples/git-sample/thetaMode/data%T.json \ + --outconfig ' \ + json.attribute.mode = \"short\" \n\ + json.dataset.mode = \"template\"' \ " WORKING_DIRECTORY ${openPMD_RUNTIME_OUTPUT_DIRECTORY} ) From 7983763a4e558ce1d6077db32721218e4b7e3efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Oct 2023 10:50:28 +0200 Subject: [PATCH 13/28] Less intrusive warnings, allow disabling them --- CMakeLists.txt | 2 +- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 17 +++++- src/IO/JSON/JSONIOHandlerImpl.cpp | 52 +++++++++++++------ 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 87cf2f860e..e5272ae2de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1330,7 +1330,7 @@ if(openPMD_BUILD_TESTING) --outfile ../samples/git-sample/thetaMode/data%T.json \ --outconfig ' \ json.attribute.mode = \"short\" \n\ - json.dataset.mode = \"template\"' \ + json.dataset.mode = \"template_no_warn\"' \ " WORKING_DIRECTORY ${openPMD_RUNTIME_OUTPUT_DIRECTORY} ) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 0a5a9779fd..68d0635fd1 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -294,9 +294,22 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl IOMode m_mode = IOMode::Dataset; SpecificationVia m_IOModeSpecificationVia = SpecificationVia::DefaultValue; + bool m_printedSkippedWriteWarningAlready = false; - std::pair - retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + struct DatasetMode + { + IOMode m_IOMode; + SpecificationVia m_specificationVia; + bool m_skipWarnings; + + template + operator std::tuple() + { + return std::tuple{ + m_IOMode, m_specificationVia, m_skipWarnings}; + } + }; + DatasetMode retrieveDatasetMode(openPMD::json::TracingJSON &config) const; /////////////////////// // Attribute IO mode // diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index a89d75c41c..6b924f4ee2 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -249,11 +249,12 @@ namespace } } // namespace -auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) - const -> std::pair +auto JSONIOHandlerImpl::retrieveDatasetMode( + openPMD::json::TracingJSON &config) const -> DatasetMode { - IOMode res = m_mode; - SpecificationVia res_2 = SpecificationVia::DefaultValue; + IOMode ioMode = m_mode; + SpecificationVia specificationVia = SpecificationVia::DefaultValue; + bool skipWarnings = false; if (auto [configLocation, maybeConfig] = getBackendConfig(config); maybeConfig.has_value()) { @@ -275,13 +276,19 @@ auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) auto mode = modeOption.value(); if (mode == "dataset") { - res = IOMode::Dataset; - res_2 = SpecificationVia::Manually; + ioMode = IOMode::Dataset; + specificationVia = SpecificationVia::Manually; } else if (mode == "template") { - res = IOMode::Template; - res_2 = SpecificationVia::Manually; + ioMode = IOMode::Template; + specificationVia = SpecificationVia::Manually; + } + else if (mode == "template_no_warn") + { + ioMode = IOMode::Template; + specificationVia = SpecificationVia::Manually; + skipWarnings = true; } else { @@ -293,7 +300,7 @@ auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config) } } } - return std::make_pair(res, res_2); + return DatasetMode{ioMode, specificationVia, skipWarnings}; } auto JSONIOHandlerImpl::retrieveAttributeMode( @@ -380,7 +387,9 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { - std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); + std::tie( + m_mode, m_IOModeSpecificationVia, m_printedSkippedWriteWarningAlready) = + retrieveDatasetMode(config); std::tie(m_attributeMode, m_attributeModeSpecificationVia) = retrieveAttributeMode(config); @@ -403,7 +412,9 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { - std::tie(m_mode, m_IOModeSpecificationVia) = retrieveDatasetMode(config); + std::tie( + m_mode, m_IOModeSpecificationVia, m_printedSkippedWriteWarningAlready) = + retrieveDatasetMode(config); std::tie(m_attributeMode, m_attributeModeSpecificationVia) = retrieveAttributeMode(config); @@ -558,7 +569,13 @@ void JSONIOHandlerImpl::createDataset( parameter.options, /* considerFiles = */ false); // Retrieves mode from dataset-specific configuration, falls back to global // value if not defined - IOMode localMode = retrieveDatasetMode(config).first; + auto [localMode, _, skipWarnings] = retrieveDatasetMode(config); + (void)_; + // No use in introducing logic to skip warnings only for one particular + // dataset. If warnings are skipped, then they are skipped consistently. + // Use |= since `false` is the default value and we don't wish to reset + // the flag. + m_printedSkippedWriteWarningAlready |= skipWarnings; parameter.warnUnusedParameters( config, @@ -1194,9 +1211,14 @@ void JSONIOHandlerImpl::writeDataset( case IOMode::Dataset: break; case IOMode::Template: - std::cerr << "[JSON/TOML backend: Warning] Trying to write data to a " - "template dataset. Will skip." - << std::endl; + if (!m_printedSkippedWriteWarningAlready) + { + std::cerr + << "[JSON/TOML backend: Warning] Trying to write data to a " + "template dataset. Will skip." + << std::endl; + m_printedSkippedWriteWarningAlready = true; + } return; } From f413d755d2d3dfe75575592ecd80e2b4d98f6140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Oct 2023 11:29:09 +0200 Subject: [PATCH 14/28] TOML: Use short modes by default --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 2 ++ src/IO/JSON/JSONIOHandlerImpl.cpp | 34 ++++++++++++------- test/SerialIOTest.cpp | 25 ++++++++++---- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 68d0635fd1..fc369047ec 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -180,6 +180,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl std::string originalExtension); #endif + void init(openPMD::json::TracingJSON config); + ~JSONIOHandlerImpl() override; void diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 6b924f4ee2..b986da8cde 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -387,18 +387,7 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { - std::tie( - m_mode, m_IOModeSpecificationVia, m_printedSkippedWriteWarningAlready) = - retrieveDatasetMode(config); - std::tie(m_attributeMode, m_attributeModeSpecificationVia) = - retrieveAttributeMode(config); - - if (auto [_, backendConfig] = getBackendConfig(config); - backendConfig.has_value()) - { - (void)_; - warnUnusedJson(backendConfig.value()); - } + init(std::move(config)); } #if openPMD_HAVE_MPI @@ -412,6 +401,26 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_fileFormat{format} , m_originalExtension{std::move(originalExtension)} { + init(std::move(config)); +} +#endif + +void JSONIOHandlerImpl::init(openPMD::json::TracingJSON config) +{ + // set the defaults + switch (m_fileFormat) + { + case FileFormat::Json: + // @todo take the switch to openPMD 2.0 as a chance to switch to + // short attribute mode as a default here + m_attributeMode = AttributeMode::Long; + m_mode = IOMode::Dataset; + break; + case FileFormat::Toml: + m_attributeMode = AttributeMode::Short; + m_mode = IOMode::Template; + break; + } std::tie( m_mode, m_IOModeSpecificationVia, m_printedSkippedWriteWarningAlready) = retrieveDatasetMode(config); @@ -425,7 +434,6 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( warnUnusedJson(backendConfig.value()); } } -#endif JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index fbbee51c06..a3e068d6b1 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1288,8 +1288,12 @@ inline void dtype_test( "../samples/dtype_test." + backend, Access::CREATE, activateTemplateMode.value()) - : Series("../samples/dtype_test." + backend, Access::CREATE); - + : + // test TOML long attribute mode by default + Series( + "../samples/dtype_test." + backend, + Access::CREATE, + R"({"toml":{"attribute":{"mode":"long"}}})"); char c = 'c'; s.setAttribute("char", c); unsigned char uc = 'u'; @@ -1905,7 +1909,7 @@ inline void fileBased_write_test(const std::string &backend) Series o = Series( "../samples/subdir/serial_fileBased_write%03T." + backend, Access::CREATE, - jsonCfg); + json::merge(jsonCfg, R"({"toml":{"dataset":{"mode":"dataset"}}})")); REQUIRE( auxiliary::replace_all(o.myPath().filePath(), "\\", "/") == auxiliary::replace_all( @@ -7563,7 +7567,10 @@ void groupbased_read_write(std::string const &ext) std::string filename = "../samples/groupbased_read_write." + ext; { - Series write(filename, Access::CREATE); + Series write( + filename, + Access::CREATE, + R"({"toml":{"dataset":{"mode":"dataset"}}})"); auto E_x = write.iterations[0].meshes["E"]["x"]; auto E_y = write.iterations[0].meshes["E"]["y"]; E_x.resetDataset(ds); @@ -7578,7 +7585,10 @@ void groupbased_read_write(std::string const &ext) } { - Series write(filename, Access::READ_WRITE); + Series write( + filename, + Access::READ_WRITE, + R"({"toml":{"dataset":{"mode":"dataset"}}})"); // create a new iteration auto E_x = write.iterations[1].meshes["E"]["x"]; E_x.resetDataset(ds); @@ -7624,7 +7634,10 @@ void groupbased_read_write(std::string const &ext) // check that truncation works correctly { - Series write(filename, Access::CREATE); + Series write( + filename, + Access::CREATE, + R"({"toml":{"dataset":{"mode":"dataset"}}})"); // create a new iteration auto E_x = write.iterations[2].meshes["E"]["x"]; E_x.resetDataset(ds); From 429ade5b99df81b4d2a4d12710f68ec2f3d55338 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 12:48:52 +0000 Subject: [PATCH 15/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/IO/JSON/JSONIOHandlerImpl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index b986da8cde..9aac51f29c 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -393,7 +393,8 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( #if openPMD_HAVE_MPI JSONIOHandlerImpl::JSONIOHandlerImpl( AbstractIOHandler *handler, - MPI_Comm comm,openPMD::json::TracingJSON config, + MPI_Comm comm, + openPMD::json::TracingJSON config, FileFormat format, std::string originalExtension) : AbstractIOHandlerImpl(handler) From c309e890d9e1ea143d831873d85b7b24efb3f446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 24 Nov 2023 16:16:31 +0100 Subject: [PATCH 16/28] Documentation --- docs/source/backends/json.rst | 36 +++++++++++++++++++++++---- docs/source/details/backendconfig.rst | 23 ++++++++++++++--- docs/source/details/json.json | 10 ++++++++ 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 docs/source/details/json.json diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index bbae92aaf6..0f0d9510c9 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -38,20 +38,46 @@ when working with the JSON backend. Datasets and groups have the same namespace, meaning that there may not be a subgroup and a dataset with the same name contained in one group. -Any **openPMD dataset** is a JSON object with three keys: +Datasets +........ - * ``attributes``: Attributes associated with the dataset. May be ``null`` or not present if no attributes are associated with the dataset. - * ``datatype``: A string describing the type of the stored data. - * ``data`` A nested array storing the actual data in row-major manner. +Datasets can be stored in two modes, either as actual datasets or as dataset templates. +The mode is selected by the :ref:`JSON/TOML parameter` ``json.dataset.mode`` (resp. ``toml.dataset.mode``) with possible values ``["dataset", "template"]`` (default: ``"dataset"``). + +Stored as an actual dataset, an **openPMD dataset** is a JSON object with three JSON keys: + + * ``datatype`` (required): A string describing the type of the stored data. + * ``data`` (required): A nested array storing the actual data in row-major manner. The data needs to be consistent with the fields ``datatype`` and ``extent``. Checking whether this key points to an array can be (and is internally) used to distinguish groups from datasets. + * ``attributes``: Attributes associated with the dataset. May be ``null`` or not present if no attributes are associated with the dataset. + +Stored as a **dataset template**, an openPMD dataset is represented by three JSON keys: + +* ``datatype`` (required): As above. +* ``extent`` (required): A list of integers, describing the extent of the dataset. +* ``attributes``: As above. -**Attributes** are stored as a JSON object with a key for each attribute. +This mode stores only the dataset metadata. +Chunk load/store operations are ignored. + +Attributes +.......... + +In order to avoid name clashes, attributes are generally stored within a separate subgroup ``attributes``. + +Attributes can be stored in two formats. +The format is selected by the :ref:`JSON/TOML parameter` ``json.attribute.mode`` (resp. ``toml.attribute.mode``) with possible values ``["long", "short"]`` (default: ``"long"`` in openPMD 1.*, ``"short"`` in openPMD >= 2.0). + +Attributes in **long format** store the datatype explicitly, by representing attributes as JSON objects. Every such attribute is itself a JSON object with two keys: * ``datatype``: A string describing the type of the value. * ``value``: The actual value of type ``datatype``. +Attributes in **short format** are stored as just the simple value corresponding with the attribute. +Since JSON/TOML values are pretty-printed into a human-readable format, byte-level type details can be lost when reading those values again later on (e.g. the distinction between different integer types). + TOML File Format ---------------- diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index f6d15a7ac8..cf78d9cdea 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -104,6 +104,8 @@ The key ``rank_table`` allows specifying the creation of a **rank table**, used Configuration Structure per Backend ----------------------------------- +Please refer to the respective backends' documentations for further information on their configuration. + .. _backendconfig-adios2: ADIOS2 @@ -231,8 +233,21 @@ The parameters eligible for being passed to flush calls may be configured global .. _backendconfig-other: -Other backends -^^^^^^^^^^^^^^ +JSON/TOML +^^^^^^^^^ -Do currently not read the configuration string. -Please refer to the respective backends' documentations for further information on their configuration. +A full configuration of the JSON backend: + +.. literalinclude:: json.json + :language: json + +The TOML backend is configured analogously, replacing the ``"json"`` key with ``"toml"``. + +All keys found under ``hdf5.dataset`` are applicable globally as well as per dataset. +Explanation of the single keys: + +* ``json.dataset.mode`` / ``toml.dataset.mode``: One of ``"dataset"`` (default) or ``"template"``. + In "dataset" mode, the dataset will be written as an n-dimensional (recursive) array, padded with nulls (JSON) or zeroes (TOML) for missing values. + In "template" mode, only the dataset metadata (type, extent and attributes) are stored and no chunks can be written or read. +* ``json.attribute.mode`` / ``toml.attribute.mode``: One of ``"long"`` (default in openPMD 1.*) or ``"short"`` (default in openPMD 2.*). + The long format explicitly encodes the attribute type in the dataset on disk, the short format only writes the actual attribute as a JSON/TOML value, requiring readers to recover the type. diff --git a/docs/source/details/json.json b/docs/source/details/json.json new file mode 100644 index 0000000000..c1491f7245 --- /dev/null +++ b/docs/source/details/json.json @@ -0,0 +1,10 @@ +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +} From e5f117780af6ac96f48b61dbb6d34702660b4130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 24 Nov 2023 17:47:57 +0100 Subject: [PATCH 17/28] Short mode in default in openPMD >= 2. --- include/openPMD/IO/IOTask.hpp | 1 + include/openPMD/RecordComponent.tpp | 51 ++++++++++++++++++++++++++--- src/IO/JSON/JSONIOHandlerImpl.cpp | 7 ++++ src/Iteration.cpp | 1 + src/Series.cpp | 1 + 5 files changed, 57 insertions(+), 4 deletions(-) diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 5589db6923..7c6735f44f 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -142,6 +142,7 @@ struct OPENPMDAPI_EXPORT Parameter } std::string name = ""; + std::string openPMDversion; // @todo: Maybe move this to AbstractIOHandler }; template <> diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 542503e806..42c29994a2 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -21,6 +21,8 @@ #pragma once +#include "openPMD/Datatype.hpp" +#include "openPMD/Error.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/Span.hpp" #include "openPMD/auxiliary/Memory.hpp" @@ -93,12 +95,38 @@ inline std::shared_ptr RecordComponent::loadChunk(Offset o, Extent e) #endif } +namespace detail +{ + template + struct do_convert + { + template + static std::optional call(Attribute &attr) + { + if constexpr (std::is_convertible_v) + { + return std::make_optional(attr.get()); + } + else + { + return std::nullopt; + } + } + + static constexpr char const *errorMsg = "is_conversible"; + }; +} // namespace detail + template inline void RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) { Datatype dtype = determineDatatype(data); - if (dtype != getDatatype()) + /* + * For constant components, we implement type conversion, so there is + * a separate check further below. + */ + if (dtype != getDatatype() && !constant()) if (!isSameInteger(getDatatype()) && !isSameFloatingPoint(getDatatype()) && !isSameComplexFloatingPoint(getDatatype()) && @@ -160,10 +188,25 @@ RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) for (auto const &dimensionSize : extent) numPoints *= dimensionSize; - T value = rc.m_constantValue.get(); + std::optional val = + switchNonVectorType>( + /* from = */ getDatatype(), rc.m_constantValue); - T *raw_ptr = data.get(); - std::fill(raw_ptr, raw_ptr + numPoints, value); + if (val.has_value()) + { + T *raw_ptr = data.get(); + std::fill(raw_ptr, raw_ptr + numPoints, *val); + } + else + { + std::string const data_type_str = datatypeToString(getDatatype()); + std::string const requ_type_str = + datatypeToString(determineDatatype()); + std::string err_msg = + "Type conversion during chunk loading not possible! "; + err_msg += "Data: " + data_type_str + "; Load as: " + requ_type_str; + throw error::WrongAPIUsage(err_msg); + } } else { diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 9aac51f29c..cfa213eb86 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -461,6 +461,13 @@ void JSONIOHandlerImpl::createFile( access::write(m_handler->m_backendAccess), "[JSON] Creating a file in read-only mode is not possible."); + if (m_attributeModeSpecificationVia == SpecificationVia::DefaultValue) + { + m_attributeMode = parameters.openPMDversion >= "2." + ? AttributeMode::Short + : AttributeMode::Long; + } + if (!writable->written) { std::string name = parameters.name + m_originalExtension; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 931a1f9e3d..1684bfd709 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -227,6 +227,7 @@ void Iteration::flushFileBased( /* create file */ Parameter fCreate; fCreate.name = filename; + fCreate.openPMDversion = s.openPMD(); IOHandler()->enqueue(IOTask(&s.writable(), fCreate)); /* diff --git a/src/Series.cpp b/src/Series.cpp index 792be0555f..802036de20 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1490,6 +1490,7 @@ void Series::flushGorVBased( } Parameter fCreate; fCreate.name = series.m_name; + fCreate.openPMDversion = openPMD(); IOHandler()->enqueue(IOTask(this, fCreate)); flushRankTable(); From 183c1a16a767a61e2300a151f365d94bc78d9069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 19 Mar 2024 11:38:59 +0100 Subject: [PATCH 18/28] Short value by default in TOML --- docs/source/backends/json.rst | 2 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 0f0d9510c9..bba6ca5df0 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -67,7 +67,7 @@ Attributes In order to avoid name clashes, attributes are generally stored within a separate subgroup ``attributes``. Attributes can be stored in two formats. -The format is selected by the :ref:`JSON/TOML parameter` ``json.attribute.mode`` (resp. ``toml.attribute.mode``) with possible values ``["long", "short"]`` (default: ``"long"`` in openPMD 1.*, ``"short"`` in openPMD >= 2.0). +The format is selected by the :ref:`JSON/TOML parameter` ``json.attribute.mode`` (resp. ``toml.attribute.mode``) with possible values ``["long", "short"]`` (default: ``"long"`` for JSON in openPMD 1.*, ``"short"`` otherwise, i.e. generally in openPMD 2.*, but always in TOML). Attributes in **long format** store the datatype explicitly, by representing attributes as JSON objects. Every such attribute is itself a JSON object with two keys: diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index cfa213eb86..1f0f4008a2 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -463,9 +463,18 @@ void JSONIOHandlerImpl::createFile( if (m_attributeModeSpecificationVia == SpecificationVia::DefaultValue) { - m_attributeMode = parameters.openPMDversion >= "2." - ? AttributeMode::Short - : AttributeMode::Long; + switch (m_fileFormat) + { + + case FileFormat::Json: + m_attributeMode = parameters.openPMDversion >= "2." + ? AttributeMode::Short + : AttributeMode::Long; + break; + case FileFormat::Toml: + m_attributeMode = AttributeMode::Short; + break; + } } if (!writable->written) From 395cd10f942830096a5593c21f166a4591bc0cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 19 Mar 2024 13:21:16 +0100 Subject: [PATCH 19/28] Store the openPMD version information in the IOHandler --- include/openPMD/IO/AbstractIOHandler.hpp | 1 + include/openPMD/IO/IOTask.hpp | 1 - src/IO/JSON/JSONIOHandlerImpl.cpp | 2 +- src/Iteration.cpp | 1 - src/Series.cpp | 1 - 5 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 649252a877..e8e55457eb 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -201,6 +201,7 @@ class AbstractIOHandler { friend class Series; friend class ADIOS2IOHandlerImpl; + friend class JSONIOHandlerImpl; friend class detail::ADIOS2File; private: diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 7c6735f44f..5589db6923 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -142,7 +142,6 @@ struct OPENPMDAPI_EXPORT Parameter } std::string name = ""; - std::string openPMDversion; // @todo: Maybe move this to AbstractIOHandler }; template <> diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 1f0f4008a2..67de0366ad 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -467,7 +467,7 @@ void JSONIOHandlerImpl::createFile( { case FileFormat::Json: - m_attributeMode = parameters.openPMDversion >= "2." + m_attributeMode = m_handler->m_standard >= OpenpmdStandard::v_2_0_0 ? AttributeMode::Short : AttributeMode::Long; break; diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 1684bfd709..931a1f9e3d 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -227,7 +227,6 @@ void Iteration::flushFileBased( /* create file */ Parameter fCreate; fCreate.name = filename; - fCreate.openPMDversion = s.openPMD(); IOHandler()->enqueue(IOTask(&s.writable(), fCreate)); /* diff --git a/src/Series.cpp b/src/Series.cpp index 802036de20..792be0555f 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1490,7 +1490,6 @@ void Series::flushGorVBased( } Parameter fCreate; fCreate.name = series.m_name; - fCreate.openPMDversion = openPMD(); IOHandler()->enqueue(IOTask(this, fCreate)); flushRankTable(); From d4b6f889b4c284520ab0d4793d3ceccde57c46fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 26 Mar 2024 12:39:26 +0100 Subject: [PATCH 20/28] Fixes --- examples/14_toml_template.cpp | 4 ++++ src/IO/JSON/JSONIOHandlerImpl.cpp | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp index 284db9ee84..f595e9b10f 100644 --- a/examples/14_toml_template.cpp +++ b/examples/14_toml_template.cpp @@ -20,6 +20,10 @@ void write() std::string config = R"( { "iteration_encoding": "variable_based", + "json": { + "dataset": {"mode": "template"}, + "attribute": {"mode": "short"} + }, "toml": { "dataset": {"mode": "template"}, "attribute": {"mode": "short"} diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 67de0366ad..8c7a563110 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -649,14 +649,16 @@ void JSONIOHandlerImpl::createDataset( break; } case IOMode::Template: - if (parameter.extent != Extent{0}) + if (parameter.extent != Extent{0} && + parameter.dtype != Datatype::UNDEFINED) { dset["extent"] = parameter.extent; } else { // no-op - // If extent is empty, don't bother writing it + // If extent is empty or no datatype is defined, don't bother + // writing it } break; } From afa01651ed24d3cdd6d6d29ce76298bd82fa16be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 7 Jun 2024 14:33:33 +0200 Subject: [PATCH 21/28] Adapt test to recent rebase Reading the chunk table requires NOT using template mode, otherwise the string just consists of '\0' bytes. --- test/SerialIOTest.cpp | 63 ++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index a3e068d6b1..e80a5b8778 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1607,28 +1607,18 @@ struct ReadFromAnyType } }; -inline void write_test(const std::string &backend) +inline void write_test( + const std::string &backend, + std::string jsonCfg = "{}", + bool test_rank_table = true) { -#ifdef _WIN32 - std::string jsonCfg = "{}"; -#else - std::string jsonCfg = R"({"rank_table": "posix_hostname"})"; +#ifndef _WIN32 + jsonCfg = json::merge(jsonCfg, R"({"rank_table": "posix_hostname"})"); chunk_assignment::RankMeta compare{ {0, host_info::byMethod( host_info::methodFromStringDescription("posix_hostname", false))}}; #endif - jsonCfg = json::merge(jsonCfg, R"( -{ - "json": { - "dataset": { - "mode": "template" - }, - "attribute": { - "mode": "short" - } - } -})"); Series o = Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg); @@ -1745,7 +1735,10 @@ inline void write_test(const std::string &backend) variantTypeDataset); #ifndef _WIN32 - REQUIRE(read.rankTable(/* collective = */ false) == compare); + if (test_rank_table) + { + REQUIRE(read.rankTable(/* collective = */ false) == compare); + } #endif } @@ -1753,7 +1746,41 @@ TEST_CASE("write_test", "[serial]") { for (auto const &t : testedFileExtensions()) { - write_test(t); + if (t == "json") + { + write_test( + "template." + t, + R"( +{ + "json": { + "dataset": { + "mode": "template" + }, + "attribute": { + "mode": "short" + } + } +})", + false); + write_test( + t, + R"( +{ + "json": { + "dataset": { + "mode": "dataset" + }, + "attribute": { + "mode": "short" + } + } +})", + true); + } + else + { + write_test(t); + } Series list{"../samples/serial_write." + t, Access::READ_ONLY}; helper::listSeries(list); } From 50ea53c687b62d5630caf0866b7f769c14f9c860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 5 Aug 2024 11:43:54 +0200 Subject: [PATCH 22/28] toml11 4.0 compatibility --- src/IO/JSON/JSONIOHandlerImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 8c7a563110..ed0d90e068 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -152,7 +152,7 @@ namespace auto asToml = openPMD::json::jsonToToml(shadow); std::cerr << "Warning: parts of the backend configuration for " "JSON/TOML backend remain unused:\n" - << asToml << std::endl; + << json::format_toml(asToml) << std::endl; break; } } From 61a84c6d8858dd72d8eb4f030dfb4fcac5663e47 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:31:19 +0000 Subject: [PATCH 23/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/SerialIOTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index e80a5b8778..9c06ac0e3e 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1,8 +1,8 @@ // expose private and protected members for invasive testing -#include "openPMD/auxiliary/JSON.hpp" #include "openPMD/ChunkInfo_internal.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" +#include "openPMD/auxiliary/JSON.hpp" #if openPMD_USE_INVASIVE_TESTS #define OPENPMD_private public: #define OPENPMD_protected public: From d5b35e268af9e742f3131833b8de87737cf9ebc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 11 Dec 2024 17:40:12 +0100 Subject: [PATCH 24/28] wip: cleanup --- examples/14_toml_template.cpp | 4 ++-- include/openPMD/Dataset.hpp | 14 ++++++++++---- src/IO/JSON/JSONIOHandlerImpl.cpp | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/examples/14_toml_template.cpp b/examples/14_toml_template.cpp index f595e9b10f..29d19fb53a 100644 --- a/examples/14_toml_template.cpp +++ b/examples/14_toml_template.cpp @@ -50,7 +50,7 @@ void write() * Don't specify datatype and extent for this one to indicate that this * information is not yet known. */ - E["z"].resetDataset({openPMD::Datatype::UNDEFINED}); + E["z"].resetDataset({}); ds.extent = {10}; @@ -100,7 +100,7 @@ void read() openPMD::Series read( "../samples/tomlTemplate." + backendEnding(), openPMD::Access::READ_LINEAR); - read.readIterations(); // @todo change to read.parseBase() + read.parseBase(); openPMD::helper::listSeries(read); } diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index a610ce6a67..ad1125d9d3 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -41,18 +41,24 @@ class Dataset public: enum : std::uint64_t { - JOINED_DIMENSION = std::numeric_limits::max() + JOINED_DIMENSION = std::numeric_limits::max(), + UNDEFINED_EXTENT = std::numeric_limits::max() - 1 }; - Dataset(Datatype, Extent = {1}, std::string options = "{}"); + Dataset(Datatype, Extent, std::string options = "{}"); /** * @brief Constructor that sets the datatype to undefined. * - * Helpful for resizing datasets, since datatypes need not be given twice. + * Helpful for: + * + * 1. Resizing datasets, since datatypes need not be given twice. + * 2. Initializing datasets as undefined, as used by template mode in the + * JSON/TOML backend. In this case, the default (undefined) specification + * for the Extent may be used. * */ - Dataset(Extent); + Dataset(Extent = {UNDEFINED_EXTENT}); Dataset &extend(Extent newExtent); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index ed0d90e068..3336bedc5e 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -650,7 +650,7 @@ void JSONIOHandlerImpl::createDataset( } case IOMode::Template: if (parameter.extent != Extent{0} && - parameter.dtype != Datatype::UNDEFINED) + parameter.extent[0] != Dataset::UNDEFINED_EXTENT) { dset["extent"] = parameter.extent; } @@ -2012,7 +2012,7 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) } } - if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue && + if (m_attributeModeSpecificationVia == SpecificationVia::DefaultValue && openpmd_internal.contains(JSONDefaults::AttributeMode)) { auto modeOption = openPMD::json::asLowerCaseStringDynamic( From 346c37e91d4912e3b7c5323fa26a5ad077d8b6a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Dec 2024 14:43:46 +0100 Subject: [PATCH 25/28] wip: cleanup --- docs/source/backends/json.rst | 7 +- docs/source/details/backendconfig.rst | 6 +- include/openPMD/Dataset.hpp | 16 ++ include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 44 ++-- src/IO/JSON/JSONIOHandlerImpl.cpp | 200 +++++++++--------- 5 files changed, 154 insertions(+), 119 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index bba6ca5df0..9b618c9e03 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -54,9 +54,10 @@ Stored as an actual dataset, an **openPMD dataset** is a JSON object with three Stored as a **dataset template**, an openPMD dataset is represented by three JSON keys: -* ``datatype`` (required): As above. -* ``extent`` (required): A list of integers, describing the extent of the dataset. -* ``attributes``: As above. + * ``datatype`` (required): As above. + * ``extent`` (required): A list of integers, describing the extent of the dataset. + This replaces the ``data`` key from the non-template representation. + * ``attributes``: As above. This mode stores only the dataset metadata. Chunk load/store operations are ignored. diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index cf78d9cdea..8b60d21a90 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -243,11 +243,11 @@ A full configuration of the JSON backend: The TOML backend is configured analogously, replacing the ``"json"`` key with ``"toml"``. -All keys found under ``hdf5.dataset`` are applicable globally as well as per dataset. +All keys found under ``json.dataset`` are applicable globally as well as per dataset. Explanation of the single keys: * ``json.dataset.mode`` / ``toml.dataset.mode``: One of ``"dataset"`` (default) or ``"template"``. In "dataset" mode, the dataset will be written as an n-dimensional (recursive) array, padded with nulls (JSON) or zeroes (TOML) for missing values. - In "template" mode, only the dataset metadata (type, extent and attributes) are stored and no chunks can be written or read. -* ``json.attribute.mode`` / ``toml.attribute.mode``: One of ``"long"`` (default in openPMD 1.*) or ``"short"`` (default in openPMD 2.*). + In "template" mode, only the dataset metadata (type, extent and attributes) are stored and no chunks can be written or read (i.e. write/read operations will be skipped). +* ``json.attribute.mode`` / ``toml.attribute.mode``: One of ``"long"`` (default in openPMD 1.*) or ``"short"`` (default in openPMD 2.* and generally in TOML). The long format explicitly encodes the attribute type in the dataset on disk, the short format only writes the actual attribute as a JSON/TOML value, requiring readers to recover the type. diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index ad1125d9d3..d79380105a 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -41,7 +41,23 @@ class Dataset public: enum : std::uint64_t { + /** + * Setting one dimension of the extent as JOINED_DIMENSION means that + * the extent along that dimension will be defined by the sum of all + * parallel processes' contributions. + * Only one dimension can be joined. For store operations, the offset + * should be an empty array and the extent should give the actual + * extent of the chunk (i.e. the number of joined elements along the + * joined dimension, equal to the global extent in all other + * dimensions). For more details, refer to + * docs/source/usage/workflow.rst. + */ JOINED_DIMENSION = std::numeric_limits::max(), + /** + * Some backends (i.e. JSON and TOML in template mode) support the + * creation of dataset with undefined datatype and extent. + * The extent should be given as {UNDEFINED_EXTENT} for that. + */ UNDEFINED_EXTENT = std::numeric_limits::max() - 1 }; diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index fc369047ec..38966e3b82 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -267,6 +267,10 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl */ FileFormat m_fileFormat{}; + /* + * Under which key do we find the backend configuration? + * -> "json" for the JSON backend, "toml" for the TOML backend. + */ std::string backendConfigKey() const; /* @@ -278,6 +282,10 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl std::string m_originalExtension; + /* + * Was the config value explicitly user-chosen, or are we still working with + * defaults? + */ enum class SpecificationVia { DefaultValue, @@ -288,19 +296,21 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // Dataset IO mode // ///////////////////// - enum class IOMode + enum class DatasetMode { Dataset, Template }; - IOMode m_mode = IOMode::Dataset; - SpecificationVia m_IOModeSpecificationVia = SpecificationVia::DefaultValue; - bool m_printedSkippedWriteWarningAlready = false; + // IOMode m_mode{}; + // SpecificationVia m_IOModeSpecificationVia = + // SpecificationVia::DefaultValue; bool m_printedSkippedWriteWarningAlready + // = false; - struct DatasetMode + struct DatasetMode_s { - IOMode m_IOMode; + // Initialized in init() + DatasetMode m_mode{}; SpecificationVia m_specificationVia; bool m_skipWarnings; @@ -308,10 +318,11 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl operator std::tuple() { return std::tuple{ - m_IOMode, m_specificationVia, m_skipWarnings}; + m_mode, m_specificationVia, m_skipWarnings}; } }; - DatasetMode retrieveDatasetMode(openPMD::json::TracingJSON &config) const; + DatasetMode_s m_datasetMode; + DatasetMode_s retrieveDatasetMode(openPMD::json::TracingJSON &config) const; /////////////////////// // Attribute IO mode // @@ -323,11 +334,16 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl Long }; - AttributeMode m_attributeMode = AttributeMode::Long; - SpecificationVia m_attributeModeSpecificationVia = - SpecificationVia::DefaultValue; + struct AttributeMode_s + { + // Will be modified in init() based on the openPMD version and the + // active file format (JSON/TOML) + AttributeMode m_mode{}; + SpecificationVia m_specificationVia = SpecificationVia::DefaultValue; + }; + AttributeMode_s m_attributeMode; - std::pair + AttributeMode_s retrieveAttributeMode(openPMD::json::TracingJSON &config) const; // HELPER FUNCTIONS @@ -376,7 +392,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // essentially: m_i = \prod_{j=0}^{i-1} extent_j static Extent getMultiplicators(Extent const &extent); - static std::pair getExtent(nlohmann::json &j); + static std::pair getExtent(nlohmann::json &j); // remove single '/' in the beginning and end of a string static std::string removeSlashes(std::string); @@ -434,7 +450,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // check whether the json reference contains a valid dataset template - IOMode verifyDataset(Param const ¶meters, nlohmann::json &); + DatasetMode verifyDataset(Param const ¶meters, nlohmann::json &); static nlohmann::json platformSpecifics(); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 3336bedc5e..9b671c097e 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -69,7 +69,7 @@ namespace JSONDefaults { using const_str = char const *const; constexpr const_str openpmd_internal = "__openPMD_internal"; - constexpr const_str IOMode = "dataset_mode"; + constexpr const_str DatasetMode = "dataset_mode"; constexpr const_str AttributeMode = "attribute_mode"; } // namespace JSONDefaults @@ -103,7 +103,11 @@ namespace } #endif - static constexpr char const *errorMsg = "JSON default value"; + template + static nlohmann::json call() + { + return 0; + } }; /* @@ -250,11 +254,13 @@ namespace } // namespace auto JSONIOHandlerImpl::retrieveDatasetMode( - openPMD::json::TracingJSON &config) const -> DatasetMode + openPMD::json::TracingJSON &config) const -> DatasetMode_s { - IOMode ioMode = m_mode; - SpecificationVia specificationVia = SpecificationVia::DefaultValue; - bool skipWarnings = false; + // start with / copy from current config + auto res = m_datasetMode; + DatasetMode &ioMode = res.m_mode; + SpecificationVia &specificationVia = res.m_specificationVia; + bool &skipWarnings = res.m_skipWarnings; if (auto [configLocation, maybeConfig] = getBackendConfig(config); maybeConfig.has_value()) { @@ -276,17 +282,17 @@ auto JSONIOHandlerImpl::retrieveDatasetMode( auto mode = modeOption.value(); if (mode == "dataset") { - ioMode = IOMode::Dataset; + ioMode = DatasetMode::Dataset; specificationVia = SpecificationVia::Manually; } else if (mode == "template") { - ioMode = IOMode::Template; + ioMode = DatasetMode::Template; specificationVia = SpecificationVia::Manually; } else if (mode == "template_no_warn") { - ioMode = IOMode::Template; + ioMode = DatasetMode::Template; specificationVia = SpecificationVia::Manually; skipWarnings = true; } @@ -300,15 +306,16 @@ auto JSONIOHandlerImpl::retrieveDatasetMode( } } } - return DatasetMode{ioMode, specificationVia, skipWarnings}; + return res; } auto JSONIOHandlerImpl::retrieveAttributeMode( - openPMD::json::TracingJSON &config) const - -> std::pair + openPMD::json::TracingJSON &config) const -> AttributeMode_s { - AttributeMode res = m_attributeMode; - SpecificationVia res_2 = SpecificationVia::DefaultValue; + // start with / copy from current config + auto res = m_attributeMode; + AttributeMode &mode = res.m_mode; + SpecificationVia &specificationVia = res.m_specificationVia; if (auto [configLocation, maybeConfig] = getBackendConfig(config); maybeConfig.has_value()) { @@ -327,28 +334,28 @@ auto JSONIOHandlerImpl::retrieveAttributeMode( "Invalid value of non-string type (accepted values are " "'dataset' and 'template'."); } - auto mode = modeOption.value(); - if (mode == "short") + auto modeCfg = modeOption.value(); + if (modeCfg == "short") { - res = AttributeMode::Short; - res_2 = SpecificationVia::Manually; + mode = AttributeMode::Short; + specificationVia = SpecificationVia::Manually; } - else if (mode == "long") + else if (modeCfg == "long") { - res = AttributeMode::Long; - res_2 = SpecificationVia::Manually; + mode = AttributeMode::Long; + specificationVia = SpecificationVia::Manually; } else { throw error::BackendConfigSchema( {configLocation, "attribute", "mode"}, - "Invalid value: '" + mode + + "Invalid value: '" + modeCfg + "' (accepted values are 'short' and 'long'."); } } } } - return std::make_pair(res, res_2); + return res; } std::string JSONIOHandlerImpl::backendConfigKey() const @@ -412,21 +419,21 @@ void JSONIOHandlerImpl::init(openPMD::json::TracingJSON config) switch (m_fileFormat) { case FileFormat::Json: - // @todo take the switch to openPMD 2.0 as a chance to switch to - // short attribute mode as a default here - m_attributeMode = AttributeMode::Long; - m_mode = IOMode::Dataset; + m_attributeMode.m_mode = + m_handler->m_standard >= OpenpmdStandard::v_2_0_0 + ? AttributeMode::Short + : AttributeMode::Long; + m_datasetMode.m_mode = DatasetMode::Dataset; break; case FileFormat::Toml: - m_attributeMode = AttributeMode::Short; - m_mode = IOMode::Template; + m_attributeMode.m_mode = AttributeMode::Short; + m_datasetMode.m_mode = DatasetMode::Dataset; break; } - std::tie( - m_mode, m_IOModeSpecificationVia, m_printedSkippedWriteWarningAlready) = - retrieveDatasetMode(config); - std::tie(m_attributeMode, m_attributeModeSpecificationVia) = - retrieveAttributeMode(config); + + // now modify according to config + m_datasetMode = retrieveDatasetMode(config); + m_attributeMode = retrieveAttributeMode(config); if (auto [_, backendConfig] = getBackendConfig(config); backendConfig.has_value()) @@ -461,22 +468,6 @@ void JSONIOHandlerImpl::createFile( access::write(m_handler->m_backendAccess), "[JSON] Creating a file in read-only mode is not possible."); - if (m_attributeModeSpecificationVia == SpecificationVia::DefaultValue) - { - switch (m_fileFormat) - { - - case FileFormat::Json: - m_attributeMode = m_handler->m_standard >= OpenpmdStandard::v_2_0_0 - ? AttributeMode::Short - : AttributeMode::Long; - break; - case FileFormat::Toml: - m_attributeMode = AttributeMode::Short; - break; - } - } - if (!writable->written) { std::string name = parameters.name + m_originalExtension; @@ -594,13 +585,14 @@ void JSONIOHandlerImpl::createDataset( parameter.options, /* considerFiles = */ false); // Retrieves mode from dataset-specific configuration, falls back to global // value if not defined + DatasetMode_s dm; auto [localMode, _, skipWarnings] = retrieveDatasetMode(config); (void)_; // No use in introducing logic to skip warnings only for one particular // dataset. If warnings are skipped, then they are skipped consistently. // Use |= since `false` is the default value and we don't wish to reset // the flag. - m_printedSkippedWriteWarningAlready |= skipWarnings; + m_datasetMode.m_skipWarnings |= skipWarnings; parameter.warnUnusedParameters( config, @@ -628,7 +620,7 @@ void JSONIOHandlerImpl::createDataset( switch (localMode) { - case IOMode::Dataset: { + case DatasetMode::Dataset: { auto extent = parameter.extent; switch (parameter.dtype) { @@ -641,14 +633,22 @@ void JSONIOHandlerImpl::createDataset( default: break; } - // TOML does not support nulls, so initialize with zero - dset["data"] = initializeNDArray( - extent, - m_fileFormat == FileFormat::Json ? std::optional{} - : parameter.dtype); + if (parameter.extent.size() == 1 && + parameter.extent[0] == Dataset::UNDEFINED_EXTENT) + { + dset["data"] = std::vector(0); + } + else + { + // TOML does not support nulls, so initialize with zero + dset["data"] = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional{} + : parameter.dtype); + } break; } - case IOMode::Template: + case DatasetMode::Template: if (parameter.extent != Extent{0} && parameter.extent[0] != Dataset::UNDEFINED_EXTENT) { @@ -700,7 +700,7 @@ void JSONIOHandlerImpl::extendDataset( refreshFileFromParent(writable); auto &j = obtainJsonContents(writable); - IOMode localIOMode; + DatasetMode localIOMode; try { Extent datasetExtent; @@ -724,7 +724,7 @@ void JSONIOHandlerImpl::extendDataset( switch (localIOMode) { - case IOMode::Dataset: { + case DatasetMode::Dataset: { auto extent = parameters.extent; auto datatype = stringToDatatype(j["datatype"].get()); switch (datatype) @@ -749,7 +749,7 @@ void JSONIOHandlerImpl::extendDataset( j["data"] = newData; } break; - case IOMode::Template: { + case DatasetMode::Template: { j["extent"] = parameters.extent; } break; @@ -1235,16 +1235,16 @@ void JSONIOHandlerImpl::writeDataset( switch (verifyDataset(parameters, j)) { - case IOMode::Dataset: + case DatasetMode::Dataset: break; - case IOMode::Template: - if (!m_printedSkippedWriteWarningAlready) + case DatasetMode::Template: + if (!m_datasetMode.m_skipWarnings) { std::cerr << "[JSON/TOML backend: Warning] Trying to write data to a " "template dataset. Will skip." << std::endl; - m_printedSkippedWriteWarningAlready = true; + m_datasetMode.m_skipWarnings = true; } return; } @@ -1283,7 +1283,7 @@ void JSONIOHandlerImpl::writeAttribute( } nlohmann::json value; switchType(parameter.dtype, value, parameter.resource); - switch (m_attributeMode) + switch (m_attributeMode.m_mode) { case AttributeMode::Long: (*jsonVal)[filePosition->id]["attributes"][parameter.name] = { @@ -1326,18 +1326,18 @@ void JSONIOHandlerImpl::readDataset( refreshFileFromParent(writable); setAndGetFilePosition(writable); auto &j = obtainJsonContents(writable); - IOMode localMode = verifyDataset(parameters, j); + DatasetMode localMode = verifyDataset(parameters, j); switch (localMode) { - case IOMode::Template: + case DatasetMode::Template: std::cerr << "[Warning] Cannot read chunks in Template mode of JSON " "backend. Will fill with zeroes instead." << std::endl; switchNonVectorType( parameters.dtype, parameters.data.get(), parameters.extent); return; - case IOMode::Dataset: + case DatasetMode::Dataset: try { switchType(parameters.dtype, j["data"], parameters); @@ -1803,13 +1803,13 @@ Extent JSONIOHandlerImpl::getMultiplicators(Extent const &extent) } auto JSONIOHandlerImpl::getExtent(nlohmann::json &j) - -> std::pair + -> std::pair { Extent res; - IOMode ioMode; + DatasetMode ioMode; if (j.contains("data")) { - ioMode = IOMode::Dataset; + ioMode = DatasetMode::Dataset; nlohmann::json *ptr = &j["data"]; while (ptr->is_array()) { @@ -1831,12 +1831,12 @@ auto JSONIOHandlerImpl::getExtent(nlohmann::json &j) } else if (j.contains("extent")) { - ioMode = IOMode::Template; + ioMode = DatasetMode::Template; res = j["extent"].get(); } else { - ioMode = IOMode::Template; + ioMode = DatasetMode::Template; res = {0}; } return std::make_pair(std::move(res), ioMode); @@ -1981,38 +1981,40 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) auto const &openpmd_internal = res->at(JSONDefaults::openpmd_internal); // Init dataset mode according to file's default - if (m_IOModeSpecificationVia == SpecificationVia::DefaultValue && - openpmd_internal.contains(JSONDefaults::IOMode)) + if (m_datasetMode.m_specificationVia == + SpecificationVia::DefaultValue && + openpmd_internal.contains(JSONDefaults::DatasetMode)) { auto modeOption = openPMD::json::asLowerCaseStringDynamic( - openpmd_internal.at(JSONDefaults::IOMode)); + openpmd_internal.at(JSONDefaults::DatasetMode)); if (!modeOption.has_value()) { std::cerr << "[JSON/TOML backend] Warning: Invalid value of " "non-string type at internal meta table for entry '" - << JSONDefaults::IOMode << "'. Will ignore and continue." - << std::endl; + << JSONDefaults::DatasetMode + << "'. Will ignore and continue.\n"; } else if (modeOption.value() == "dataset") { - m_mode = IOMode::Dataset; + m_datasetMode.m_mode = DatasetMode::Dataset; } else if (modeOption.value() == "template") { - m_mode = IOMode::Template; + m_datasetMode.m_mode = DatasetMode::Template; } else { std::cerr << "[JSON/TOML backend] Warning: Invalid value '" << modeOption.value() << "' at internal meta table for entry '" - << JSONDefaults::IOMode - << "'. Will ignore and continue." << std::endl; + << JSONDefaults::DatasetMode + << "'. Will ignore and continue." << '\n'; } } - if (m_attributeModeSpecificationVia == SpecificationVia::DefaultValue && + if (m_attributeMode.m_specificationVia == + SpecificationVia::DefaultValue && openpmd_internal.contains(JSONDefaults::AttributeMode)) { auto modeOption = openPMD::json::asLowerCaseStringDynamic( @@ -2023,23 +2025,23 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) << "[JSON/TOML backend] Warning: Invalid value of " "non-string type at internal meta table for entry '" << JSONDefaults::AttributeMode - << "'. Will ignore and continue." << std::endl; + << "'. Will ignore and continue." << '\n'; } else if (modeOption.value() == "long") { - m_attributeMode = AttributeMode::Long; + m_attributeMode.m_mode = AttributeMode::Long; } else if (modeOption.value() == "short") { - m_attributeMode = AttributeMode::Short; + m_attributeMode.m_mode = AttributeMode::Short; } else { std::cerr << "[JSON/TOML backend] Warning: Invalid value '" << modeOption.value() << "' at internal meta table for entry '" - << JSONDefaults::IOMode - << "'. Will ignore and continue." << std::endl; + << JSONDefaults::DatasetMode + << "'. Will ignore and continue." << '\n'; } } } @@ -2068,20 +2070,20 @@ auto JSONIOHandlerImpl::putJsonContents( return it; } - switch (m_mode) + switch (m_datasetMode.m_mode) { - case IOMode::Dataset: + case DatasetMode::Dataset: (*it->second)["platform_byte_widths"] = platformSpecifics(); - (*it->second)[JSONDefaults::openpmd_internal][JSONDefaults::IOMode] = - "dataset"; + (*it->second)[JSONDefaults::openpmd_internal] + [JSONDefaults::DatasetMode] = "dataset"; break; - case IOMode::Template: - (*it->second)[JSONDefaults::openpmd_internal][JSONDefaults::IOMode] = - "template"; + case DatasetMode::Template: + (*it->second)[JSONDefaults::openpmd_internal] + [JSONDefaults::DatasetMode] = "template"; break; } - switch (m_attributeMode) + switch (m_attributeMode.m_mode) { case AttributeMode::Short: (*it->second)[JSONDefaults::openpmd_internal] @@ -2314,13 +2316,13 @@ bool JSONIOHandlerImpl::isGroup(nlohmann::json::const_iterator const &it) template auto JSONIOHandlerImpl::verifyDataset( - Param const ¶meters, nlohmann::json &j) -> IOMode + Param const ¶meters, nlohmann::json &j) -> DatasetMode { VERIFY_ALWAYS( isDataset(j), "[JSON] Specified dataset does not exist or is not a dataset."); - IOMode res; + DatasetMode res; try { Extent datasetExtent; From be3543af61a51c8248ff93e5e6674392802c7d70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Dec 2024 17:16:09 +0100 Subject: [PATCH 26/28] Cleanup --- include/openPMD/RecordComponent.tpp | 4 + src/IO/JSON/JSONIOHandlerImpl.cpp | 279 +++++++++++++++++++--------- 2 files changed, 192 insertions(+), 91 deletions(-) diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 42c29994a2..7beaae8b9d 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -125,6 +125,10 @@ RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) /* * For constant components, we implement type conversion, so there is * a separate check further below. + * This is especially useful for the short-attribute representation in the + * JSON/TOML backends as they might implicitly turn a LONG into an INT in a + * constant component. The frontend needs to catch such edge cases. + * Ref. `if (constant())` branch. */ if (dtype != getDatatype() && !constant()) if (!isSameInteger(getDatatype()) && diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 9b671c097e..0cf82ebd5e 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -25,7 +25,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" -#include "openPMD/IO/Access.hpp" +#include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Memory.hpp" @@ -150,13 +150,13 @@ namespace case openPMD::json::SupportedLanguages::JSON: std::cerr << "Warning: parts of the backend configuration for " "JSON/TOML backend remain unused:\n" - << shadow << std::endl; + << shadow << '\n'; break; case openPMD::json::SupportedLanguages::TOML: { auto asToml = openPMD::json::jsonToToml(shadow); std::cerr << "Warning: parts of the backend configuration for " "JSON/TOML backend remain unused:\n" - << json::format_toml(asToml) << std::endl; + << json::format_toml(asToml) << '\n'; break; } } @@ -419,10 +419,11 @@ void JSONIOHandlerImpl::init(openPMD::json::TracingJSON config) switch (m_fileFormat) { case FileFormat::Json: - m_attributeMode.m_mode = - m_handler->m_standard >= OpenpmdStandard::v_2_0_0 - ? AttributeMode::Short - : AttributeMode::Long; + // Set the attribute mode to Long for now, needs to be evaluated + // again when creating a new file, since the openPMD version might + // be specified via Series::setOpenPMD() after initialization of the + // JSON backend. + m_attributeMode.m_mode = AttributeMode::Long; m_datasetMode.m_mode = DatasetMode::Dataset; break; case FileFormat::Toml: @@ -468,6 +469,25 @@ void JSONIOHandlerImpl::createFile( access::write(m_handler->m_backendAccess), "[JSON] Creating a file in read-only mode is not possible."); + /* + * Need to resolve this later than init() since the openPMD version might be + * specified after the creation of the IOHandler. + */ + if (m_attributeMode.m_specificationVia == SpecificationVia::DefaultValue) + { + switch (m_fileFormat) + { + case FileFormat::Json: + m_attributeMode.m_mode = + m_handler->m_standard >= OpenpmdStandard::v_2_0_0 + ? AttributeMode::Short + : AttributeMode::Long; + break; + default: + break; + } + } + if (!writable->written) { std::string name = parameters.name + m_originalExtension; @@ -658,7 +678,8 @@ void JSONIOHandlerImpl::createDataset( { // no-op // If extent is empty or no datatype is defined, don't bother - // writing it + // writing it. + // The datatype is written above anyway. } break; } @@ -1243,7 +1264,7 @@ void JSONIOHandlerImpl::writeDataset( std::cerr << "[JSON/TOML backend: Warning] Trying to write data to a " "template dataset. Will skip." - << std::endl; + << '\n'; m_datasetMode.m_skipWarnings = true; } return; @@ -1333,7 +1354,7 @@ void JSONIOHandlerImpl::readDataset( case DatasetMode::Template: std::cerr << "[Warning] Cannot read chunks in Template mode of JSON " "backend. Will fill with zeroes instead." - << std::endl; + << '\n'; switchNonVectorType( parameters.dtype, parameters.data.get(), parameters.extent); return; @@ -1344,8 +1365,11 @@ void JSONIOHandlerImpl::readDataset( } catch (json::basic_json::type_error &) { - throw std::runtime_error( - "[JSON] The given path does not contain a valid dataset."); + throw error::ReadError( + error::AffectedObject::Dataset, + error::Reason::UnexpectedContent, + "JSON", + "The given path does not contain a valid dataset."); } break; } @@ -1353,6 +1377,18 @@ void JSONIOHandlerImpl::readDataset( namespace { + /* + * While the short attribute representation is more easily human-readable + * (and ultimately also closer to the idea of JSON), this means that + * recovering the actual datatype of an attribute is now more difficult. + * The functions in this anonymous namespace take care of doing that. + */ + + /* + * Input: Element type `T` that has already been resolved and a JSON value + * `j` containing a flat array with elements of type `T`. + * Output: An openPMD Attribute containing that array. + */ template Attribute recoverVectorAttributeFromJson(nlohmann::json const &j) { @@ -1369,7 +1405,10 @@ namespace std::is_same_v)) { /* - * The frontend must deal with wrong type reports here. + * The JSON value does not contain enough information to distinguish + * ARRAY_DOUBLE_7 from other VECTOR types. Return the array type if + * it applies, the frontend must deal with correctly converting + * to vector types when needed. */ std::array res; for (size_t i = 0; i < 7; ++i) @@ -1390,6 +1429,17 @@ namespace } } + /* + * Input: A JSON array whose first element has been found to be some numeric + * type. + * + * We now need to decide, if the array has type unsigned, integer or + * float. All elements need to be inspected for this since the first element + * might be `1`, but the third might be `-3.14`, and we need a datatype + * generic enough to represent all elements. + * + * Output: That datatype as instance of the nlohmann::json::value_t enum. + */ nlohmann::json::value_t unifyNumericType(nlohmann::json const &j) { if (!j.is_array() || j.empty()) @@ -1435,6 +1485,107 @@ namespace return res; } + /* Input: A JSON array `j`, additionally its name for use in error messages + * Output: The array as an openPMD Attribute with adequate recovered + * datatype + */ + Attribute recoverVectorAttributeFromJson( + nlohmann::json const &j, std::string const &nameForErrorMessages) + { + if (j.empty()) + { +#if 0 // probably no need to warn here + std::cerr << "Cannot recover datatype of empty vector without " + "explicit type annotation for attribute '" + << nameForErrorMessages + << "'. Will continue with VEC_INT datatype." + << '\n'; +#endif + /* + * Since an empty array's datatype cannot be recovered without + * type annotations, we need to use some type. + * In that case, use integers. + */ + return std::vector{}; + } + + auto valueType = j[0].type(); + /* + * If the vector is of numeric type, it might happen that the + * first entry is an integer, but a later entry is a float. + * We need to pick the most generic datatype in that case. + */ + if (valueType == nlohmann::json::value_t::number_float || + valueType == nlohmann::json::value_t::number_unsigned || + valueType == nlohmann::json::value_t::number_integer) + { + valueType = unifyNumericType(j); + } + switch (valueType) + { + case nlohmann::json::value_t::null: + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "JSON", + " Attribute must not be null: '" + nameForErrorMessages + "'."); + case nlohmann::json::value_t::object: + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "JSON", + " Invalid contained datatype (object) " + "inside vector-type attribute: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::array: + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "JSON", + " Invalid contained datatype (array) " + "inside vector-type attribute: '" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::string: + return recoverVectorAttributeFromJson(j); + case nlohmann::json::value_t::boolean: + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "JSON", + " Attribute must not be vector of bool: " + "'" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::number_integer: + return recoverVectorAttributeFromJson< + nlohmann::json::number_integer_t>(j); + case nlohmann::json::value_t::number_unsigned: + return recoverVectorAttributeFromJson< + nlohmann::json::number_unsigned_t>(j); + case nlohmann::json::value_t::number_float: + return recoverVectorAttributeFromJson< + nlohmann::json::number_float_t>(j); + case nlohmann::json::value_t::binary: + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "JSON", + " Attribute must not have binary type: " + "'" + + nameForErrorMessages + "'."); + case nlohmann::json::value_t::discarded: + throw std::runtime_error( + "Internal JSON parser datatype leaked into JSON " + "value."); + } + throw std::runtime_error("Unreachable!"); + } + + /* + * Read a shorthand-type JSON attribute into an openPMD attribute, + * recovering the datatype from the JSON value. + * Note that precise datatype-preserving roundtrips are not possible due to + * JSON not encoding byte-level type details. + */ Attribute recoverAttributeFromJson( nlohmann::json const &j, std::string const &nameForErrorMessages) { @@ -1442,82 +1593,21 @@ namespace switch (j.type()) { case nlohmann::json::value_t::null: - throw std::runtime_error( - "[JSON backend] Attribute must not be null: '" + - nameForErrorMessages + "'."); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "JSON", + "Attribute must not be null: '" + nameForErrorMessages + "'."); case nlohmann::json::value_t::object: - throw std::runtime_error( - "[JSON backend] Shorthand-style attribute must not be an " + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "JSON", + "Shorthand-style attribute must not be an " "object: '" + - nameForErrorMessages + "'."); + nameForErrorMessages + "'."); case nlohmann::json::value_t::array: - if (j.empty()) - { - std::cerr << "Cannot recover datatype of empty vector without " - "explicit type annotation for attribute '" - << nameForErrorMessages - << "'. Will continue with VEC_INT datatype." - << std::endl; - return std::vector{}; - } - else - { - auto valueType = j[0].type(); - /* - * If the vector is of numeric type, it might happen that the - * first entry is an integer, but a later entry is a float. - * We need to pick the most generic datatype in that case. - */ - if (valueType == nlohmann::json::value_t::number_float || - valueType == nlohmann::json::value_t::number_unsigned || - valueType == nlohmann::json::value_t::number_integer) - { - valueType = unifyNumericType(j); - } - switch (valueType) - { - case nlohmann::json::value_t::null: - throw std::runtime_error( - "[JSON backend] Attribute must not be null: '" + - nameForErrorMessages + "'."); - case nlohmann::json::value_t::object: - throw std::runtime_error( - "[JSON backend] Invalid contained datatype (object) " - "inside vector-type attribute: '" + - nameForErrorMessages + "'."); - case nlohmann::json::value_t::array: - throw std::runtime_error( - "[JSON backend] Invalid contained datatype (array) " - "inside vector-type attribute: '" + - nameForErrorMessages + "'."); - case nlohmann::json::value_t::string: - return recoverVectorAttributeFromJson(j); - case nlohmann::json::value_t::boolean: - throw std::runtime_error( - "[JSON backend] Attribute must not be vector of bool: " - "'" + - nameForErrorMessages + "'."); - case nlohmann::json::value_t::number_integer: - return recoverVectorAttributeFromJson< - nlohmann::json::number_integer_t>(j); - case nlohmann::json::value_t::number_unsigned: - return recoverVectorAttributeFromJson< - nlohmann::json::number_unsigned_t>(j); - case nlohmann::json::value_t::number_float: - return recoverVectorAttributeFromJson< - nlohmann::json::number_float_t>(j); - case nlohmann::json::value_t::binary: - throw std::runtime_error( - "[JSON backend] Attribute must not have binary type: " - "'" + - nameForErrorMessages + "'."); - case nlohmann::json::value_t::discarded: - throw std::runtime_error( - "Internal JSON parser datatype leaked into JSON " - "value."); - } - throw std::runtime_error("Unreachable!"); - } + return recoverVectorAttributeFromJson(j, nameForErrorMessages); case nlohmann::json::value_t::string: return j.get(); case nlohmann::json::value_t::boolean: @@ -1529,9 +1619,12 @@ namespace case nlohmann::json::value_t::number_float: return j.get(); case nlohmann::json::value_t::binary: - throw std::runtime_error( - "[JSON backend] Attribute must not have binary type: '" + - nameForErrorMessages + "'."); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "JSON", + " Attribute must not have binary type: '" + + nameForErrorMessages + "'."); case nlohmann::json::value_t::discarded: throw std::runtime_error( "Internal JSON parser datatype leaked into JSON value."); @@ -1837,7 +1930,7 @@ auto JSONIOHandlerImpl::getExtent(nlohmann::json &j) else { ioMode = DatasetMode::Template; - res = {0}; + res = {Dataset::UNDEFINED_EXTENT}; } return std::make_pair(std::move(res), ioMode); } @@ -1980,7 +2073,10 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) { auto const &openpmd_internal = res->at(JSONDefaults::openpmd_internal); - // Init dataset mode according to file's default + // Init dataset mode according to file's default. + // Note that dataset parsing will expect and properly deal with both + // representations. The mode to be detected here will determine the the + // layout of newly created datasets, e.g. in READ_WRITE or APPEND mode. if (m_datasetMode.m_specificationVia == SpecificationVia::DefaultValue && openpmd_internal.contains(JSONDefaults::DatasetMode)) @@ -2013,6 +2109,7 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) } } + // Same for attribute mode if (m_attributeMode.m_specificationVia == SpecificationVia::DefaultValue && openpmd_internal.contains(JSONDefaults::AttributeMode)) From 7f81e62400bb426edf153745c67b999bb6a7e197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 13 Dec 2024 19:16:19 +0100 Subject: [PATCH 27/28] Extensive testing --- src/IO/JSON/JSONIOHandlerImpl.cpp | 8 +- test/JSONTest.cpp | 191 ++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 6 deletions(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 0cf82ebd5e..a432737188 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -653,12 +653,8 @@ void JSONIOHandlerImpl::createDataset( default: break; } - if (parameter.extent.size() == 1 && - parameter.extent[0] == Dataset::UNDEFINED_EXTENT) - { - dset["data"] = std::vector(0); - } - else + if (parameter.extent.size() != 1 || + parameter.extent[0] != Dataset::UNDEFINED_EXTENT) { // TOML does not support nulls, so initialize with zero dset["data"] = initializeNDArray( diff --git a/test/JSONTest.cpp b/test/JSONTest.cpp index 161f1fa3a3..78ffc6e01e 100644 --- a/test/JSONTest.cpp +++ b/test/JSONTest.cpp @@ -1,6 +1,7 @@ #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/Error.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/helper/list_series.hpp" #include "openPMD/openPMD.hpp" #include @@ -9,6 +10,7 @@ #include #include #include +#include #include #include @@ -306,3 +308,192 @@ TEST_CASE("variableBasedModifiedSnapshot", "[auxiliary]") testRead(std::vector{1, 2, 3, 4, 5}); } + +namespace auxiliary +{ +template +void test_matrix_impl(Callable &callable, AccumulatorTuple tuple) +{ + std::apply(callable, std::move(tuple)); +} + +template < + typename Callable, + typename AccumulatorTuple, + typename Arg, + typename... Args> +void test_matrix_impl( + Callable &callable, + AccumulatorTuple tuple, + std::vector const &arg, + std::vector const &...args) +{ + for (auto &val : arg) + { + test_matrix_impl( + callable, std::tuple_cat(tuple, std::tuple{val}), args...); + } +} + +template +void test_matrix(Callable &&callable, std::vector const &...matrix) +{ + test_matrix_impl(callable, std::tuple<>(), matrix...); +} +} // namespace auxiliary + +void json_short_modes( + std::optional short_attributes, + std::optional template_datasets, + std::string const &standardVersion, + std::string const &backend, + unsigned int *name_counter) +{ + nlohmann::json config = nlohmann::json::object(); + if (short_attributes.has_value()) + { + config[backend]["attribute"]["mode"] = + *short_attributes ? "short" : "long"; + } + if (template_datasets.has_value()) + { + config[backend]["dataset"]["mode"] = + *template_datasets ? "template" : "dataset"; + } + std::string name = "../samples/json_short_modes/test" + + std::to_string((*name_counter)++) + "." + backend; + + auto config_str = [&]() { + std::stringstream res; + res << config; + return res.str(); + }(); + Series output(name, Access::CREATE, config_str); + output.setOpenPMD(standardVersion); + auto iteration = output.writeIterations()[0]; + + auto default_configured = iteration.meshes["default_configured"]; + Dataset ds1(Datatype::INT, {5}); + default_configured.resetDataset(ds1); + + auto explicitly_templated = iteration.meshes["explicitly_templated"]; + Dataset ds2 = ds1; + ds2.options = backend + R"(.dataset.mode = "template")"; + explicitly_templated.resetDataset(ds2); + + auto explicitly_not_templated = + iteration.meshes["explicitly_not_templated"]; + Dataset ds3 = ds1; + ds3.options = backend + R"(.dataset.mode = "dataset")"; + explicitly_not_templated.resetDataset(ds3); + + auto undefined_dataset = iteration.meshes["undefined_dataset"]; + Dataset d4(Datatype::UNDEFINED, {Dataset::UNDEFINED_EXTENT}); + undefined_dataset.resetDataset(d4); + + output.close(); + + bool expect_template_datasets = template_datasets.value_or(false); + bool expect_short_attributes = short_attributes.value_or( + backend == "toml" || standardVersion == "2.0.0"); + + nlohmann::json resulting_dataset = [&]() { + std::fstream handle; + handle.open(name, std::ios_base::binary | std::ios_base::in); + if (backend == "json") + { + nlohmann::json res; + handle >> res; + return res; + } + else + { + auto toml_val = toml::parse(handle, name); + return json::tomlToJson(toml_val); + } + }(); + + if (expect_short_attributes) + { + REQUIRE( + resulting_dataset["attributes"]["openPMD"] == + nlohmann::json::string_t{standardVersion}); + REQUIRE( + resulting_dataset["__openPMD_internal"]["attribute_mode"] == + nlohmann::json::string_t{"short"}); + } + else + { + REQUIRE( + resulting_dataset["attributes"]["openPMD"] == + nlohmann::json{{"datatype", "STRING"}, {"value", standardVersion}}); + REQUIRE( + resulting_dataset["__openPMD_internal"]["attribute_mode"] == + nlohmann::json::string_t{"long"}); + } + + auto verify_full_dataset = [&](nlohmann::json const &j) { + REQUIRE(j["datatype"] == "INT"); + if (backend == "json") + { + nlohmann::json null; + REQUIRE( + j["data"] == + nlohmann::json::array_t{null, null, null, null, null}); + } + else + { + REQUIRE(j["data"] == nlohmann::json::array_t{0, 0, 0, 0, 0}); + } + REQUIRE(j.size() == 3); + }; + auto verify_template_dataset = [](nlohmann::json const &j) { + REQUIRE(j["datatype"] == "INT"); + REQUIRE(j["extent"] == nlohmann::json::array_t{5}); + REQUIRE(j.size() == 3); + }; + + // Undefined datasets write neither `extent` nor `data` key, so they are + // not distinguished between template and nontemplate mode. + REQUIRE( + resulting_dataset["data"]["0"]["meshes"]["undefined_dataset"] + ["datatype"] == nlohmann::json::string_t{"UNDEFINED"}); + REQUIRE( + resulting_dataset["data"]["0"]["meshes"]["undefined_dataset"].size() == + 2); + if (expect_template_datasets) + { + REQUIRE( + resulting_dataset["__openPMD_internal"]["dataset_mode"] == + nlohmann::json::string_t{"template"}); + verify_template_dataset( + resulting_dataset["data"]["0"]["meshes"]["default_configured"]); + } + else + { + REQUIRE( + resulting_dataset["__openPMD_internal"]["dataset_mode"] == + nlohmann::json::string_t{"dataset"}); + verify_full_dataset( + resulting_dataset["data"]["0"]["meshes"]["default_configured"]); + } + verify_template_dataset( + resulting_dataset["data"]["0"]["meshes"]["explicitly_templated"]); + verify_full_dataset( + resulting_dataset["data"]["0"]["meshes"]["explicitly_not_templated"]); + + Series read(name, Access::READ_ONLY); + helper::listSeries(read); +} + +TEST_CASE("json_short_modes") +{ + unsigned int name_counter = 0; + ::auxiliary::test_matrix( + &json_short_modes, + std::vector>{std::nullopt, true, false}, + std::vector>{std::nullopt, true, false}, + std::vector{getStandardDefault(), getStandardMaximum()}, + std::vector{"json", "toml"}, + std::vector{&name_counter}); +} From 985a59f1d9f801953df3db8e6fc04459abdefa81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 16 Dec 2024 10:58:32 +0100 Subject: [PATCH 28/28] CI fixes --- test/JSONTest.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/JSONTest.cpp b/test/JSONTest.cpp index 78ffc6e01e..067919bd89 100644 --- a/test/JSONTest.cpp +++ b/test/JSONTest.cpp @@ -338,6 +338,8 @@ void test_matrix_impl( template void test_matrix(Callable &&callable, std::vector const &...matrix) { + // no std::forward, callable is called multiple times, so the impl takes + // a simple reference test_matrix_impl(callable, std::tuple<>(), matrix...); } } // namespace auxiliary @@ -378,13 +380,14 @@ void json_short_modes( auto explicitly_templated = iteration.meshes["explicitly_templated"]; Dataset ds2 = ds1; - ds2.options = backend + R"(.dataset.mode = "template")"; + ds2.options = + R"({")" + backend + R"(": {"dataset": {"mode": "template"}}})"; explicitly_templated.resetDataset(ds2); auto explicitly_not_templated = iteration.meshes["explicitly_not_templated"]; Dataset ds3 = ds1; - ds3.options = backend + R"(.dataset.mode = "dataset")"; + ds3.options = R"({")" + backend + R"(": {"dataset": {"mode": "dataset"}}})"; explicitly_not_templated.resetDataset(ds3); auto undefined_dataset = iteration.meshes["undefined_dataset"]; @@ -445,11 +448,13 @@ void json_short_modes( { REQUIRE(j["data"] == nlohmann::json::array_t{0, 0, 0, 0, 0}); } + // `data` key, `datatype` key, and `attributes` key REQUIRE(j.size() == 3); }; auto verify_template_dataset = [](nlohmann::json const &j) { REQUIRE(j["datatype"] == "INT"); REQUIRE(j["extent"] == nlohmann::json::array_t{5}); + // `extent` key, `datatype` key, and `attributes` key REQUIRE(j.size() == 3); }; @@ -494,6 +499,12 @@ TEST_CASE("json_short_modes") std::vector>{std::nullopt, true, false}, std::vector>{std::nullopt, true, false}, std::vector{getStandardDefault(), getStandardMaximum()}, - std::vector{"json", "toml"}, + std::vector{ + "json" +#if !__NVCOMPILER // see https://github.com/ToruNiina/toml11/issues/205 + , + "toml" +#endif + }, std::vector{&name_counter}); }