From 30c98f5521e24f89ea35bc59f196ae1a46e58adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 10 May 2023 17:56:02 +0200 Subject: [PATCH 01/10] TOML backend --- include/openPMD/IO/Format.hpp | 1 + include/openPMD/IO/JSON/JSONIOHandler.hpp | 7 +- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 45 +++- include/openPMD/auxiliary/TypeTraits.hpp | 16 ++ src/Format.cpp | 4 + src/IO/AbstractIOHandlerHelper.cpp | 15 +- src/IO/JSON/JSONIOHandler.cpp | 10 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 223 +++++++++++++----- src/Series.cpp | 3 +- src/auxiliary/JSON.cpp | 8 +- src/config.cpp | 3 + test/CoreTest.cpp | 2 +- test/SerialIOTest.cpp | 43 +++- test/python/unittest/API/APITest.py | 5 +- 14 files changed, 289 insertions(+), 96 deletions(-) diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index 43ec4d04a1..858da29a40 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -35,6 +35,7 @@ enum class Format ADIOS2_SST, ADIOS2_SSC, JSON, + TOML, DUMMY }; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 0cdc6f3c36..452098137e 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -29,7 +29,12 @@ namespace openPMD class JSONIOHandler : public AbstractIOHandler { public: - JSONIOHandler(std::string path, Access at); + JSONIOHandler( + std::string path, + Access at, + openPMD::json::TracingJSON config, + JSONIOHandlerImpl::FileFormat, + std::string originalExtension); ~JSONIOHandler() override; diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 7f10f62cd9..c935647665 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -26,8 +26,10 @@ #include "openPMD/IO/Access.hpp" #include "openPMD/IO/JSON/JSONFilePosition.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/config.hpp" +#include #include #include @@ -153,7 +155,17 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl using json = nlohmann::json; public: - explicit JSONIOHandlerImpl(AbstractIOHandler *); + enum class FileFormat + { + Json, + Toml + }; + + explicit JSONIOHandlerImpl( + AbstractIOHandler *, + openPMD::json::TracingJSON config, + FileFormat, + std::string originalExtension); ~JSONIOHandlerImpl() override; @@ -229,15 +241,25 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // files that have logically, but not physically been written to std::unordered_set m_dirty; + /* + * Is set by constructor. + */ + FileFormat m_fileFormat{}; + + std::string m_originalExtension; + // HELPER FUNCTIONS - // will use the IOHandler to retrieve the correct directory - // shared pointer to circumvent the fact that c++ pre 17 does - // not enforce (only allow) copy elision in return statements - std::shared_ptr getFilehandle( - File, - Access access); //, Access - // m_frontendAccess=this->m_handler->m_frontendAccess); + // will use the IOHandler to retrieve the correct directory. + // first tuple element will be the underlying opened file handle. + // if Access is read mode, then the second tuple element will be the istream + // casted to precision std::numeric_limits::digits10 + 1, else null. + // if Access is write mode, then the second tuple element will be the + // ostream casted to precision std::numeric_limits::digits10 + 1, + // else null. first tuple element needs to be a pointer, since the casted + // streams are references only. + std::tuple, std::istream *, std::ostream *> + getFilehandle(File, Access access); // full operating system path of the given file std::string fullPath(File); @@ -272,15 +294,13 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // essentially: m_i = \prod_{j=0}^{i-1} extent_j static Extent getMultiplicators(Extent const &extent); - static nlohmann::json initializeNDArray(Extent const &extent); - static Extent getExtent(nlohmann::json &j); // remove single '/' in the beginning and end of a string static std::string removeSlashes(std::string); template - static bool hasKey(nlohmann::json &, KeyT &&key); + static bool hasKey(nlohmann::json const &, KeyT &&key); // make sure that the given path exists in proper form in // the passed json value @@ -366,7 +386,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl struct AttributeReader { template - static void call(nlohmann::json &, Parameter &); + static void + call(nlohmann::json const &, Parameter &); static constexpr char const *errorMsg = "JSON: writeAttribute"; }; diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index 923cfb5be2..3e5a36774e 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -24,6 +24,7 @@ #include "openPMD/auxiliary/UniquePtr.hpp" #include +#include #include // size_t #include #include @@ -56,6 +57,18 @@ namespace detail static constexpr bool value = true; }; + template + struct IsComplex + { + static constexpr bool value = false; + }; + + template + struct IsComplex> + { + static constexpr bool value = true; + }; + template struct IsPointer { @@ -114,6 +127,9 @@ using IsPointer_t = typename detail::IsPointer::type; template inline constexpr bool IsContiguousContainer_v = IsVector_v || IsArray_v; +template +inline constexpr bool IsComplex_v = detail::IsComplex::value; + namespace { // see https://en.cppreference.com/w/cpp/language/if diff --git a/src/Format.cpp b/src/Format.cpp index d5a8acf5f3..8a6ead832a 100644 --- a/src/Format.cpp +++ b/src/Format.cpp @@ -43,6 +43,8 @@ Format determineFormat(std::string const &filename) return Format::ADIOS2_SSC; if (auxiliary::ends_with(filename, ".json")) return Format::JSON; + if (auxiliary::ends_with(filename, ".toml")) + return Format::TOML; // Format might still be specified via JSON return Format::DUMMY; @@ -66,6 +68,8 @@ std::string suffix(Format f) return ".ssc"; case Format::JSON: return ".json"; + case Format::TOML: + return ".toml"; default: return ""; } diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 4cd74a4de2..c6cd69f8a5 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -194,7 +194,20 @@ std::unique_ptr createIOHandler( std::move(originalExtension)); case Format::JSON: return constructIOHandler( - "JSON", path, access); + "JSON", + path, + access, + std::move(options), + JSONIOHandlerImpl::FileFormat::Json, + std::move(originalExtension)); + case Format::TOML: + return constructIOHandler( + "JSON", + path, + access, + std::move(options), + JSONIOHandlerImpl::FileFormat::Toml, + std::move(originalExtension)); default: throw std::runtime_error( "Unknown file format! Did you specify a file ending? Specified " diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 10ffce9927..7eb8a57278 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -25,8 +25,14 @@ namespace openPMD { JSONIOHandler::~JSONIOHandler() = default; -JSONIOHandler::JSONIOHandler(std::string path, Access at) - : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}} +JSONIOHandler::JSONIOHandler( + std::string path, + Access at, + openPMD::json::TracingJSON jsonCfg, + JSONIOHandlerImpl::FileFormat format, + std::string originalExtension) + : AbstractIOHandler{path, at} + , m_impl{this, std::move(jsonCfg), format, std::move(originalExtension)} {} std::future JSONIOHandler::flush(internal::ParsedFlushParams &) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index bec6cebb71..0911be3c4d 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -26,8 +26,12 @@ #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/backend/Writable.hpp" +#include + +#include #include #include #include @@ -54,9 +58,82 @@ namespace openPMD throw std::runtime_error((TEXT)); \ } -JSONIOHandlerImpl::JSONIOHandlerImpl(AbstractIOHandler *handler) +namespace +{ + struct DefaultValue + { + template + static nlohmann::json call() + { + if constexpr (auxiliary::IsComplex_v) + { + return typename T::value_type{}; + } + else + { + return T{}; + } +#if defined(__INTEL_COMPILER) +/* + * ICPC has trouble with if constexpr, thinking that return statements are + * missing afterwards. Deactivate the warning. + * Note that putting a statement here will not help to fix this since it will + * then complain about unreachable code. + * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + */ +#pragma warning(disable : 1011) + } +#pragma warning(default : 1011) +#else + } +#endif + + static constexpr char const *errorMsg = "JSON default value"; + }; + + /* + * If initializeWithDefaultValue contains a datatype, then the dataset ought + * to be initialized with the zero value of that dataset. + * Otherwise with null. + */ + nlohmann::json initializeNDArray( + Extent const &extent, + std::optional initializeWithDefaultValue) + { + // idea: begin from the innermost shale and copy the result into the + // outer shales + nlohmann::json accum = initializeWithDefaultValue.has_value() + ? switchNonVectorType( + initializeWithDefaultValue.value()) + : nlohmann::json(); + nlohmann::json old; + auto *accum_ptr = &accum; + auto *old_ptr = &old; + for (auto it = extent.rbegin(); it != extent.rend(); it++) + { + std::swap(old_ptr, accum_ptr); + *accum_ptr = nlohmann::json::array(); + for (Extent::value_type i = 0; i < *it; i++) + { + (*accum_ptr)[i] = *old_ptr; // copy boi + } + } + return *accum_ptr; + } +} // namespace + +JSONIOHandlerImpl::JSONIOHandlerImpl( + AbstractIOHandler *handler, + openPMD::json::TracingJSON config, + FileFormat format, + std::string originalExtension) : AbstractIOHandlerImpl(handler) -{} + , m_fileFormat{format} + , m_originalExtension{std::move(originalExtension)} +{ + // Currently unused + (void)config; +} JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; @@ -80,11 +157,7 @@ void JSONIOHandlerImpl::createFile( if (!writable->written) { - std::string name = parameters.name; - if (!auxiliary::ends_with(name, ".json")) - { - name += ".json"; - } + std::string name = parameters.name + m_originalExtension; auto res_pair = getPossiblyExisting(name); auto fullPathToFile = fullPath(std::get<0>(res_pair)); @@ -205,20 +278,23 @@ void JSONIOHandlerImpl::createDataset( setAndGetFilePosition(writable, name); auto &dset = jsonVal[name]; dset["datatype"] = datatypeToString(parameter.dtype); + auto extent = parameter.extent; switch (parameter.dtype) { case Datatype::CFLOAT: case Datatype::CDOUBLE: case Datatype::CLONG_DOUBLE: { - auto complexExtent = parameter.extent; - complexExtent.push_back(2); - dset["data"] = initializeNDArray(complexExtent); + extent.push_back(2); break; } default: - dset["data"] = initializeNDArray(parameter.extent); break; } + // TOML does not support nulls, so initialize with zero + dset["data"] = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional() + : parameter.dtype); writable->written = true; m_dirty.emplace(file); } @@ -276,27 +352,28 @@ void JSONIOHandlerImpl::extendDataset( throw std::runtime_error( "[JSON] The specified location contains no valid dataset"); } - switch (stringToDatatype(j["datatype"].get())) + auto extent = parameters.extent; + auto datatype = stringToDatatype(j["datatype"].get()); + switch (datatype) { case Datatype::CFLOAT: case Datatype::CDOUBLE: case Datatype::CLONG_DOUBLE: { - // @todo test complex resizing - auto complexExtent = parameters.extent; - complexExtent.push_back(2); - nlohmann::json newData = initializeNDArray(complexExtent); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + extent.push_back(2); break; } default: - nlohmann::json newData = initializeNDArray(parameters.extent); - nlohmann::json &oldData = j["data"]; - mergeInto(newData, oldData); - j["data"] = newData; + // nothing to do break; } + // TOML does not support nulls, so initialize with zero + nlohmann::json newData = initializeNDArray( + extent, + m_fileFormat == FileFormat::Json ? std::optional() + : datatype); + nlohmann::json &oldData = j["data"]; + mergeInto(newData, oldData); + j["data"] = newData; writable->written = true; } @@ -521,11 +598,7 @@ void JSONIOHandlerImpl::openFile( "Supplied directory is not valid: " + m_handler->directory); } - std::string name = parameter.name; - if (!auxiliary::ends_with(name, ".json")) - { - name += ".json"; - } + std::string name = parameter.name + m_originalExtension; auto file = std::get<0>(getPossiblyExisting(name)); @@ -844,7 +917,8 @@ void JSONIOHandlerImpl::readAttribute( "[JSON] Attributes have to be written before reading.") refreshFileFromParent(writable); auto name = removeSlashes(parameters.name); - auto &jsonLoc = obtainJsonContents(writable)["attributes"]; + auto const &jsonContents = obtainJsonContents(writable); + auto const &jsonLoc = jsonContents["attributes"]; setAndGetFilePosition(writable); std::string error_msg("[JSON] No such attribute '"); if (!hasKey(jsonLoc, name)) @@ -919,7 +993,12 @@ void JSONIOHandlerImpl::listAttributes( "[JSON] Attributes have to be written before reading.") refreshFileFromParent(writable); auto filePosition = setAndGetFilePosition(writable); - auto &j = obtainJsonContents(writable)["attributes"]; + auto const &jsonContents = obtainJsonContents(writable); + if (!jsonContents.contains("attributes")) + { + return; + } + auto const &j = jsonContents["attributes"]; for (auto it = j.begin(); it != j.end(); it++) { parameters.attributes->push_back(it.key()); @@ -932,14 +1011,16 @@ void JSONIOHandlerImpl::deregister( m_files.erase(writable); } -std::shared_ptr -JSONIOHandlerImpl::getFilehandle(File fileName, Access access) +auto JSONIOHandlerImpl::getFilehandle(File fileName, Access access) + -> std::tuple, std::istream *, std::ostream *> { VERIFY_ALWAYS( fileName.valid(), "[JSON] Tried opening a file that has been overwritten or deleted.") auto path = fullPath(std::move(fileName)); - auto fs = std::make_shared(); + auto fs = std::make_unique(); + std::istream *istream = nullptr; + std::ostream *ostream = nullptr; if (access::write(access)) { /* @@ -949,14 +1030,31 @@ JSONIOHandlerImpl::getFilehandle(File fileName, Access access) * equivalent, but the openPMD frontend exposes no reading * functionality in APPEND mode. */ - fs->open(path, std::ios_base::out | std::ios_base::trunc); + std::ios_base::openmode openmode = + std::ios_base::out | std::ios_base::trunc; + if (m_fileFormat == FileFormat::Toml) + { + openmode |= std::ios_base::binary; + } + fs->open(path, openmode); + ostream = + &(*fs << std::setprecision( + std::numeric_limits::digits10 + 1)); } else { - fs->open(path, std::ios_base::in); + std::ios_base::openmode openmode = std::ios_base::in; + if (m_fileFormat == FileFormat::Toml) + { + openmode |= std::ios_base::binary; + } + fs->open(path, openmode); + istream = + &(*fs >> + std::setprecision(std::numeric_limits::digits10 + 1)); } VERIFY(fs->good(), "[JSON] Failed opening a file '" + path + "'"); - return fs; + return std::make_tuple(std::move(fs), istream, ostream); } std::string JSONIOHandlerImpl::fullPath(File fileName) @@ -1048,26 +1146,6 @@ Extent JSONIOHandlerImpl::getMultiplicators(Extent const &extent) return res; } -nlohmann::json JSONIOHandlerImpl::initializeNDArray(Extent const &extent) -{ - // idea: begin from the innermost shale and copy the result into the - // outer shales - nlohmann::json accum; - nlohmann::json old; - auto *accum_ptr = &accum; - auto *old_ptr = &old; - for (auto it = extent.rbegin(); it != extent.rend(); it++) - { - std::swap(old_ptr, accum_ptr); - *accum_ptr = nlohmann::json{}; - for (Extent::value_type i = 0; i < *it; i++) - { - (*accum_ptr)[i] = *old_ptr; // copy boi - } - } - return *accum_ptr; -} - Extent JSONIOHandlerImpl::getExtent(nlohmann::json &j) { Extent res; @@ -1106,7 +1184,7 @@ std::string JSONIOHandlerImpl::removeSlashes(std::string s) } template -bool JSONIOHandlerImpl::hasKey(nlohmann::json &j, KeyT &&key) +bool JSONIOHandlerImpl::hasKey(nlohmann::json const &j, KeyT &&key) { return j.find(std::forward(key)) != j.end(); } @@ -1166,9 +1244,18 @@ std::shared_ptr JSONIOHandlerImpl::obtainJsonContents(File file) return it->second; } // read from file - auto fh = getFilehandle(file, Access::READ_ONLY); + auto [fh, fh_with_precision, _] = getFilehandle(file, Access::READ_ONLY); std::shared_ptr res = std::make_shared(); - *fh >> *res; + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision >> *res; + break; + case FileFormat::Toml: + *res = + openPMD::json::tomlToJson(toml::parse(*fh_with_precision, *file)); + break; + } VERIFY(fh->good(), "[JSON] Failed reading from a file."); m_jsonVals.emplace(file, res); return res; @@ -1192,9 +1279,21 @@ void JSONIOHandlerImpl::putJsonContents( auto it = m_jsonVals.find(filename); if (it != m_jsonVals.end()) { - auto fh = getFilehandle(filename, Access::CREATE); + auto [fh, _, fh_with_precision] = + getFilehandle(filename, Access::CREATE); (*it->second)["platform_byte_widths"] = platformSpecifics(); - *fh << *it->second << std::endl; + + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision << *it->second << std::endl; + break; + case FileFormat::Toml: + *fh_with_precision << openPMD::json::jsonToToml(*it->second) + << std::endl; + break; + } + VERIFY(fh->good(), "[JSON] Failed writing data to disk.") m_jsonVals.erase(it); if (unsetDirty) @@ -1399,7 +1498,7 @@ void JSONIOHandlerImpl::AttributeWriter::call( template void JSONIOHandlerImpl::AttributeReader::call( - nlohmann::json &json, Parameter ¶meters) + nlohmann::json const &json, Parameter ¶meters) { JsonToCpp jtc; *parameters.resource = jtc(json); diff --git a/src/Series.cpp b/src/Series.cpp index 4a22330179..59a96efdb7 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2180,7 +2180,8 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) std::map const backendDescriptors{ {"hdf5", Format::HDF5}, {"adios2", Format::ADIOS2_BP}, - {"json", Format::JSON}}; + {"json", Format::JSON}, + {"toml", Format::TOML}}; std::string backend; getJsonOptionLowerCase(options, "backend", backend); if (!backend.empty()) diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index c04e672ae6..168cab7bf6 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -200,6 +200,7 @@ namespace return nlohmann::json(); // null } + // @todo maybe generalize error type throw error::BackendConfigSchema( currentPath, "Unexpected datatype in TOML configuration. This is probably a " @@ -215,7 +216,8 @@ namespace switch (val.type()) { case nlohmann::json::value_t::null: - return toml::value(); + throw error::BackendConfigSchema( + currentPath, "TOML does not support null values."); case nlohmann::json::value_t::object: { toml::value::table_type res; for (auto pair = val.begin(); pair != val.end(); ++pair) @@ -247,7 +249,7 @@ namespace case nlohmann::json::value_t::number_unsigned: return val.get(); case nlohmann::json::value_t::number_float: - return val.get(); + return (long double)val.get(); case nlohmann::json::value_t::binary: return val.get(); case nlohmann::json::value_t::discarded: @@ -501,7 +503,7 @@ std::optional asLowerCaseStringDynamic(nlohmann::json const &value) std::vector backendKeys() { - return {"adios2", "json", "hdf5"}; + return {"adios2", "json", "toml", "hdf5"}; } void warnGlobalUnusedOptions(TracingJSON const &config) diff --git a/src/config.cpp b/src/config.cpp index a44925287a..ff1a0cb471 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -28,11 +28,13 @@ #include #include +// @todo add TOML here std::map openPMD::getVariants() { return std::map{ {"mpi", bool(openPMD_HAVE_MPI)}, {"json", true}, + {"toml", true}, {"hdf5", bool(openPMD_HAVE_HDF5)}, {"adios1", false}, {"adios2", bool(openPMD_HAVE_ADIOS2)}}; @@ -42,6 +44,7 @@ std::vector openPMD::getFileExtensions() { std::vector fext; fext.emplace_back("json"); + fext.emplace_back("toml"); #if openPMD_HAVE_ADIOS2 fext.emplace_back("bp"); #endif diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 224596864e..d660e29ec4 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1051,7 +1051,7 @@ TEST_CASE("no_file_ending", "[core]") Access::CREATE, R"({"backend": "json"})"); } - REQUIRE(auxiliary::file_exists("../samples/no_extension_specified.json")); + REQUIRE(auxiliary::file_exists("../samples/no_extension_specified")); } TEST_CASE("backend_via_json", "[core]") diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 11297c7595..f6fe3f81da 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -81,20 +81,20 @@ std::vector testedFileExtensions() allExtensions.begin(), allExtensions.end(), []([[maybe_unused]] std::string const &ext) { -#if openPMD_HAVE_ADIOS2 -#define HAS_ADIOS_2_9 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209) -#if HAS_ADIOS_2_9 +#if openPMD_HAS_ADIOS_2_9 // sst and ssc need a receiver for testing // bp5 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp5"; + // toml parsing is very slow and its implementation is equivalent to + // the json backend, so it is only activated for selected tests + return ext == "sst" || ext == "ssc" || ext == "bp5" || + ext == "toml"; #else + // toml parsing is very slow and its implementation is equivalent to + // the json backend, so it is only activated for selected tests // sst and ssc need a receiver for testing // bp4 is already tested via bp - return ext == "sst" || ext == "ssc" || ext == "bp4"; -#endif -#undef HAS_ADIOS_2_9 -#else - return false; + return ext == "sst" || ext == "ssc" || ext == "bp4" || + ext == "toml"; #endif }); return {allExtensions.begin(), newEnd}; @@ -1447,7 +1447,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("short").dtype == Datatype::SHORT); REQUIRE(s.getAttribute("int").dtype == Datatype::INT); REQUIRE(s.getAttribute("long").dtype == Datatype::LONG); - REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + } REQUIRE(s.getAttribute("ushort").dtype == Datatype::USHORT); REQUIRE(s.getAttribute("uint").dtype == Datatype::UINT); REQUIRE(s.getAttribute("ulong").dtype == Datatype::ULONG); @@ -1459,7 +1462,10 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("vecShort").dtype == Datatype::VEC_SHORT); REQUIRE(s.getAttribute("vecInt").dtype == Datatype::VEC_INT); REQUIRE(s.getAttribute("vecLong").dtype == Datatype::VEC_LONG); - REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); + } REQUIRE(s.getAttribute("vecUShort").dtype == Datatype::VEC_USHORT); REQUIRE(s.getAttribute("vecUInt").dtype == Datatype::VEC_UINT); REQUIRE(s.getAttribute("vecULong").dtype == Datatype::VEC_ULONG); @@ -1508,6 +1514,11 @@ TEST_CASE("dtype_test", "[serial]") { dtype_test(t); } + /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + dtype_test("toml"); } inline void write_test(const std::string &backend) @@ -2106,6 +2117,11 @@ TEST_CASE("fileBased_write_test", "[serial]") { fileBased_write_test(t); } + /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + fileBased_write_test("toml"); } inline void sample_write_thetaMode(std::string file_ending) @@ -7405,4 +7421,9 @@ TEST_CASE("groupbased_read_write", "[serial]") } } } + /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + groupbased_read_write("toml"); } diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 0aa71cf7a8..72b099c5ee 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -359,7 +359,8 @@ def attributeRoundTrip(self, file_ending): # c_types self.assertEqual(series.get_attribute("byte_c"), 30) self.assertEqual(series.get_attribute("ubyte_c"), 50) - if file_ending != "json": # TODO: returns [100] instead of 100 in json + # TODO: returns [100] instead of 100 in json + if file_ending != "json" and file_ending != "toml": self.assertEqual(chr(series.get_attribute("char_c")), 'd') self.assertEqual(series.get_attribute("int16_c"), 2) self.assertEqual(series.get_attribute("int32_c"), 3) @@ -1824,7 +1825,7 @@ def testIterator(self): self.makeIteratorRoundTrip(b, backend_filesupport[b]) def makeAvailableChunksRoundTrip(self, ext): - if ext == "h5": + if ext == "h5" or ext == "toml": return name = "../samples/available_chunks_python." + ext write = io.Series( From 3cd65e69a560aa1822d688eb08ee5f7e7dff291f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 10 May 2023 17:34:44 +0200 Subject: [PATCH 02/10] Add documentation for TOML --- docs/source/backends/json.rst | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 752497aa09..1132b8b24c 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -1,10 +1,10 @@ .. _backends-json: -JSON -==== +JSON/TOML +========= -openPMD supports writing to and reading from JSON files. -The JSON backend is always available. +openPMD supports writing to and reading from JSON and TOML files. +The JSON and TOML backends are always available. JSON File Format @@ -43,9 +43,17 @@ Every such attribute is itself a JSON object with two keys: * ``datatype``: A string describing the type of the value. * ``value``: The actual value of type ``datatype``. +TOML File Format +---------------- + +A TOML file uses the file ending ``.toml``. The TOML backend is chosen by creating a ``Series`` object with a filename that has this file ending. + +The TOML backend internally works with JSON datasets and converts to/from TOML during I/O. +As a result, data layout and usage are equivalent to the JSON backend. -Restrictions ------------- + +JSON Restrictions +----------------- For creation of JSON serializations (i.e. writing), the restrictions of the JSON backend are equivalent to those of the `JSON library by Niels Lohmann `_ @@ -77,6 +85,20 @@ The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved an A parallel (i.e. MPI) implementation is *not* available. +TOML Restrictions +----------------- + +Note that the JSON datatype-specific restrictions do not automatically hold for TOML, as those affect only the representation on disk, not the internal representation. + +TOML supports most numeric types, up to long double and long long types. +Special floating point values such as NaN are also support. + +TOML does not support null values. + +The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved and must not be used for base/mesh/particles path, records and their components. + +A parallel (i.e. MPI) implementation is *not* available. + Example ------- From 914092154c63bfe11955c739b63b2a3c74ab7960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 11 May 2023 12:08:44 +0200 Subject: [PATCH 03/10] Fixes for long double and long integer types --- docs/source/backends/json.rst | 2 +- test/SerialIOTest.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 1132b8b24c..3e9c66ec7a 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -90,7 +90,7 @@ TOML Restrictions Note that the JSON datatype-specific restrictions do not automatically hold for TOML, as those affect only the representation on disk, not the internal representation. -TOML supports most numeric types, up to long double and long long types. +TOML supports most numeric types, with the support for long double and long integer types being platform-defined. Special floating point values such as NaN are also support. TOML does not support null values. diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index f6fe3f81da..4fe6b63977 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1231,7 +1231,8 @@ TEST_CASE("particle_patches", "[serial]") inline void dtype_test(const std::string &backend) { - bool test_long_double = (backend != "json") || sizeof(long double) <= 8; + bool test_long_double = + (backend != "json" && backend != "toml") || sizeof(long double) <= 8; bool test_long_long = (backend != "json") || sizeof(long long) <= 8; { Series s = Series("../samples/dtype_test." + backend, Access::CREATE); From c3ab89d2f04ac7544242e80c70d729bc35f3893f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 11 May 2023 17:54:19 +0200 Subject: [PATCH 04/10] Only run TOML tests if TOML is available TOML is not shown as available on NVIDIA compilers --- test/SerialIOTest.cpp | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 4fe6b63977..2f2c54ff49 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1515,11 +1515,15 @@ TEST_CASE("dtype_test", "[serial]") { dtype_test(t); } - /* - * TOML backend is not generally tested for performance reasons, opt in to - * testing it here. - */ - dtype_test("toml"); + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + dtype_test("toml"); + } } inline void write_test(const std::string &backend) @@ -2118,11 +2122,15 @@ TEST_CASE("fileBased_write_test", "[serial]") { fileBased_write_test(t); } - /* - * TOML backend is not generally tested for performance reasons, opt in to - * testing it here. - */ - fileBased_write_test("toml"); + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + fileBased_write_test("toml"); + } } inline void sample_write_thetaMode(std::string file_ending) @@ -7422,9 +7430,13 @@ TEST_CASE("groupbased_read_write", "[serial]") } } } - /* - * TOML backend is not generally tested for performance reasons, opt in to - * testing it here. - */ - groupbased_read_write("toml"); + if (auto extensions = getFileExtensions(); + std::find(extensions.begin(), extensions.end(), "toml") != + extensions.end()) + { /* + * TOML backend is not generally tested for performance reasons, opt in to + * testing it here. + */ + groupbased_read_write("toml"); + } } From 1bfe6a11c59fe2a3f751ba9650c7895055c38378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 23 May 2023 14:53:52 +0200 Subject: [PATCH 05/10] Deactivate long double entirely for JSON/TOML --- test/SerialIOTest.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 2f2c54ff49..962ca636aa 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1231,8 +1231,7 @@ TEST_CASE("particle_patches", "[serial]") inline void dtype_test(const std::string &backend) { - bool test_long_double = - (backend != "json" && backend != "toml") || sizeof(long double) <= 8; + bool test_long_double = backend != "json" && backend != "toml"; bool test_long_long = (backend != "json") || sizeof(long long) <= 8; { Series s = Series("../samples/dtype_test." + backend, Access::CREATE); From 761da4ce80ddc6669f509ebac1fe86253ab4d858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 23 May 2023 16:08:57 +0200 Subject: [PATCH 06/10] CI fix: unused variable --- src/IO/JSON/JSONIOHandlerImpl.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 0911be3c4d..73d50366f7 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -1245,6 +1245,7 @@ std::shared_ptr JSONIOHandlerImpl::obtainJsonContents(File file) } // read from file auto [fh, fh_with_precision, _] = getFilehandle(file, Access::READ_ONLY); + (void)_; std::shared_ptr res = std::make_shared(); switch (m_fileFormat) { @@ -1281,6 +1282,7 @@ void JSONIOHandlerImpl::putJsonContents( { auto [fh, _, fh_with_precision] = getFilehandle(filename, Access::CREATE); + (void)_; (*it->second)["platform_byte_widths"] = platformSpecifics(); switch (m_fileFormat) From 48af3193d9b560e9064f0aadbdf0a91b1ec94c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 20 Mar 2023 12:26:10 +0100 Subject: [PATCH 07/10] Hide/deactivate/warn Toml backend on nvcc compilers https://github.com/ToruNiina/toml11/issues/205 --- src/config.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/config.cpp b/src/config.cpp index ff1a0cb471..89a824500c 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -31,20 +31,28 @@ // @todo add TOML here std::map openPMD::getVariants() { + // clang-format off return std::map{ {"mpi", bool(openPMD_HAVE_MPI)}, {"json", true}, +// https://github.com/ToruNiina/toml11/issues/205 +#if !defined(__NVCOMPILER_MAJOR__) || __NVCOMPILER_MAJOR__ >= 23 {"toml", true}, +#endif {"hdf5", bool(openPMD_HAVE_HDF5)}, {"adios1", false}, {"adios2", bool(openPMD_HAVE_ADIOS2)}}; + // clang-format on } std::vector openPMD::getFileExtensions() { std::vector fext; fext.emplace_back("json"); +// https://github.com/ToruNiina/toml11/issues/205 +#if !defined(__NVCOMPILER_MAJOR__) || __NVCOMPILER_MAJOR__ >= 23 fext.emplace_back("toml"); +#endif #if openPMD_HAVE_ADIOS2 fext.emplace_back("bp"); #endif From f087215ba7bc0e76ecd9ed403e8495364ac1ed52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 17 Aug 2023 15:42:31 +0200 Subject: [PATCH 08/10] Usage notes for JSON / TOML --- docs/source/backends/json.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 3e9c66ec7a..17389cc09c 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -6,6 +6,14 @@ JSON/TOML openPMD supports writing to and reading from JSON and TOML files. The JSON and TOML backends are always available. +.. note:: + + Both the JSON and the TOML backends are not intended for large-scale use. + + The JSON backend is mainly intended for prototyping and learning, or similar workflows where setting up a large IO backend such as HDF5 or ADIOS2 is perceived as obstructive. It can also be used for small datasets that need to be stored in text format rather than binary. + + The TOML backend is intended for handwritten configuration files. The motivation for its addition to the openPMD-api was the handwritten specification for physical quantities along the openPMD standard within the context of experiment workflows. + JSON File Format ---------------- From 9b3860c677337fea193e969a2fc327c534b8a807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 17 Aug 2023 15:44:37 +0200 Subject: [PATCH 09/10] Update comment in test/python/unittest/API/APITest.py Co-authored-by: Axel Huebl --- test/python/unittest/API/APITest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 72b099c5ee..a510098c8d 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -359,7 +359,7 @@ def attributeRoundTrip(self, file_ending): # c_types self.assertEqual(series.get_attribute("byte_c"), 30) self.assertEqual(series.get_attribute("ubyte_c"), 50) - # TODO: returns [100] instead of 100 in json + # TODO: returns [100] instead of 100 in json/toml if file_ending != "json" and file_ending != "toml": self.assertEqual(chr(series.get_attribute("char_c")), 'd') self.assertEqual(series.get_attribute("int16_c"), 2) From 7d85f2273d3a3ccc3c5a8cfcbb4b4dd0137f633e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 17 Aug 2023 17:32:20 +0200 Subject: [PATCH 10/10] Update documentation text Co-authored-by: Axel Huebl --- docs/source/backends/json.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 17389cc09c..48ec6b1f44 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -8,11 +8,12 @@ The JSON and TOML backends are always available. .. note:: - Both the JSON and the TOML backends are not intended for large-scale use. + Both the JSON and the TOML backends are not intended for large-scale data I/O. The JSON backend is mainly intended for prototyping and learning, or similar workflows where setting up a large IO backend such as HDF5 or ADIOS2 is perceived as obstructive. It can also be used for small datasets that need to be stored in text format rather than binary. - The TOML backend is intended for handwritten configuration files. The motivation for its addition to the openPMD-api was the handwritten specification for physical quantities along the openPMD standard within the context of experiment workflows. + The TOML backend is intended for exchanging the *structure* of a data series without its "heavy" data fields. + For instance, one can easily create and exchange human-readable, machine-actionable data configurations for experiments and simulations. JSON File Format