diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index 66133881c8..39be86ab43 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -42,7 +42,8 @@ namespace openPMD enum class Datatype : int { CHAR, - UCHAR, // SCHAR, + UCHAR, + SCHAR, SHORT, INT, LONG, @@ -74,6 +75,7 @@ enum class Datatype : int VEC_CFLOAT, VEC_CDOUBLE, VEC_CLONG_DOUBLE, + VEC_SCHAR, VEC_STRING, ARR_DBL_7, @@ -124,6 +126,10 @@ inline constexpr Datatype determineDatatype() { return DT::UCHAR; } + else if (decay_equiv::value) + { + return DT::SCHAR; + } else if (decay_equiv::value) { return DT::SHORT; @@ -208,6 +214,10 @@ inline constexpr Datatype determineDatatype() { return DT::VEC_UCHAR; } + else if (decay_equiv>::value) + { + return DT::VEC_SCHAR; + } else if (decay_equiv>::value) { return DT::VEC_USHORT; @@ -276,6 +286,10 @@ inline constexpr Datatype determineDatatype(std::shared_ptr) { return DT::UCHAR; } + else if (decay_equiv::value) + { + return DT::SCHAR; + } else if (decay_equiv::value) { return DT::SHORT; @@ -360,6 +374,10 @@ inline constexpr Datatype determineDatatype(std::shared_ptr) { return DT::VEC_UCHAR; } + else if (decay_equiv>::value) + { + return DT::VEC_SCHAR; + } else if (decay_equiv>::value) { return DT::VEC_USHORT; @@ -434,9 +452,9 @@ inline size_t toBytes(Datatype d) case DT::UCHAR: case DT::VEC_UCHAR: return sizeof(unsigned char); - // case DT::SCHAR: - // case DT::VEC_SCHAR: - // return sizeof(signed char); + case DT::SCHAR: + case DT::VEC_SCHAR: + return sizeof(signed char); case DT::SHORT: case DT::VEC_SHORT: return sizeof(short); @@ -776,36 +794,6 @@ inline bool isSame(openPMD::Datatype const d, openPMD::Datatype const e) return false; } -namespace detail -{ - template - struct BasicDatatypeHelper - { - Datatype m_dt = determineDatatype(); - }; - - template - struct BasicDatatypeHelper> - { - Datatype m_dt = BasicDatatypeHelper{}.m_dt; - }; - - template - struct BasicDatatypeHelper> - { - Datatype m_dt = BasicDatatypeHelper{}.m_dt; - }; - - struct BasicDatatype - { - template - static Datatype call(); - - template - static Datatype call(); - }; -} // namespace detail - /** * @brief basicDatatype Strip openPMD Datatype of std::vector, std::array et. * al. diff --git a/include/openPMD/DatatypeHelpers.hpp b/include/openPMD/DatatypeHelpers.hpp index f9332d2ddb..07f3c7fc37 100644 --- a/include/openPMD/DatatypeHelpers.hpp +++ b/include/openPMD/DatatypeHelpers.hpp @@ -110,6 +110,8 @@ auto switchType(Datatype dt, Args &&...args) case Datatype::UCHAR: return Action::template call( std::forward(args)...); + case Datatype::SCHAR: + return Action::template call(std::forward(args)...); case Datatype::SHORT: return Action::template call(std::forward(args)...); case Datatype::INT: @@ -164,6 +166,9 @@ auto switchType(Datatype dt, Args &&...args) case Datatype::VEC_UCHAR: return Action::template call >( std::forward(args)...); + case Datatype::VEC_SCHAR: + return Action::template call >( + std::forward(args)...); case Datatype::VEC_USHORT: return Action::template call >( std::forward(args)...); @@ -241,6 +246,8 @@ auto switchNonVectorType(Datatype dt, Args &&...args) case Datatype::UCHAR: return Action::template call( std::forward(args)...); + case Datatype::SCHAR: + return Action::template call(std::forward(args)...); case Datatype::SHORT: return Action::template call(std::forward(args)...); case Datatype::INT: diff --git a/include/openPMD/Datatype_internal.hpp b/include/openPMD/Datatype_internal.hpp new file mode 100644 index 0000000000..64e9061c54 --- /dev/null +++ b/include/openPMD/Datatype_internal.hpp @@ -0,0 +1,114 @@ +/* Copyright 2022 Franz Poeschel, Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/auxiliary/TypeTraits.hpp" + +#include +#include +#include + +namespace openPMD +{ +namespace detail +{ + template + struct BasicDatatype + { + using DT = typename DoDetermineDatatype::DT_enum; + + template + constexpr static DT call() + { + if constexpr (auxiliary::IsVector_v) + { + return DoDetermineDatatype::template call< + typename T::value_type>(); + } + else if constexpr (auxiliary::IsArray_v) + { + return DoDetermineDatatype::template call< + typename T::value_type>(); + } + else + { + return DoDetermineDatatype::template call(); + } +#if defined(__INTEL_COMPILER) +/* + * ICPC has trouble with if constexpr, thinking that return statements are + * missing afterwards. Deactivate the warning. + * Note that putting a statement here will not help to fix this since it will + * then complain about unreachable code. + * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + */ +#pragma warning(disable : 1011) + } +#pragma warning(default : 1011) +#else + } +#endif + + constexpr static char const *errorMsg = + "basicDatatype: received unknown datatype."; + }; + + template + struct ToVectorType + { + using DT = typename DoDetermineDatatype::DT_enum; + + template + constexpr static DT call() + { + if constexpr (auxiliary::IsVector_v) + { + return DoDetermineDatatype::template call(); + } + else if constexpr (auxiliary::IsArray_v) + { + return DoDetermineDatatype::template call< + std::vector>(); + } + else + { + return DoDetermineDatatype::template call>(); + } +#if defined(__INTEL_COMPILER) +/* + * ICPC has trouble with if constexpr, thinking that return statements are + * missing afterwards. Deactivate the warning. + * Note that putting a statement here will not help to fix this since it will + * then complain about unreachable code. + * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + */ +#pragma warning(disable : 1011) + } +#pragma warning(default : 1011) +#else + } +#endif + + constexpr static char const *errorMsg = + "toVectorType: received unknown datatype."; + }; +} // namespace detail +} // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp index 838725da94..c4e99d773f 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp @@ -86,6 +86,8 @@ inline ADIOS_DATATYPES getBP1DataType(Datatype dtype) { case DT::CHAR: case DT::VEC_CHAR: + case DT::SCHAR: + case DT::VEC_SCHAR: return adios_byte; case DT::UCHAR: case DT::VEC_UCHAR: diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 593c937eba..87a06f1e27 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -50,13 +50,13 @@ namespace detail }; template - struct ToDatatypeHelper > + struct ToDatatypeHelper> { static std::string type(); }; template - struct ToDatatypeHelper > + struct ToDatatypeHelper> { static std::string type(); }; @@ -149,6 +149,8 @@ auto switchAdios2AttributeType(Datatype dt, Args &&...args) case Datatype::UCHAR: return Action::template call( std::forward(args)...); + case Datatype::SCHAR: + return Action::template call(std::forward(args)...); case Datatype::SHORT: return Action::template call(std::forward(args)...); case Datatype::INT: @@ -175,10 +177,10 @@ auto switchAdios2AttributeType(Datatype dt, Args &&...args) case Datatype::LONG_DOUBLE: return Action::template call(std::forward(args)...); case Datatype::CFLOAT: - return Action::template call >( + return Action::template call>( std::forward(args)...); case Datatype::CDOUBLE: - return Action::template call >( + return Action::template call>( std::forward(args)...); // missing std::complex< long double > type in ADIOS2 v2.6.0 // case Datatype::CLONG_DOUBLE: @@ -227,6 +229,8 @@ auto switchAdios2VariableType(Datatype dt, Args &&...args) case Datatype::UCHAR: return Action::template call( std::forward(args)...); + case Datatype::SCHAR: + return Action::template call(std::forward(args)...); case Datatype::SHORT: return Action::template call(std::forward(args)...); case Datatype::INT: @@ -253,10 +257,10 @@ auto switchAdios2VariableType(Datatype dt, Args &&...args) case Datatype::LONG_DOUBLE: return Action::template call(std::forward(args)...); case Datatype::CFLOAT: - return Action::template call >( + return Action::template call>( std::forward(args)...); case Datatype::CDOUBLE: - return Action::template call >( + return Action::template call>( std::forward(args)...); // missing std::complex< long double > type in ADIOS2 v2.6.0 // case Datatype::CLONG_DOUBLE: diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index aed11094e3..18a4aad192 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -575,7 +575,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, T value); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); @@ -614,7 +614,7 @@ namespace detail "attribute types"); } - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string, std::shared_ptr) @@ -647,7 +647,7 @@ namespace detail "vector attribute types"); } - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string, std::shared_ptr) @@ -675,7 +675,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, const std::vector &value); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); @@ -713,7 +713,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, const std::vector &vec); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); @@ -751,7 +751,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, const std::array &value); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); @@ -816,7 +816,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, bool value); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp index 75c2613674..c2d9da0cbc 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -31,6 +31,7 @@ #include #include "openPMD/Datatype.hpp" +#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" namespace openPMD { @@ -150,13 +151,6 @@ namespace detail } AttributeLocation const &location = it->second; Datatype determinedDatatype = determineDatatype(); - if (std::is_same::value) - { - // workaround: we use Datatype::CHAR to represent ADIOS2 signed char - // (ADIOS2 does not have chars with unspecified signed-ness - // anyway) - determinedDatatype = Datatype::CHAR; - } if (location.dt != determinedDatatype) { std::stringstream errorMsg; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 844aaea12f..ae58e787e1 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -295,8 +295,6 @@ class Iteration : public Attributable * @brief End an IO step on the IO file (or file-like object) * containing this iteration. In case of group-based iteration * layout, this will be the complete Series. - * - * @return AdvanceStatus */ void endStep(); diff --git a/include/openPMD/auxiliary/Memory.hpp b/include/openPMD/auxiliary/Memory.hpp index 8752f0a4da..11a47d215f 100644 --- a/include/openPMD/auxiliary/Memory.hpp +++ b/include/openPMD/auxiliary/Memory.hpp @@ -133,6 +133,11 @@ namespace auxiliary data = new unsigned char[numPoints]; del = [](void *p) { delete[] static_cast(p); }; break; + case DT::VEC_SCHAR: + case DT::SCHAR: + data = new signed char[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; case DT::BOOL: data = new bool[numPoints]; del = [](void *p) { delete[] static_cast(p); }; diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index f83b99a2ae..0ac26d9dec 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -53,7 +53,8 @@ class Attribute : public auxiliary::Variant< Datatype, char, - unsigned char, // signed char, + unsigned char, + signed char, short, int, long, @@ -85,6 +86,7 @@ class Attribute std::vector >, std::vector >, std::vector >, + std::vector, std::vector, std::array, bool> diff --git a/include/openPMD/binding/python/Numpy.hpp b/include/openPMD/binding/python/Numpy.hpp index f0fc7c1b84..f28ac8dbc2 100644 --- a/include/openPMD/binding/python/Numpy.hpp +++ b/include/openPMD/binding/python/Numpy.hpp @@ -136,6 +136,8 @@ inline pybind11::dtype dtype_to_numpy(Datatype const dt) { case DT::CHAR: case DT::VEC_CHAR: + case DT::SCHAR: + case DT::VEC_SCHAR: case DT::STRING: case DT::VEC_STRING: return pybind11::dtype("b"); diff --git a/src/Datatype.cpp b/src/Datatype.cpp index 683cfbb06f..71565b3d52 100644 --- a/src/Datatype.cpp +++ b/src/Datatype.cpp @@ -20,6 +20,7 @@ */ #include "openPMD/Datatype.hpp" #include "openPMD/DatatypeHelpers.hpp" +#include "openPMD/Datatype_internal.hpp" #include #include @@ -46,6 +47,9 @@ std::ostream &operator<<(std::ostream &os, openPMD::Datatype const &d) case DT::UCHAR: os << "UCHAR"; break; + case DT::SCHAR: + os << "SCHAR"; + break; case DT::SHORT: os << "SHORT"; break; @@ -139,6 +143,9 @@ std::ostream &operator<<(std::ostream &os, openPMD::Datatype const &d) case DT::VEC_CLONG_DOUBLE: os << "VEC_CLONG_DOUBLE"; break; + case DT::VEC_SCHAR: + os << "VEC_SCHAR"; + break; case DT::VEC_STRING: os << "VEC_STRING"; break; @@ -161,6 +168,7 @@ Datatype stringToDatatype(std::string s) static std::unordered_map m{ {"CHAR", Datatype::CHAR}, {"UCHAR", Datatype::UCHAR}, + {"SCHAR", Datatype::SCHAR}, {"SHORT", Datatype::SHORT}, {"INT", Datatype::INT}, {"LONG", Datatype::LONG}, @@ -192,6 +200,7 @@ Datatype stringToDatatype(std::string s) {"VEC_CFLOAT", Datatype::VEC_CFLOAT}, {"VEC_CDOUBLE", Datatype::VEC_CDOUBLE}, {"VEC_CLONG_DOUBLE", Datatype::VEC_CLONG_DOUBLE}, + {"VEC_SCHAR", Datatype::VEC_SCHAR}, {"VEC_STRING", Datatype::VEC_STRING}, {"ARR_DBL_7", Datatype::ARR_DBL_7}, {"BOOL", Datatype::BOOL}, @@ -216,67 +225,71 @@ std::string datatypeToString(openPMD::Datatype dt) } std::vector openPMD_Datatypes{ - Datatype::CHAR, Datatype::UCHAR, Datatype::SHORT, - Datatype::INT, Datatype::LONG, Datatype::LONGLONG, - Datatype::USHORT, Datatype::UINT, Datatype::ULONG, - Datatype::ULONGLONG, Datatype::FLOAT, Datatype::DOUBLE, - Datatype::LONG_DOUBLE, Datatype::CFLOAT, Datatype::CDOUBLE, - Datatype::CLONG_DOUBLE, Datatype::STRING, Datatype::VEC_CHAR, - Datatype::VEC_SHORT, Datatype::VEC_INT, Datatype::VEC_LONG, - Datatype::VEC_LONGLONG, Datatype::VEC_UCHAR, Datatype::VEC_USHORT, - Datatype::VEC_UINT, Datatype::VEC_ULONG, Datatype::VEC_ULONGLONG, - Datatype::VEC_FLOAT, Datatype::VEC_DOUBLE, Datatype::VEC_LONG_DOUBLE, - Datatype::VEC_CFLOAT, Datatype::VEC_CDOUBLE, Datatype::VEC_CLONG_DOUBLE, - Datatype::VEC_STRING, Datatype::ARR_DBL_7, Datatype::BOOL, + Datatype::CHAR, + Datatype::UCHAR, + Datatype::SCHAR, + Datatype::SHORT, + Datatype::INT, + Datatype::LONG, + Datatype::LONGLONG, + Datatype::USHORT, + Datatype::UINT, + Datatype::ULONG, + Datatype::ULONGLONG, + Datatype::FLOAT, + Datatype::DOUBLE, + Datatype::LONG_DOUBLE, + Datatype::CFLOAT, + Datatype::CDOUBLE, + Datatype::CLONG_DOUBLE, + Datatype::STRING, + Datatype::VEC_CHAR, + Datatype::VEC_SHORT, + Datatype::VEC_INT, + Datatype::VEC_LONG, + Datatype::VEC_LONGLONG, + Datatype::VEC_UCHAR, + Datatype::VEC_USHORT, + Datatype::VEC_UINT, + Datatype::VEC_ULONG, + Datatype::VEC_ULONGLONG, + Datatype::VEC_FLOAT, + Datatype::VEC_DOUBLE, + Datatype::VEC_LONG_DOUBLE, + Datatype::VEC_CFLOAT, + Datatype::VEC_CDOUBLE, + Datatype::VEC_CLONG_DOUBLE, + Datatype::VEC_SCHAR, + Datatype::VEC_STRING, + Datatype::ARR_DBL_7, + Datatype::BOOL, Datatype::UNDEFINED}; -Datatype basicDatatype(Datatype dt) +namespace { - return switchType(dt); -} + struct DoDetermineDatatype + { + /* + * Suppress wrong compiler warnings. + * The typedef is needed in instantiation. + */ + using DT_enum [[maybe_unused]] = Datatype; -Datatype toVectorType(Datatype dt) -{ - auto initializer = []() { - std::map res; - for (Datatype d : openPMD_Datatypes) + template + static constexpr Datatype call() { - if (d == Datatype::ARR_DBL_7 || d == Datatype::UNDEFINED) - continue; - Datatype basic = basicDatatype(d); - if (basic == d) - continue; - res[basic] = d; + return determineDatatype(); } - return res; }; - static auto map(initializer()); - auto it = map.find(dt); - if (it != map.end()) - { - return it->second; - } - else - { - std::cerr << "Encountered non-basic type " << dt << ", aborting." - << std::endl; - throw std::runtime_error("toVectorType: passed non-basic type."); - } -} +} // namespace -namespace detail +Datatype basicDatatype(Datatype dt) { - template - Datatype BasicDatatype::call() - { - static auto res = BasicDatatypeHelper{}.m_dt; - return res; - } + return switchType>(dt); +} - template - Datatype BasicDatatype::call() - { - throw std::runtime_error("basicDatatype: received unknown datatype."); - } -} // namespace detail +Datatype toVectorType(Datatype dt) +{ + return switchType>(dt); +} } // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index 9a4b645c0f..d4c08408ce 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -22,6 +22,8 @@ #include "openPMD/config.hpp" #if openPMD_HAVE_ADIOS2 #include "openPMD/Datatype.hpp" +#include "openPMD/DatatypeHelpers.hpp" +#include "openPMD/Datatype_internal.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include @@ -35,7 +37,7 @@ std::string ToDatatypeHelper::type() } template -std::string ToDatatypeHelper >::type() +std::string ToDatatypeHelper>::type() { return @@ -43,7 +45,7 @@ std::string ToDatatypeHelper >::type() } template -std::string ToDatatypeHelper >::type() +std::string ToDatatypeHelper>::type() { return @@ -72,7 +74,7 @@ Datatype fromADIOS2Type(std::string const &dt, bool verbose) static std::map map{ {"string", Datatype::STRING}, {"char", Datatype::CHAR}, - {"signed char", Datatype::CHAR}, + {"signed char", Datatype::SCHAR}, {"unsigned char", Datatype::UCHAR}, {"short", Datatype::SHORT}, {"unsigned short", Datatype::USHORT}, @@ -87,11 +89,11 @@ Datatype fromADIOS2Type(std::string const &dt, bool verbose) {"long double", Datatype::LONG_DOUBLE}, {"float complex", Datatype::CFLOAT}, {"double complex", Datatype::CDOUBLE}, - {"long double complex", - Datatype::CLONG_DOUBLE}, // does not exist as of 2.7.0 but might come - // later + // does not exist as of 2.7.0 but might come later + // {"long double complex", + // Datatype::CLONG_DOUBLE}, {"uint8_t", Datatype::UCHAR}, - {"int8_t", Datatype::CHAR}, + {"int8_t", Datatype::SCHAR}, {"uint16_t", determineDatatype()}, {"int16_t", determineDatatype()}, {"uint32_t", determineDatatype()}, @@ -217,7 +219,8 @@ Datatype attributeInfo( } else if ( shape.size() == 2 && - (basicType == Datatype::CHAR || basicType == Datatype::UCHAR)) + (basicType == Datatype::CHAR || basicType == Datatype::SCHAR || + basicType == Datatype::UCHAR)) { return Datatype::VEC_STRING; } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 072c5cd285..2e53700bea 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -166,10 +166,10 @@ void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) m_schema = auxiliary::getEnvNum("OPENPMD2_ADIOS2_SCHEMA", m_schema); } -std::optional > +std::optional> ADIOS2IOHandlerImpl::getOperators(json::TracingJSON cfg) { - using ret_t = std::optional >; + using ret_t = std::optional>; std::vector res; if (!cfg.json().contains("dataset")) { @@ -222,7 +222,7 @@ ADIOS2IOHandlerImpl::getOperators(json::TracingJSON cfg) return std::make_optional(std::move(res)); } -std::optional > +std::optional> ADIOS2IOHandlerImpl::getOperators() { return getOperators(m_config); @@ -1436,14 +1436,13 @@ namespace detail auto attr = IO.InquireAttribute(metaAttr); if (attr.Data().size() == 1 && attr.Data()[0] == 1) { - AttributeTypes::readAttribute( + return AttributeTypes::readAttribute( preloadedAttributes, name, resource); - return determineDatatype(); } } } - AttributeTypes::readAttribute(preloadedAttributes, name, resource); - return determineDatatype(); + return AttributeTypes::readAttribute( + preloadedAttributes, name, resource); } template @@ -1756,7 +1755,7 @@ namespace detail } template - void AttributeTypes::readAttribute( + Datatype AttributeTypes::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -1771,10 +1770,11 @@ namespace detail std::to_string(attr.shape.size()) + "D: " + name); } *resource = *attr.data; + return determineDatatype(); } template - void AttributeTypes >::createAttribute( + void AttributeTypes>::createAttribute( adios2::IO &IO, adios2::Engine &engine, detail::BufferedAttributeWrite ¶ms, @@ -1797,7 +1797,7 @@ namespace detail } template - void AttributeTypes >::readAttribute( + Datatype AttributeTypes>::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -1812,9 +1812,10 @@ namespace detail std::vector res(attr.shape[0]); std::copy_n(attr.data, attr.shape[0], res.data()); *resource = std::move(res); + return determineDatatype>(); } - void AttributeTypes >::createAttribute( + void AttributeTypes>::createAttribute( adios2::IO &IO, adios2::Engine &engine, detail::BufferedAttributeWrite ¶ms, @@ -1859,7 +1860,7 @@ namespace detail attr, params.bufferForVecString.data(), adios2::Mode::Deferred); } - void AttributeTypes >::readAttribute( + Datatype AttributeTypes>::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -1935,9 +1936,10 @@ namespace detail *resource = res; }; /* - * If writing char variables in ADIOS2, they might become either int8_t - * or uint8_t on disk depending on the platform. - * So allow reading from both types. + * If writing char variables in ADIOS2, they might become either int8_t, + * uint8_t or char on disk depending on platform, ADIOS2 version and + * ADIOS2 engine. + * So allow reading from all these types. */ switch (preloadedAttributes.attributeType(name)) { @@ -1946,10 +1948,10 @@ namespace detail * ADIOS2 does not have an explicit char type, * we don't have an explicit schar type. * Until this is fixed, we use CHAR to represent ADIOS signed char. + * @todo revisit this workaround and its necessity */ case Datatype::CHAR: { - using schar_t = signed char; - loadFromDatatype(schar_t{}); + loadFromDatatype(char{}); break; } case Datatype::UCHAR: { @@ -1957,16 +1959,21 @@ namespace detail loadFromDatatype(uchar_t{}); break; } + case Datatype::SCHAR: { + using schar_t = signed char; + loadFromDatatype(schar_t{}); + break; + } default: { throw std::runtime_error( - "[ADIOS2] Expecting 2D ADIOS variable of " - "type signed or unsigned char."); + "[ADIOS2] Expecting 2D ADIOS variable of any char type."); } } + return Datatype::VEC_STRING; } template - void AttributeTypes >::createAttribute( + void AttributeTypes>::createAttribute( adios2::IO &IO, adios2::Engine &engine, detail::BufferedAttributeWrite ¶ms, @@ -1988,7 +1995,7 @@ namespace detail } template - void AttributeTypes >::readAttribute( + Datatype AttributeTypes>::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -2005,6 +2012,7 @@ namespace detail std::array res; std::copy_n(attr.data, n, res.data()); *resource = std::move(res); + return determineDatatype>(); } void AttributeTypes::createAttribute( @@ -2019,7 +2027,7 @@ namespace detail IO, engine, params, toRep(value)); } - void AttributeTypes::readAttribute( + Datatype AttributeTypes::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -2035,6 +2043,7 @@ namespace detail } *resource = fromRep(*attr.data); + return Datatype::BOOL; } void BufferedGet::run(BufferedActions &ba) diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index 5e30ecabb3..7bed2f15c7 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -108,6 +108,9 @@ void CommonADIOS1IOHandlerImpl::flush_attribute( case DT::VEC_UCHAR: nelems = att.get >().size(); break; + case DT::VEC_SCHAR: + nelems = att.get >().size(); + break; case DT::VEC_USHORT: nelems = att.get >().size(); break; @@ -156,6 +159,11 @@ void CommonADIOS1IOHandlerImpl::flush_attribute( *ptr = att.get(); break; } + case DT::SCHAR: { + auto ptr = reinterpret_cast(values.get()); + *ptr = att.get(); + break; + } case DT::SHORT: { auto ptr = reinterpret_cast(values.get()); *ptr = att.get(); @@ -274,6 +282,13 @@ void CommonADIOS1IOHandlerImpl::flush_attribute( ptr[i] = vec[i]; break; } + case DT::VEC_SCHAR: { + auto ptr = reinterpret_cast(values.get()); + auto const &vec = att.get >(); + for (size_t i = 0; i < vec.size(); ++i) + ptr[i] = vec[i]; + break; + } case DT::VEC_USHORT: { auto ptr = reinterpret_cast(values.get()); auto const &vec = att.get >(); @@ -1149,6 +1164,7 @@ void CommonADIOS1IOHandlerImpl::readDataset( case DT::ULONGLONG: case DT::CHAR: case DT::UCHAR: + case DT::SCHAR: case DT::BOOL: break; case DT::UNDEFINED: diff --git a/src/IO/HDF5/HDF5Auxiliary.cpp b/src/IO/HDF5/HDF5Auxiliary.cpp index f77ddc68d2..33201d3ab5 100644 --- a/src/IO/HDF5/HDF5Auxiliary.cpp +++ b/src/IO/HDF5/HDF5Auxiliary.cpp @@ -62,6 +62,9 @@ hid_t openPMD::GetH5DataType::operator()(Attribute const &att) case DT::UCHAR: case DT::VEC_UCHAR: return H5Tcopy(H5T_NATIVE_UCHAR); + case DT::SCHAR: + case DT::VEC_SCHAR: + return H5Tcopy(H5T_NATIVE_SCHAR); case DT::SHORT: case DT::VEC_SHORT: return H5Tcopy(H5T_NATIVE_SHORT); @@ -145,6 +148,7 @@ hid_t openPMD::getH5DataSpace(Attribute const &att) { case DT::CHAR: case DT::UCHAR: + case DT::SCHAR: case DT::SHORT: case DT::INT: case DT::LONG: @@ -198,6 +202,12 @@ hid_t openPMD::getH5DataSpace(Attribute const &att) H5Sset_extent_simple(vec_t_id, 1, dims, nullptr); return vec_t_id; } + case DT::VEC_SCHAR: { + hid_t vec_t_id = H5Screate(H5S_SIMPLE); + hsize_t dims[1] = {att.get >().size()}; + H5Sset_extent_simple(vec_t_id, 1, dims, nullptr); + return vec_t_id; + } case DT::VEC_USHORT: { hid_t vec_t_id = H5Screate(H5S_SIMPLE); hsize_t dims[1] = {att.get >().size()}; diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 51a013fca8..ae4e6588aa 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -1247,6 +1247,7 @@ void HDF5IOHandlerImpl::writeDataset( case DT::ULONGLONG: case DT::CHAR: case DT::UCHAR: + case DT::SCHAR: case DT::BOOL: status = H5Dwrite( dataset_id, @@ -1376,6 +1377,11 @@ void HDF5IOHandlerImpl::writeAttribute( status = H5Awrite(attribute_id, dataType, &u); break; } + case DT::SCHAR: { + auto u = att.get(); + status = H5Awrite(attribute_id, dataType, &u); + break; + } case DT::SHORT: { auto i = att.get(); status = H5Awrite(attribute_id, dataType, &i); @@ -1476,6 +1482,12 @@ void HDF5IOHandlerImpl::writeAttribute( dataType, att.get >().data()); break; + case DT::VEC_SCHAR: + status = H5Awrite( + attribute_id, + dataType, + att.get >().data()); + break; case DT::VEC_USHORT: status = H5Awrite( attribute_id, diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 9e7a93e6ea..bfa024a786 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -287,6 +287,20 @@ void RecordComponent::read() readBase(); } +namespace +{ + struct MakeConstant + { + template + static void call(RecordComponent rc, Attribute const &attr) + { + rc.makeConstant(attr.get()); + } + + static constexpr char const *errorMsg = "Unexpected constant datatype"; + }; +} // namespace + void RecordComponent::readBase() { using DT = Datatype; @@ -302,62 +316,7 @@ void RecordComponent::readBase() Attribute a(*aRead.resource); DT dtype = *aRead.dtype; written() = false; - switch (dtype) - { - case DT::LONG_DOUBLE: - makeConstant(a.get()); - break; - case DT::DOUBLE: - makeConstant(a.get()); - break; - case DT::FLOAT: - makeConstant(a.get()); - break; - case DT::CLONG_DOUBLE: - makeConstant(a.get >()); - break; - case DT::CDOUBLE: - makeConstant(a.get >()); - break; - case DT::CFLOAT: - makeConstant(a.get >()); - break; - case DT::SHORT: - makeConstant(a.get()); - break; - case DT::INT: - makeConstant(a.get()); - break; - case DT::LONG: - makeConstant(a.get()); - break; - case DT::LONGLONG: - makeConstant(a.get()); - break; - case DT::USHORT: - makeConstant(a.get()); - break; - case DT::UINT: - makeConstant(a.get()); - break; - case DT::ULONG: - makeConstant(a.get()); - break; - case DT::ULONGLONG: - makeConstant(a.get()); - break; - case DT::CHAR: - makeConstant(a.get()); - break; - case DT::UCHAR: - makeConstant(a.get()); - break; - case DT::BOOL: - makeConstant(a.get()); - break; - default: - throw std::runtime_error("Unexpected constant datatype"); - } + switchNonVectorType(dtype, *this, a); written() = true; aRead.name = "shape"; diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index b73c850409..aa2f600da1 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -333,6 +333,12 @@ void Attributable::readAttributes(ReadMode mode) a.get(), internal::SetAttributeMode::WhileReadingAttributes); break; + case DT::SCHAR: + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); + break; case DT::SHORT: setAttributeImpl( att, @@ -510,6 +516,12 @@ void Attributable::readAttributes(ReadMode mode) a.get > >(), internal::SetAttributeMode::WhileReadingAttributes); break; + case DT::VEC_SCHAR: + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); + break; case DT::VEC_STRING: setAttributeImpl( att, diff --git a/src/binding/python/PatchRecordComponent.cpp b/src/binding/python/PatchRecordComponent.cpp index cecd57b252..a734e1d6b9 100644 --- a/src/binding/python/PatchRecordComponent.cpp +++ b/src/binding/python/PatchRecordComponent.cpp @@ -21,6 +21,7 @@ #include #include +#include "openPMD/DatatypeHelpers.hpp" #include "openPMD/auxiliary/ShareRaw.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" @@ -29,6 +30,20 @@ namespace py = pybind11; using namespace openPMD; +namespace +{ +struct Prc_Load +{ + template + static void call(PatchRecordComponent &prc, py::array &a) + { + prc.load(shareRaw((T *)a.mutable_data())); + } + + static constexpr char const *errorMsg = "Datatype not known in 'load'!"; +}; +} // namespace + void init_PatchRecordComponent(py::module &m) { py::class_( @@ -49,44 +64,7 @@ void init_PatchRecordComponent(py::module &m) auto const dtype = dtype_to_numpy(prc.getDatatype()); auto a = py::array(dtype, prc.getExtent()[0]); - if (prc.getDatatype() == Datatype::CHAR) - prc.load(shareRaw((char *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::UCHAR) - prc.load( - shareRaw((unsigned char *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::SHORT) - prc.load(shareRaw((short *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::INT) - prc.load(shareRaw((int *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::LONG) - prc.load(shareRaw((long *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::LONGLONG) - prc.load( - shareRaw((long long *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::USHORT) - prc.load( - shareRaw((unsigned short *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::UINT) - prc.load( - shareRaw((unsigned int *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::ULONG) - prc.load( - shareRaw((unsigned long *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::ULONGLONG) - prc.load( - shareRaw((unsigned long long *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::LONG_DOUBLE) - prc.load( - shareRaw((long double *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::DOUBLE) - prc.load(shareRaw((double *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::FLOAT) - prc.load(shareRaw((float *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::BOOL) - prc.load(shareRaw((bool *)a.mutable_data())); - else - throw std::runtime_error( - std::string("Datatype not known in 'load'!")); + switchNonVectorType(prc.getDatatype(), prc, a); return a; }) diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 272b6b27cd..c2c1acfebb 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -558,6 +558,8 @@ void load_chunk( load_data((char)0); else if (r.getDatatype() == Datatype::UCHAR) load_data((unsigned char)0); + else if (r.getDatatype() == Datatype::SCHAR) + load_data((signed char)0); else if (r.getDatatype() == Datatype::SHORT) load_data((short)0); else if (r.getDatatype() == Datatype::INT) @@ -664,6 +666,8 @@ inline void load_chunk( load_data(char()); else if (r.getDatatype() == Datatype::UCHAR) load_data((unsigned char)0); + else if (r.getDatatype() == Datatype::SCHAR) + load_data((signed char)0); else if (r.getDatatype() == Datatype::SHORT) load_data(short()); else if (r.getDatatype() == Datatype::INT) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 155d4fc58d..6846ee0f14 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1237,6 +1237,8 @@ inline void dtype_test(const std::string &backend) s.setAttribute("char", c); unsigned char uc = 'u'; s.setAttribute("uchar", uc); + signed char sc = 's'; + s.setAttribute("schar", sc); int16_t i16 = 16; s.setAttribute("int16", i16); int32_t i32 = 32; @@ -1268,7 +1270,9 @@ inline void dtype_test(const std::string &backend) "vecInt64", std::vector({9223372036854775806, 9223372036854775807})); s.setAttribute( - "vecUchar", std::vector({'u', 'c', 'h', 'a', 'r'})); + "vecUchar", std::vector({'u', 'c', 'h', 'a', 'r'})); + s.setAttribute( + "vecSchar", std::vector({'s', 'c', 'h', 'a', 'r'})); s.setAttribute("vecUint16", std::vector({65534u, 65535u})); s.setAttribute( "vecUint32", std::vector({4294967294u, 4294967295u})); @@ -1349,6 +1353,7 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("char").get() == 'c'); REQUIRE(s.getAttribute("uchar").get() == 'u'); + REQUIRE(s.getAttribute("schar").get() == 's'); REQUIRE(s.getAttribute("int16").get() == 16); REQUIRE(s.getAttribute("int32").get() == 32); REQUIRE(s.getAttribute("int64").get() == 64); @@ -1375,8 +1380,11 @@ inline void dtype_test(const std::string &backend) s.getAttribute("vecInt64").get >() == std::vector({9223372036854775806, 9223372036854775807})); REQUIRE( - s.getAttribute("vecUchar").get >() == - std::vector({'u', 'c', 'h', 'a', 'r'})); + s.getAttribute("vecUchar").get >() == + std::vector({'u', 'c', 'h', 'a', 'r'})); + REQUIRE( + s.getAttribute("vecSchar").get >() == + std::vector({'s', 'c', 'h', 'a', 'r'})); REQUIRE( s.getAttribute("vecUint16").get >() == std::vector({65534u, 65535u})); @@ -2292,7 +2300,7 @@ TEST_CASE("deletion_test", "[serial]") { for (auto const &t : testedFileExtensions()) { - if (t == "bp") + if (t == "bp" || t == "bp4" || t == "bp5") { continue; // deletion not implemented in ADIOS1 backend } @@ -5794,7 +5802,7 @@ void append_mode( } { Series read(filename, Access::READ_ONLY); - if (variableBased) + if (variableBased || extension == "bp5") { // in variable-based encodings, iterations are not parsed ahead of // time but as they go @@ -5825,7 +5833,7 @@ TEST_CASE("append_mode", "[serial]") { for (auto const &t : testedFileExtensions()) { - if (t == "bp") + if (t == "bp" || t == "bp4" || t == "bp5") { std::string jsonConfigOld = R"END( {