highfive/bits/H5DataType_misc.hpp

0001 /*
0002  *  Copyright (c), 2017, Adrien Devresse <adrien.devresse@epfl.ch>
0003  *
0004  *  Distributed under the Boost Software License, Version 1.0.
0005  *    (See accompanying file LICENSE_1_0.txt or copy at
0006  *          http://www.boost.org/LICENSE_1_0.txt)
0007  *
0008  */
0009 #pragma once
0010
0011 #include <string>
0012 #include <complex>
0013 #include <cstring>
0014 #if HIGHFIVE_CXX_STD >= 17
0015 #include <cstddef>
0016 #endif
0017
0018 #include <H5Ppublic.h>
0019
0020 #include "H5Inspector_misc.hpp"
0021 #include "h5t_wrapper.hpp"
0022 #include "h5i_wrapper.hpp"
0023
0024 namespace HighFive {
0025
0026 namespace {  // unnamed
0027 inline DataTypeClass convert_type_class(const H5T_class_t& tclass);
0028 inline std::string type_class_string(DataTypeClass);
0029 inline hid_t create_string(std::size_t length);
0030 }  // namespace
0031
0032 inline bool DataType::empty() const noexcept {
0033     return _hid == H5I_INVALID_HID;
0034 }
0035
0036 inline DataTypeClass DataType::getClass() const {
0037     return convert_type_class(detail::h5t_get_class(_hid));
0038 }
0039
0040 inline size_t DataType::getSize() const {
0041     return detail::h5t_get_size(_hid);
0042 }
0043
0044 inline bool DataType::operator==(const DataType& other) const {
0045     return detail::h5t_equal(_hid, other._hid) > 0;
0046 }
0047
0048 inline bool DataType::operator!=(const DataType& other) const {
0049     return !(*this == other);
0050 }
0051
0052 inline bool DataType::isVariableStr() const {
0053     return detail::h5t_is_variable_str(_hid) > 0;
0054 }
0055
0056 inline bool DataType::isFixedLenStr() const {
0057     return getClass() == DataTypeClass::String && !isVariableStr();
0058 }
0059
0060 inline bool DataType::isReference() const {
0061     return detail::h5t_equal(_hid, H5T_STD_REF_OBJ) > 0;
0062 }
0063
0064 inline StringType DataType::asStringType() const {
0065     if (getClass() != DataTypeClass::String) {
0066         throw DataTypeException("Invalid conversion to StringType.");
0067     }
0068
0069     if (isValid()) {
0070         detail::h5i_inc_ref(_hid);
0071     }
0072
0073     return StringType(_hid);
0074 }
0075
0076 inline std::string DataType::string() const {
0077     return type_class_string(getClass()) + std::to_string(getSize() * 8);
0078 }
0079
0080 inline StringPadding StringType::getPadding() const {
0081     return StringPadding(detail::h5t_get_strpad(_hid));
0082 }
0083
0084 inline CharacterSet StringType::getCharacterSet() const {
0085     return CharacterSet(detail::h5t_get_cset(_hid));
0086 }
0087
0088 inline FixedLengthStringType::FixedLengthStringType(size_t size,
0089                                                     StringPadding padding,
0090                                                     CharacterSet character_set) {
0091     if (size == 0 && padding == StringPadding::NullTerminated) {
0092         throw DataTypeException(
0093             "Fixed-length, null-terminated need at least one byte to store the null-character.");
0094     }
0095
0096     _hid = detail::h5t_copy(H5T_C_S1);
0097
0098     detail::h5t_set_size(_hid, hsize_t(size));
0099     detail::h5t_set_cset(_hid, H5T_cset_t(character_set));
0100     detail::h5t_set_strpad(_hid, H5T_str_t(padding));
0101 }
0102
0103 inline VariableLengthStringType::VariableLengthStringType(CharacterSet character_set) {
0104     _hid = detail::h5t_copy(H5T_C_S1);
0105
0106     detail::h5t_set_size(_hid, H5T_VARIABLE);
0107     detail::h5t_set_cset(_hid, H5T_cset_t(character_set));
0108 }
0109
0110 // char mapping
0111 template <>
0112 inline AtomicType<char>::AtomicType() {
0113     _hid = detail::h5t_copy(H5T_NATIVE_CHAR);
0114 }
0115
0116 template <>
0117 inline AtomicType<signed char>::AtomicType() {
0118     _hid = detail::h5t_copy(H5T_NATIVE_SCHAR);
0119 }
0120
0121 template <>
0122 inline AtomicType<unsigned char>::AtomicType() {
0123     _hid = detail::h5t_copy(H5T_NATIVE_UCHAR);
0124 }
0125
0126 // short mapping
0127 template <>
0128 inline AtomicType<short>::AtomicType() {
0129     _hid = detail::h5t_copy(H5T_NATIVE_SHORT);
0130 }
0131
0132 template <>
0133 inline AtomicType<unsigned short>::AtomicType() {
0134     _hid = detail::h5t_copy(H5T_NATIVE_USHORT);
0135 }
0136
0137 // integer mapping
0138 template <>
0139 inline AtomicType<int>::AtomicType() {
0140     _hid = detail::h5t_copy(H5T_NATIVE_INT);
0141 }
0142
0143 template <>
0144 inline AtomicType<unsigned>::AtomicType() {
0145     _hid = detail::h5t_copy(H5T_NATIVE_UINT);
0146 }
0147
0148 // long mapping
0149 template <>
0150 inline AtomicType<long>::AtomicType() {
0151     _hid = detail::h5t_copy(H5T_NATIVE_LONG);
0152 }
0153
0154 template <>
0155 inline AtomicType<unsigned long>::AtomicType() {
0156     _hid = detail::h5t_copy(H5T_NATIVE_ULONG);
0157 }
0158
0159 // long long mapping
0160 template <>
0161 inline AtomicType<long long>::AtomicType() {
0162     _hid = detail::h5t_copy(H5T_NATIVE_LLONG);
0163 }
0164
0165 template <>
0166 inline AtomicType<unsigned long long>::AtomicType() {
0167     _hid = detail::h5t_copy(H5T_NATIVE_ULLONG);
0168 }
0169
0170 // half-float, float, double and long double mapping
0171 template <>
0172 inline AtomicType<float>::AtomicType() {
0173     _hid = detail::h5t_copy(H5T_NATIVE_FLOAT);
0174 }
0175
0176 template <>
0177 inline AtomicType<double>::AtomicType() {
0178     _hid = detail::h5t_copy(H5T_NATIVE_DOUBLE);
0179 }
0180
0181 template <>
0182 inline AtomicType<long double>::AtomicType() {
0183     _hid = detail::h5t_copy(H5T_NATIVE_LDOUBLE);
0184 }
0185
0186 // std string
0187 template <>
0188 inline AtomicType<std::string>::AtomicType() {
0189     _hid = create_string(H5T_VARIABLE);
0190 }
0191
0192 #if HIGHFIVE_CXX_STD >= 17
0193 // std byte
0194 template <>
0195 inline AtomicType<std::byte>::AtomicType() {
0196     _hid = detail::h5t_copy(H5T_NATIVE_B8);
0197 }
0198 #endif
0199
0200 // Fixed-Length strings
0201 // require class specialization templated for the char length
0202 template <size_t StrLen>
0203 class AtomicType<char[StrLen]>: public DataType {
0204   public:
0205     inline AtomicType()
0206         : DataType(create_string(StrLen)) {}
0207 };
0208
0209 template <size_t StrLen>
0210 class AtomicType<deprecated::FixedLenStringArray<StrLen>>: public DataType {
0211   public:
0212     inline AtomicType()
0213         : DataType(create_string(StrLen)) {}
0214 };
0215
0216 template <typename T>
0217 class AtomicType<std::complex<T>>: public DataType {
0218   public:
0219     inline AtomicType()
0220         : DataType(
0221               CompoundType({{"r", create_datatype<T>(), 0}, {"i", create_datatype<T>(), sizeof(T)}},
0222                            sizeof(std::complex<T>))) {
0223         static_assert(std::is_arithmetic<T>::value,
0224                       "std::complex accepts only floating point and integral numbers.");
0225     }
0226 };
0227
0228 // For boolean we act as h5py
0229 inline EnumType<details::Boolean> create_enum_boolean() {
0230     return {{"FALSE", details::Boolean::HighFiveFalse}, {"TRUE", details::Boolean::HighFiveTrue}};
0231 }
0232
0233 // Other cases not supported. Fail early with a user message
0234 template <typename T>
0235 AtomicType<T>::AtomicType() {
0236     static_assert(details::inspector<T>::recursive_ndim == 0,
0237                   "Atomic types cant be arrays, except for char[] (fixed-length strings)");
0238     static_assert(details::inspector<T>::recursive_ndim > 0, "Type not supported");
0239 }
0240
0241
0242 namespace deprecated {
0243 template <std::size_t N>
0244 inline FixedLenStringArray<N>::FixedLenStringArray(const char array[][N], std::size_t length) {
0245     datavec.resize(length);
0246     std::memcpy(datavec[0].data(), array[0].data(), N * length);
0247 }
0248
0249 template <std::size_t N>
0250 inline FixedLenStringArray<N>::FixedLenStringArray(const std::string* iter_begin,
0251                                                    const std::string* iter_end) {
0252     datavec.reserve(static_cast<std::size_t>(iter_end - iter_begin));
0253     for (std::string const* it = iter_begin; it != iter_end; ++it) {
0254         push_back(*it);
0255     }
0256 }
0257
0258 template <std::size_t N>
0259 inline FixedLenStringArray<N>::FixedLenStringArray(const std::vector<std::string>& vec)
0260     : FixedLenStringArray(vec.data(), vec.data() + vec.size()) {}
0261
0262 template <std::size_t N>
0263 inline FixedLenStringArray<N>::FixedLenStringArray(
0264     const std::initializer_list<std::string>& init_list)
0265     : FixedLenStringArray(init_list.begin(), init_list.end()) {}
0266
0267 template <std::size_t N>
0268 inline void FixedLenStringArray<N>::push_back(const std::string& src) {
0269     datavec.emplace_back();
0270     const size_t length = std::min(N - 1, src.length());
0271     std::memcpy(datavec.back().data(), src.c_str(), length);
0272     datavec.back()[length] = 0;
0273 }
0274
0275 template <std::size_t N>
0276 inline void FixedLenStringArray<N>::push_back(const std::array<char, N>& src) {
0277     datavec.emplace_back();
0278     std::copy(src.begin(), src.end(), datavec.back().data());
0279 }
0280
0281 template <std::size_t N>
0282 inline std::string FixedLenStringArray<N>::getString(std::size_t i) const {
0283     return std::string(datavec[i].data());
0284 }
0285 }  // namespace deprecated
0286
0287 // Internal
0288 // Reference mapping
0289 template <>
0290 inline AtomicType<Reference>::AtomicType() {
0291     _hid = detail::h5t_copy(H5T_STD_REF_OBJ);
0292 }
0293
0294 inline size_t find_first_atomic_member_size(hid_t hid) {
0295     // Recursive exit condition
0296     if (detail::h5t_get_class(hid) == H5T_COMPOUND) {
0297         auto number_of_members = detail::h5t_get_nmembers(hid);
0298         if (number_of_members == -1) {
0299             throw DataTypeException("Cannot get members of CompoundType with hid: " +
0300                                     std::to_string(hid));
0301         }
0302         if (number_of_members == 0) {
0303             throw DataTypeException("No members defined for CompoundType with hid: " +
0304                                     std::to_string(hid));
0305         }
0306
0307         auto member_type = detail::h5t_get_member_type(hid, 0);
0308         auto size = find_first_atomic_member_size(member_type);
0309         detail::h5t_close(member_type);
0310         return size;
0311     } else if (detail::h5t_get_class(hid) == H5T_STRING) {
0312         return 1;
0313     }
0314     return detail::h5t_get_size(hid);
0315 }
0316
0317 // Calculate the padding required to align an element of a struct
0318 // For padding see explanation here: https://en.cppreference.com/w/cpp/language/object#Alignment
0319 // It is to compute padding following last element inserted inside a struct
0320 // 1) We want to push back an element padded to the structure
0321 // 'current_size' is the size of the structure before adding the new element.
0322 // 'member_size' the size of the element we want to add.
0323 // 2) We want to compute the final padding for the global structure
0324 // 'current_size' is the size of the whole structure without final padding
0325 // 'member_size' is the maximum size of all element of the struct
0326 //
0327 // The basic formula is only to know how much we need to add to 'current_size' to fit
0328 // 'member_size'.
0329 // And at the end, we do another computation because the end padding, should fit the biggest
0330 // element of the struct.
0331 //
0332 // As we are with `size_t` element, we need to compute everything inside R+
0333 #define _H5_STRUCT_PADDING(current_size, member_size)                                \
0334     (((member_size) >= (current_size))                                               \
0335          ? (((member_size) - (current_size)) % (member_size))                        \
0336          : ((((member_size) - (((current_size) - (member_size)) % (member_size)))) % \
0337             (member_size)))
0338
0339 inline void CompoundType::create(size_t size) {
0340     if (size == 0) {
0341         size_t current_size = 0, max_atomic_size = 0;
0342
0343         // Do a first pass to find the total size of the compound datatype
0344         for (auto& member: members) {
0345             size_t member_size = detail::h5t_get_size(member.base_type.getId());
0346
0347             if (member_size == 0) {
0348                 throw DataTypeException("Cannot get size of DataType with hid: " +
0349                                         std::to_string(member.base_type.getId()));
0350             }
0351
0352             size_t first_atomic_size = find_first_atomic_member_size(member.base_type.getId());
0353
0354             // Set the offset of this member within the struct according to the
0355             // standard alignment rules. The c++ standard specifies that:
0356             // > objects have an alignment requirement of which their size is a multiple
0357             member.offset = current_size + _H5_STRUCT_PADDING(current_size, first_atomic_size);
0358
0359             // Set the current size to the end of the new member
0360             current_size = member.offset + member_size;
0361
0362             // Keep track of the highest atomic member size because it's needed
0363             // for the padding of the complete compound type.
0364             max_atomic_size = std::max(max_atomic_size, first_atomic_size);
0365         }
0366
0367         size = current_size + _H5_STRUCT_PADDING(current_size, max_atomic_size);
0368     }
0369
0370     // Create the HDF5 type
0371     _hid = detail::h5t_create(H5T_COMPOUND, size);
0372
0373     // Loop over all the members and insert them into the datatype
0374     for (const auto& member: members) {
0375         detail::h5t_insert(_hid, member.name.c_str(), member.offset, member.base_type.getId());
0376     }
0377 }
0378
0379 #undef _H5_STRUCT_PADDING
0380
0381 inline void CompoundType::commit(const Object& object, const std::string& name) const {
0382     detail::h5t_commit2(
0383         object.getId(), name.c_str(), getId(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
0384 }
0385
0386 template <typename T>
0387 inline void EnumType<T>::create() {
0388     // Create the HDF5 type
0389     _hid = detail::h5t_enum_create(AtomicType<typename std::underlying_type<T>::type>{}.getId());
0390
0391     // Loop over all the members and insert them into the datatype
0392     for (const auto& member: members) {
0393         detail::h5t_enum_insert(_hid, member.name.c_str(), &(member.value));
0394     }
0395 }
0396
0397 template <typename T>
0398 inline void EnumType<T>::commit(const Object& object, const std::string& name) const {
0399     detail::h5t_commit2(
0400         object.getId(), name.c_str(), getId(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
0401 }
0402
0403 namespace {
0404
0405 inline hid_t create_string(size_t length) {
0406     hid_t _hid = detail::h5t_copy(H5T_C_S1);
0407     detail::h5t_set_size(_hid, length);
0408     detail::h5t_set_cset(_hid, H5T_CSET_UTF8);
0409     return _hid;
0410 }
0411
0412
0413 inline DataTypeClass convert_type_class(const H5T_class_t& tclass) {
0414     switch (tclass) {
0415     case H5T_TIME:
0416         return DataTypeClass::Time;
0417     case H5T_INTEGER:
0418         return DataTypeClass::Integer;
0419     case H5T_FLOAT:
0420         return DataTypeClass::Float;
0421     case H5T_STRING:
0422         return DataTypeClass::String;
0423     case H5T_BITFIELD:
0424         return DataTypeClass::BitField;
0425     case H5T_OPAQUE:
0426         return DataTypeClass::Opaque;
0427     case H5T_COMPOUND:
0428         return DataTypeClass::Compound;
0429     case H5T_REFERENCE:
0430         return DataTypeClass::Reference;
0431     case H5T_ENUM:
0432         return DataTypeClass::Enum;
0433     case H5T_VLEN:
0434         return DataTypeClass::VarLen;
0435     case H5T_ARRAY:
0436         return DataTypeClass::Array;
0437     case H5T_NO_CLASS:
0438     case H5T_NCLASSES:
0439     default:
0440         return DataTypeClass::Invalid;
0441     }
0442 }
0443
0444
0445 inline std::string type_class_string(DataTypeClass tclass) {
0446     switch (tclass) {
0447     case DataTypeClass::Time:
0448         return "Time";
0449     case DataTypeClass::Integer:
0450         return "Integer";
0451     case DataTypeClass::Float:
0452         return "Float";
0453     case DataTypeClass::String:
0454         return "String";
0455     case DataTypeClass::BitField:
0456         return "BitField";
0457     case DataTypeClass::Opaque:
0458         return "Opaque";
0459     case DataTypeClass::Compound:
0460         return "Compound";
0461     case DataTypeClass::Reference:
0462         return "Reference";
0463     case DataTypeClass::Enum:
0464         return "Enum";
0465     case DataTypeClass::VarLen:
0466         return "Varlen";
0467     case DataTypeClass::Array:
0468         return "Array";
0469     default:
0470         return "(Invalid)";
0471     }
0472 }
0473
0474 }  // unnamed namespace
0475
0476
0477 /// \brief Create a DataType instance representing type T
0478 template <typename T>
0479 inline DataType create_datatype() {
0480     return AtomicType<T>();
0481 }
0482
0483
0484 /// \brief Create a DataType instance representing type T and perform a sanity check on its size
0485 template <typename T>
0486 inline DataType create_and_check_datatype() {
0487     DataType t = create_datatype<T>();
0488     if (t.empty()) {
0489         throw DataTypeException("Type given to create_and_check_datatype is not valid");
0490     }
0491
0492     // Skip check if the base type is a variable length string
0493     if (t.isVariableStr()) {
0494         return t;
0495     }
0496
0497     // Check that the size of the template type matches the size that HDF5 is
0498     // expecting.
0499     if (t.isReference() || t.isFixedLenStr()) {
0500         return t;
0501     }
0502     if (sizeof(T) != t.getSize()) {
0503         std::ostringstream ss;
0504         ss << "Size of array type " << sizeof(T) << " != that of memory datatype " << t.getSize()
0505            << std::endl;
0506         throw DataTypeException(ss.str());
0507     }
0508
0509     return t;
0510 }
0511
0512 }  // namespace HighFive
0513 HIGHFIVE_REGISTER_TYPE(HighFive::details::Boolean, HighFive::create_enum_boolean)
0514
0515 namespace HighFive {
0516
0517 template <>
0518 inline DataType create_datatype<bool>() {
0519     return create_datatype<HighFive::details::Boolean>();
0520 }
0521
0522 }  // namespace HighFive
0523
0524 #ifdef H5_USE_HALF_FLOAT
0525 #include <highfive/half_float.hpp>
0526 #endif