root/ROOT/RNTupleTypes.hxx

0001 /// \file ROOT/RNTupleTypes.hxx
0002 /// \ingroup NTuple
0003 /// \author Jakob Blomer <jblomer@cern.ch>
0004 /// \date 2018-10-04
0005
0006 /*************************************************************************
0007  * Copyright (C) 1995-2020, Rene Brun and Fons Rademakers.               *
0008  * All rights reserved.                                                  *
0009  *                                                                       *
0010  * For the licensing terms see $ROOTSYS/LICENSE.                         *
0011  * For the list of contributors see $ROOTSYS/README/CREDITS.             *
0012  *************************************************************************/
0013
0014 #ifndef ROOT_RNTupleTypes
0015 #define ROOT_RNTupleTypes
0016
0017 #include <ROOT/RConfig.hxx>
0018
0019 #include <cstddef>
0020 #include <cstdint>
0021 #include <limits>
0022 #include <ostream>
0023 #include <type_traits>
0024 #include <variant>
0025
0026 namespace ROOT {
0027
0028 /// Helper types to present an offset column as array of collection sizes.
0029 /// See RField<RNTupleCardinality<SizeT>> for details.
0030 template <typename SizeT>
0031 struct RNTupleCardinality {
0032    static_assert(std::is_same_v<SizeT, std::uint32_t> || std::is_same_v<SizeT, std::uint64_t>,
0033                  "RNTupleCardinality is only supported with std::uint32_t or std::uint64_t template parameters");
0034
0035    using ValueType = SizeT;
0036
0037    RNTupleCardinality() : fValue(0) {}
0038    explicit constexpr RNTupleCardinality(ValueType value) : fValue(value) {}
0039    RNTupleCardinality &operator=(const ValueType value)
0040    {
0041       fValue = value;
0042       return *this;
0043    }
0044    operator ValueType() const { return fValue; }
0045
0046    ValueType fValue;
0047 };
0048
0049 // clang-format off
0050 /**
0051 \class ROOT::ENTupleColumnType
0052 \ingroup NTuple
0053 \brief The available trivial, native content types of a column
0054
0055 More complex types, such as classes, get translated into columns of such simple types by the RField.
0056 When changed, remember to update
0057   - RColumnElement::Generate()
0058   - RColumnElement::GetTypeName()
0059   - RColumnElement::GetValidBitRange()
0060   - RColumnElement template specializations / packing & unpacking
0061   - If necessary, endianess handling for the packing + unit test in ntuple_endian
0062   - RNTupleSerializer::[Des|S]erializeColumnType
0063 */
0064 // clang-format on
0065 enum class ENTupleColumnType {
0066    kUnknown = 0,
0067    // type for root columns of (nested) collections; offsets are relative to the current cluster
0068    kIndex64,
0069    kIndex32,
0070    // 96 bit column that is a pair of a kIndex64 and a 32bit dispatch tag to a column ID;
0071    // used to serialize std::variant.
0072    kSwitch,
0073    kByte,
0074    kChar,
0075    kBit,
0076    kReal64,
0077    kReal32,
0078    kReal16,
0079    kInt64,
0080    kUInt64,
0081    kInt32,
0082    kUInt32,
0083    kInt16,
0084    kUInt16,
0085    kInt8,
0086    kUInt8,
0087    kSplitIndex64,
0088    kSplitIndex32,
0089    kSplitReal64,
0090    kSplitReal32,
0091    kSplitInt64,
0092    kSplitUInt64,
0093    kSplitInt32,
0094    kSplitUInt32,
0095    kSplitInt16,
0096    kSplitUInt16,
0097    kReal32Trunc,
0098    kReal32Quant,
0099    kMax,
0100 };
0101
0102 /// The fields in the RNTuple data model tree can carry different structural information about the type system.
0103 /// Collection fields have an offset column and subfields with arbitrary cardinality, record fields have no
0104 /// materialization on the primitive column layer and an arbitrary number of subfields. Plain fields are either
0105 /// leafs (e.g., `float`) or "wrapper fields" with exactly one child that has the same cardinality
0106 /// (number of elements in the data set modulo field repetitions) as the parent (e.g., std::atomic<T>).
0107 // IMPORTANT: if you add members, remember to change the related `operator<<` below.
0108 enum class ENTupleStructure : std::uint16_t {
0109    kInvalid,
0110    kPlain,
0111    kCollection,
0112    kRecord,
0113    kVariant,
0114    kStreamer,
0115    kUnknown,
0116
0117    // for backwards compatibility
0118    kLeaf R__DEPRECATED(6, 42, "use instead ROOT::ENTupleStructure::kPlain") = kPlain
0119 };
0120
0121 inline std::ostream &operator<<(std::ostream &os, ENTupleStructure structure)
0122 {
0123    static const char *const names[] = {"Invalid", "Plain", "Collection", "Record", "Variant", "Streamer", "Unknown"};
0124    static_assert((std::size_t)ENTupleStructure::kUnknown + 1 == std::size(names));
0125
0126    if (R__likely(static_cast<std::size_t>(structure) <= std::size(names)))
0127       os << names[static_cast<std::uint16_t>(structure)];
0128    else
0129       os << "(invalid)";
0130    return os;
0131 }
0132
0133 /// Integer type long enough to hold the maximum number of entries in a column
0134 using NTupleSize_t = std::uint64_t;
0135 constexpr NTupleSize_t kInvalidNTupleIndex = std::uint64_t(-1);
0136
0137 /// Distriniguishes elements of the same type within a descriptor, e.g. different fields
0138 using DescriptorId_t = std::uint64_t;
0139 constexpr DescriptorId_t kInvalidDescriptorId = std::uint64_t(-1);
0140
0141 /// Addresses a column element or field item relative to a particular cluster, instead of a global NTupleSize_t index
0142 class RNTupleLocalIndex {
0143 private:
0144    ROOT::DescriptorId_t fClusterId = ROOT::kInvalidDescriptorId;
0145    ROOT::NTupleSize_t fIndexInCluster = ROOT::kInvalidNTupleIndex;
0146
0147 public:
0148    RNTupleLocalIndex() = default;
0149    RNTupleLocalIndex(const RNTupleLocalIndex &other) = default;
0150    RNTupleLocalIndex &operator=(const RNTupleLocalIndex &other) = default;
0151    constexpr RNTupleLocalIndex(ROOT::DescriptorId_t clusterId, ROOT::NTupleSize_t indexInCluster)
0152       : fClusterId(clusterId), fIndexInCluster(indexInCluster)
0153    {
0154    }
0155
0156    RNTupleLocalIndex operator+(ROOT::NTupleSize_t off) const
0157    {
0158       return RNTupleLocalIndex(fClusterId, fIndexInCluster + off);
0159    }
0160
0161    RNTupleLocalIndex operator-(ROOT::NTupleSize_t off) const
0162    {
0163       return RNTupleLocalIndex(fClusterId, fIndexInCluster - off);
0164    }
0165
0166    RNTupleLocalIndex operator*(ROOT::NTupleSize_t repetitionFactor) const
0167    {
0168       return RNTupleLocalIndex(fClusterId, fIndexInCluster * repetitionFactor);
0169    }
0170
0171    RNTupleLocalIndex operator++(int) /* postfix */
0172    {
0173       auto r = *this;
0174       fIndexInCluster++;
0175       return r;
0176    }
0177
0178    RNTupleLocalIndex &operator++() /* prefix */
0179    {
0180       ++fIndexInCluster;
0181       return *this;
0182    }
0183
0184    bool operator==(RNTupleLocalIndex other) const
0185    {
0186       return fClusterId == other.fClusterId && fIndexInCluster == other.fIndexInCluster;
0187    }
0188
0189    bool operator!=(RNTupleLocalIndex other) const { return !(*this == other); }
0190
0191    ROOT::DescriptorId_t GetClusterId() const { return fClusterId; }
0192    ROOT::NTupleSize_t GetIndexInCluster() const { return fIndexInCluster; }
0193 };
0194
0195 /// RNTupleLocator payload that is common for object stores using 64bit location information.
0196 /// This might not contain the full location of the content. In particular, for page locators this information may be
0197 /// used in conjunction with the cluster and column ID.
0198 class RNTupleLocatorObject64 {
0199 private:
0200    std::uint64_t fLocation = 0;
0201
0202 public:
0203    RNTupleLocatorObject64() = default;
0204    explicit RNTupleLocatorObject64(std::uint64_t location) : fLocation(location) {}
0205    bool operator==(const RNTupleLocatorObject64 &other) const { return fLocation == other.fLocation; }
0206    std::uint64_t GetLocation() const { return fLocation; }
0207 };
0208
0209 /// Generic information about the physical location of data. Values depend on the concrete storage type.  E.g.,
0210 /// for a local file `fPosition` might be a 64bit file offset. Referenced objects on storage can be compressed
0211 /// and therefore we need to store their actual size.
0212 class RNTupleLocator {
0213 public:
0214    /// Values for the _Type_ field in non-disk locators.  Serializable types must have the MSb == 0; see
0215    /// `doc/BinaryFormatSpecification.md` for details
0216    enum ELocatorType : std::uint8_t {
0217       // The kTypeFile locator may translate to an on-disk standard locator (type 0x00) or a large locator (type 0x01),
0218       // if the size of the referenced data block is >2GB
0219       kTypeFile = 0x00,
0220       kTypeDAOS = 0x02,
0221
0222       kLastSerializableType = 0x7f,
0223       kTypePageZero = kLastSerializableType + 1,
0224       kTypeUnknown,
0225    };
0226
0227 private:
0228    std::uint64_t fNBytesOnStorage = 0;
0229    /// Simple on-disk locators consisting of a 64-bit offset use variant type `uint64_t`; extended locators have
0230    /// `fPosition.index()` > 0
0231    std::variant<std::uint64_t, RNTupleLocatorObject64> fPosition{};
0232    /// For non-disk locators, the value for the _Type_ field. This makes it possible to have different type values even
0233    /// if the payload structure is identical.
0234    ELocatorType fType = kTypeFile;
0235    /// Reserved for use by concrete storage backends
0236    std::uint8_t fReserved = 0;
0237
0238 public:
0239    RNTupleLocator() = default;
0240
0241    bool operator==(const RNTupleLocator &other) const
0242    {
0243       return fPosition == other.fPosition && fNBytesOnStorage == other.fNBytesOnStorage && fType == other.fType;
0244    }
0245
0246    std::uint64_t GetNBytesOnStorage() const { return fNBytesOnStorage; }
0247    ELocatorType GetType() const { return fType; }
0248    std::uint8_t GetReserved() const { return fReserved; }
0249
0250    void SetNBytesOnStorage(std::uint64_t nBytesOnStorage) { fNBytesOnStorage = nBytesOnStorage; }
0251    void SetType(ELocatorType type) { fType = type; }
0252    void SetReserved(std::uint8_t reserved) { fReserved = reserved; }
0253
0254    template <typename T>
0255    T GetPosition() const
0256    {
0257       return std::get<T>(fPosition);
0258    }
0259
0260    template <typename T>
0261    void SetPosition(T position)
0262    {
0263       fPosition = position;
0264    }
0265 };
0266
0267 namespace Internal {
0268
0269 /// The in-memory representation of a 32bit or 64bit on-disk index column. Wraps the integer in a
0270 /// named type so that templates can distinguish between integer data columns and index columns.
0271 class RColumnIndex {
0272 public:
0273    using ValueType = std::uint64_t;
0274
0275 private:
0276    ValueType fValue = 0;
0277
0278 public:
0279    RColumnIndex() = default;
0280    explicit constexpr RColumnIndex(ValueType value) : fValue(value) {}
0281    RColumnIndex &operator=(const ValueType value)
0282    {
0283       fValue = value;
0284       return *this;
0285    }
0286    RColumnIndex &operator+=(const ValueType value)
0287    {
0288       fValue += value;
0289       return *this;
0290    }
0291    RColumnIndex operator++(int)
0292    {
0293       auto result = *this;
0294       fValue++;
0295       return result;
0296    }
0297    operator ValueType() const { return fValue; }
0298 };
0299
0300 /// Holds the index and the tag of a kSwitch column
0301 class RColumnSwitch {
0302 private:
0303    ROOT::NTupleSize_t fIndex;
0304    std::uint32_t fTag = 0;
0305
0306 public:
0307    RColumnSwitch() = default;
0308    RColumnSwitch(ROOT::NTupleSize_t index, std::uint32_t tag) : fIndex(index), fTag(tag) {}
0309    ROOT::NTupleSize_t GetIndex() const { return fIndex; }
0310    std::uint32_t GetTag() const { return fTag; }
0311 };
0312
0313 inline constexpr ENTupleColumnType kTestFutureColumnType =
0314    static_cast<ENTupleColumnType>(std::numeric_limits<std::underlying_type_t<ENTupleColumnType>>::max() - 1);
0315
0316 inline constexpr ROOT::ENTupleStructure kTestFutureFieldStructure =
0317    static_cast<ROOT::ENTupleStructure>(std::numeric_limits<std::underlying_type_t<ROOT::ENTupleStructure>>::max() - 1);
0318
0319 inline constexpr RNTupleLocator::ELocatorType kTestLocatorType = static_cast<RNTupleLocator::ELocatorType>(0x7e);
0320 static_assert(kTestLocatorType < RNTupleLocator::ELocatorType::kLastSerializableType);
0321
0322 } // namespace Internal
0323 } // namespace ROOT
0324
0325 #endif