root/ROOT/RNTupleSerialize.hxx

0001 /// \file ROOT/RNTupleSerialize.hxx
0002 /// \ingroup NTuple
0003 /// \author Jakob Blomer <jblomer@cern.ch>
0004 /// \author Javier Lopez-Gomez <javier.lopez.gomez@cern.ch>
0005 /// \date 2021-08-02
0006
0007 /*************************************************************************
0008  * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers.               *
0009  * All rights reserved.                                                  *
0010  *                                                                       *
0011  * For the licensing terms see $ROOTSYS/LICENSE.                         *
0012  * For the list of contributors see $ROOTSYS/README/CREDITS.             *
0013  *************************************************************************/
0014
0015 #ifndef ROOT_RNTupleSerialize
0016 #define ROOT_RNTupleSerialize
0017
0018 #include <ROOT/RError.hxx>
0019 #include <ROOT/RNTupleTypes.hxx>
0020 #include <ROOT/RSpan.hxx>
0021
0022 #include <Rtypes.h>
0023
0024 #include <cstdint>
0025 #include <limits>
0026 #include <map>
0027 #include <string>
0028 #include <unordered_map>
0029 #include <vector>
0030
0031 class TVirtualStreamerInfo;
0032
0033 namespace ROOT {
0034
0035 class RNTupleDescriptor;
0036 class RClusterDescriptor;
0037 enum class EExtraTypeInfoIds;
0038
0039 namespace Experimental {
0040 class RNTupleAttrSetDescriptor;
0041 namespace Internal {
0042 class RNTupleAttrSetDescriptorBuilder;
0043 } // namespace Internal
0044 } // namespace Experimental
0045
0046 namespace Internal {
0047
0048 class RClusterDescriptorBuilder;
0049 class RNTupleDescriptorBuilder;
0050
0051 // clang-format off
0052 /**
0053 \class ROOT::Internal::RNTupleSerializer
0054 \ingroup NTuple
0055 \brief A helper class for serializing and deserialization of the RNTuple binary format
0056
0057 All serialization and deserialization routines return the number of bytes processed (written or read).
0058
0059 The serialization routines can be called with a nullptr buffer, in which case only the size required to perform
0060 a serialization is returned. Deserialization routines must be called with a buffer that is sufficiently large.
0061
0062 Deserialization errors throw exceptions. Only when indicated or when passed as a parameter is the buffer size checked.
0063 */
0064 // clang-format on
0065 class RNTupleSerializer {
0066    static RResult<std::vector<ROOT::Internal::RClusterDescriptorBuilder>>
0067    DeserializePageListRaw(const void *buffer, std::uint64_t bufSize, ROOT::DescriptorId_t clusterGroupId,
0068                           const RNTupleDescriptor &desc);
0069
0070 public:
0071    static constexpr std::uint16_t kEnvelopeTypeHeader = 0x01;
0072    static constexpr std::uint16_t kEnvelopeTypeFooter = 0x02;
0073    static constexpr std::uint16_t kEnvelopeTypePageList = 0x03;
0074
0075    static constexpr std::uint16_t kFlagRepetitiveField = 0x01;
0076    static constexpr std::uint16_t kFlagProjectedField = 0x02;
0077    static constexpr std::uint16_t kFlagHasTypeChecksum = 0x04;
0078    static constexpr std::uint16_t kFlagIsSoACollection = 0x08;
0079
0080    static constexpr std::uint16_t kFlagDeferredColumn = 0x01;
0081    static constexpr std::uint16_t kFlagHasValueRange = 0x02;
0082
0083    static constexpr ROOT::DescriptorId_t kZeroFieldId = std::uint64_t(-2);
0084
0085    static constexpr int64_t kSuppressedColumnMarker = std::numeric_limits<std::int64_t>::min();
0086
0087    // In the page sink and the streamer field, the seen streamer infos are stored in a map
0088    // with the unique streamer info number being the key. Sorted by unique number.
0089    using StreamerInfoMap_t = std::map<Int_t, TVirtualStreamerInfo *>;
0090
0091    struct REnvelopeLink {
0092       std::uint64_t fLength = 0;
0093       RNTupleLocator fLocator;
0094    };
0095
0096    struct RClusterSummary {
0097       std::uint64_t fFirstEntry = 0;
0098       std::uint64_t fNEntries = 0;
0099       std::uint8_t fFlags = 0;
0100    };
0101
0102    struct RClusterGroup {
0103       std::uint64_t fMinEntry = 0;
0104       std::uint64_t fEntrySpan = 0;
0105       std::uint32_t fNClusters = 0;
0106       REnvelopeLink fPageListEnvelopeLink;
0107    };
0108
0109    /// The serialization context is used for the piecewise serialization of a descriptor.  During header serialization,
0110    /// the mapping of in-memory field and column IDs to on-disk IDs is built so that it can be used for the
0111    /// footer serialization in a second step.
0112    class RContext {
0113    private:
0114       std::uint64_t fHeaderSize = 0;
0115       std::uint64_t fHeaderXxHash3 = 0;
0116       std::map<ROOT::DescriptorId_t, ROOT::DescriptorId_t> fMem2OnDiskFieldIDs;
0117       std::map<ROOT::DescriptorId_t, ROOT::DescriptorId_t> fMem2OnDiskColumnIDs;
0118       std::map<ROOT::DescriptorId_t, ROOT::DescriptorId_t> fMem2OnDiskClusterIDs;
0119       std::map<ROOT::DescriptorId_t, ROOT::DescriptorId_t> fMem2OnDiskClusterGroupIDs;
0120       std::vector<ROOT::DescriptorId_t> fOnDisk2MemFieldIDs;
0121       std::vector<ROOT::DescriptorId_t> fOnDisk2MemColumnIDs;
0122       std::vector<ROOT::DescriptorId_t> fOnDisk2MemClusterIDs;
0123       std::vector<ROOT::DescriptorId_t> fOnDisk2MemClusterGroupIDs;
0124
0125    public:
0126       void SetHeaderSize(std::uint64_t size) { fHeaderSize = size; }
0127       std::uint64_t GetHeaderSize() const { return fHeaderSize; }
0128       void SetHeaderXxHash3(std::uint64_t xxhash3) { fHeaderXxHash3 = xxhash3; }
0129       std::uint64_t GetHeaderXxHash3() const { return fHeaderXxHash3; }
0130       /// Map an in-memory field ID to its on-disk counterpart. It is allowed to call this function multiple times for
0131       /// the same `memId`, in which case the return value is the on-disk ID assigned on the first call.
0132       ROOT::DescriptorId_t MapFieldId(ROOT::DescriptorId_t memId)
0133       {
0134          auto onDiskId = fOnDisk2MemFieldIDs.size();
0135          const auto &p = fMem2OnDiskFieldIDs.try_emplace(memId, onDiskId);
0136          if (p.second)
0137             fOnDisk2MemFieldIDs.push_back(memId);
0138          return (*p.first).second;
0139       }
0140       /// Map an in-memory column ID to its on-disk counterpart. It is allowed to call this function multiple times for
0141       /// the same `memId`, in which case the return value is the on-disk ID assigned on the first call.
0142       /// Note that we only map physical column IDs.  Logical column IDs of alias columns are shifted before the
0143       /// serialization of the extension header.  Also, we only need to query physical column IDs for the page list
0144       /// serialization.
0145       ROOT::DescriptorId_t MapPhysicalColumnId(ROOT::DescriptorId_t memId)
0146       {
0147          auto onDiskId = fOnDisk2MemColumnIDs.size();
0148          const auto &p = fMem2OnDiskColumnIDs.try_emplace(memId, onDiskId);
0149          if (p.second)
0150             fOnDisk2MemColumnIDs.push_back(memId);
0151          return (*p.first).second;
0152       }
0153       ROOT::DescriptorId_t MapClusterId(ROOT::DescriptorId_t memId)
0154       {
0155          auto onDiskId = fOnDisk2MemClusterIDs.size();
0156          fMem2OnDiskClusterIDs[memId] = onDiskId;
0157          fOnDisk2MemClusterIDs.push_back(memId);
0158          return onDiskId;
0159       }
0160       ROOT::DescriptorId_t MapClusterGroupId(ROOT::DescriptorId_t memId)
0161       {
0162          auto onDiskId = fOnDisk2MemClusterGroupIDs.size();
0163          fMem2OnDiskClusterGroupIDs[memId] = onDiskId;
0164          fOnDisk2MemClusterGroupIDs.push_back(memId);
0165          return onDiskId;
0166       }
0167       /// Map in-memory field and column IDs to their on-disk counterparts. This function is unconditionally called
0168       /// during header serialization.  This function must be manually called after an incremental schema update as page
0169       /// list serialization requires all columns to be mapped.
0170       void MapSchema(const RNTupleDescriptor &desc, bool forHeaderExtension);
0171
0172       ROOT::DescriptorId_t GetOnDiskFieldId(ROOT::DescriptorId_t memId) const { return fMem2OnDiskFieldIDs.at(memId); }
0173       ROOT::DescriptorId_t GetOnDiskColumnId(ROOT::DescriptorId_t memId) const
0174       {
0175          return fMem2OnDiskColumnIDs.at(memId);
0176       }
0177       ROOT::DescriptorId_t GetOnDiskClusterId(ROOT::DescriptorId_t memId) const
0178       {
0179          return fMem2OnDiskClusterIDs.at(memId);
0180       }
0181       ROOT::DescriptorId_t GetOnDiskClusterGroupId(ROOT::DescriptorId_t memId) const
0182       {
0183          return fMem2OnDiskClusterGroupIDs.at(memId);
0184       }
0185       ROOT::DescriptorId_t GetMemFieldId(ROOT::DescriptorId_t onDiskId) const { return fOnDisk2MemFieldIDs[onDiskId]; }
0186       ROOT::DescriptorId_t GetMemColumnId(ROOT::DescriptorId_t onDiskId) const
0187       {
0188          return fOnDisk2MemColumnIDs[onDiskId];
0189       }
0190       ROOT::DescriptorId_t GetMemClusterId(ROOT::DescriptorId_t onDiskId) const
0191       {
0192          return fOnDisk2MemClusterIDs[onDiskId];
0193       }
0194       ROOT::DescriptorId_t GetMemClusterGroupId(ROOT::DescriptorId_t onDiskId) const
0195       {
0196          return fOnDisk2MemClusterGroupIDs[onDiskId];
0197       }
0198
0199       /// Return a vector containing the in-memory field ID for each on-disk counterpart, in order, i.e. the `i`-th
0200       /// value corresponds to the in-memory field ID for `i`-th on-disk ID
0201       const std::vector<ROOT::DescriptorId_t> &GetOnDiskFieldList() const { return fOnDisk2MemFieldIDs; }
0202    };
0203
0204    /// Writes a XxHash-3 64bit checksum of the byte range given by data and length.
0205    static std::uint32_t
0206    SerializeXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3, void *buffer);
0207    /// Expects an xxhash3 checksum in the 8 bytes following data + length and verifies it.
0208    static RResult<void> VerifyXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3);
0209    static RResult<void> VerifyXxHash3(const unsigned char *data, std::uint64_t length);
0210
0211    static std::uint32_t SerializeInt16(std::int16_t val, void *buffer);
0212    static std::uint32_t DeserializeInt16(const void *buffer, std::int16_t &val);
0213    static std::uint32_t SerializeUInt16(std::uint16_t val, void *buffer);
0214    static std::uint32_t DeserializeUInt16(const void *buffer, std::uint16_t &val);
0215
0216    static std::uint32_t SerializeInt32(std::int32_t val, void *buffer);
0217    static std::uint32_t DeserializeInt32(const void *buffer, std::int32_t &val);
0218    static std::uint32_t SerializeUInt32(std::uint32_t val, void *buffer);
0219    static std::uint32_t DeserializeUInt32(const void *buffer, std::uint32_t &val);
0220
0221    static std::uint32_t SerializeInt64(std::int64_t val, void *buffer);
0222    static std::uint32_t DeserializeInt64(const void *buffer, std::int64_t &val);
0223    static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer);
0224    static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val);
0225
0226    static std::uint32_t SerializeString(const std::string &val, void *buffer);
0227    static RResult<std::uint32_t> DeserializeString(const void *buffer, std::uint64_t bufSize, std::string &val);
0228
0229    /// While we could just interpret the enums as ints, we make the translation explicit
0230    /// in order to avoid accidentally changing the on-disk numbers when adjusting the enum classes.
0231    static RResult<std::uint32_t> SerializeFieldStructure(ROOT::ENTupleStructure structure, void *buffer);
0232    static RResult<std::uint32_t> SerializeColumnType(ROOT::ENTupleColumnType type, void *buffer);
0233    static RResult<std::uint32_t> SerializeExtraTypeInfoId(ROOT::EExtraTypeInfoIds id, void *buffer);
0234    static RResult<std::uint32_t> DeserializeFieldStructure(const void *buffer, ROOT::ENTupleStructure &structure);
0235    static RResult<std::uint32_t> DeserializeColumnType(const void *buffer, ROOT::ENTupleColumnType &type);
0236    static RResult<std::uint32_t> DeserializeExtraTypeInfoId(const void *buffer, ROOT::EExtraTypeInfoIds &id);
0237
0238    static std::uint32_t SerializeEnvelopePreamble(std::uint16_t envelopeType, void *buffer);
0239    static RResult<std::uint32_t> SerializeEnvelopePostscript(unsigned char *envelope, std::uint64_t size);
0240    static RResult<std::uint32_t>
0241    SerializeEnvelopePostscript(unsigned char *envelope, std::uint64_t size, std::uint64_t &xxhash3);
0242    // The bufSize must include the 8 bytes for the final xxhash3 checksum.
0243    static RResult<std::uint32_t>
0244    DeserializeEnvelope(const void *buffer, std::uint64_t bufSize, std::uint16_t expectedType);
0245    static RResult<std::uint32_t>
0246    DeserializeEnvelope(const void *buffer, std::uint64_t bufSize, std::uint16_t expectedType, std::uint64_t &xxhash3);
0247
0248    static std::uint32_t SerializeRecordFramePreamble(void *buffer);
0249    static std::uint32_t SerializeListFramePreamble(std::uint32_t nitems, void *buffer);
0250    static RResult<std::uint32_t> SerializeFramePostscript(void *frame, std::uint64_t size);
0251    static RResult<std::uint32_t>
0252    DeserializeFrameHeader(const void *buffer, std::uint64_t bufSize, std::uint64_t &frameSize, std::uint32_t &nitems);
0253    static RResult<std::uint32_t>
0254    DeserializeFrameHeader(const void *buffer, std::uint64_t bufSize, std::uint64_t &frameSize);
0255
0256    // An empty flags vector will be serialized as a single, zero feature flag
0257    // The most significant bit in every flag is reserved and must _not_ be set
0258    static RResult<std::uint32_t> SerializeFeatureFlags(const std::vector<std::uint64_t> &flags, void *buffer);
0259    static RResult<std::uint32_t>
0260    DeserializeFeatureFlags(const void *buffer, std::uint64_t bufSize, std::vector<std::uint64_t> &flags);
0261
0262    static RResult<std::uint32_t> SerializeLocator(const RNTupleLocator &locator, void *buffer);
0263    static RResult<std::uint32_t> SerializeEnvelopeLink(const REnvelopeLink &envelopeLink, void *buffer);
0264    static RResult<std::uint32_t> DeserializeLocator(const void *buffer, std::uint64_t bufSize, RNTupleLocator &locator);
0265    static RResult<std::uint32_t>
0266    DeserializeEnvelopeLink(const void *buffer, std::uint64_t bufSize, REnvelopeLink &envelopeLink);
0267
0268    static RResult<std::uint32_t> SerializeClusterSummary(const RClusterSummary &clusterSummary, void *buffer);
0269    static RResult<std::uint32_t> SerializeClusterGroup(const RClusterGroup &clusterGroup, void *buffer);
0270    static RResult<std::uint32_t>
0271    DeserializeClusterSummary(const void *buffer, std::uint64_t bufSize, RClusterSummary &clusterSummary);
0272    static RResult<std::uint32_t>
0273    DeserializeClusterGroup(const void *buffer, std::uint64_t bufSize, RClusterGroup &clusterGroup);
0274
0275    /// Serialize the schema description in `desc` into `buffer`. If `forHeaderExtension` is true, serialize only the
0276    /// fields and columns tagged as part of the header extension (see `RNTupleDescriptorBuilder::BeginHeaderExtension`).
0277    static RResult<std::uint32_t> SerializeSchemaDescription(void *buffer, const RNTupleDescriptor &desc,
0278                                                             const RContext &context, bool forHeaderExtension = false);
0279    static RResult<std::uint32_t> DeserializeSchemaDescription(const void *buffer, std::uint64_t bufSize,
0280                                                               ROOT::Internal::RNTupleDescriptorBuilder &descBuilder);
0281
0282    static RResult<std::uint32_t>
0283    SerializeAttributeSet(const Experimental::RNTupleAttrSetDescriptor &attrSetDesc, void *buffer);
0284    static RResult<std::uint32_t>
0285    DeserializeAttributeSet(const void *buffer, std::uint64_t bufSize,
0286                            Experimental::Internal::RNTupleAttrSetDescriptorBuilder &attrSetDescBld);
0287
0288    static RResult<RContext> SerializeHeader(void *buffer, const RNTupleDescriptor &desc);
0289    static RResult<std::uint32_t> SerializePageList(void *buffer, const RNTupleDescriptor &desc,
0290                                                    std::span<ROOT::DescriptorId_t> physClusterIDs,
0291                                                    const RContext &context);
0292    static RResult<std::uint32_t> SerializeFooter(void *buffer, const RNTupleDescriptor &desc, const RContext &context);
0293
0294    static RResult<void>
0295    DeserializeHeader(const void *buffer, std::uint64_t bufSize, ROOT::Internal::RNTupleDescriptorBuilder &descBuilder);
0296    static RResult<void>
0297    DeserializeFooter(const void *buffer, std::uint64_t bufSize, ROOT::Internal::RNTupleDescriptorBuilder &descBuilder);
0298
0299    enum class EDescriptorDeserializeMode {
0300       /// Deserializes the descriptor as-is without performing any additional fixup. The produced descriptor is
0301       /// unsuitable for reading or writing, but it's a faithful representation of the on-disk information.
0302       kRaw,
0303       /// Deserializes the descriptor and performs fixup on the suppressed column ranges. This produces a descriptor
0304       /// that is suitable for writing, but not reading.
0305       kForWriting,
0306       /// Deserializes the descriptor and performs fixup on the suppressed column ranges and on clusters, taking
0307       /// into account the header extension. This produces a descriptor that is suitable for reading.
0308       kForReading,
0309    };
0310    // The clusters vector must be initialized with the cluster summaries corresponding to the page list
0311    static RResult<void> DeserializePageList(const void *buffer, std::uint64_t bufSize,
0312                                             ROOT::DescriptorId_t clusterGroupId, RNTupleDescriptor &desc,
0313                                             EDescriptorDeserializeMode mode);
0314
0315    // Helper functions to (de-)serialize the streamer info type extra information
0316    static std::string SerializeStreamerInfos(const StreamerInfoMap_t &infos);
0317    static RResult<StreamerInfoMap_t> DeserializeStreamerInfos(const std::string &extraTypeInfoContent);
0318 }; // class RNTupleSerializer
0319
0320 } // namespace Internal
0321 } // namespace ROOT
0322
0323 #endif // ROOT_RNTupleSerialize