root/ROOT/RPageStorageDaos.hxx

0001 /// \file ROOT/RPageStorageDaos.hxx
0002 /// \ingroup NTuple
0003 /// \author Javier Lopez-Gomez <j.lopez@cern.ch>
0004 /// \date 2020-11-03
0005 /// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
0006 /// is welcome!
0007
0008 /*************************************************************************
0009  * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers.               *
0010  * All rights reserved.                                                  *
0011  *                                                                       *
0012  * For the licensing terms see $ROOTSYS/LICENSE.                         *
0013  * For the list of contributors see $ROOTSYS/README/CREDITS.             *
0014  *************************************************************************/
0015
0016 #ifndef ROOT_RPageStorageDaos
0017 #define ROOT_RPageStorageDaos
0018
0019 #include <ROOT/RError.hxx>
0020 #include <ROOT/RPageStorage.hxx>
0021 #include <ROOT/RNTuple.hxx>
0022 #include <ROOT/RNTupleSerialize.hxx>
0023 #include <ROOT/RNTupleZip.hxx>
0024 #include <string_view>
0025
0026 #include <array>
0027 #include <atomic>
0028 #include <cstdio>
0029 #include <memory>
0030 #include <string>
0031 #include <optional>
0032
0033 namespace ROOT {
0034
0035 namespace Internal {
0036 class RCluster;
0037 } // namespace Internal
0038
0039 namespace Experimental {
0040 namespace Internal {
0041 using ntuple_index_t = std::uint32_t;
0042 class RDaosPool;
0043 class RDaosContainer;
0044 class RPageAllocatorHeap;
0045
0046 // clang-format off
0047 /**
0048 \class ROOT::Experimental::Internal::RDaosNTupleAnchor
0049 \ingroup NTuple
0050 \brief Entry point for an RNTuple in a DAOS container. It encodes essential
0051 information to read the ntuple; currently, it contains (un)compressed size of
0052 the header/footer blobs and the object class for user data OIDs.
0053 The length of a serialized anchor cannot be greater than the value returned by the `GetSize` function.
0054 */
0055 // clang-format on
0056 struct RDaosNTupleAnchor {
0057    /// Allows for evolving the struct in future versions
0058    std::uint64_t fVersionAnchor = 1;
0059    /// Version of the binary format supported by the writer
0060    std::uint16_t fVersionEpoch = RNTuple::kVersionEpoch;
0061    std::uint16_t fVersionMajor = RNTuple::kVersionMajor;
0062    std::uint16_t fVersionMinor = RNTuple::kVersionMinor;
0063    std::uint16_t fVersionPatch = RNTuple::kVersionPatch;
0064    /// The size of the compressed ntuple header
0065    std::uint32_t fNBytesHeader = 0;
0066    /// The size of the uncompressed ntuple header
0067    std::uint32_t fLenHeader = 0;
0068    /// The size of the compressed ntuple footer
0069    std::uint32_t fNBytesFooter = 0;
0070    /// The size of the uncompressed ntuple footer
0071    std::uint32_t fLenFooter = 0;
0072    /// The object class for user data OIDs, e.g. `SX`
0073    std::string fObjClass{};
0074
0075    bool operator==(const RDaosNTupleAnchor &other) const
0076    {
0077       return fVersionAnchor == other.fVersionAnchor && fVersionEpoch == other.fVersionEpoch &&
0078              fVersionMajor == other.fVersionMajor && fVersionMinor == other.fVersionMinor &&
0079              fVersionPatch == other.fVersionPatch && fNBytesHeader == other.fNBytesHeader &&
0080              fLenHeader == other.fLenHeader && fNBytesFooter == other.fNBytesFooter && fLenFooter == other.fLenFooter &&
0081              fObjClass == other.fObjClass;
0082    }
0083
0084    std::uint32_t Serialize(void *buffer) const;
0085    RResult<std::uint32_t> Deserialize(const void *buffer, std::uint32_t bufSize);
0086
0087    static std::uint32_t GetSize();
0088 }; // struct RDaosNTupleAnchor
0089
0090 // clang-format off
0091 /**
0092 \class ROOT::Experimental::Internal::RPageSinkDaos
0093 \ingroup NTuple
0094 \brief Storage provider that writes ntuple pages to into a DAOS container
0095
0096 Currently, an object is allocated for ntuple metadata (anchor/header/footer).
0097 Objects can correspond to pages or clusters of pages depending on the RNTuple-DAOS mapping strategy.
0098 */
0099 // clang-format on
0100 class RPageSinkDaos : public ROOT::Internal::RPagePersistentSink {
0101 private:
0102    /// \brief Underlying DAOS container. An internal `std::shared_ptr` keep the pool connection alive.
0103    /// ISO C++ ensures the correct destruction order, i.e., `~RDaosContainer` is invoked first
0104    /// (which calls `daos_cont_close()`; the destructor for the `std::shared_ptr<RDaosPool>` is invoked
0105    /// after (which calls `daos_pool_disconect()`).
0106    std::unique_ptr<RDaosContainer> fDaosContainer;
0107    /// Page identifier for the next committed page; it is automatically incremented in `CommitSealedPageImpl()`
0108    std::atomic<std::uint64_t> fPageId{0};
0109    /// Cluster group counter for the next committed cluster pagelist; incremented in `CommitClusterGroupImpl()`
0110    std::atomic<std::uint64_t> fClusterGroupId{0};
0111    /// \brief A URI to a DAOS pool of the form 'daos://pool-label/container-label'
0112    std::string fURI;
0113    /// Tracks the number of bytes committed to the current cluster
0114    std::uint64_t fNBytesCurrentCluster{0};
0115
0116    RDaosNTupleAnchor fNTupleAnchor;
0117    ntuple_index_t fNTupleIndex{0};
0118
0119 protected:
0120    using RPagePersistentSink::InitImpl;
0121    void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final;
0122    RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const ROOT::Internal::RPage &page) final;
0123    RNTupleLocator
0124    CommitSealedPageImpl(ROOT::DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final;
0125    std::vector<RNTupleLocator>
0126    CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges, const std::vector<bool> &mask) final;
0127    std::uint64_t StageClusterImpl() final;
0128    RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final;
0129    using RPagePersistentSink::CommitDatasetImpl;
0130    ROOT::Internal::RNTupleLink CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) final;
0131    void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader);
0132    void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter);
0133    void WriteNTupleAnchor();
0134
0135 public:
0136    RPageSinkDaos(std::string_view ntupleName, std::string_view uri, const ROOT::RNTupleWriteOptions &options);
0137    ~RPageSinkDaos() override;
0138
0139    std::unique_ptr<ROOT::Internal::RPageSink>
0140    CloneAsHidden(std::string_view name, const ROOT::RNTupleWriteOptions &opts) const final;
0141 }; // class RPageSinkDaos
0142
0143 // clang-format off
0144 /**
0145 \class ROOT::Experimental::Internal::RPageSourceDaos
0146 \ingroup NTuple
0147 \brief Storage provider that reads ntuple pages from a DAOS container
0148 */
0149 // clang-format on
0150 class RPageSourceDaos : public ROOT::Internal::RPageSource {
0151 private:
0152    ntuple_index_t fNTupleIndex{0};
0153
0154    /// The last cluster from which a page got loaded.  Points into fClusterPool->fPool
0155    ROOT::Internal::RCluster *fCurrentCluster = nullptr;
0156    /// A container that stores object data (header/footer, pages, etc.)
0157    std::unique_ptr<RDaosContainer> fDaosContainer;
0158    /// A URI to a DAOS pool of the form 'daos://pool-label/container-label'
0159    std::string fURI;
0160
0161    ROOT::Internal::RNTupleDescriptorBuilder fDescriptorBuilder;
0162
0163    ROOT::Internal::RPageRef
0164    LoadPageImpl(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo, ROOT::NTupleSize_t idxInCluster) final;
0165
0166 protected:
0167    void LoadStructureImpl() final {}
0168    ROOT::RNTupleDescriptor AttachImpl(ROOT::Internal::RNTupleSerializer::EDescriptorDeserializeMode mode) final;
0169    /// The cloned page source creates a new connection to the pool/container.
0170    std::unique_ptr<RPageSource> CloneImpl() const final;
0171
0172 public:
0173    RPageSourceDaos(std::string_view ntupleName, std::string_view uri, const ROOT::RNTupleReadOptions &options);
0174    ~RPageSourceDaos() override;
0175
0176    void
0177    LoadSealedPage(ROOT::DescriptorId_t physicalColumnId, RNTupleLocalIndex localIndex, RSealedPage &sealedPage) final;
0178
0179    std::vector<std::unique_ptr<ROOT::Internal::RCluster>>
0180    LoadClusters(std::span<ROOT::Internal::RCluster::RKey> clusterKeys) final;
0181
0182    /// Return the object class used for user data OIDs in this ntuple.
0183    std::string GetObjectClass() const;
0184
0185    void LoadStreamerInfo() final;
0186
0187    std::unique_ptr<RPageSource> OpenWithDifferentAnchor(const ROOT::Internal::RNTupleLink &anchorLink,
0188                                                         const ROOT::RNTupleReadOptions &options = {}) final;
0189 }; // class RPageSourceDaos
0190
0191 } // namespace Internal
0192
0193 } // namespace Experimental
0194 } // namespace ROOT
0195
0196 #endif