root/ROOT/RPageStorageFile.hxx

0001 /// \file ROOT/RPageStorageFile.hxx
0002 /// \ingroup NTuple
0003 /// \author Jakob Blomer <jblomer@cern.ch>
0004 /// \date 2019-11-21
0005
0006 /*************************************************************************
0007  * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers.               *
0008  * All rights reserved.                                                  *
0009  *                                                                       *
0010  * For the licensing terms see $ROOTSYS/LICENSE.                         *
0011  * For the list of contributors see $ROOTSYS/README/CREDITS.             *
0012  *************************************************************************/
0013
0014 #ifndef ROOT_RPageStorageFile
0015 #define ROOT_RPageStorageFile
0016
0017 #include <ROOT/RMiniFile.hxx>
0018 #include <ROOT/RNTuple.hxx>
0019 #include <ROOT/RNTupleSerialize.hxx>
0020 #include <ROOT/RNTupleZip.hxx>
0021 #include <ROOT/RPageStorage.hxx>
0022 #include <ROOT/RRawFile.hxx>
0023 #include <string_view>
0024
0025 #include <array>
0026 #include <cstdio>
0027 #include <memory>
0028 #include <optional>
0029 #include <string>
0030 #include <utility>
0031
0032 class TDirectory;
0033
0034 namespace ROOT {
0035 class RNTuple; // for making RPageSourceFile a friend of RNTuple
0036 class RNTupleLocator;
0037
0038 namespace Experimental {
0039 class RFile;
0040 }
0041
0042 namespace Internal {
0043 class RRawFile;
0044 class RPageAllocatorHeap;
0045
0046 // clang-format off
0047 /**
0048 \class ROOT::Internal::RPageSinkFile
0049 \ingroup NTuple
0050 \brief Storage provider that write ntuple pages into a file
0051
0052 The written file can be either in ROOT format or in RNTuple bare format.
0053 */
0054 // clang-format on
0055 class RPageSinkFile : public RPagePersistentSink {
0056 private:
0057    // A set of pages to be committed together in a vector write.
0058    // Currently we assume they're all sequential (although they may span multiple ranges).
0059    struct CommitBatch {
0060       /// The list of pages to commit
0061       std::vector<const RSealedPage *> fSealedPages;
0062       /// Total size in bytes of the batch
0063       size_t fSize;
0064       /// Total uncompressed size of the elements in the page batch
0065       size_t fBytesPacked;
0066    };
0067
0068    std::unique_ptr<ROOT::Internal::RNTupleFileWriter> fWriter;
0069    /// Number of bytes committed to storage in the current cluster
0070    std::uint64_t fNBytesCurrentCluster = 0;
0071    /// On UpdateSchema(), the new class fields register the corresponding streamer info here so that the
0072    /// streamer info records in the file can be properly updated on dataset commit
0073    ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t fInfosOfClassFields;
0074
0075    RPageSinkFile(std::string_view ntupleName, const ROOT::RNTupleWriteOptions &options);
0076    RPageSinkFile(std::unique_ptr<ROOT::Internal::RNTupleFileWriter> writer, const ROOT::RNTupleWriteOptions &options);
0077
0078    /// We pass bytesPacked so that TFile::ls() reports a reasonable value for the compression ratio of the corresponding
0079    /// key. It is not strictly necessary to write and read the sealed page.
0080    RNTupleLocator WriteSealedPage(const RPageStorage::RSealedPage &sealedPage, std::size_t bytesPacked);
0081
0082    /// Subroutine of CommitSealedPageVImpl, used to perform a vector write of the (multi-)range of pages
0083    /// contained in `batch`. The locators for the written pages are appended to `locators`.
0084    /// This procedure also updates some internal metrics of the page sink, hence it's not const.
0085    /// `batch` gets reset to size 0 after the writing is done (but its begin and end are not updated).
0086    void CommitBatchOfPages(CommitBatch &batch, std::vector<RNTupleLocator> &locators);
0087
0088 protected:
0089    using RPagePersistentSink::InitImpl;
0090    void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final;
0091    RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) override;
0092    RNTupleLocator
0093    CommitSealedPageImpl(ROOT::DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final;
0094    std::vector<RNTupleLocator>
0095    CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges, const std::vector<bool> &mask) final;
0096    std::uint64_t StageClusterImpl() final;
0097    RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final;
0098    using RPagePersistentSink::CommitDatasetImpl;
0099    RNTupleLink CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) final;
0100
0101 public:
0102    RPageSinkFile(std::string_view ntupleName, std::string_view path, const ROOT::RNTupleWriteOptions &options);
0103    RPageSinkFile(std::string_view ntupleName, TDirectory &fileOrDirectory, const ROOT::RNTupleWriteOptions &options);
0104    RPageSinkFile(std::string_view ntupleName, ROOT::Experimental::RFile &file, std::string_view ntupleDir,
0105                  const ROOT::RNTupleWriteOptions &options);
0106    RPageSinkFile(const RPageSinkFile &) = delete;
0107    RPageSinkFile &operator=(const RPageSinkFile &) = delete;
0108    RPageSinkFile(RPageSinkFile &&) = default;
0109    RPageSinkFile &operator=(RPageSinkFile &&) = default;
0110    ~RPageSinkFile() override;
0111
0112    void UpdateSchema(const ROOT::Internal::RNTupleModelChangeset &changeset, ROOT::NTupleSize_t firstEntry) final;
0113
0114    std::unique_ptr<RPageSink>
0115    CloneAsHidden(std::string_view name, const ROOT::RNTupleWriteOptions &opts) const override;
0116 }; // class RPageSinkFile
0117
0118 // clang-format off
0119 /**
0120 \class ROOT::Internal::RPageSourceFile
0121 \ingroup NTuple
0122 \brief Storage provider that reads ntuple pages from a file
0123 */
0124 // clang-format on
0125 class RPageSourceFile : public RPageSource {
0126    friend class ROOT::RNTuple;
0127
0128 private:
0129    /// Holds the uncompressed header and footer
0130    struct RStructureBuffer {
0131       std::unique_ptr<unsigned char[]> fBuffer; ///< single buffer for both header and footer
0132       void *fPtrHeader = nullptr;               ///< either nullptr or points into fBuffer
0133       void *fPtrFooter = nullptr;               ///< either nullptr or points into fBuffer
0134
0135       /// Called at the end of Attach(), i.e. when the header and footer are processed
0136       void Reset()
0137       {
0138          RStructureBuffer empty;
0139          std::swap(empty, *this);
0140       }
0141    };
0142
0143    /// Either provided by CreateFromAnchor, or read from the ROOT file given the ntuple name
0144    std::optional<RNTuple> fAnchor;
0145    /// The last cluster from which a page got loaded.  Points into fClusterPool->fPool
0146    ROOT::Internal::RCluster *fCurrentCluster = nullptr;
0147    /// An RRawFile is used to request the necessary byte ranges from a local or a remote file
0148    std::unique_ptr<RRawFile> fFile;
0149    /// Takes the fFile to read ntuple blobs from it
0150    ROOT::Internal::RMiniFileReader fReader;
0151    /// The descriptor is created from the header and footer either in AttachImpl or in CreateFromAnchor
0152    RNTupleDescriptorBuilder fDescriptorBuilder;
0153    /// Populated by LoadStructureImpl(), reset at the end of Attach()
0154    RStructureBuffer fStructureBuffer;
0155
0156    RPageSourceFile(std::string_view ntupleName, const ROOT::RNTupleReadOptions &options);
0157
0158    /// Helper function for LoadClusters: it prepares the memory buffer (page map) and the
0159    /// read requests for a given cluster and columns.  The reead requests are appended to
0160    /// the provided vector.  This way, requests can be collected for multiple clusters before
0161    /// sending them to RRawFile::ReadV().
0162    std::unique_ptr<ROOT::Internal::RCluster>
0163    PrepareSingleCluster(const ROOT::Internal::RCluster::RKey &clusterKey, std::vector<RRawFile::RIOVec> &readRequests);
0164
0165 protected:
0166    void LoadStructureImpl() final;
0167    ROOT::RNTupleDescriptor AttachImpl(RNTupleSerializer::EDescriptorDeserializeMode mode) final;
0168    /// The cloned page source creates a new raw file and reader and opens its own file descriptor to the data.
0169    std::unique_ptr<RPageSource> CloneImpl() const final;
0170
0171    RPageRef
0172    LoadPageImpl(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo, ROOT::NTupleSize_t idxInCluster) final;
0173
0174 public:
0175    RPageSourceFile(std::string_view ntupleName, std::string_view path, const ROOT::RNTupleReadOptions &options);
0176    RPageSourceFile(std::string_view ntupleName, std::unique_ptr<RRawFile> file,
0177                    const ROOT::RNTupleReadOptions &options);
0178    /// Used from the RNTuple class to build a datasource if the anchor is already available.
0179    /// Requires the RNTuple object to be streamed from a file.
0180    static std::unique_ptr<RPageSourceFile>
0181    CreateFromAnchor(const RNTuple &anchor, const ROOT::RNTupleReadOptions &options = ROOT::RNTupleReadOptions());
0182
0183    RPageSourceFile(const RPageSourceFile &) = delete;
0184    RPageSourceFile &operator=(const RPageSourceFile &) = delete;
0185    RPageSourceFile(RPageSourceFile &&) = delete;
0186    RPageSourceFile &operator=(RPageSourceFile &&) = delete;
0187    ~RPageSourceFile() override;
0188
0189    std::unique_ptr<RPageSource> OpenWithDifferentAnchor(const ROOT::Internal::RNTupleLink &anchorLink,
0190                                                         const ROOT::RNTupleReadOptions &options = {}) final;
0191
0192    void
0193    LoadSealedPage(ROOT::DescriptorId_t physicalColumnId, RNTupleLocalIndex localIndex, RSealedPage &sealedPage) final;
0194
0195    std::vector<std::unique_ptr<ROOT::Internal::RCluster>>
0196    LoadClusters(std::span<ROOT::Internal::RCluster::RKey> clusterKeys) final;
0197
0198    void LoadStreamerInfo() final;
0199 }; // class RPageSourceFile
0200
0201 } // namespace Internal
0202 } // namespace ROOT
0203
0204 #endif