include/root/TFileMerger.h

0001 // @(#)root/io:$Id$
0002 // Author: Andreas Peters + Fons Rademakers   26/5/2005
0003
0004 /*************************************************************************
0005  * Copyright (C) 1995-2005, Rene Brun and Fons Rademakers.               *
0006  * All rights reserved.                                                  *
0007  *                                                                       *
0008  * For the licensing terms see $ROOTSYS/LICENSE.                         *
0009  * For the list of contributors see $ROOTSYS/README/CREDITS.             *
0010  *************************************************************************/
0011
0012 #ifndef ROOT_TFileMerger
0013 #define ROOT_TFileMerger
0014
0015 #include "TObject.h"
0016 #include "TList.h"
0017 #include "TString.h"
0018 #include "TStopwatch.h"
0019 #include <string>
0020
0021 class TFile;
0022 class TDirectory;
0023 class THashList;
0024 class TKey;
0025
0026 namespace ROOT {
0027 class TIOFeatures;
0028 }  // namespace ROOT
0029
0030 class TFileMerger : public TObject {
0031 public:
0032    enum class EErrorBehavior {
0033       /// The merging process will stop and yield failure when encountering invalid objects
0034       kFailOnError,
0035       /// The merging process will skip invalid objects and continue
0036       kSkipOnError
0037    };
0038
0039 private:
0040    using TIOFeatures = ROOT::TIOFeatures;
0041
0042    TFileMerger(const TFileMerger&) = delete;
0043    TFileMerger& operator=(const TFileMerger&) = delete;
0044
0045 protected:
0046    TStopwatch     fWatch;                     ///< Stop watch to measure file copy speed
0047    TList          fFileList;                  ///< A list the file (TFile*) which shall be merged
0048    TFile         *fOutputFile{nullptr};       ///< The outputfile for merging
0049    TString        fOutputFilename;            ///< The name of the outputfile for merging
0050    Bool_t         fFastMethod{kTRUE};         ///< True if using Fast merging algorithm (default)
0051    Bool_t         fNoTrees{kFALSE};           ///< True if Trees should not be merged (default is kFALSE)
0052    Bool_t         fExplicitCompLevel{kFALSE}; ///< True if the user explicitly requested a compression level change (default kFALSE)
0053    Bool_t         fCompressionChange{kFALSE}; ///< True if the output and input have different compression level (default kFALSE)
0054    Int_t          fPrintLevel{0};             ///< How much information to print out at run time
0055    TString        fMergeOptions;              ///< Options (in string format) to be passed down to the Merge functions
0056    TIOFeatures   *fIOFeatures{nullptr};       ///< IO features to use in the output file.
0057    TString        fMsgPrefix{"TFileMerger"};  ///< Prefix to be used when printing informational message (default TFileMerger)
0058    EErrorBehavior fErrBehavior = EErrorBehavior::kFailOnError; ///< What to do in case of errors during merging
0059
0060    Int_t          fMaxOpenedFiles;            ///< Maximum number of files opened at the same time by the TFileMerger
0061    Bool_t         fLocal;                     ///< Makes local copies of merging files if True (default is kTRUE)
0062    Bool_t         fHistoOneGo;                ///< Merger histos in one go (default is kTRUE)
0063    TString        fObjectNames;               ///< List of object names to be either merged exclusively or skipped
0064    TList          fMergeList;                 ///< list of TObjString containing the name of the files need to be merged
0065    TList          fExcessFiles;               ///<! List of TObjString containing the name of the files not yet added to fFileList due to user or system limitation on the max number of files opened.
0066
0067    bool fOutFileWasExplicitlyClosed = false; ///<! the user has called CloseOutputFile(), so we shouldn't error out in RecursiveRemove
0068
0069    Bool_t         OpenExcessFiles();
0070    virtual Bool_t AddFile(TFile *source, Bool_t own, Bool_t cpProgress);
0071    virtual Bool_t MergeRecursive(TDirectory *target, TList *sourcelist, Int_t type = kRegular | kAll);
0072
0073    virtual Bool_t MergeOne(TDirectory *target, TList *sourcelist, Int_t type,
0074                 TFileMergeInfo &info, TString &oldkeyname, THashList &allNames, Bool_t &status, Bool_t &onlyListed,
0075                 const TString &path,
0076                 TDirectory *current_sourcedir, TFile *current_file,
0077                 TKey *key, TObject *obj, TIter &nextkey);
0078 public:
0079    /// Type of the partial merge
0080    enum EPartialMergeType {
0081       kRegular      = 0,             ///< Normal merge, overwriting the output file.
0082       kIncremental  = BIT(1),        ///< Merge the input file with the content of the output file (if already existing).
0083       kResetable    = BIT(2),        ///< Only the objects with a MergeAfterReset member function.
0084       kNonResetable = BIT(3),        ///< Only the objects without a MergeAfterReset member function.
0085       kDelayWrite   = BIT(4),        ///< Delay the TFile write (to reduce the number of write when reusing the file)
0086
0087       kAll            = BIT(2)|BIT(3),      ///< Merge all type of objects (default)
0088       kAllIncremental = kIncremental | kAll, ///< Merge incrementally all type of objects.
0089
0090       kOnlyListed     = BIT(5),        ///< Only the objects specified in fObjectNames list
0091       kSkipListed     = BIT(6),        ///< Skip objects specified in fObjectNames list
0092       kKeepCompression= BIT(7)         ///< Keep compression level unchanged for each input files
0093    };
0094
0095    TFileMerger(Bool_t isLocal = kTRUE, Bool_t histoOneGo = kTRUE);
0096    ~TFileMerger() override;
0097
0098    Int_t       GetPrintLevel() const { return fPrintLevel; }
0099    void        SetPrintLevel(Int_t level) { fPrintLevel = level; }
0100    Bool_t      HasCompressionChange() const { return fCompressionChange; }
0101    const char *GetOutputFileName() const { return fOutputFilename; }
0102    TList      *GetMergeList() { return &fMergeList; }
0103    TFile      *GetOutputFile() const { return fOutputFile; }
0104    Int_t       GetMaxOpenedFiles() const { return fMaxOpenedFiles; }
0105    void        SetMaxOpenedFiles(Int_t newmax);
0106    const char *GetMsgPrefix() const { return fMsgPrefix; }
0107    void        SetMsgPrefix(const char *prefix);
0108    const char *GetMergeOptions() { return fMergeOptions; }
0109    void        SetMergeOptions(const TString &options) { fMergeOptions = options; }
0110    void        SetMergeOptions(const std::string_view &options) { fMergeOptions = options; }
0111    void        SetIOFeatures(ROOT::TIOFeatures &features) { fIOFeatures = &features; }
0112    void        AddObjectNames(const char *name) {fObjectNames += name; fObjectNames += " ";}
0113    const char *GetObjectNames() const {return fObjectNames.Data();}
0114    void        ClearObjectNames() {fObjectNames.Clear();}
0115    void        CloseOutputFile();
0116
0117     //--- file management interface
0118    virtual Bool_t SetCWD(const char * /*path*/) { MayNotUse("SetCWD"); return kFALSE; }
0119    virtual const char *GetCWD() { MayNotUse("GetCWD"); return nullptr; }
0120
0121    //--- file merging interface
0122    virtual void   Reset();
0123    virtual Bool_t AddFile(const char *url, Bool_t cpProgress = kTRUE);
0124    virtual Bool_t AddFile(TFile *source, Bool_t cpProgress = kTRUE);
0125    virtual Bool_t AddAdoptFile(TFile *source, Bool_t cpProgress = kTRUE);
0126    virtual Bool_t OutputFile(const char *url, Bool_t force);
0127    virtual Bool_t OutputFile(const char *url, Bool_t force, Int_t compressionLevel);
0128    virtual Bool_t OutputFile(const char *url, const char *mode = "RECREATE");
0129    virtual Bool_t OutputFile(const char *url, const char *mode, Int_t compressionLevel);
0130    virtual Bool_t OutputFile(std::unique_ptr<TFile> file);
0131    virtual void   PrintFiles(Option_t *options);
0132    virtual Bool_t Merge(Bool_t = kTRUE);
0133    virtual Bool_t PartialMerge(Int_t type = kAll | kIncremental);
0134    virtual void   SetFastMethod(Bool_t fast=kTRUE)  {fFastMethod = fast;}
0135            Bool_t GetNotrees() const { return fNoTrees; }
0136    virtual void   SetNotrees(Bool_t notrees=kFALSE) {fNoTrees = notrees;}
0137            void   RecursiveRemove(TObject *obj) override;
0138            /// Determines how the merging process should behave when encontering invalid objects.
0139            /// By default the merging will be aborted.
0140            /// \sa EErrorBehavior
0141            void SetErrorBehavior(EErrorBehavior errBehavior) { fErrBehavior = errBehavior; }
0142
0143            ClassDefOverride(TFileMerger, 6) // File copying and merging services
0144 };
0145
0146 #endif
0147