Root/src/RootFileHasher.cpp

0001 // This file is part of the ACTS project.
0002 //
0003 // Copyright (C) 2016 CERN for the benefit of the ACTS project
0004 //
0005 // This Source Code Form is subject to the terms of the Mozilla Public
0006 // License, v. 2.0. If a copy of the MPL was not distributed with this
0007 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
0008
0009 #include "ActsExamples/Io/Root/RootFileHasher.hpp"
0010
0011 #include "ActsExamples/Io/Root/detail/RootFileHasherHelpers.hpp"
0012
0013 #include <algorithm>
0014 #include <array>
0015 #include <cstddef>
0016 #include <cstdint>
0017 #include <format>
0018 #include <iostream>
0019 #include <memory>
0020 #include <set>
0021 #include <span>
0022 #include <stdexcept>
0023 #include <vector>
0024
0025 #include <TBranch.h>
0026 #include <TFile.h>
0027 #include <TKey.h>
0028 #include <TLeaf.h>
0029 #include <TObjArray.h>
0030 #include <TTree.h>
0031 #include <TTreeReader.h>
0032 #include <TTreeReaderArray.h>
0033 #include <TTreeReaderValue.h>
0034
0035 #if defined(ACTS_ROOT_FILE_HASHER_USE_BOOST_HASH2)
0036 #include <boost/hash2/sha2.hpp>
0037 #else
0038 #include <TMD5.h>
0039 #endif
0040
0041 namespace ActsExamples {
0042
0043 namespace {
0044
0045 template <typename H>
0046 concept HasBoostUpdate =
0047     requires(H h, const unsigned char* data, std::size_t size) {
0048       { h.update(data, size) } -> std::same_as<void>;
0049     };
0050
0051 template <typename H>
0052 concept HasFinalize = requires(H h, unsigned char* data) { h.Final(data); };
0053
0054 /// Incremental hasher backed by Boost.Hash2 SHA-256 (preferred) or ROOT's
0055 /// TMD5 (fallback for Boost < 1.86). Only the backend member and the two
0056 /// method bodies differ between the two configurations.
0057 template <typename H, std::size_t digestSize>
0058 class HasherT {
0059  public:
0060   using HasherType = H;
0061   static constexpr std::size_t kDigestSize = digestSize;
0062   using Digest = std::array<unsigned char, kDigestSize>;
0063
0064   void update(std::span<const std::byte> data) {
0065     if constexpr (HasBoostUpdate<H>) {
0066       m_hasher.update(reinterpret_cast<const unsigned char*>(data.data()),
0067                       data.size());
0068     } else {
0069       m_hasher.Update(reinterpret_cast<const UChar_t*>(data.data()),
0070                       data.size());
0071     }
0072   }
0073
0074   Digest finalize() {
0075     Digest digest{};
0076
0077     if constexpr (HasFinalize<H>) {
0078       m_hasher.Final(digest.data());
0079     } else {
0080       boost::hash2::digest<kDigestSize> result = m_hasher.result();
0081       std::copy(result.data(), result.data() + kDigestSize, digest.begin());
0082     }
0083     return digest;
0084   }
0085
0086  private:
0087   HasherType m_hasher;
0088 };
0089
0090 #if defined(ACTS_ROOT_FILE_HASHER_USE_BOOST_HASH2)
0091 using Hasher = HasherT<boost::hash2::sha2_256, 32>;
0092 #else
0093 using Hasher = HasherT<TMD5, 16>;
0094 #endif
0095
0096 using Digest = typename Hasher::Digest;
0097
0098 /// Compute the digest of a contiguous byte buffer.
0099 Digest digestOf(std::span<const std::byte> data) {
0100   Hasher hasher;
0101   hasher.update(data);
0102   return hasher.finalize();
0103 }
0104
0105 /// Render a digest as a lowercase hex string.
0106 std::string toHex(const Digest& digest) {
0107   static const char* const hexDigits = "0123456789abcdef";
0108   std::string out;
0109   out.reserve(2 * digest.size());
0110   for (unsigned char byte : digest) {
0111     out.push_back(hexDigits[byte >> 4]);
0112     out.push_back(hexDigits[byte & 0x0f]);
0113   }
0114   return out;
0115 }
0116
0117 /// Append the raw object representation of `x` to a byte buffer.
0118 template <typename T>
0119 void appendBytes(std::vector<std::byte>& out, const T& x) {
0120   const auto* p = reinterpret_cast<const std::byte*>(&x);
0121   out.insert(out.end(), p, p + sizeof(T));
0122 }
0123
0124 /// Type-erased reader appending the raw bytes of one branch's current-entry
0125 /// value(s) to a buffer.
0126 struct IBranchReader {
0127   virtual ~IBranchReader() = default;
0128   /// Append the bytes of the current entry to `out`.
0129   virtual void append(std::vector<std::byte>& out) = 0;
0130   /// Whether the underlying branch was matched successfully. Only meaningful
0131   /// after the first entry has been loaded.
0132   virtual bool valid() const = 0;
0133 };
0134
0135 template <typename T>
0136 bool setupOk(const T& proxy) {
0137   return proxy.GetSetupStatus() ==
0138          ROOT::Internal::TTreeReaderValueBase::kSetupMatch;
0139 }
0140
0141 /// Reader for a fundamental scalar branch.
0142 template <typename T>
0143 struct ScalarReader final : IBranchReader {
0144   TTreeReaderValue<T> value;
0145   ScalarReader(TTreeReader& reader, const char* name) : value(reader, name) {}
0146   bool valid() const override { return setupOk(value); }
0147   void append(std::vector<std::byte>& out) override {
0148     appendBytes(out, *value);
0149   }
0150 };
0151
0152 /// Reader for a 1D array branch (`std::vector<T>` or a C-style array leaf).
0153 template <typename T>
0154 struct ArrayReader final : IBranchReader {
0155   TTreeReaderArray<T> array;
0156   ArrayReader(TTreeReader& reader, const char* name) : array(reader, name) {}
0157   bool valid() const override { return setupOk(array); }
0158   void append(std::vector<std::byte>& out) override {
0159     for (T x : array) {
0160       appendBytes(out, x);
0161     }
0162   }
0163 };
0164
0165 /// Reader for a nested `std::vector<std::vector<T>>` branch.
0166 template <typename T>
0167 struct NestedReader final : IBranchReader {
0168   TTreeReaderValue<std::vector<std::vector<T>>> value;
0169   NestedReader(TTreeReader& reader, const char* name) : value(reader, name) {}
0170   bool valid() const override { return setupOk(value); }
0171   void append(std::vector<std::byte>& out) override {
0172     for (const auto& inner : *value) {
0173       for (T x : inner) {
0174         appendBytes(out, x);
0175       }
0176     }
0177   }
0178 };
0179
0180 enum class Kind { Scalar, Array, Nested };
0181
0182 template <typename T>
0183 std::unique_ptr<IBranchReader> makeReader(Kind kind, TTreeReader& reader,
0184                                           const char* name) {
0185   switch (kind) {
0186     case Kind::Scalar:
0187       return std::make_unique<ScalarReader<T>>(reader, name);
0188     case Kind::Array:
0189       return std::make_unique<ArrayReader<T>>(reader, name);
0190     case Kind::Nested:
0191       return std::make_unique<NestedReader<T>>(reader, name);
0192   }
0193   return nullptr;
0194 }
0195
0196 /// Map a (ROOT or canonical C++) element type name to a typed reader. The
0197 /// instantiated C++ type must match the branch's stored type for ROOT to bind
0198 /// the reader, so we keep the names aligned. Returns nullptr for unsupported
0199 /// types.
0200 std::unique_ptr<IBranchReader> dispatch(const std::string& type, Kind kind,
0201                                         TTreeReader& reader, const char* name) {
0202   if (type == "Float_t" || type == "float") {
0203     return makeReader<float>(kind, reader, name);
0204   }
0205   if (type == "Double_t" || type == "double") {
0206     return makeReader<double>(kind, reader, name);
0207   }
0208   if (type == "Int_t" || type == "int") {
0209     return makeReader<int>(kind, reader, name);
0210   }
0211   if (type == "UInt_t" || type == "unsigned int" || type == "unsigned") {
0212     return makeReader<unsigned int>(kind, reader, name);
0213   }
0214   if (type == "Long64_t" || type == "long long") {
0215     return makeReader<long long>(kind, reader, name);
0216   }
0217   if (type == "ULong64_t" || type == "unsigned long long") {
0218     return makeReader<unsigned long long>(kind, reader, name);
0219   }
0220   if (type == "Long_t" || type == "long") {
0221     return makeReader<long>(kind, reader, name);
0222   }
0223   if (type == "ULong_t" || type == "unsigned long") {
0224     return makeReader<unsigned long>(kind, reader, name);
0225   }
0226   if (type == "Short_t" || type == "short") {
0227     return makeReader<short>(kind, reader, name);
0228   }
0229   if (type == "UShort_t" || type == "unsigned short") {
0230     return makeReader<unsigned short>(kind, reader, name);
0231   }
0232   if (type == "Char_t" || type == "char") {
0233     return makeReader<char>(kind, reader, name);
0234   }
0235   if (type == "UChar_t" || type == "unsigned char") {
0236     return makeReader<unsigned char>(kind, reader, name);
0237   }
0238   if (type == "Bool_t" || type == "bool") {
0239     return makeReader<bool>(kind, reader, name);
0240   }
0241   return nullptr;
0242 }
0243
0244 struct BranchInfo {
0245   std::string name;
0246   Kind kind = Kind::Scalar;
0247   std::string elementType;
0248 };
0249
0250 /// Determine the kind and element type of every branch in a tree.
0251 std::vector<BranchInfo> describeBranches(TTree& tree) {
0252   std::vector<BranchInfo> infos;
0253   TObjArray* branches = tree.GetListOfBranches();
0254   if (branches == nullptr) {
0255     return infos;
0256   }
0257   for (int i = 0; i < branches->GetEntriesFast(); ++i) {
0258     auto* branch = dynamic_cast<TBranch*>(branches->At(i));
0259     if (branch == nullptr) {
0260       continue;
0261     }
0262     BranchInfo info;
0263     info.name = branch->GetName();
0264
0265     std::string className = branch->GetClassName();
0266     if (!className.empty()) {
0267       // STL collection branch.
0268       std::string arg = detail::firstTemplateArg(className);
0269       if (arg.rfind("vector<", 0) == 0) {
0270         info.kind = Kind::Nested;
0271         info.elementType = detail::firstTemplateArg(arg);
0272       } else {
0273         info.kind = Kind::Array;
0274         info.elementType = arg;
0275       }
0276     } else {
0277       // Fundamental leaf branch.
0278       auto* leaf = dynamic_cast<TLeaf*>(branch->GetListOfLeaves()->At(0));
0279       if (leaf == nullptr) {
0280         continue;
0281       }
0282       info.elementType = leaf->GetTypeName();
0283       bool isArray =
0284           (leaf->GetLeafCount() != nullptr) || (leaf->GetLenStatic() > 1);
0285       info.kind = isArray ? Kind::Array : Kind::Scalar;
0286     }
0287     infos.push_back(std::move(info));
0288   }
0289   std::sort(
0290       infos.begin(), infos.end(),
0291       [](const BranchInfo& a, const BranchInfo& b) { return a.name < b.name; });
0292   return infos;
0293 }
0294
0295 struct Reader {
0296   std::string name;
0297   std::unique_ptr<IBranchReader> reader;
0298   std::unique_ptr<Hasher> hasher;  // only used in non-order-invariant mode
0299 };
0300
0301 /// Compute the digest of a single tree.
0302 Digest hashTree(TTree& tree, bool orderInvariant) {
0303   std::vector<BranchInfo> infos = describeBranches(tree);
0304
0305   TTreeReader treeReader(&tree);
0306   std::vector<Reader> readers;
0307   for (const BranchInfo& info : infos) {
0308     auto branchReader =
0309         dispatch(info.elementType, info.kind, treeReader, info.name.c_str());
0310     if (branchReader == nullptr) {
0311       std::cerr << "RootFileHasher: unsupported type '" << info.elementType
0312                 << "' for branch '" << info.name << "' (skipped)" << std::endl;
0313       continue;
0314     }
0315     readers.push_back({info.name, std::move(branchReader),
0316                        orderInvariant ? nullptr : std::make_unique<Hasher>()});
0317   }
0318
0319   std::vector<Digest> rowDigests;
0320   std::vector<std::byte> buffer;
0321   bool pruned = false;
0322   while (treeReader.Next()) {
0323     if (!pruned) {
0324       // Drop branches that could not be bound (e.g. unmatched type). This can
0325       // only be checked once an entry has been loaded.
0326       std::vector<Reader> kept;
0327       for (auto& r : readers) {
0328         if (r.reader->valid()) {
0329           kept.push_back(std::move(r));
0330         } else {
0331           std::cerr << "RootFileHasher: branch '" << r.name
0332                     << "' could not be read (skipped)" << std::endl;
0333         }
0334       }
0335       readers = std::move(kept);
0336       pruned = true;
0337     }
0338
0339     if (orderInvariant) {
0340       buffer.clear();
0341       for (auto& r : readers) {
0342         r.reader->append(buffer);
0343       }
0344       rowDigests.push_back(digestOf(buffer));
0345     } else {
0346       for (auto& r : readers) {
0347         buffer.clear();
0348         r.reader->append(buffer);
0349         r.hasher->update(buffer);
0350       }
0351     }
0352   }
0353
0354   Hasher treeHash;
0355   std::string names;
0356   for (const auto& r : readers) {
0357     names += r.name;
0358   }
0359   treeHash.update(std::as_bytes(std::span(names)));
0360
0361   if (orderInvariant) {
0362     std::sort(rowDigests.begin(), rowDigests.end());
0363     for (const Digest& d : rowDigests) {
0364       treeHash.update(std::as_bytes(std::span(d)));
0365     }
0366   } else {
0367     for (auto& r : readers) {
0368       Digest d = r.hasher->finalize();
0369       treeHash.update(std::as_bytes(std::span(d)));
0370     }
0371   }
0372
0373   return treeHash.finalize();
0374 }
0375
0376 }  // namespace
0377
0378 std::string hashRootFile(const std::filesystem::path& path,
0379                          bool orderInvariant) {
0380   TFile file(path.c_str(), "READ");
0381   if (file.IsZombie()) {
0382     throw std::runtime_error(
0383         std::format("RootFileHasher: could not open '{}'", path.string()));
0384   }
0385
0386   // Collect unique top-level key names in sorted order. Every key name is
0387   // folded into the hash (so structural changes are detected even for files
0388   // that only contain histograms), and the numeric content of TTrees is hashed
0389   // on top.
0390   std::set<std::string> keyNames;
0391   TList* keys = file.GetListOfKeys();
0392   if (keys != nullptr) {
0393     for (int i = 0; i < keys->GetEntries(); ++i) {
0394       auto* key = dynamic_cast<TKey*>(keys->At(i));
0395       if (key != nullptr) {
0396         keyNames.insert(key->GetName());
0397       }
0398     }
0399   }
0400
0401   Hasher global;
0402   for (const std::string& name : keyNames) {
0403     global.update(std::as_bytes(std::span(name)));
0404
0405     auto* tree = dynamic_cast<TTree*>(file.Get(name.c_str()));
0406     if (tree == nullptr) {
0407       continue;
0408     }
0409     Digest treeDigest = hashTree(*tree, orderInvariant);
0410     global.update(std::as_bytes(std::span(treeDigest)));
0411   }
0412
0413   return toHex(global.finalize());
0414 }
0415
0416 }  // namespace ActsExamples