ROOT/RDF/RFilter.hxx

0001 // Author: Enrico Guiraud, Danilo Piparo CERN  09/2018
0002
0003 /*************************************************************************
0004  * Copyright (C) 1995-2018, Rene Brun and Fons Rademakers.               *
0005  * All rights reserved.                                                  *
0006  *                                                                       *
0007  * For the licensing terms see $ROOTSYS/LICENSE.                         *
0008  * For the list of contributors see $ROOTSYS/README/CREDITS.             *
0009  *************************************************************************/
0010
0011 #ifndef ROOT_RFILTER
0012 #define ROOT_RFILTER
0013
0014 #include "ROOT/RDF/ColumnReaderUtils.hxx"
0015 #include "ROOT/RDF/RColumnReaderBase.hxx"
0016 #include "ROOT/RDF/RCutFlowReport.hxx"
0017 #include "ROOT/RDF/Utils.hxx"
0018 #include "ROOT/RDF/RFilterBase.hxx"
0019 #include "ROOT/RDF/RLoopManager.hxx"
0020 #include "ROOT/TypeTraits.hxx"
0021 #include "RtypesCore.h"
0022
0023 #include <algorithm>
0024 #include <cassert>
0025 #include <memory>
0026 #include <string>
0027 #include <unordered_map>
0028 #include <utility> // std::index_sequence
0029 #include <vector>
0030
0031 namespace ROOT {
0032
0033 namespace Internal {
0034 namespace RDF {
0035 using namespace ROOT::Detail::RDF;
0036
0037 // fwd decl for RFilter
0038 namespace GraphDrawing {
0039 std::shared_ptr<GraphNode>
0040 CreateFilterNode(const RFilterBase *filterPtr, std::unordered_map<void *, std::shared_ptr<GraphNode>> &visitedMap);
0041
0042 std::shared_ptr<GraphNode> AddDefinesToGraph(std::shared_ptr<GraphNode> node, const RColumnRegister &colRegister,
0043                                              const std::vector<std::string> &prevNodeDefines,
0044                                              std::unordered_map<void *, std::shared_ptr<GraphNode>> &visitedMap);
0045 } // ns GraphDrawing
0046
0047 } // ns RDF
0048 } // ns Internal
0049
0050 namespace Detail {
0051 namespace RDF {
0052 using namespace ROOT::TypeTraits;
0053 namespace RDFGraphDrawing = ROOT::Internal::RDF::GraphDrawing;
0054 class RJittedFilter;
0055
0056 template <typename FilterF, typename PrevNodeRaw>
0057 class R__CLING_PTRCHECK(off) RFilter final : public RFilterBase {
0058    using ColumnTypes_t = typename CallableTraits<FilterF>::arg_types;
0059    using TypeInd_t = std::make_index_sequence<ColumnTypes_t::list_size>;
0060    // If the PrevNode is a RJittedFilter, treat it as a more generic RFilterBase: when dealing with systematic
0061    // variations we'll have a RJittedFilter node for the nominal case but other "universes" will use concrete filters,
0062    // so we normalize the "previous node type" to the base type RFilterBase.
0063    using PrevNode_t = std::conditional_t<std::is_same<PrevNodeRaw, RJittedFilter>::value, RFilterBase, PrevNodeRaw>;
0064
0065    FilterF fFilter;
0066    /// Column readers per slot and per input column
0067    std::vector<std::array<RColumnReaderBase *, ColumnTypes_t::list_size>> fValues;
0068    const std::shared_ptr<PrevNode_t> fPrevNodePtr;
0069    PrevNode_t &fPrevNode;
0070
0071 public:
0072    RFilter(FilterF f, const ROOT::RDF::ColumnNames_t &columns, std::shared_ptr<PrevNode_t> pd,
0073            const RDFInternal::RColumnRegister &colRegister, std::string_view name = "",
0074            const std::string &variationName = "nominal")
0075       : RFilterBase(pd->GetLoopManagerUnchecked(), name, pd->GetLoopManagerUnchecked()->GetNSlots(), colRegister,
0076                     columns, pd->GetVariations(), variationName),
0077         fFilter(std::move(f)), fValues(pd->GetLoopManagerUnchecked()->GetNSlots()), fPrevNodePtr(std::move(pd)),
0078         fPrevNode(*fPrevNodePtr)
0079    {
0080       fLoopManager->Register(this);
0081    }
0082
0083    // Rule of five
0084
0085    RFilter(const RFilter &) = delete;
0086    RFilter &operator=(const RFilter &) = delete;
0087    RFilter(RFilter &&) = delete;
0088    RFilter &operator=(RFilter &&) = delete;
0089    ~RFilter() final
0090    {
0091       // must Deregister objects from the RLoopManager here, before the fPrevNode data member is destroyed:
0092       // otherwise if fPrevNode is the RLoopManager, it will be destroyed before the calls to Deregister happen.
0093       fLoopManager->Deregister(this);
0094    }
0095
0096    bool CheckFilters(unsigned int slot, Long64_t entry) final
0097    {
0098       if (entry != fLastCheckedEntry[slot * RDFInternal::CacheLineStep<Long64_t>()]) {
0099          if (!fPrevNode.CheckFilters(slot, entry)) {
0100             // a filter upstream returned false, cache the result
0101             fLastResult[slot * RDFInternal::CacheLineStep<int>()] = false;
0102          } else {
0103             // evaluate this filter, cache the result
0104             auto passed = CheckFilterHelper(slot, entry, ColumnTypes_t{}, TypeInd_t{});
0105             passed ? ++fAccepted[slot * RDFInternal::CacheLineStep<ULong64_t>()]
0106                    : ++fRejected[slot * RDFInternal::CacheLineStep<ULong64_t>()];
0107             fLastResult[slot * RDFInternal::CacheLineStep<int>()] = passed;
0108          }
0109          fLastCheckedEntry[slot * RDFInternal::CacheLineStep<Long64_t>()] = entry;
0110       }
0111       return fLastResult[slot * RDFInternal::CacheLineStep<int>()];
0112    }
0113
0114    template <typename ColType>
0115    auto GetValueChecked(unsigned int slot, std::size_t readerIdx, Long64_t entry) -> ColType &
0116    {
0117       if (auto *val = fValues[slot][readerIdx]->template TryGet<ColType>(entry))
0118          return *val;
0119
0120       throw std::out_of_range{"RDataFrame: Filter could not retrieve value for column '" + fColumnNames[readerIdx] +
0121                               "' for entry " + std::to_string(entry) +
0122                               ". You can use the DefaultValueFor operation to provide a default value, or "
0123                               "FilterAvailable/FilterMissing to discard/keep entries with missing values instead."};
0124    }
0125
0126    template <typename... ColTypes, std::size_t... S>
0127    bool CheckFilterHelper(unsigned int slot, Long64_t entry, TypeList<ColTypes...>, std::index_sequence<S...>)
0128    {
0129       return fFilter(GetValueChecked<ColTypes>(slot, S, entry)...);
0130       // avoid unused parameter warnings (gcc 12.1)
0131       (void)slot;
0132       (void)entry;
0133    }
0134
0135    void InitSlot(TTreeReader *r, unsigned int slot) final
0136    {
0137       RDFInternal::RColumnReadersInfo info{fColumnNames, fColRegister, fIsDefine.data(), *fLoopManager};
0138       fValues[slot] = RDFInternal::GetColumnReaders(slot, r, ColumnTypes_t{}, info, fVariation);
0139       fLastCheckedEntry[slot * RDFInternal::CacheLineStep<Long64_t>()] = -1;
0140    }
0141
0142    // recursive chain of `Report`s
0143    void Report(ROOT::RDF::RCutFlowReport &rep) const final { PartialReport(rep); }
0144
0145    void PartialReport(ROOT::RDF::RCutFlowReport &rep) const final
0146    {
0147       fPrevNode.PartialReport(rep);
0148       FillReport(rep);
0149    }
0150
0151    void StopProcessing() final
0152    {
0153       ++fNStopsReceived;
0154       if (fNStopsReceived == fNChildren)
0155          fPrevNode.StopProcessing();
0156    }
0157
0158    void IncrChildrenCount() final
0159    {
0160       ++fNChildren;
0161       // propagate "children activation" upstream. named filters do the propagation via `TriggerChildrenCount`.
0162       if (fNChildren == 1 && fName.empty())
0163          fPrevNode.IncrChildrenCount();
0164    }
0165
0166    void TriggerChildrenCount() final
0167    {
0168       assert(!fName.empty()); // this method is to only be called on named filters
0169       fPrevNode.IncrChildrenCount();
0170    }
0171
0172    void AddFilterName(std::vector<std::string> &filters) final
0173    {
0174       fPrevNode.AddFilterName(filters);
0175       auto name = (HasName() ? fName : "Unnamed Filter");
0176       filters.push_back(name);
0177    }
0178
0179    /// Clean-up operations to be performed at the end of a task.
0180    void FinalizeSlot(unsigned int slot) final { fValues[slot].fill(nullptr); }
0181
0182    std::shared_ptr<RDFGraphDrawing::GraphNode>
0183    GetGraph(std::unordered_map<void *, std::shared_ptr<RDFGraphDrawing::GraphNode>> &visitedMap) final
0184    {
0185       // Recursively call for the previous node.
0186       auto prevNode = fPrevNode.GetGraph(visitedMap);
0187       const auto &prevColumns = prevNode->GetDefinedColumns();
0188
0189       auto thisNode = RDFGraphDrawing::CreateFilterNode(this, visitedMap);
0190
0191       /* If the returned node is not new, there is no need to perform any other operation.
0192        * This is a likely scenario when building the entire graph in which branches share
0193        * some nodes. */
0194       if (!thisNode->IsNew()) {
0195          return thisNode;
0196       }
0197
0198       auto upmostNode = AddDefinesToGraph(thisNode, fColRegister, prevColumns, visitedMap);
0199
0200       // Keep track of the columns defined up to this point.
0201       thisNode->AddDefinedColumns(fColRegister.GenerateColumnNames());
0202
0203       upmostNode->SetPrevNode(prevNode);
0204       return thisNode;
0205    }
0206
0207    /// Return a clone of this Filter that works with values in the variationName "universe".
0208    std::shared_ptr<RNodeBase> GetVariedFilter(const std::string &variationName) final
0209    {
0210       // Only the nominal filter should be asked to produce varied filters
0211       assert(fVariation == "nominal");
0212       // nobody should ask for a varied filter for the nominal variation: they can just
0213       // use the nominal filter!
0214       assert(variationName != "nominal");
0215       // nobody should ask for a varied filter for a variation on which this filter does not depend:
0216       // they can just use the nominal filter.
0217       assert(RDFInternal::IsStrInVec(variationName, fVariations));
0218
0219       auto it = fVariedFilters.find(variationName);
0220       if (it != fVariedFilters.end())
0221          return it->second;
0222
0223       auto prevNode = fPrevNodePtr;
0224       if (static_cast<RNodeBase *>(fPrevNodePtr.get()) != static_cast<RNodeBase *>(fLoopManager) &&
0225           RDFInternal::IsStrInVec(variationName, prevNode->GetVariations()))
0226          prevNode = std::static_pointer_cast<PrevNode_t>(prevNode->GetVariedFilter(variationName));
0227
0228       // the varied filters get a copy of the callable object.
0229       // TODO document this
0230       auto variedFilter = std::unique_ptr<RFilterBase>(
0231          new RFilter(fFilter, fColumnNames, std::move(prevNode), fColRegister, fName, variationName));
0232       auto e = fVariedFilters.insert({variationName, std::move(variedFilter)});
0233       return e.first->second;
0234    }
0235 };
0236
0237 } // ns RDF
0238 } // ns Detail
0239 } // ns ROOT
0240
0241 #endif // ROOT_RFILTER