include/Geant4/G4MPIToolsManager.icc

0001 //
0002 // ********************************************************************
0003 // * License and Disclaimer                                           *
0004 // *                                                                  *
0005 // * The  Geant4 software  is  copyright of the Copyright Holders  of *
0006 // * the Geant4 Collaboration.  It is provided  under  the terms  and *
0007 // * conditions of the Geant4 Software License,  included in the file *
0008 // * LICENSE and available at  http://cern.ch/geant4/license .  These *
0009 // * include a list of copyright holders.                             *
0010 // *                                                                  *
0011 // * Neither the authors of this software system, nor their employing *
0012 // * institutes,nor the agencies providing financial support for this *
0013 // * work  make  any representation or  warranty, express or implied, *
0014 // * regarding  this  software system or assume any liability for its *
0015 // * use.  Please see the license in the file  LICENSE  and URL above *
0016 // * for the full disclaimer and the limitation of liability.         *
0017 // *                                                                  *
0018 // * This  code  implementation is the result of  the  scientific and *
0019 // * technical work of the GEANT4 collaboration.                      *
0020 // * By using,  copying,  modifying or  distributing the software (or *
0021 // * any work based  on the software)  you  agree  to acknowledge its *
0022 // * use  in  resulting  scientific  publications,  and indicate your *
0023 // * acceptance of all terms of the Geant4 Software license.          *
0024 // ********************************************************************
0025 //
0026
0027 // The manager class for MPI applications.
0028
0029 // Author: Ivana Hrivnacova, 25/06/2015  (ivana@ipno.in2p3.fr)
0030
0031 using std::to_string;
0032
0033 //_____________________________________________________________________________
0034 template <typename HT>
0035 inline
0036 G4bool G4MPIToolsManager::Send(
0037   G4int nofActiveT, const std::vector<std::pair<HT*, G4HnInformation*>>& hnVector)
0038 {
0039   auto result = true;
0040
0041   // send object to destination rank
0042   // G4cout << "Begin send for " << nofActiveT << G4endl;
0043   fHmpi->beg_send(nofActiveT);
0044
0045   // pack objects
0046   for (const auto& [ht, info] : hnVector) {
0047     // skip sending if activation is enabled and HT is inactivated
0048     // or if histogram was deleted
0049     if ( ( fState.GetIsActivation() && ( ! info->GetActivation() ) ) ||
0050         ( info->GetDeleted() ) ) continue;
0051     // pack histogram for sending
0052     // G4cout << "Packed " << i << "th T" << G4endl;
0053     result &= fHmpi->pack(*ht);
0054   }
0055
0056   //G4cout << "Go to send all " << G4endl;
0057   if ( ! fHmpi->send(fHmpi->rank()) ) {
0058     G4Analysis::Warn(
0059       "Rank: " + to_string(fHmpi->rank()) + " : can't send histos.",
0060       fkClass, "Send");
0061     return false;
0062   }
0063
0064   return result;
0065 }
0066
0067 //_____________________________________________________________________________
0068 template <typename HT>
0069 inline
0070 G4bool G4MPIToolsManager::Receive(
0071   G4int nofActiveT, const std::vector<std::pair<HT*, G4HnInformation*>>& hnVector)
0072 {
0073   G4int commSize;
0074   G4bool result = fHmpi->comm_size(commSize);
0075   if ( ! result ) {
0076     G4Analysis::Warn(
0077       "Failed to get MPI commander size.\nMerging will not be performed.",
0078       fkClass, "Receive");
0079     return false;
0080   }
0081
0082   // get objects from source ranks
0083   for (G4int srank = 0; srank < commSize; ++srank) {
0084
0085     // skip destination rank
0086     if ( srank == fHmpi->rank() ) continue;
0087
0088     // get objects from this source rank
0089     // G4cout << "Go to wait_histos " << srank << G4endl;
0090     using class_pointer = std::pair<std::string,void*>;
0091     std::vector<class_pointer> hs;
0092     if ( ! fHmpi->wait_histos(srank, hs) ) {
0093       G4Analysis::Warn(
0094         "Wait_histos from " + to_string(srank) + " : failed.",
0095         fkClass, "Receive");
0096       return false;
0097     }
0098
0099     // check that we got the right number of objects
0100     if ( G4int(hs.size()) != nofActiveT ) {
0101       G4Analysis::Warn(
0102         "srank: " + to_string(srank) + " : got " + to_string(hs.size()) +
0103         " objects, while " + to_string(nofActiveT) +" were expected.",
0104         fkClass, "Receive");
0105       return false;
0106     }
0107
0108     // merge the objects to destination rank
0109     G4int counter = 0;
0110     for (const auto& [ht, info] : hnVector) {
0111       // skip sending if activation is enabled and HT is inactivated
0112       if ( ( fState.GetIsActivation() && ( ! info->GetActivation() ) ) ) continue;
0113       // merge histograms
0114       auto newHt = static_cast<HT*>(hs[counter++].second);
0115       ht->add(*newHt);
0116     }
0117   }
0118   return true;
0119 }
0120
0121
0122 //_____________________________________________________________________________
0123 template <typename HT>
0124 inline
0125 G4bool G4MPIToolsManager::Merge(
0126   const std::vector<std::pair<HT*, G4HnInformation*>>& hnVector)
0127 {
0128   if ( hnVector.empty() ) return true;
0129
0130   // Get number of objects to be sent
0131   G4int nofActiveT = 0;
0132   if ( fState.GetIsActivation() ) {
0133     // only activated histograms will be treated
0134     for (const auto& htn : hnVector) {
0135       auto activation = htn.second->GetActivation();
0136       if ( activation ) ++nofActiveT;
0137     }
0138   } else {
0139       nofActiveT = G4int(hnVector.size());
0140   }
0141
0142   if ( ! nofActiveT ) return true;
0143
0144   G4int commRank;
0145   if ( ! fHmpi->comm_rank(commRank) ) {
0146     G4Analysis::Warn(
0147       "Failed to get MPI commander rank.\nMerging will not be performed.",
0148       fkClass, "Merge");
0149     return false;
0150   }
0151
0152   auto result = true;
0153
0154   if ( commRank != fHmpi->rank() ) {
0155     fState.Message(G4Analysis::kVL3, "mpi send", "Hn|Pn",
0156       "on rank "  + to_string(commRank) +
0157       " destination rank: " + to_string(fHmpi->rank()));
0158
0159     result &= Send(nofActiveT, hnVector);
0160
0161     fState.Message(G4Analysis::kVL1, "mpi send", "Hn|Pn",
0162       "on rank "  + to_string(commRank) +
0163       " destination rank: " + to_string(fHmpi->rank()));
0164
0165   } else {
0166     fState.Message(G4Analysis::kVL3, "mpi wait_histos", "Hn|Pn",
0167       "on rank "  + to_string(commRank) +
0168       " destination rank: " + to_string(fHmpi->rank()));
0169
0170     result &= Receive(nofActiveT, hnVector);
0171
0172     fState.Message(G4Analysis::kVL1, "mpi wait_histos", "Hn|Pn",
0173       "on rank "  + to_string(commRank) +
0174       " destination rank: " + to_string(fHmpi->rank()));
0175   }
0176   return result;
0177 }