exMPI04/src/RunActionMaster.cc

0001 //
0002 // ********************************************************************
0003 // * License and Disclaimer                                           *
0004 // *                                                                  *
0005 // * The  Geant4 software  is  copyright of the Copyright Holders  of *
0006 // * the Geant4 Collaboration.  It is provided  under  the terms  and *
0007 // * conditions of the Geant4 Software License,  included in the file *
0008 // * LICENSE and available at  http://cern.ch/geant4/license .  These *
0009 // * include a list of copyright holders.                             *
0010 // *                                                                  *
0011 // * Neither the authors of this software system, nor their employing *
0012 // * institutes,nor the agencies providing financial support for this *
0013 // * work  make  any representation or  warranty, express or implied, *
0014 // * regarding  this  software system or assume any liability for its *
0015 // * use.  Please see the license in the file  LICENSE  and URL above *
0016 // * for the full disclaimer and the limitation of liability.         *
0017 // *                                                                  *
0018 // * This  code  implementation is the result of  the  scientific and *
0019 // * technical work of the GEANT4 collaboration.                      *
0020 // * By using,  copying,  modifying or  distributing the software (or *
0021 // * any work based  on the software)  you  agree  to acknowledge its *
0022 // * use  in  resulting  scientific  publications,  and indicate your *
0023 // * acceptance of all terms of the Geant4 Software license.          *
0024 // ********************************************************************
0025 //
0026 //
0027 /// @file RunActionMaster.cc
0028 /// @brief Describe run actions
0029
0030 #include "RunActionMaster.hh"
0031
0032 #include "Analysis.hh"
0033 #include "RunMerger.hh"
0034
0035 #include "G4MPImanager.hh"
0036 #include "G4Threading.hh"
0037
0038 #include <stdio.h>
0039 #ifndef G4MULTITHREADED
0040 #  include "G4RootMpiAnalysisManager.hh"
0041 using G4AnalysisManager = G4RootMpiAnalysisManager;
0042 #else
0043 #  include "G4RootAnalysisManager.hh"
0044 using G4AnalysisManager = G4RootAnalysisManager;
0045 #endif
0046 #include "Run.hh"
0047
0048 #include "G4Filesystem.hh"
0049 #include "G4MPIhistoMerger.hh"
0050 #include "G4MPIntupleMerger.hh"
0051 #include "G4MPIscorerMerger.hh"
0052
0053 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0054 RunActionMaster::RunActionMaster(G4bool useNtuple, G4bool mergeNtuple)
0055   : G4UserRunAction(), fMPIntupleMerger(nullptr)
0056 {
0057 #ifndef G4MULTITHREADED
0058   if (mergeNtuple && G4MPImanager::GetManager()->GetTotalSize() >= 2) {
0059     // Activate MPI ntuple merging
0060     // The merger must be created before creating G4AnalysisManager:
0061     // (= the first call to G4AnalysisManager::Instance())
0062     // and deleted only at the end of program
0063     G4int nofReducedNtupleFiles = 0;
0064     // Multiple reduced ntuple files are not yet supported
0065     G4bool rowWise = false;
0066     G4bool rowMode = true;
0067     fMPIntupleMerger = new G4MPIntupleMerger(nofReducedNtupleFiles, rowWise, rowMode);
0068   }
0069 #endif
0070
0071   // Book analysis in ctor
0072   Analysis* myana = Analysis::GetAnalysis();
0073   myana->SetUseNtuple(useNtuple);
0074   myana->SetMergeNtuple(mergeNtuple);
0075   G4cout << "Book analysis on master on rank: " << G4MPImanager::GetManager()->GetRank() << G4endl;
0076   ;
0077   myana->Book();
0078 }
0079
0080 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0081 G4Run* RunActionMaster::GenerateRun()
0082 {
0083   return new Run;
0084 }
0085
0086 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0087 RunActionMaster::~RunActionMaster()
0088 {
0089   delete fMPIntupleMerger;
0090 }
0091
0092 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0093 void RunActionMaster::BeginOfRunAction(const G4Run*)
0094 {
0095   G4cout << "RunActionMaster::BeginOfRunAction" << G4endl;
0096
0097   Analysis* myana = Analysis::GetAnalysis();
0098
0099   G4int rank = G4MPImanager::GetManager()->GetRank();
0100   std::ostringstream fname;
0101   fname << "dose-rank" << rank;
0102   myana->OpenFile(fname.str());
0103   // OpenFile triggeres creating collecting/sending ntuples objects;
0104   // must be called at BeginOfRunAction
0105 }
0106
0107 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0108 void RunActionMaster::EndOfRunAction(const G4Run* arun)
0109 {
0110   G4cout << "RunActionMaster::EndOfRunAction" << G4endl;
0111
0112   // This is executed by master thread only. Worker threads have already
0113   // merged their results into this master threads.
0114
0115   // We are going to merge the results via MPI for:
0116   //  1. User-defined "Run" object
0117   //  2. Command line scorers, if exists
0118   //  3. G4Analysis objects
0119
0120   // For debugging purposes in the following we write out results twice:
0121   // BEFORE and AFTER the merging, so that rank 0 actually
0122   // writes two files, the one called "*rank000*" will contain the partial
0123   // results only from rank #0,
0124   // the file *merged* contains the reduction from all ranks.
0125   // It should be very easy to adapt this code
0126
0127   const G4int rank = G4MPImanager::GetManager()->GetRank();
0128
0129   G4cout << "=====================================================" << G4endl;
0130   G4cout << "Start EndOfRunAction for master thread in rank: " << rank << G4endl;
0131   G4cout << "=====================================================" << G4endl;
0132
0133   if (!G4MPImanager::GetManager()->IsExtraWorker()) {
0134     G4cout << "Go to merge on processing workers" << G4endl;
0135
0136     // Merging of G4Run object:
0137     // All ranks > 0 merge to rank #0
0138     RunMerger rm(static_cast<const Run*>(arun), G4MPImanager::kRANK_MASTER, 5);
0139     G4int ver = 0;  // Use 4 for lots of info
0140     if (rank == 0 && ver == 0) ver = 1;
0141
0142     if (ver > 1) {
0143       G4cout << "Before merge the Run object has test counter value: "
0144              << static_cast<const Run*>(arun)->GetCounter() << G4endl;
0145     }
0146     rm.SetVerbosity(ver);
0147     rm.Merge();
0148     if (ver > 1) {
0149       G4cout << "After merge the Run object has test counter value: "
0150              << static_cast<const Run*>(arun)->GetCounter() << " (with 1 thread== number of ranks)"
0151              << G4endl;
0152     }
0153
0154     // Merge of scorers
0155     ver = 0;
0156     if (G4ScoringManager::GetScoringManagerIfExist()) {
0157       const auto scor = G4ScoringManager::GetScoringManager();
0158       G4MPIscorerMerger sm(scor);
0159       sm.SetVerbosity(ver);
0160       // Debug!
0161       auto debugme = [&scor]() {
0162         for (size_t idx = 0; idx < scor->GetNumberOfMesh(); ++idx) {
0163           const auto m = scor->GetMesh(idx);
0164           const auto map = m->GetScoreMap();
0165           std::for_each(
0166             map.begin(), map.end(), [](const G4VScoringMesh::MeshScoreMap::value_type& e) {
0167               G4cout << e.first << "(" << e.second << "):" << G4endl;
0168               const auto data = e.second->GetMap();
0169               for (auto it = data->begin(); it != data->end(); ++it) {
0170                 G4cout << it->first
0171                        << " => G4StatDouble(n,sum_w,sum_w2,sum_wx,sum_wx2): " << it->second->n()
0172                        << " " << it->second->sum_w() << " " << it->second->sum_w2() << " "
0173                        << it->second->sum_wx() << " " << it->second->sum_wx2() << G4endl;
0174               }
0175             });
0176         }
0177       };
0178       // Debug!
0179       if (ver > 4) {
0180         G4cout << "Before merging: Meshes dump" << G4endl;
0181         debugme();
0182       }
0183       // Write partial scorers from single ranks *before* merging
0184       // Do not rely on UI command to write out scorers, because rank-specific
0185       // files will have same file name: need to add rank # to file name
0186       if (true) {
0187         for (size_t idx = 0; idx < scor->GetNumberOfMesh(); ++idx) {
0188           const auto m = scor->GetMesh(idx);
0189           const auto& mn = m->GetWorldName();
0190           std::ostringstream fname;
0191           fname << "scorer-" << mn << "-rank" << rank << ".csv";
0192           scor->DumpAllQuantitiesToFile(mn, fname.str());
0193         }
0194       }
0195
0196       // Now reduce all scorers to rank #0
0197       sm.Merge();
0198
0199       // Debug!
0200       if (ver > 4) {
0201         G4cout << "After merging: Meshes dump" << G4endl;
0202         debugme();
0203       }
0204       // For rank #0 write out the merged files
0205       if (rank == 0) {
0206         for (size_t idx = 0; idx < scor->GetNumberOfMesh(); ++idx) {
0207           const auto m = scor->GetMesh(idx);
0208           const auto& mn = m->GetWorldName();
0209           std::ostringstream fname;
0210           fname << "scorer-" << mn << "-merged.csv";
0211           scor->DumpAllQuantitiesToFile(mn, fname.str());
0212         }
0213       }
0214     }
0215
0216     // Save histograms *before* MPI merging for rank #0
0217     // Can be done only if MPI merging is not activated
0218     if (rank == 0 && (fMPIntupleMerger == nullptr)) {
0219       Analysis* myana = Analysis::GetAnalysis();
0220       myana->Save();
0221       myana->Close(false);  // close without resetting histograms
0222       // Rename file in another file
0223       G4fs::rename("dose-rank0.root", "dose-rank0-part.root");
0224     }
0225
0226     // Merge of g4analysis objects
0227     G4cout << "Go to merge histograms " << G4endl;
0228     ver = 1;
0229     G4MPIhistoMerger hm(G4AnalysisManager::Instance());
0230     hm.SetVerbosity(ver);
0231     hm.Merge();
0232     G4cout << "Done merge histograms " << G4endl;
0233   }
0234   else {
0235     G4cout << "Skip merging on the extra worker" << G4endl;
0236   }
0237
0238   // Save g4analysis objects to a file
0239   Analysis* myana = Analysis::GetAnalysis();
0240   myana->OpenFile("dose-rank0.root");
0241   myana->Save();
0242   myana->Close();
0243
0244   if (rank == 0) {
0245     // Rename files
0246     G4fs::rename("dose-rank0.root", "dose-merged.root");
0247     if (fMPIntupleMerger == nullptr) {
0248       G4fs::rename("dose-rank0-part.root", "dose-rank0.root");
0249     }
0250   }
0251
0252   G4cout << "===================================================" << G4endl;
0253   G4cout << "End EndOfRunAction for master thread in rank: " << rank << G4endl;
0254   G4cout << "===================================================" << G4endl;
0255 }