exMPI03/src/RunActionMaster.cc

0001 //
0002 // ********************************************************************
0003 // * License and Disclaimer                                           *
0004 // *                                                                  *
0005 // * The  Geant4 software  is  copyright of the Copyright Holders  of *
0006 // * the Geant4 Collaboration.  It is provided  under  the terms  and *
0007 // * conditions of the Geant4 Software License,  included in the file *
0008 // * LICENSE and available at  http://cern.ch/geant4/license .  These *
0009 // * include a list of copyright holders.                             *
0010 // *                                                                  *
0011 // * Neither the authors of this software system, nor their employing *
0012 // * institutes,nor the agencies providing financial support for this *
0013 // * work  make  any representation or  warranty, express or implied, *
0014 // * regarding  this  software system or assume any liability for its *
0015 // * use.  Please see the license in the file  LICENSE  and URL above *
0016 // * for the full disclaimer and the limitation of liability.         *
0017 // *                                                                  *
0018 // * This  code  implementation is the result of  the  scientific and *
0019 // * technical work of the GEANT4 collaboration.                      *
0020 // * By using,  copying,  modifying or  distributing the software (or *
0021 // * any work based  on the software)  you  agree  to acknowledge its *
0022 // * use  in  resulting  scientific  publications,  and indicate your *
0023 // * acceptance of all terms of the Geant4 Software license.          *
0024 // ********************************************************************
0025 //
0026 //
0027 /// @file RunActionMaster.cc
0028 /// @brief Describe run actions
0029
0030 #include "RunActionMaster.hh"
0031
0032 #include "Analysis.hh"
0033 #include "Run.hh"
0034 #include "RunMerger.hh"
0035
0036 #include "G4AnalysisManager.hh"
0037 #include "G4MPIhistoMerger.hh"  // New code with use of g4analysis
0038 #include "G4MPImanager.hh"
0039 #include "G4MPIscorerMerger.hh"
0040 #include "G4Threading.hh"
0041
0042 #include <stdio.h>
0043
0044 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0045 RunActionMaster::RunActionMaster() {}
0046
0047 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0048 G4Run* RunActionMaster::GenerateRun()
0049 {
0050   return new Run;
0051 }
0052
0053 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0054 RunActionMaster::~RunActionMaster() {}
0055
0056 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0057 void RunActionMaster::BeginOfRunAction(const G4Run*)
0058 {
0059   Analysis* myana = Analysis::GetAnalysis();
0060   myana->Clear();
0061   myana->Book();
0062 }
0063
0064 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
0065 void RunActionMaster::EndOfRunAction(const G4Run* arun)
0066 {
0067   // This is executed by master thread only. Worker threads have already
0068   // merged their results into this master threads.
0069
0070   // We are going to merge the results via MPI for:
0071   //  1. User-defined "Run" object
0072   //  2. Command line scorers, if exists
0073   //  3. G4Analysis objects
0074
0075   // For debugging purposes in the following we write out results twice:
0076   // BEFORE and AFTER the merging, so that rank 0 actually
0077   // writes two files, the one called "*rank000*" will contain the partial
0078   // results only from rank #0,
0079   // the file *merged* contains the reduction from all ranks.
0080   // It should be very easy to adapt this code
0081
0082   const G4int rank = G4MPImanager::GetManager()->GetRank();
0083
0084   G4cout << "=====================================================" << G4endl;
0085   G4cout << "Start EndOfRunAction for master thread in rank: " << rank << G4endl;
0086   G4cout << "=====================================================" << G4endl;
0087
0088   // Merging of G4Run object:
0089   // All ranks > 0 merge to rank #0
0090   RunMerger rm(static_cast<const Run*>(arun));
0091   G4int ver = 0;  // Use 4 for lots of info
0092   if (rank == 0 && ver == 0) ver = 1;
0093
0094   if (ver > 1) {
0095     G4cout << "Before merge the Run object has test counter value: "
0096            << static_cast<const Run*>(arun)->GetCounter() << G4endl;
0097   }
0098   rm.SetVerbosity(ver);
0099   rm.Merge();
0100   if (ver > 1) {
0101     G4cout << "After merge the Run object has test counter value: "
0102            << static_cast<const Run*>(arun)->GetCounter() << " (with 1 thread== number of ranks)"
0103            << G4endl;
0104   }
0105
0106   // Merge of scorers
0107   ver = 0;
0108   if (G4ScoringManager::GetScoringManagerIfExist()) {
0109     const auto scor = G4ScoringManager::GetScoringManager();
0110     G4MPIscorerMerger sm(scor);
0111     sm.SetVerbosity(ver);
0112     // Debug!
0113     auto debugme = [&scor]() {
0114       for (size_t idx = 0; idx < scor->GetNumberOfMesh(); ++idx) {
0115         const auto m = scor->GetMesh(idx);
0116         const auto map = m->GetScoreMap();
0117         std::for_each(
0118           map.begin(), map.end(), [](const G4VScoringMesh::MeshScoreMap::value_type& e) {
0119             G4cout << e.first << "(" << e.second << "):" << G4endl;
0120             const auto data = e.second->GetMap();
0121             for (auto it = data->begin(); it != data->end(); ++it) {
0122               G4cout << it->first
0123                      << " => G4StatDouble(n,sum_w,sum_w2,sum_wx,sum_wx2): " << it->second->n()
0124                      << " " << it->second->sum_w() << " " << it->second->sum_w2() << " "
0125                      << it->second->sum_wx() << " " << it->second->sum_wx2() << G4endl;
0126             }
0127           });
0128       }
0129     };
0130     // Debug!
0131     if (ver > 4) {
0132       G4cout << "Before merging: Meshes dump" << G4endl;
0133       debugme();
0134     }
0135     // Write partial scorers from single ranks *before* merging
0136     // Do not rely on UI command to write out scorers, because rank-specific
0137     // files will have same file name: need to add rank # to file name
0138     if (true) {
0139       for (size_t idx = 0; idx < scor->GetNumberOfMesh(); ++idx) {
0140         const auto m = scor->GetMesh(idx);
0141         const auto& mn = m->GetWorldName();
0142         std::ostringstream fname;
0143         fname << "scorer-" << mn << "-rank" << rank << ".csv";
0144         scor->DumpAllQuantitiesToFile(mn, fname.str());
0145       }
0146     }
0147
0148     // Now reduce all scorers to rank #0
0149     sm.Merge();
0150
0151     // Debug!
0152     if (ver > 4) {
0153       G4cout << "After merging: Meshes dump" << G4endl;
0154       debugme();
0155     }
0156     // For rank #0 write out the merged files
0157     if (rank == 0) {
0158       for (size_t idx = 0; idx < scor->GetNumberOfMesh(); ++idx) {
0159         const auto m = scor->GetMesh(idx);
0160         const auto& mn = m->GetWorldName();
0161         std::ostringstream fname;
0162         fname << "scorer-" << mn << "-merged.csv";
0163         scor->DumpAllQuantitiesToFile(mn, fname.str());
0164       }
0165     }
0166   }
0167
0168   // Save histograms *before* MPI merging for rank #0
0169   if (rank == 0) {
0170     G4String fname("dose-rank0");
0171     Analysis* myana = Analysis::GetAnalysis();
0172     myana->Save(fname);
0173     myana->Close(false);  // close file withour resetting data
0174   }
0175   // Merge of g4analysis objects
0176   ver = 0;
0177   G4MPIhistoMerger hm(G4AnalysisManager::Instance());
0178   hm.SetVerbosity(ver);
0179   hm.Merge();
0180
0181   // Save g4analysis objects to a file
0182   // NB: It is important that the save is done *after* MPI-merging of histograms
0183
0184   // One can save all ranks or just rank0, chane the if
0185   if (true /*rank == 0*/) {
0186     std::ostringstream fname;
0187     fname << "dose-rank" << rank;
0188     if (rank == 0) {
0189       fname.str("dose-merged");
0190     }
0191     Analysis* myana = Analysis::GetAnalysis();
0192     myana->Save(fname.str());
0193   }
0194   Analysis* myana = Analysis::GetAnalysis();
0195   myana->Close();
0196
0197   G4cout << "===================================================" << G4endl;
0198   G4cout << "End EndOfRunAction for master thread in rank: " << rank << G4endl;
0199   G4cout << "===================================================" << G4endl;
0200 }