root/Minuit2/MPIProcess.h

0001 // @(#)root/minuit2:$Id$
0002 // Author: A. Lazzaro 2009
0003 /***************************************************************************
0004  * Package: Minuit2                                                        *
0005  *    File: $Id$         *
0006  *  Author: Alfio Lazzaro, alfio.lazzaro@mi.infn.it                        *
0007  *                                                                         *
0008  * Copyright: (C) 2008 by Universita' and INFN, Milan                      *
0009  ***************************************************************************/
0010
0011 #ifndef ROOT_Minuit2_MPIProcess
0012 #define ROOT_Minuit2_MPIProcess
0013
0014 // disable MPI calls
0015 //#define MPIPROC
0016
0017 #include "Minuit2/MnMatrix.h"
0018
0019 #ifdef MPIPROC
0020 #include "mpi.h"
0021 #include <iostream>
0022 #endif
0023
0024 namespace ROOT {
0025
0026 namespace Minuit2 {
0027
0028 class MPITerminate {
0029 public:
0030    ~MPITerminate()
0031    {
0032 #ifdef MPIPROC
0033       if (MPI::Is_initialized() && !(MPI::Is_finalized())) {
0034          std::cout << "Info --> MPITerminate:: End MPI on #" << MPI::COMM_WORLD.Get_rank() << " processor" << std::endl;
0035
0036          MPI::Finalize();
0037       }
0038 #endif
0039    }
0040 };
0041
0042 class MPIProcess {
0043 public:
0044    MPIProcess(unsigned int nelements, unsigned int indexComm);
0045    ~MPIProcess();
0046
0047    inline unsigned int NumElements4JobIn() const { return fNumElements4JobIn; }
0048    inline unsigned int NumElements4JobOut() const { return fNumElements4JobOut; }
0049
0050    inline unsigned int NumElements4Job(unsigned int rank) const
0051    {
0052       return NumElements4JobIn() + ((rank < NumElements4JobOut()) ? 1 : 0);
0053    }
0054
0055    inline unsigned int StartElementIndex() const
0056    {
0057       return ((fRank < NumElements4JobOut()) ? (fRank * NumElements4Job(fRank))
0058                                              : (fNelements - (fSize - fRank) * NumElements4Job(fRank)));
0059    }
0060
0061    inline unsigned int EndElementIndex() const { return StartElementIndex() + NumElements4Job(fRank); }
0062
0063    inline unsigned int GetMPISize() const { return fSize; }
0064    inline unsigned int GetMPIRank() const { return fRank; }
0065
0066    bool SyncVector(ROOT::Minuit2::MnAlgebraicVector &mnvector);
0067    bool SyncSymMatrixOffDiagonal(ROOT::Minuit2::MnAlgebraicSymMatrix &mnmatrix);
0068
0069    static unsigned int GetMPIGlobalRank()
0070    {
0071       StartMPI();
0072       return fgGlobalRank;
0073    }
0074    static unsigned int GetMPIGlobalSize()
0075    {
0076       StartMPI();
0077       return fgGlobalSize;
0078    }
0079    static inline void StartMPI()
0080    {
0081 #ifdef MPIPROC
0082       if (!(MPI::Is_initialized())) {
0083          MPI::Init();
0084          std::cout << "Info --> MPIProcess::StartMPI: Start MPI on #" << MPI::COMM_WORLD.Get_rank() << " processor"
0085                    << std::endl;
0086       }
0087       fgGlobalSize = MPI::COMM_WORLD.Get_size();
0088       fgGlobalRank = MPI::COMM_WORLD.Get_rank();
0089 #endif
0090    }
0091
0092    static void TerminateMPI()
0093    {
0094 #ifdef MPIPROC
0095       if (fgCommunicators[0] != 0 && fgCommunicators[1] != 0) {
0096          delete fgCommunicators[0];
0097          fgCommunicators[0] = 0;
0098          fgIndicesComm[0] = 0;
0099          delete fgCommunicators[1];
0100          fgCommunicators[1] = 0;
0101          fgIndicesComm[1] = 0;
0102       }
0103
0104       MPITerminate();
0105
0106 #endif
0107    }
0108
0109    static bool SetCartDimension(unsigned int dimX, unsigned int dimY);
0110    static bool SetDoFirstMPICall(bool doFirstMPICall = true);
0111
0112    inline void SumReduce(const double &sub, double &total)
0113    {
0114       total = sub;
0115
0116 #ifdef MPIPROC
0117       if (fSize > 1) {
0118          fgCommunicator->Allreduce(&sub, &total, 1, MPI::DOUBLE, MPI::SUM);
0119       }
0120 #endif
0121    }
0122
0123 private:
0124 #ifdef MPIPROC
0125    void MPISyncVector(double *ivector, int svector, double *ovector);
0126 #endif
0127
0128 private:
0129    unsigned int fNelements;
0130    unsigned int fSize;
0131    unsigned int fRank;
0132
0133    static unsigned int fgGlobalSize;
0134    static unsigned int fgGlobalRank;
0135
0136    static unsigned int fgCartSizeX;
0137    static unsigned int fgCartSizeY;
0138    static unsigned int fgCartDimension;
0139    static bool fgNewCart;
0140
0141    unsigned int fNumElements4JobIn;
0142    unsigned int fNumElements4JobOut;
0143
0144 #ifdef MPIPROC
0145    static MPI::Intracomm *fgCommunicator;
0146    static int fgIndexComm;                    // maximum 2 communicators, so index can be 0 and 1
0147    static MPI::Intracomm *fgCommunicators[2]; // maximum 2 communicators
0148    static unsigned int fgIndicesComm[2];
0149 #endif
0150 };
0151
0152 } // namespace Minuit2
0153 } // namespace ROOT
0154
0155 #endif