mpi/collectives/all_to_all.hpp

0001 // Copyright (C) 2005, 2006 Douglas Gregor.
0002
0003 // Use, modification and distribution is subject to the Boost Software
0004 // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
0005 // http://www.boost.org/LICENSE_1_0.txt)
0006
0007 // Message Passing Interface 1.1 -- Section 4.8. All-to-all
0008 #ifndef BOOST_MPI_ALL_TO_ALL_HPP
0009 #define BOOST_MPI_ALL_TO_ALL_HPP
0010
0011 #include <boost/mpi/exception.hpp>
0012 #include <boost/mpi/datatype.hpp>
0013 #include <vector>
0014 #include <boost/mpi/packed_oarchive.hpp>
0015 #include <boost/mpi/packed_iarchive.hpp>
0016 #include <boost/mpi/communicator.hpp>
0017 #include <boost/mpi/environment.hpp>
0018 #include <boost/assert.hpp>
0019 #include <boost/mpi/collectives_fwd.hpp>
0020 #include <boost/mpi/allocator.hpp>
0021
0022 namespace boost { namespace mpi {
0023
0024 namespace detail {
0025   // We're performing an all-to-all with a type that has an
0026   // associated MPI datatype, so we'll use MPI_Alltoall to do all of
0027   // the work.
0028   template<typename T>
0029   void
0030   all_to_all_impl(const communicator& comm, const T* in_values, int n,
0031                   T* out_values, mpl::true_)
0032   {
0033     MPI_Datatype type = get_mpi_datatype<T>(*in_values);
0034     BOOST_MPI_CHECK_RESULT(MPI_Alltoall,
0035                            (const_cast<T*>(in_values), n, type,
0036                             out_values, n, type, comm));
0037   }
0038
0039   // We're performing an all-to-all with a type that does not have an
0040   // associated MPI datatype, so we'll need to serialize
0041   // it.
0042   template<typename T>
0043   void
0044   all_to_all_impl(const communicator& comm, const T* in_values, int n,
0045                   T* out_values, mpl::false_)
0046   {
0047     int size = comm.size();
0048     int rank = comm.rank();
0049
0050     // The amount of data to be sent to each process
0051     std::vector<int> send_sizes(size);
0052
0053     // The displacements for each outgoing value.
0054     std::vector<int> send_disps(size);
0055
0056     // The buffer that will store all of the outgoing values
0057     std::vector<char, allocator<char> > outgoing;
0058
0059     // Pack the buffer with all of the outgoing values.
0060     for (int dest = 0; dest < size; ++dest) {
0061       // Keep track of the displacements
0062       send_disps[dest] = outgoing.size();
0063
0064       // Our own value will never be transmitted, so don't pack it.
0065       if (dest != rank) {
0066         packed_oarchive oa(comm, outgoing);
0067         for (int i = 0; i < n; ++i)
0068           oa << in_values[dest * n + i];
0069       }
0070
0071       // Keep track of the sizes
0072       send_sizes[dest] = outgoing.size() - send_disps[dest];
0073     }
0074
0075     // Determine how much data each process will receive.
0076     std::vector<int> recv_sizes(size);
0077     all_to_all(comm, send_sizes, recv_sizes);
0078
0079     // Prepare a buffer to receive the incoming data.
0080     std::vector<int> recv_disps(size);
0081     int sum = 0;
0082     for (int src = 0; src < size; ++src) {
0083       recv_disps[src] = sum;
0084       sum += recv_sizes[src];
0085     }
0086     std::vector<char, allocator<char> > incoming(sum > 0? sum : 1);
0087
0088     // Make sure we don't try to reference an empty vector
0089     if (outgoing.empty())
0090       outgoing.push_back(0);
0091
0092     // Transmit the actual data
0093     BOOST_MPI_CHECK_RESULT(MPI_Alltoallv,
0094                            (detail::c_data(outgoing), detail::c_data(send_sizes),
0095                             detail::c_data(send_disps), MPI_PACKED,
0096                             detail::c_data(incoming), detail::c_data(recv_sizes),
0097                             detail::c_data(recv_disps), MPI_PACKED,
0098                             comm));
0099
0100     // Deserialize data from the iarchive
0101     for (int src = 0; src < size; ++src) {
0102       if (src == rank)
0103         std::copy(in_values + src * n, in_values + (src + 1) * n,
0104                   out_values + src * n);
0105       else {
0106         packed_iarchive ia(comm, incoming, boost::archive::no_header,
0107                            recv_disps[src]);
0108         for (int i = 0; i < n; ++i)
0109           ia >> out_values[src * n + i];
0110       }
0111     }
0112   }
0113 } // end namespace detail
0114
0115 template<typename T>
0116 inline void
0117 all_to_all(const communicator& comm, const T* in_values, T* out_values)
0118 {
0119   detail::all_to_all_impl(comm, in_values, 1, out_values, is_mpi_datatype<T>());
0120 }
0121
0122 template<typename T>
0123 void
0124 all_to_all(const communicator& comm, const std::vector<T>& in_values,
0125            std::vector<T>& out_values)
0126 {
0127   BOOST_ASSERT((int)in_values.size() == comm.size());
0128   out_values.resize(comm.size());
0129   ::boost::mpi::all_to_all(comm, detail::c_data(in_values), detail::c_data(out_values));
0130 }
0131
0132 template<typename T>
0133 inline void
0134 all_to_all(const communicator& comm, const T* in_values, int n, T* out_values)
0135 {
0136   detail::all_to_all_impl(comm, in_values, n, out_values, is_mpi_datatype<T>());
0137 }
0138
0139 template<typename T>
0140 void
0141 all_to_all(const communicator& comm, const std::vector<T>& in_values, int n,
0142            std::vector<T>& out_values)
0143 {
0144   BOOST_ASSERT((int)in_values.size() == comm.size() * n);
0145   out_values.resize(comm.size() * n);
0146   ::boost::mpi::all_to_all(comm, detail::c_data(in_values), n, detail::c_data(out_values));
0147 }
0148
0149 } } // end namespace boost::mpi
0150
0151 #endif // BOOST_MPI_ALL_TO_ALL_HPP