Back to home page

EIC code displayed by LXR

 
 

    


File indexing completed on 2025-01-30 09:46:55

0001 // Copyright (C) 2005, 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
0002 // Copyright (C) 2016 K. Noel Belcourt <kbelco -at- sandia.gov>.
0003 
0004 // Use, modification and distribution is subject to the Boost Software
0005 // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
0006 // http://www.boost.org/LICENSE_1_0.txt)
0007 
0008 /** @file communicator.hpp
0009  *
0010  *  This header defines the @c communicator class, which is the basis
0011  *  of all communication within Boost.MPI, and provides point-to-point
0012  *  communication operations.
0013  */
0014 #ifndef BOOST_MPI_COMMUNICATOR_HPP
0015 #define BOOST_MPI_COMMUNICATOR_HPP
0016 
0017 #include <boost/assert.hpp>
0018 #include <boost/mpi/config.hpp>
0019 #include <boost/mpi/exception.hpp>
0020 #include <boost/optional.hpp>
0021 #include <boost/shared_ptr.hpp>
0022 #include <boost/mpi/datatype.hpp>
0023 #include <boost/mpi/nonblocking.hpp>
0024 #include <boost/static_assert.hpp>
0025 #include <utility>
0026 #include <iterator>
0027 #include <stdexcept> // for std::range_error
0028 #include <vector>
0029 
0030 // For (de-)serializing sends and receives
0031 #include <boost/mpi/packed_oarchive.hpp>
0032 #include <boost/mpi/packed_iarchive.hpp>
0033 
0034 // For (de-)serializing skeletons and content
0035 #include <boost/mpi/skeleton_and_content_fwd.hpp>
0036 
0037 #include <boost/mpi/detail/point_to_point.hpp>
0038 #include <boost/mpi/status.hpp>
0039 #include <boost/mpi/request.hpp>
0040 
0041 #ifdef BOOST_MSVC
0042 #  pragma warning(push)
0043 #  pragma warning(disable : 4800) // forcing to bool 'true' or 'false'
0044 #endif
0045 
0046 namespace boost { namespace mpi {
0047 
0048 /**
0049  * @brief A constant representing "any process."
0050  *
0051  * This constant may be used for the @c source parameter of @c receive
0052  * operations to indicate that a message may be received from any
0053  * source.
0054  */
0055 const int any_source = MPI_ANY_SOURCE;
0056 
0057 /**
0058  * @brief A constant representing "any tag."
0059  *
0060  * This constant may be used for the @c tag parameter of @c receive
0061  * operations to indicate that a @c send with any tag will be matched
0062  * by the receive.
0063  */
0064 const int any_tag = MPI_ANY_TAG;
0065 
0066 /**
0067  * @brief Enumeration used to describe how to adopt a C @c MPI_Comm into
0068  * a Boost.MPI communicator.
0069  *
0070  * The values for this enumeration determine how a Boost.MPI
0071  * communicator will behave when constructed with an MPI
0072  * communicator. The options are:
0073  *
0074  *   - @c comm_duplicate: Duplicate the MPI_Comm communicator to
0075  *   create a new communicator (e.g., with MPI_Comm_dup). This new
0076  *   MPI_Comm communicator will be automatically freed when the
0077  *   Boost.MPI communicator (and all copies of it) is destroyed.
0078  *
0079  *   - @c comm_take_ownership: Take ownership of the communicator. It
0080  *   will be freed automatically when all of the Boost.MPI
0081  *   communicators go out of scope. This option must not be used with
0082  *   MPI_COMM_WORLD.
0083  *
0084  *   - @c comm_attach: The Boost.MPI communicator will reference the
0085  *   existing MPI communicator but will not free it when the Boost.MPI
0086  *   communicator goes out of scope. This option should only be used
0087  *   when the communicator is managed by the user or MPI library
0088  *   (e.g., MPI_COMM_WORLD).
0089  */
0090 enum comm_create_kind { comm_duplicate, comm_take_ownership, comm_attach };
0091 
0092 /**
0093  * INTERNAL ONLY
0094  * 
0095  * Forward declaration of @c group needed for the @c group
0096  * constructor and accessor.
0097  */
0098 class group;
0099 
0100 /**
0101  * INTERNAL ONLY
0102  *
0103  * Forward declaration of @c intercommunicator needed for the "cast"
0104  * from a communicator to an intercommunicator.
0105  */
0106 class intercommunicator;
0107 
0108 /**
0109  * INTERNAL ONLY
0110  *
0111  * Forward declaration of @c graph_communicator needed for the "cast"
0112  * from a communicator to a graph communicator.
0113  */
0114 class graph_communicator;
0115 
0116 /**
0117  * INTERNAL ONLY
0118  *
0119  * Forward declaration of @c cartesian_communicator needed for the "cast"
0120  * from a communicator to a cartesian communicator.
0121  */
0122 class cartesian_communicator;
0123 
0124 /**
0125  * @brief A communicator that permits communication and
0126  * synchronization among a set of processes.
0127  *
0128  * The @c communicator class abstracts a set of communicating
0129  * processes in MPI. All of the processes that belong to a certain
0130  * communicator can determine the size of the communicator, their rank
0131  * within the communicator, and communicate with any other processes
0132  * in the communicator.
0133  */
0134 class BOOST_MPI_DECL communicator
0135 {
0136  public:
0137   /**
0138    * Build a new Boost.MPI communicator for @c MPI_COMM_WORLD.
0139    *
0140    * Constructs a Boost.MPI communicator that attaches to @c
0141    * MPI_COMM_WORLD. This is the equivalent of constructing with
0142    * @c (MPI_COMM_WORLD, comm_attach).
0143    */
0144   communicator();
0145 
0146   /**
0147    * Build a new Boost.MPI communicator based on the MPI communicator
0148    * @p comm.
0149    *
0150    * @p comm may be any valid MPI communicator. If @p comm is
0151    * MPI_COMM_NULL, an empty communicator (that cannot be used for
0152    * communication) is created and the @p kind parameter is
0153    * ignored. Otherwise, the @p kind parameters determines how the
0154    * Boost.MPI communicator will be related to @p comm:
0155    *
0156    *   - If @p kind is @c comm_duplicate, duplicate @c comm to create
0157    *   a new communicator. This new communicator will be freed when
0158    *   the Boost.MPI communicator (and all copies of it) is destroyed.
0159    *   This option is only permitted if @p comm is a valid MPI
0160    *   intracommunicator or if the underlying MPI implementation
0161    *   supports MPI 2.0 (which supports duplication of
0162    *   intercommunicators).
0163    *
0164    *   - If @p kind is @c comm_take_ownership, take ownership of @c
0165    *   comm. It will be freed automatically when all of the Boost.MPI
0166    *   communicators go out of scope. This option must not be used
0167    *   when @c comm is MPI_COMM_WORLD.
0168    *
0169    *   - If @p kind is @c comm_attach, this Boost.MPI communicator
0170    *   will reference the existing MPI communicator @p comm but will
0171    *   not free @p comm when the Boost.MPI communicator goes out of
0172    *   scope. This option should only be used when the communicator is
0173    *   managed by the user or MPI library (e.g., MPI_COMM_WORLD).
0174    */
0175   communicator(const MPI_Comm& comm, comm_create_kind kind);
0176 
0177   /**
0178    * Build a new Boost.MPI communicator based on a subgroup of another
0179    * MPI communicator.
0180    *
0181    * This routine will construct a new communicator containing all of
0182    * the processes from communicator @c comm that are listed within
0183    * the group @c subgroup. Equivalent to @c MPI_Comm_create.
0184    *
0185    * @param comm An MPI communicator.
0186    *
0187    * @param subgroup A subgroup of the MPI communicator, @p comm, for
0188    * which we will construct a new communicator.
0189    */
0190   communicator(const communicator& comm, const boost::mpi::group& subgroup);
0191 
0192   /**
0193    * @brief Determine the rank of the executing process in a
0194    * communicator.
0195    *
0196    * This routine is equivalent to @c MPI_Comm_rank.
0197    *
0198    *   @returns The rank of the process in the communicator, which
0199    *   will be a value in [0, size())
0200    */
0201   int rank() const;
0202 
0203   /**
0204    * @brief Determine the number of processes in a communicator.
0205    *
0206    * This routine is equivalent to @c MPI_Comm_size.
0207    *
0208    *   @returns The number of processes in the communicator.
0209    */
0210   int size() const;
0211 
0212   /**
0213    * This routine constructs a new group whose members are the
0214    * processes within this communicator. Equivalent to
0215    * calling @c MPI_Comm_group.
0216    */
0217   boost::mpi::group group() const;
0218 
0219   // ----------------------------------------------------------------
0220   // Point-to-point communication
0221   // ----------------------------------------------------------------
0222 
0223   /**
0224    *  @brief Send data to another process.
0225    *
0226    *  This routine executes a potentially blocking send with tag @p tag
0227    *  to the process with rank @p dest. It can be received by the
0228    *  destination process with a matching @c recv call.
0229    *
0230    *  The given @p value must be suitable for transmission over
0231    *  MPI. There are several classes of types that meet these
0232    *  requirements:
0233    *
0234    *    - Types with mappings to MPI data types: If @c
0235    *    is_mpi_datatype<T> is convertible to @c mpl::true_, then @p
0236    *    value will be transmitted using the MPI data type
0237    *    @c get_mpi_datatype<T>(). All primitive C++ data types that have
0238    *    MPI equivalents, e.g., @c int, @c float, @c char, @c double,
0239    *    etc., have built-in mappings to MPI data types. You may turn a
0240    *    Serializable type with fixed structure into an MPI data type by
0241    *    specializing @c is_mpi_datatype for your type.
0242    *
0243    *    - Serializable types: Any type that provides the @c serialize()
0244    *    functionality required by the Boost.Serialization library can be
0245    *    transmitted and received.
0246    *
0247    *    - Packed archives and skeletons: Data that has been packed into
0248    *    an @c mpi::packed_oarchive or the skeletons of data that have
0249    *    been backed into an @c mpi::packed_skeleton_oarchive can be
0250    *    transmitted, but will be received as @c mpi::packed_iarchive and
0251    *    @c mpi::packed_skeleton_iarchive, respectively, to allow the
0252    *    values (or skeletons) to be extracted by the destination process.
0253    *
0254    *    - Content: Content associated with a previously-transmitted
0255    *    skeleton can be transmitted by @c send and received by @c
0256    *    recv. The receiving process may only receive content into the
0257    *    content of a value that has been constructed with the matching
0258    *    skeleton.
0259    *
0260    *  For types that have mappings to an MPI data type (including the
0261    *  concent of a type), an invocation of this routine will result in
0262    *  a single MPI_Send call. For variable-length data, e.g.,
0263    *  serialized types and packed archives, two messages will be sent
0264    *  via MPI_Send: one containing the length of the data and the
0265    *  second containing the data itself.
0266    * 
0267    *  Std::vectors of MPI data type
0268    *  are considered variable size, e.g. their number of elements is 
0269    *  unknown and must be transmited (although the serialization process
0270    *  is skipped). You can use the array specialized versions of 
0271    *  communication methods is both sender and receiver know the vector 
0272    *  size.
0273    *  
0274    *  Note that the transmission mode for variable-length data is an 
0275    *  implementation detail that is subject to change.
0276    *
0277    *  @param dest The rank of the remote process to which the data
0278    *  will be sent.
0279    *
0280    *  @param tag The tag that will be associated with this message. Tags
0281    *  may be any integer between zero and an implementation-defined
0282    *  upper limit. This limit is accessible via @c environment::max_tag().
0283    *
0284    *  @param value The value that will be transmitted to the
0285    *  receiver. The type @c T of this value must meet the aforementioned
0286    *  criteria for transmission. 
0287    */
0288   template<typename T>
0289   void send(int dest, int tag, const T& value) const;
0290 
0291   template<typename T, typename A>
0292   void send(int dest, int tag, const std::vector<T,A>& value) const;
0293 
0294   /**
0295    *  @brief Send the skeleton of an object.
0296    *
0297    *  This routine executes a potentially blocking send with tag @p
0298    *  tag to the process with rank @p dest. It can be received by the
0299    *  destination process with a matching @c recv call. This variation
0300    *  on @c send will be used when a send of a skeleton is explicitly
0301    *  requested via code such as:
0302    *
0303    *  @code
0304    *    comm.send(dest, tag, skeleton(object));
0305    *  @endcode
0306    *
0307    *  The semantics of this routine are equivalent to that of sending
0308    *  a @c packed_skeleton_oarchive storing the skeleton of the @c
0309    *  object.
0310    *
0311    *  @param dest The rank of the remote process to which the skeleton
0312    *  will be sent.
0313    *
0314    *  @param tag The tag that will be associated with this message. Tags
0315    *  may be any integer between zero and an implementation-defined
0316    *  upper limit. This limit is accessible via @c environment::max_tag().
0317    *
0318    *  @param proxy The @c skeleton_proxy containing a reference to the
0319    *  object whose skeleton will be transmitted.
0320    *
0321    */
0322   template<typename T>
0323   void send(int dest, int tag, const skeleton_proxy<T>& proxy) const;
0324 
0325   /**
0326    *  @brief Send an array of values to another process.
0327    *
0328    *  This routine executes a potentially blocking send of an array of
0329    *  data with tag @p tag to the process with rank @p dest. It can be
0330    *  received by the destination process with a matching array @c
0331    *  recv call.
0332    *
0333    *  If @c T is an MPI datatype, an invocation of this routine will
0334    *  be mapped to a single call to MPI_Send, using the datatype @c
0335    *  get_mpi_datatype<T>().
0336    *
0337    *  @param dest The process rank of the remote process to which
0338    *  the data will be sent.
0339    *
0340    *  @param tag The tag that will be associated with this message. Tags
0341    *  may be any integer between zero and an implementation-defined
0342    *  upper limit. This limit is accessible via @c environment::max_tag().
0343    *
0344    *  @param values The array of values that will be transmitted to the
0345    *  receiver. The type @c T of these values must be mapped to an MPI
0346    *  data type.
0347    *
0348    *  @param n The number of values stored in the array. The destination
0349    *  process must call receive with at least this many elements to
0350    *  correctly receive the message.
0351    */
0352   template<typename T>
0353   void send(int dest, int tag, const T* values, int n) const;
0354 
0355   /**
0356    *  @brief Send a message to another process without any data.
0357    *
0358    *  This routine executes a potentially blocking send of a message
0359    *  to another process. The message contains no extra data, and can
0360    *  therefore only be received by a matching call to @c recv().
0361    *
0362    *  @param dest The process rank of the remote process to which
0363    *  the message will be sent.
0364    *
0365    *  @param tag The tag that will be associated with this message. Tags
0366    *  may be any integer between zero and an implementation-defined
0367    *  upper limit. This limit is accessible via @c environment::max_tag().
0368    *
0369    */
0370   void send(int dest, int tag) const;
0371 
0372   /**
0373    * @brief Receive data from a remote process.
0374    *
0375    * This routine blocks until it receives a message from the process @p
0376    * source with the given @p tag. The type @c T of the @p value must be
0377    * suitable for transmission over MPI, which includes serializable
0378    * types, types that can be mapped to MPI data types (including most
0379    * built-in C++ types), packed MPI archives, skeletons, and content
0380    * associated with skeletons; see the documentation of @c send for a
0381    * complete description.
0382    *
0383    *   @param source The process that will be sending data. This will
0384    *   either be a process rank within the communicator or the
0385    *   constant @c any_source, indicating that we can receive the
0386    *   message from any process.
0387    *
0388    *   @param tag The tag that matches a particular kind of message sent
0389    *   by the source process. This may be any tag value permitted by @c
0390    *   send. Alternatively, the argument may be the constant @c any_tag,
0391    *   indicating that this receive matches a message with any tag.
0392    *
0393    *   @param value Will contain the value of the message after a
0394    *   successful receive. The type of this value must match the value
0395    *   transmitted by the sender, unless the sender transmitted a packed
0396    *   archive or skeleton: in these cases, the sender transmits a @c
0397    *   packed_oarchive or @c packed_skeleton_oarchive and the
0398    *   destination receives a @c packed_iarchive or @c
0399    *   packed_skeleton_iarchive, respectively.
0400    *
0401    *   @returns Information about the received message.
0402    */
0403   template<typename T>
0404   status recv(int source, int tag, T& value) const;
0405 
0406   template<typename T, typename A>
0407   status recv(int source, int tag, std::vector<T,A>& value) const;
0408 
0409   /**
0410    *  @brief Receive a skeleton from a remote process.
0411    *
0412    *  This routine blocks until it receives a message from the process @p
0413    *  source with the given @p tag containing a skeleton.
0414    *
0415    *  @param source The process that will be sending data. This will
0416    *  either be a process rank within the communicator or the constant
0417    *  @c any_source, indicating that we can receive the message from
0418    *  any process.
0419    *
0420    *  @param tag The tag that matches a particular kind of message
0421    *  sent by the source process. This may be any tag value permitted
0422    *  by @c send. Alternatively, the argument may be the constant @c
0423    *  any_tag, indicating that this receive matches a message with any
0424    *  tag.
0425    *
0426    *  @param proxy The @c skeleton_proxy containing a reference to the
0427    *  object that will be reshaped to match the received skeleton.
0428    *
0429    *  @returns Information about the received message.
0430    */
0431   template<typename T>
0432   status recv(int source, int tag, const skeleton_proxy<T>& proxy) const;
0433 
0434   /**
0435    *  @brief Receive a skeleton from a remote process.
0436    *
0437    *  This routine blocks until it receives a message from the process @p
0438    *  source with the given @p tag containing a skeleton.
0439    *
0440    *  @param source The process that will be sending data. This will
0441    *  either be a process rank within the communicator or the constant
0442    *  @c any_source, indicating that we can receive the message from
0443    *  any process.
0444    *
0445    *  @param tag The tag that matches a particular kind of message
0446    *  sent by the source process. This may be any tag value permitted
0447    *  by @c send. Alternatively, the argument may be the constant @c
0448    *  any_tag, indicating that this receive matches a message with any
0449    *  tag.
0450    *
0451    *  @param proxy The @c skeleton_proxy containing a reference to the
0452    *  object that will be reshaped to match the received skeleton.
0453    *
0454    *  @returns Information about the received message.
0455    */
0456   template<typename T>
0457   status recv(int source, int tag, skeleton_proxy<T>& proxy) const;
0458 
0459   /**
0460    * @brief Receive an array of values from a remote process.
0461    *
0462    * This routine blocks until it receives an array of values from the
0463    * process @p source with the given @p tag. If the type @c T is 
0464    *
0465    *   @param source The process that will be sending data. This will
0466    *   either be a process rank within the communicator or the
0467    *   constant @c any_source, indicating that we can receive the
0468    *   message from any process.
0469    *
0470    *   @param tag The tag that matches a particular kind of message sent
0471    *   by the source process. This may be any tag value permitted by @c
0472    *   send. Alternatively, the argument may be the constant @c any_tag,
0473    *   indicating that this receive matches a message with any tag.
0474    *
0475    *   @param values Will contain the values in the message after a
0476    *   successful receive. The type of these elements must match the
0477    *   type of the elements transmitted by the sender.
0478    *
0479    *   @param n The number of values that can be stored into the @p
0480    *   values array. This shall not be smaller than the number of
0481    *   elements transmitted by the sender.
0482    *
0483    *   @throws std::range_error if the message to be received contains
0484    *   more than @p n values.
0485    *
0486    *   @returns Information about the received message.
0487    */
0488   template<typename T>
0489   status recv(int source, int tag, T* values, int n) const;
0490 
0491   /**
0492    *  @brief Receive a message from a remote process without any data.
0493    *
0494    *  This routine blocks until it receives a message from the process
0495    *  @p source with the given @p tag.
0496    *
0497    *  @param source The process that will be sending the message. This
0498    *  will either be a process rank within the communicator or the
0499    *  constant @c any_source, indicating that we can receive the
0500    *  message from any process.
0501    *
0502    *  @param tag The tag that matches a particular kind of message
0503    *  sent by the source process. This may be any tag value permitted
0504    *  by @c send. Alternatively, the argument may be the constant @c
0505    *  any_tag, indicating that this receive matches a message with any
0506    *  tag.
0507    *
0508    *  @returns Information about the received message.
0509    */
0510   status recv(int source, int tag) const;
0511 
0512   /** @brief Send a message to remote process and receive another message
0513    *  from another process.
0514    */
0515   template<typename T>
0516   status sendrecv(int dest, int stag, const T& sval, int src, int rtag, T& rval) const;
0517   
0518   /**
0519    *  @brief Send a message to a remote process without blocking.
0520    *
0521    *  The @c isend method is functionality identical to the @c send
0522    *  method and transmits data in the same way, except that @c isend
0523    *  will not block while waiting for the data to be
0524    *  transmitted. Instead, a request object will be immediately
0525    *  returned, allowing one to query the status of the communication
0526    *  or wait until it has completed.
0527    *
0528    *  @param dest The rank of the remote process to which the data
0529    *  will be sent.
0530    *
0531    *  @param tag The tag that will be associated with this message. Tags
0532    *  may be any integer between zero and an implementation-defined
0533    *  upper limit. This limit is accessible via @c environment::max_tag().
0534    *
0535    *  @param value The value that will be transmitted to the
0536    *  receiver. The type @c T of this value must meet the aforementioned
0537    *  criteria for transmission. If modified before transmited, the 
0538    *  modification may or may not be transmited.
0539    *
0540    *  @returns a @c request object that describes this communication.
0541    */
0542   template<typename T>
0543   request isend(int dest, int tag, const T& value) const;
0544 
0545   /**
0546    *  @brief Send the skeleton of an object without blocking.
0547    *
0548    *  This routine is functionally identical to the @c send method for
0549    *  @c skeleton_proxy objects except that @c isend will not block
0550    *  while waiting for the data to be transmitted. Instead, a request
0551    *  object will be immediately returned, allowing one to query the
0552    *  status of the communication or wait until it has completed.  
0553    *
0554    *  The semantics of this routine are equivalent to a non-blocking
0555    *  send of a @c packed_skeleton_oarchive storing the skeleton of
0556    *  the @c object.
0557    *
0558    *  @param dest The rank of the remote process to which the skeleton
0559    *  will be sent.
0560    *
0561    *  @param tag The tag that will be associated with this message. Tags
0562    *  may be any integer between zero and an implementation-defined
0563    *  upper limit. This limit is accessible via @c environment::max_tag().
0564    *
0565    *  @param proxy The @c skeleton_proxy containing a reference to the
0566    *  object whose skeleton will be transmitted.
0567    *
0568    *  @returns a @c request object that describes this communication.
0569    */
0570   template<typename T>
0571   request isend(int dest, int tag, const skeleton_proxy<T>& proxy) const;
0572 
0573   /**
0574    *  @brief Send an array of values to another process without
0575    *  blocking.
0576    *
0577    *  This routine is functionally identical to the @c send method for
0578    *  arrays except that @c isend will not block while waiting for the
0579    *  data to be transmitted. Instead, a request object will be
0580    *  immediately returned, allowing one to query the status of the
0581    *  communication or wait until it has completed.
0582    *
0583    *  @param dest The process rank of the remote process to which
0584    *  the data will be sent.
0585    *
0586    *  @param tag The tag that will be associated with this message. Tags
0587    *  may be any integer between zero and an implementation-defined
0588    *  upper limit. This limit is accessible via @c environment::max_tag().
0589    *
0590    *  @param values The array of values that will be transmitted to the
0591    *  receiver. The type @c T of these values must be mapped to an MPI
0592    *  data type.
0593    *
0594    *  @param n The number of values stored in the array. The destination
0595    *  process must call receive with at least this many elements to
0596    *  correctly receive the message.
0597    *
0598    *  @returns a @c request object that describes this communication.
0599    */
0600   template<typename T>
0601   request isend(int dest, int tag, const T* values, int n) const;
0602 
0603   template<typename T, class A>
0604   request isend(int dest, int tag, const std::vector<T,A>& values) const;
0605 
0606   /**
0607    *  @brief Send a message to another process without any data
0608    *  without blocking.
0609    *
0610    *  This routine is functionally identical to the @c send method for
0611    *  sends with no data, except that @c isend will not block while
0612    *  waiting for the message to be transmitted. Instead, a request
0613    *  object will be immediately returned, allowing one to query the
0614    *  status of the communication or wait until it has completed.
0615    *
0616    *  @param dest The process rank of the remote process to which
0617    *  the message will be sent.
0618    *
0619    *  @param tag The tag that will be associated with this message. Tags
0620    *  may be any integer between zero and an implementation-defined
0621    *  upper limit. This limit is accessible via @c environment::max_tag().
0622    *
0623    *
0624    *  @returns a @c request object that describes this communication.
0625    */
0626   request isend(int dest, int tag) const;
0627 
0628   /**
0629    *  @brief Prepare to receive a message from a remote process.
0630    *
0631    *  The @c irecv method is functionally identical to the @c recv
0632    *  method and receive data in the same way, except that @c irecv
0633    *  will not block while waiting for data to be
0634    *  transmitted. Instead, it immediately returns a request object
0635    *  that allows one to query the status of the receive or wait until
0636    *  it has completed.
0637    *
0638    *   @param source The process that will be sending data. This will
0639    *   either be a process rank within the communicator or the
0640    *   constant @c any_source, indicating that we can receive the
0641    *   message from any process.
0642    *
0643    *   @param tag The tag that matches a particular kind of message sent
0644    *   by the source process. This may be any tag value permitted by @c
0645    *   send. Alternatively, the argument may be the constant @c any_tag,
0646    *   indicating that this receive matches a message with any tag.
0647    *
0648    *   @param value Will contain the value of the message after a
0649    *   successful receive. The type of this value must match the value
0650    *   transmitted by the sender, unless the sender transmitted a packed
0651    *   archive or skeleton: in these cases, the sender transmits a @c
0652    *   packed_oarchive or @c packed_skeleton_oarchive and the
0653    *   destination receives a @c packed_iarchive or @c
0654    *   packed_skeleton_iarchive, respectively.
0655    *
0656    *   @returns a @c request object that describes this communication.
0657    */
0658   template<typename T>
0659   request irecv(int source, int tag, T& value) const;
0660 
0661   /**
0662    * @brief Initiate receipt of an array of values from a remote process.
0663    *
0664    * This routine initiates a receive operation for an array of values
0665    * transmitted by process @p source with the given @p tag. 
0666    *
0667    *    @param source The process that will be sending data. This will
0668    *    either be a process rank within the communicator or the
0669    *    constant @c any_source, indicating that we can receive the
0670    *    message from any process.
0671    *
0672    *    @param tag The tag that matches a particular kind of message sent
0673    *    by the source process. This may be any tag value permitted by @c
0674    *    send. Alternatively, the argument may be the constant @c any_tag,
0675    *    indicating that this receive matches a message with any tag.
0676    *
0677    *    @param values Will contain the values in the message after a
0678    *    successful receive. The type of these elements must match the
0679    *    type of the elements transmitted by the sender.
0680    *
0681    *    @param n The number of values that can be stored into the @p
0682    *    values array. This shall not be smaller than the number of
0683    *    elements transmitted by the sender.
0684    *
0685    *    @returns a @c request object that describes this communication.
0686    */
0687   template<typename T>
0688   request irecv(int source, int tag, T* values, int n) const;
0689 
0690   template<typename T, typename A>
0691   request irecv(int source, int tag, std::vector<T,A>& values) const;
0692 
0693   /**
0694    *  @brief Initiate receipt of a message from a remote process that
0695    *  carries no data.
0696    *
0697    *  This routine initiates a receive operation for a message from
0698    *  process @p source with the given @p tag that carries no data.
0699    *
0700    *    @param source The process that will be sending the message. This
0701    *    will either be a process rank within the communicator or the
0702    *    constant @c any_source, indicating that we can receive the
0703    *    message from any process.
0704    *
0705    *    @param tag The tag that matches a particular kind of message
0706    *    sent by the source process. This may be any tag value permitted
0707    *    by @c send. Alternatively, the argument may be the constant @c
0708    *    any_tag, indicating that this receive matches a message with any
0709    *    tag.
0710    *
0711    *    @returns a @c request object that describes this communication.
0712    */
0713   request irecv(int source, int tag) const;
0714 
0715   /**
0716    * @brief Waits until a message is available to be received.
0717    *
0718    * This operation waits until a message matching (@p source, @p tag)
0719    * is available to be received. It then returns information about
0720    * that message. The functionality is equivalent to @c MPI_Probe. To
0721    * check if a message is available without blocking, use @c iprobe.
0722    *
0723    *   @param source Determine if there is a message available from
0724    *   this rank. If @c any_source, then the message returned may come
0725    *   from any source.
0726    *
0727    *   @param tag Determine if there is a message available with the
0728    *   given tag. If @c any_tag, then the message returned may have any
0729    *   tag.
0730    *
0731    *   @returns Returns information about the first message that
0732    *   matches the given criteria.
0733    */
0734   status probe(int source = any_source, int tag = any_tag) const;
0735 
0736   /**
0737    * @brief Determine if a message is available to be received.
0738    *
0739    * This operation determines if a message matching (@p source, @p
0740    * tag) is available to be received. If so, it returns information
0741    * about that message; otherwise, it returns immediately with an
0742    * empty optional. The functionality is equivalent to @c
0743    * MPI_Iprobe. To wait until a message is available, use @c wait.
0744    *
0745    *   @param source Determine if there is a message available from
0746    *   this rank. If @c any_source, then the message returned may come
0747    *   from any source.
0748    *
0749    *   @param tag Determine if there is a message available with the
0750    *   given tag. If @c any_tag, then the message returned may have any
0751    *   tag.
0752    *
0753    *   @returns If a matching message is available, returns
0754    *   information about that message. Otherwise, returns an empty
0755    *   @c boost::optional.
0756    */
0757   optional<status>
0758   iprobe(int source = any_source, int tag = any_tag) const;
0759 
0760 #ifdef barrier
0761   // Linux defines a function-like macro named "barrier". So, we need
0762   // to avoid expanding the macro when we define our barrier()
0763   // function. However, some C++ parsers (Doxygen, for instance) can't
0764   // handle this syntax, so we only use it when necessary.
0765   void (barrier)() const;
0766 #else
0767   /**
0768    * @brief Wait for all processes within a communicator to reach the
0769    * barrier.
0770    *
0771    * This routine is a collective operation that blocks each process
0772    * until all processes have entered it, then releases all of the
0773    * processes "simultaneously". It is equivalent to @c MPI_Barrier.
0774    */
0775   void barrier() const;
0776 #endif
0777 
0778   /** @brief Determine if this communicator is valid for
0779    * communication.
0780    *
0781    * Evaluates @c true in a boolean context if this communicator is
0782    * valid for communication, i.e., does not represent
0783    * MPI_COMM_NULL. Otherwise, evaluates @c false.
0784    */
0785   operator bool() const { return (bool)comm_ptr; }
0786 
0787   /**
0788    * @brief Access the MPI communicator associated with a Boost.MPI
0789    * communicator.
0790    *
0791    * This routine permits the implicit conversion from a Boost.MPI
0792    * communicator to an MPI communicator.
0793    *
0794    *   @returns The associated MPI communicator.
0795    */
0796   operator MPI_Comm() const;
0797 
0798   /**
0799    * Split the communicator into multiple, disjoint communicators
0800    * each of which is based on a particular color. This is a
0801    * collective operation that returns a new communicator that is a
0802    * subgroup of @p this.
0803    *
0804    *   @param color The color of this process. All processes with the
0805    *   same @p color value will be placed into the same group.
0806    *
0807    *   @param key A key value that will be used to determine the
0808    *   ordering of processes with the same color in the resulting
0809    *   communicator. If omitted, the rank of the processes in @p this
0810    *   will determine the ordering of processes in the resulting
0811    *   group.
0812    *
0813    *   @returns A new communicator containing all of the processes in
0814    *   @p this that have the same @p color.
0815    */
0816   communicator split(int color, int key) const;
0817   communicator split(int color) const;
0818 
0819   /**
0820    * Determine if the communicator is in fact an intercommunicator
0821    * and, if so, return that intercommunicator.
0822    *
0823    * @returns an @c optional containing the intercommunicator, if this
0824    * communicator is in fact an intercommunicator. Otherwise, returns
0825    * an empty @c optional.
0826    */
0827   optional<intercommunicator> as_intercommunicator() const;
0828 
0829   /**
0830    * Determine if the communicator has a graph topology and, if so,
0831    * return that @c graph_communicator. Even though the communicators
0832    * have different types, they refer to the same underlying
0833    * communication space and can be used interchangeably for
0834    * communication.
0835    *
0836    * @returns an @c optional containing the graph communicator, if this
0837    * communicator does in fact have a graph topology. Otherwise, returns
0838    * an empty @c optional.
0839    */
0840   optional<graph_communicator> as_graph_communicator() const;
0841 
0842   /**
0843    * Determines whether this communicator has a Graph topology.
0844    */
0845   bool has_graph_topology() const;
0846 
0847   /**
0848    * Determine if the communicator has a cartesian topology and, if so,
0849    * return that @c cartesian_communicator. Even though the communicators
0850    * have different types, they refer to the same underlying
0851    * communication space and can be used interchangeably for
0852    * communication.
0853    *
0854    * @returns an @c optional containing the cartesian communicator, if this
0855    * communicator does in fact have a cartesian topology. Otherwise, returns
0856    * an empty @c optional.
0857    */
0858   optional<cartesian_communicator> as_cartesian_communicator() const;
0859 
0860   /**
0861    * Determines whether this communicator has a Cartesian topology.
0862    */
0863   bool has_cartesian_topology() const;
0864 
0865   /** Abort all tasks in the group of this communicator.
0866    *
0867    *  Makes a "best attempt" to abort all of the tasks in the group of
0868    *  this communicator. Depending on the underlying MPI
0869    *  implementation, this may either abort the entire program (and
0870    *  possibly return @p errcode to the environment) or only abort
0871    *  some processes, allowing the others to continue. Consult the
0872    *  documentation for your MPI implementation. This is equivalent to
0873    *  a call to @c MPI_Abort
0874    *
0875    *  @param errcode The error code to return from aborted processes.
0876    *  @returns Will not return.
0877    */
0878   void abort(int errcode) const;
0879 
0880  protected:
0881   
0882   /**
0883    * INTERNAL ONLY
0884    *
0885    * Implementation of sendrecv for mpi type.
0886    */
0887   template<typename T>
0888   status sendrecv_impl(int dest, int stag, const T& sval, int src, int rtag, T& rval,
0889                        mpl::true_) const;
0890 
0891   /**
0892    * INTERNAL ONLY
0893    *
0894    * Implementation of sendrecv for complex types, which must be passed as archives.
0895    */
0896   template<typename T>
0897   status sendrecv_impl(int dest, int stag, const T& sval, int src, int rtag, T& rval,
0898                        mpl::false_) const;
0899 
0900   /**
0901    * INTERNAL ONLY
0902    *
0903    * Function object that frees an MPI communicator and deletes the
0904    * memory associated with it. Intended to be used as a deleter with
0905    * shared_ptr.
0906    */
0907   struct comm_free
0908   {
0909     void operator()(MPI_Comm* comm) const
0910     {
0911       BOOST_ASSERT( comm != 0 );
0912       BOOST_ASSERT(*comm != MPI_COMM_NULL);
0913       int finalized;
0914       BOOST_MPI_CHECK_RESULT(MPI_Finalized, (&finalized));
0915       if (!finalized)
0916         BOOST_MPI_CHECK_RESULT(MPI_Comm_free, (comm));
0917       delete comm;
0918     }
0919   };
0920 
0921   
0922   /**
0923    * INTERNAL ONLY
0924    *
0925    * We're sending a type that has an associated MPI datatype, so we
0926    * map directly to that datatype.
0927    */
0928   template<typename T>
0929   void send_impl(int dest, int tag, const T& value, mpl::true_) const;
0930 
0931   /**
0932    * INTERNAL ONLY
0933    *
0934    * We're sending a type that does not have an associated MPI
0935    * datatype, so it must be serialized then sent as MPI_PACKED data,
0936    * to be deserialized on the receiver side.
0937    */
0938   template<typename T>
0939   void send_impl(int dest, int tag, const T& value, mpl::false_) const;
0940 
0941   /**
0942    * INTERNAL ONLY
0943    *
0944    * We're sending an array of a type that has an associated MPI
0945    * datatype, so we map directly to that datatype.
0946    */
0947   template<typename T>
0948   void 
0949   array_send_impl(int dest, int tag, const T* values, int n, mpl::true_) const;
0950 
0951   /**
0952    * INTERNAL ONLY
0953    *
0954    * We're sending an array of a type that does not have an associated
0955    * MPI datatype, so it must be serialized then sent as MPI_PACKED
0956    * data, to be deserialized on the receiver side.
0957    */
0958   template<typename T>
0959   void 
0960   array_send_impl(int dest, int tag, const T* values, int n, 
0961                   mpl::false_) const;
0962 
0963   /**
0964    * INTERNAL ONLY
0965    *
0966    * We're sending a type that has an associated MPI datatype, so we
0967    * map directly to that datatype.
0968    */
0969   template<typename T>
0970   request isend_impl(int dest, int tag, const T& value, mpl::true_) const;
0971 
0972   /**
0973    * INTERNAL ONLY
0974    *
0975    * We're sending a type that does not have an associated MPI
0976    * datatype, so it must be serialized then sent as MPI_PACKED data,
0977    * to be deserialized on the receiver side.
0978    */
0979   template<typename T>
0980   request isend_impl(int dest, int tag, const T& value, mpl::false_) const;
0981 
0982   /**
0983    * INTERNAL ONLY
0984    *
0985    * We're sending an array of a type that has an associated MPI
0986    * datatype, so we map directly to that datatype.
0987    */
0988   template<typename T>
0989   request 
0990   array_isend_impl(int dest, int tag, const T* values, int n, 
0991                    mpl::true_) const;
0992 
0993   /**
0994    * INTERNAL ONLY
0995    *
0996    * We're sending an array of a type that does not have an associated
0997    * MPI datatype, so it must be serialized then sent as MPI_PACKED
0998    * data, to be deserialized on the receiver side.
0999    */
1000   template<typename T>
1001   request 
1002   array_isend_impl(int dest, int tag, const T* values, int n, 
1003                    mpl::false_) const;
1004 
1005   /**
1006    * INTERNAL ONLY
1007    *
1008    * We're receiving a type that has an associated MPI datatype, so we
1009    * map directly to that datatype.
1010    */
1011   template<typename T>
1012   status recv_impl(int source, int tag, T& value, mpl::true_) const;
1013 
1014   /**
1015    * INTERNAL ONLY
1016    *
1017    * We're receiving a type that does not have an associated MPI
1018    * datatype, so it must have been serialized then sent as
1019    * MPI_PACKED. We'll receive it and then deserialize.
1020    */
1021   template<typename T>
1022   status recv_impl(int source, int tag, T& value, mpl::false_) const;
1023 
1024   /**
1025    * INTERNAL ONLY
1026    *
1027    * We're receiving an array of a type that has an associated MPI
1028    * datatype, so we map directly to that datatype.
1029    */
1030   template<typename T>
1031   status 
1032   array_recv_impl(int source, int tag, T* values, int n, mpl::true_) const;
1033 
1034   /**
1035    * INTERNAL ONLY
1036    *
1037    * We're receiving a type that does not have an associated MPI
1038    * datatype, so it must have been serialized then sent as
1039    * MPI_PACKED. We'll receive it and then deserialize.
1040    */
1041   template<typename T>
1042   status 
1043   array_recv_impl(int source, int tag, T* values, int n, mpl::false_) const;
1044 
1045   /**
1046    * INTERNAL ONLY
1047    *
1048    * We're receiving a type that has an associated MPI datatype, so we
1049    * map directly to that datatype.
1050    */
1051   template<typename T>
1052   request irecv_impl(int source, int tag, T& value, mpl::true_) const;
1053 
1054   /**
1055    * INTERNAL ONLY
1056    *
1057    * We're receiving a type that does not have an associated MPI
1058    * datatype, so it must have been serialized then sent as
1059    * MPI_PACKED. We'll receive it and then deserialize.
1060    */
1061   template<typename T>
1062   request irecv_impl(int source, int tag, T& value, mpl::false_) const;
1063 
1064   /**
1065    * INTERNAL ONLY
1066    *
1067    * We're receiving a type that has an associated MPI datatype, so we
1068    * map directly to that datatype.
1069    */
1070   template<typename T>
1071   request 
1072   array_irecv_impl(int source, int tag, T* values, int n, mpl::true_) const;
1073 
1074   /**
1075    * INTERNAL ONLY
1076    *
1077    * We're receiving a type that does not have an associated MPI
1078    * datatype, so it must have been serialized then sent as
1079    * MPI_PACKED. We'll receive it and then deserialize.
1080    */
1081   template<typename T>
1082   request 
1083   array_irecv_impl(int source, int tag, T* values, int n, mpl::false_) const;
1084 
1085   // We're sending/receivig a vector with associated MPI datatype.
1086   // We need to send/recv the size and then the data and make sure 
1087   // blocking and non blocking method agrees on the format.
1088   template<typename T, typename A>
1089   request irecv_vector(int source, int tag, std::vector<T,A>& values, 
1090                        mpl::true_) const;
1091   template<typename T, class A>
1092   request isend_vector(int dest, int tag, const std::vector<T,A>& values,
1093                        mpl::true_) const;
1094   template<typename T, typename A>
1095   void send_vector(int dest, int tag, const std::vector<T,A>& value, 
1096            mpl::true_) const;
1097   template<typename T, typename A>
1098   status recv_vector(int source, int tag, std::vector<T,A>& value,
1099              mpl::true_) const;
1100   
1101   // We're sending/receivig a vector with no associated MPI datatype.
1102   // We need to send/recv it as an archive and make sure 
1103   // blocking and non blocking method agrees on the format.
1104   template<typename T, typename A>
1105   request irecv_vector(int source, int tag, std::vector<T,A>& values, 
1106                        mpl::false_) const;
1107   template<typename T, class A>
1108   request isend_vector(int dest, int tag, const std::vector<T,A>& values,
1109                        mpl::false_) const;
1110   template<typename T, typename A>
1111   void send_vector(int dest, int tag, const std::vector<T,A>& value, 
1112            mpl::false_) const;
1113   template<typename T, typename A>
1114   status recv_vector(int source, int tag, std::vector<T,A>& value,
1115              mpl::false_) const;
1116 
1117  protected:
1118   shared_ptr<MPI_Comm> comm_ptr;
1119 };
1120 
1121 /**
1122  * @brief Determines whether two communicators are identical.
1123  *
1124  * Equivalent to calling @c MPI_Comm_compare and checking whether the
1125  * result is @c MPI_IDENT.
1126  *
1127  * @returns True when the two communicators refer to the same
1128  * underlying MPI communicator.
1129  */
1130 BOOST_MPI_DECL bool operator==(const communicator& comm1, const communicator& comm2);
1131 
1132 /**
1133  * @brief Determines whether two communicators are different.
1134  *
1135  * @returns @c !(comm1 == comm2)
1136  */
1137 inline bool operator!=(const communicator& comm1, const communicator& comm2)
1138 {
1139   return !(comm1 == comm2);
1140 }
1141 
1142 }} // boost::mpi
1143 
1144 /************************************************************************
1145  * Implementation details                                               *
1146  ************************************************************************/
1147 
1148 #include <boost/mpi/detail/request_handlers.hpp>
1149 
1150 namespace boost { namespace mpi {
1151 /**
1152  * INTERNAL ONLY (using the same 'end' name might be considerd unfortunate
1153  */
1154 template<>
1155 BOOST_MPI_DECL void
1156 communicator::send<packed_oarchive>(int dest, int tag,
1157                                     const packed_oarchive& ar) const;
1158 
1159 /**
1160  * INTERNAL ONLY
1161  */
1162 template<>
1163 BOOST_MPI_DECL void
1164 communicator::send<packed_skeleton_oarchive>
1165   (int dest, int tag, const packed_skeleton_oarchive& ar) const;
1166 
1167 /**
1168  * INTERNAL ONLY
1169  */
1170 template<>
1171 BOOST_MPI_DECL void
1172 communicator::send<content>(int dest, int tag, const content& c) const;
1173 
1174 /**
1175  * INTERNAL ONLY
1176  */
1177 template<>
1178 BOOST_MPI_DECL status
1179 communicator::recv<packed_iarchive>(int source, int tag,
1180                                     packed_iarchive& ar) const;
1181 
1182 /**
1183  * INTERNAL ONLY
1184  */
1185 template<>
1186 BOOST_MPI_DECL status
1187 communicator::recv<packed_skeleton_iarchive>
1188   (int source, int tag, packed_skeleton_iarchive& ar) const;
1189 
1190 /**
1191  * INTERNAL ONLY
1192  */
1193 template<>
1194 BOOST_MPI_DECL status
1195 communicator::recv<const content>(int source, int tag,
1196                                   const content& c) const;
1197 
1198 /**
1199  * INTERNAL ONLY
1200  */
1201 template<>
1202 inline status
1203 communicator::recv<content>(int source, int tag,
1204                                   content& c) const
1205 {
1206   return recv<const content>(source,tag,c);
1207 }                                  
1208 
1209 /**
1210  * INTERNAL ONLY
1211  */
1212 template<>
1213 BOOST_MPI_DECL request
1214 communicator::isend<packed_oarchive>(int dest, int tag,
1215                                      const packed_oarchive& ar) const;
1216 
1217 /**
1218  * INTERNAL ONLY
1219  */
1220 template<>
1221 BOOST_MPI_DECL request
1222 communicator::isend<packed_skeleton_oarchive>
1223   (int dest, int tag, const packed_skeleton_oarchive& ar) const;
1224 
1225 /**
1226  * INTERNAL ONLY
1227  */
1228 template<>
1229 BOOST_MPI_DECL request
1230 communicator::isend<content>(int dest, int tag, const content& c) const;
1231 
1232 /**
1233  * INTERNAL ONLY
1234  */
1235 template<>
1236 BOOST_MPI_DECL request
1237 communicator::irecv<packed_skeleton_iarchive>
1238   (int source, int tag, packed_skeleton_iarchive& ar) const;
1239 
1240 /**
1241  * INTERNAL ONLY
1242  */
1243 template<>
1244 BOOST_MPI_DECL request
1245 communicator::irecv<const content>(int source, int tag,
1246                                    const content& c) const;
1247 
1248 /**
1249  * INTERNAL ONLY
1250  */
1251 template<>
1252 inline request
1253 communicator::irecv<content>(int source, int tag,
1254                              content& c) const
1255 {
1256   return irecv<const content>(source, tag, c);
1257 }
1258 
1259 // We're sending a type that has an associated MPI datatype, so we
1260 // map directly to that datatype.
1261 template<typename T>
1262 void
1263 communicator::send_impl(int dest, int tag, const T& value, mpl::true_) const
1264 {
1265   // received by recv or trivial handler.
1266   BOOST_MPI_CHECK_RESULT(MPI_Send,
1267                          (const_cast<T*>(&value), 1, get_mpi_datatype<T>(value),
1268                           dest, tag, MPI_Comm(*this)));
1269 }
1270 
1271 // We're sending a type that does not have an associated MPI
1272 // datatype, so it must be serialized then sent as MPI_PACKED data,
1273 // to be deserialized on the receiver side.
1274 template<typename T>
1275 void
1276 communicator::send_impl(int dest, int tag, const T& value, mpl::false_) const
1277 {
1278   packed_oarchive oa(*this);
1279   oa << value;
1280   send(dest, tag, oa);
1281 }
1282 
1283 // Single-element receive may either send the element directly or
1284 // serialize it via a buffer.
1285 template<typename T>
1286 void communicator::send(int dest, int tag, const T& value) const
1287 {
1288   this->send_impl(dest, tag, value, is_mpi_datatype<T>());
1289 }
1290 
1291 // We're sending an array of a type that has an associated MPI
1292 // datatype, so we map directly to that datatype.
1293 template<typename T>
1294 void
1295 communicator::array_send_impl(int dest, int tag, const T* values, int n,
1296                               mpl::true_) const
1297 {
1298   BOOST_MPI_CHECK_RESULT(MPI_Send,
1299                          (const_cast<T*>(values), n, 
1300                           get_mpi_datatype<T>(*values),
1301                           dest, tag, MPI_Comm(*this)));
1302 }
1303 
1304 // We're sending an array of a type that does not have an associated
1305 // MPI datatype, so it must be serialized then sent as MPI_PACKED
1306 // data, to be deserialized on the receiver side.
1307 template<typename T>
1308 void
1309 communicator::array_send_impl(int dest, int tag, const T* values, int n,
1310                               mpl::false_) const
1311 {
1312   packed_oarchive oa(*this);
1313   T const* v = values;
1314   while (v < values+n) {
1315     oa << *v++;
1316   }
1317   send(dest, tag, oa);
1318 }
1319 
1320 template<typename T, typename A>
1321 void communicator::send_vector(int dest, int tag, 
1322   const std::vector<T,A>& values, mpl::true_ primitive) const
1323 {
1324 #if defined(BOOST_MPI_USE_IMPROBE)
1325   array_send_impl(dest, tag, values.data(), values.size(), primitive);
1326 #else
1327   {
1328     // non blocking recv by legacy_dynamic_primitive_array_handler
1329     // blocking recv by recv_vector(source,tag,value,primitive)
1330     // send the vector size
1331     typename std::vector<T,A>::size_type size = values.size();
1332     send(dest, tag, size);
1333     // send the data
1334     this->array_send_impl(dest, tag, values.data(), size, primitive);
1335   }
1336 #endif
1337 }
1338 
1339 template<typename T, typename A>
1340 void communicator::send_vector(int dest, int tag, 
1341   const std::vector<T,A>& value, mpl::false_ primitive) const
1342 {
1343   this->send_impl(dest, tag, value, primitive);
1344 }
1345 
1346 template<typename T, typename A>
1347 void communicator::send(int dest, int tag, const std::vector<T,A>& value) const
1348 {
1349   send_vector(dest, tag, value, is_mpi_datatype<T>());
1350 }
1351 
1352 // Array send must send the elements directly
1353 template<typename T>
1354 void communicator::send(int dest, int tag, const T* values, int n) const
1355 {
1356   this->array_send_impl(dest, tag, values, n, is_mpi_datatype<T>());
1357 }
1358 
1359 // We're receiving a type that has an associated MPI datatype, so we
1360 // map directly to that datatype.
1361 template<typename T>
1362 status communicator::recv_impl(int source, int tag, T& value, mpl::true_) const
1363 {
1364   status stat;
1365   BOOST_MPI_CHECK_RESULT(MPI_Recv,
1366                          (const_cast<T*>(&value), 1, 
1367                           get_mpi_datatype<T>(value),
1368                           source, tag, MPI_Comm(*this), &stat.m_status));
1369   return stat;
1370 }
1371 
1372 template<typename T>
1373 status
1374 communicator::recv_impl(int source, int tag, T& value, mpl::false_) const
1375 {
1376   // Receive the message
1377   packed_iarchive ia(*this);
1378   status stat = recv(source, tag, ia);
1379 
1380   // Deserialize the data in the message
1381   ia >> value;
1382 
1383   return stat;
1384 }
1385 
1386 // Single-element receive may either receive the element directly or
1387 // deserialize it from a buffer.
1388 template<typename T>
1389 status communicator::recv(int source, int tag, T& value) const
1390 {
1391   return this->recv_impl(source, tag, value, is_mpi_datatype<T>());
1392 }
1393 
1394 template<typename T>
1395 status 
1396 communicator::array_recv_impl(int source, int tag, T* values, int n, 
1397                               mpl::true_) const
1398 {
1399   status stat;
1400   BOOST_MPI_CHECK_RESULT(MPI_Recv,
1401                          (const_cast<T*>(values), n, 
1402                           get_mpi_datatype<T>(*values),
1403                           source, tag, MPI_Comm(*this), &stat.m_status));
1404   return stat;
1405 }
1406 
1407 template<typename T>
1408 status
1409 communicator::array_recv_impl(int source, int tag, T* values, int n, 
1410                               mpl::false_) const
1411 {
1412   packed_iarchive ia(*this);
1413   status stat = recv(source, tag, ia);
1414   T* v = values;
1415   while (v != values+n) {
1416     ia >> *v++;
1417   }
1418   stat.m_count = n;
1419   return stat;
1420 }
1421 
1422 template<typename T, typename A>
1423 status communicator::recv_vector(int source, int tag, 
1424                                  std::vector<T,A>& values, mpl::true_ primitive) const
1425 {
1426 #if defined(BOOST_MPI_USE_IMPROBE)
1427   {
1428     MPI_Message msg;
1429     status stat;
1430     BOOST_MPI_CHECK_RESULT(MPI_Mprobe, (source,tag,*this,&msg,&stat.m_status));
1431     int count;
1432     BOOST_MPI_CHECK_RESULT(MPI_Get_count, (&stat.m_status,get_mpi_datatype<T>(),&count));
1433     values.resize(count);
1434     BOOST_MPI_CHECK_RESULT(MPI_Mrecv, (values.data(), count, get_mpi_datatype<T>(), &msg, &stat.m_status));
1435     return stat;
1436   }
1437 #else
1438   {
1439     // receive the vector size
1440     typename std::vector<T,A>::size_type size = 0;
1441     recv(source, tag, size);
1442     // size the vector
1443     values.resize(size);
1444     // receive the data
1445     return this->array_recv_impl(source, tag, values.data(), size, primitive);
1446   }
1447 #endif
1448 }
1449 
1450 template<typename T, typename A>
1451 status communicator::recv_vector(int source, int tag, 
1452   std::vector<T,A>& value, mpl::false_ false_type) const
1453 {
1454   return this->recv_impl(source, tag, value, false_type);
1455 }
1456 
1457 template<typename T, typename A>
1458 status communicator::recv(int source, int tag, std::vector<T,A>& value) const
1459 {
1460   return recv_vector(source, tag, value, is_mpi_datatype<T>());
1461 }
1462 
1463 // Array receive must receive the elements directly into a buffer.
1464 template<typename T>
1465 status communicator::recv(int source, int tag, T* values, int n) const
1466 {
1467   return this->array_recv_impl(source, tag, values, n, is_mpi_datatype<T>());
1468 }
1469 
1470  
1471 template<typename T>
1472 status communicator::sendrecv_impl(int dest, int stag, const T& sval, int src, int rtag, T& rval,
1473                                     mpl::true_) const
1474 {
1475   status stat;
1476   BOOST_MPI_CHECK_RESULT(MPI_Sendrecv,
1477                          (const_cast<T*>(&sval), 1, 
1478                           get_mpi_datatype<T>(sval),
1479                           dest, stag, 
1480                           &rval, 1,
1481                           get_mpi_datatype<T>(rval),
1482                           src, rtag,
1483                           MPI_Comm(*this), &stat.m_status));
1484   return stat;
1485 }
1486 
1487 template<typename T>
1488 status communicator::sendrecv_impl(int dest, int stag, const T& sval, int src, int rtag, T& rval,
1489                                    mpl::false_) const
1490 {
1491   int const SEND = 0;
1492   int const RECV = 1;
1493   request srrequests[2];
1494   srrequests[SEND] = this->isend_impl(dest, stag, sval, mpl::false_());
1495   srrequests[RECV] = this->irecv_impl(src,  rtag, rval, mpl::false_());
1496   status srstatuses[2];
1497   wait_all(srrequests, srrequests + 2, srstatuses);
1498   return srstatuses[RECV];
1499 }
1500 
1501 template<typename T>
1502 status communicator::sendrecv(int dest, int stag, const T& sval, int src, int rtag, T& rval) const
1503 {
1504   return this->sendrecv_impl(dest, stag, sval, src, rtag, rval, is_mpi_datatype<T>());
1505 }
1506 
1507 
1508 // We're sending a type that has an associated MPI datatype, so we
1509 // map directly to that datatype.
1510 template<typename T>
1511 request
1512 communicator::isend_impl(int dest, int tag, const T& value, mpl::true_) const
1513 {
1514   return request::make_trivial_send(*this, dest, tag, value);
1515 }
1516 
1517 // We're sending a type that does not have an associated MPI
1518 // datatype, so it must be serialized then sent as MPI_PACKED data,
1519 // to be deserialized on the receiver side.
1520 template<typename T>
1521 request
1522 communicator::isend_impl(int dest, int tag, const T& value, mpl::false_) const
1523 {
1524   shared_ptr<packed_oarchive> archive(new packed_oarchive(*this));
1525   *archive << value;
1526   request result = isend(dest, tag, *archive);
1527   result.preserve(archive);
1528   return result;
1529 }
1530 
1531 // Single-element receive may either send the element directly or
1532 // serialize it via a buffer.
1533 template<typename T>
1534 request communicator::isend(int dest, int tag, const T& value) const
1535 {
1536   return this->isend_impl(dest, tag, value, is_mpi_datatype<T>());
1537 }
1538 
1539 template<typename T, class A>
1540 request communicator::isend(int dest, int tag, const std::vector<T,A>& values) const
1541 {
1542   return this->isend_vector(dest, tag, values, is_mpi_datatype<T>());
1543 }
1544 
1545 template<typename T, class A>
1546 request
1547 communicator::isend_vector(int dest, int tag, const std::vector<T,A>& values,
1548                            mpl::true_ primitive) const
1549 {
1550   return request::make_dynamic_primitive_array_send(*this, dest, tag, values);
1551 }
1552 
1553 template<typename T, class A>
1554 request
1555 communicator::isend_vector(int dest, int tag, const std::vector<T,A>& values,
1556                            mpl::false_ no) const 
1557 {
1558   return this->isend_impl(dest, tag, values, no);
1559 }
1560 
1561 template<typename T>
1562 request
1563 communicator::array_isend_impl(int dest, int tag, const T* values, int n,
1564                                mpl::true_) const
1565 {
1566   return request::make_trivial_send(*this, dest, tag, values, n);
1567 }
1568 
1569 template<typename T>
1570 request
1571 communicator::array_isend_impl(int dest, int tag, const T* values, int n, 
1572                                mpl::false_) const
1573 {
1574   shared_ptr<packed_oarchive> archive(new packed_oarchive(*this));
1575   T const* v = values;
1576   while (v < values+n) {
1577     *archive << *v++;
1578   }
1579   request result = isend(dest, tag, *archive);
1580   result.preserve(archive);
1581   return result;
1582 }
1583 
1584 
1585 // Array isend must send the elements directly
1586 template<typename T>
1587 request communicator::isend(int dest, int tag, const T* values, int n) const
1588 {
1589   return array_isend_impl(dest, tag, values, n, is_mpi_datatype<T>());
1590 }
1591 
1592 // We're receiving a type that has an associated MPI datatype, so we
1593 // map directly to that datatype.
1594 template<typename T>
1595 request 
1596 communicator::irecv_impl(int source, int tag, T& value, mpl::true_) const
1597 {
1598   return request::make_trivial_recv(*this, source, tag, value);
1599 }
1600 
1601 template<typename T>
1602 request
1603 communicator::irecv_impl(int source, int tag, T& value, mpl::false_) const
1604 {
1605   return request::make_serialized(*this, source, tag, value);
1606 }
1607 
1608 template<typename T>
1609 request 
1610 communicator::irecv(int source, int tag, T& value) const
1611 {
1612   return this->irecv_impl(source, tag, value, is_mpi_datatype<T>());
1613 }
1614 
1615 template<typename T>
1616 request 
1617 communicator::array_irecv_impl(int source, int tag, T* values, int n, 
1618                                mpl::true_) const
1619 {
1620   return request::make_trivial_recv(*this, source, tag, values, n);
1621 }
1622 
1623 template<typename T>
1624 request
1625 communicator::array_irecv_impl(int source, int tag, T* values, int n, 
1626                                mpl::false_) const
1627 {
1628   return request::make_serialized_array(*this, source, tag, values, n);
1629 }
1630 
1631 template<typename T, class A>
1632 request
1633 communicator::irecv_vector(int source, int tag, std::vector<T,A>& values, 
1634                            mpl::true_ primitive) const
1635 {
1636   return request::make_dynamic_primitive_array_recv(*this, source, tag, values);
1637 }
1638 
1639 template<typename T, class A>
1640 request
1641 communicator::irecv_vector(int source, int tag, std::vector<T,A>& values, 
1642                            mpl::false_ no) const
1643 {
1644   return irecv_impl(source, tag, values, no);
1645 }
1646 
1647 template<typename T, typename A>
1648 request
1649 communicator::irecv(int source, int tag, std::vector<T,A>& values) const
1650 {
1651   return irecv_vector(source, tag, values, is_mpi_datatype<T>());
1652 }
1653 
1654 // Array receive must receive the elements directly into a buffer.
1655 template<typename T>
1656 request communicator::irecv(int source, int tag, T* values, int n) const
1657 {
1658   return this->array_irecv_impl(source, tag, values, n, is_mpi_datatype<T>());
1659 }
1660 
1661 } } // end namespace boost::mpi
1662 
1663 // If the user has already included skeleton_and_content.hpp, include
1664 // the code to send/receive skeletons and content.
1665 #ifdef BOOST_MPI_SKELETON_AND_CONTENT_HPP
1666 #  include <boost/mpi/detail/communicator_sc.hpp>
1667 #endif
1668 
1669 #ifdef BOOST_MSVC
1670 #  pragma warning(pop)
1671 #endif
1672 
1673 #endif // BOOST_MPI_COMMUNICATOR_HPP