Comm_Group_MPI.h

/****************************************************************************
* Copyright (c) 2026, CEA
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*****************************************************************************/
 
#ifndef Comm_Group_MPI_included
#define Comm_Group_MPI_included
 
#include <Comm_Group_Noparallel.h>
#include <TRUST_Ref.h>
 
class Comm_Group;
//GF comm_incl includes mpi.h but it is easier to create a workshop
// where one can change the MPI implementation by going through this intermediate file
 
#include <comm_incl.h>
 
/*! @brief : Class Comm_Group_MPI, derived from the abstract class Comm_Group.
 *
 *   This class represents a communication group based on MPI.
 *
 *   Object construction: see init_group_trio() for groupe_TRUST
 *    and init_group() for the others.
 *
 */
class Comm_Group_MPI : public Comm_Group
{
  Declare_instanciable_sans_constructeur_ni_destructeur(Comm_Group_MPI);
public:
  Comm_Group_MPI();
  ~Comm_Group_MPI() override;
  void abort() const override;
 
  void mp_collective_op(const double *x, double *resu, int n, Collective_Op op) const override;
  void mp_collective_op(const double *x, double *resu, const Collective_Op *op, int n) const override;
  void mp_collective_op(const float *x, float *resu, int n, Collective_Op op) const override;
  void mp_collective_op(const float *x, float *resu, const Collective_Op *op, int n) const override;
  void mp_collective_op(const int *x, int *resu, int n, Collective_Op op) const override;
  void mp_collective_op(const int *x, int *resu, const Collective_Op *op, int n) const override;
#if INT_is_64_ == 2
  void mp_collective_op(const trustIdType *x, trustIdType *resu, int n, Collective_Op op) const override;
  void mp_collective_op(const trustIdType *x, trustIdType *resu, const Collective_Op *op, int n) const override;
#endif
 
  void barrier(int tag) const override;
  void send_recv_start(const ArrOfInt& send_list,
                       const ArrOfInt& send_size,
                       const char * const * const send_buffers,
                       const ArrOfInt& recv_list,
                       const ArrOfInt& recv_size,
                       char * const * const recv_buffers,
                       TypeHint typehint = CHAR) const override;
  void send_recv_finish() const override;
  void send(int pe, const void *buffer, int size, int tag) const override; // Blocking send
  void recv(int pe, void *buffer, int size, int tag) const override; // Blocking receive
  void broadcast(void *buffer, int size, int pe_source) const override;
  void all_to_all(const void *src_buffer, void *dest_buffer, int data_size) const override;
  void all_gather(const void *src_buffer, void *dest_buffer, int data_size) const override;
  void gather(const void *src_buffer, void *dest_buffer, int data_size, int root) const override;
  void all_gatherv(const void *src_buffer, void *dest_buffer, int send_size, const int* recv_size, const int* displs) const override;
 
#ifdef MPI_
  void init_group_trio();
  void init_comm_on_numa_node();
  void init_comm_on_node_master();
  void free();
  void free_all();
  void all_to_allv(const void *src_buffer, int *send_data_size, int *send_data_offset,
                   void *dest_buffer, int *recv_data_size, int *recv_data_offset) const;
  static void set_trio_u_world(MPI_Comm world);
  static MPI_Comm get_trio_u_world();
  static void set_must_mpi_initialize(bool flag);
 
  void ptop_send_recv(const void * send_buf, int send_buf_size, int send_proc,
                      void * recv_buf, int recv_buf_size, int recv_proc) const;
 
  void set_must_finalize(int flag) { must_finalize_=flag; }
  MPI_Comm get_mpi_comm() const { return mpi_comm_; }
protected:
  void init_group(const ArrOfInt& pe_list) override;
  void internal_collective(const int *x, int *resu, int nx, const Collective_Op *op, int nop, int level) const;
#if INT_is_64_ == 2
  void internal_collective(const trustIdType *x, trustIdType *resu, int nx, const Collective_Op *op, int nop, int level) const;
#endif
  void internal_collective(const double *x, double *resu, int nx, const Collective_Op *op, int nop, int level) const;
  void internal_collective(const float *x, float *resu, int nx, const Collective_Op *op, int nop, int level) const;
 
private:
  // TYP_IDX = 1: int, 2: long, 3: double, 4 :float
  // [ABN] Can not use a MPI datatype directly because in OpenMPI for example this type is too complex to be used as a template parameter.
  template <typename _TYPE_, int TYP_IDX>
  void mp_collective_op_template(const _TYPE_ *x, _TYPE_ *resu, int n, Comm_Group::Collective_Op op) const;
  trustIdType mppartial_sum_impl(trustIdType x) const;
 
  // See set_must_mpi_initialize() and init_group_trio()
  static bool must_mpi_initialize_;
  // The global trio_u group is associated with this communicator
  //  (different from MPI_COMM_WORLD for coupling for example)
  static MPI_Comm trio_u_world_;
  static MPI_Status * mpi_status_;
  static MPI_Request * mpi_requests_;
  static int mpi_nrequests_;
  static int mpi_maxrequests_;
  static int current_msg_size_; // The size of data sent/received for the current send_recv_start
 
  MPI_Group mpi_group_;// Handle on the MPI group
  MPI_Comm  mpi_comm_; // Handle on the MPI communicator
 
  int must_finalize_; // Should it be done in the destructor?
  OBS_PTR(Comm_Group) groupe_pere_;
#endif
};
 
#endif