source: XIOS/dev/branch_openmp/extern/src_ep_dev/ep_dup.cpp @ 1521

#include "ep_lib.hpp"
#include <mpi.h>
#include "ep_declaration.hpp"
#include "ep_mpi.hpp"

namespace ep_lib
{

  int MPI_Comm_dup_mpi(MPI_Comm comm, MPI_Comm *newcomm)
  {
    newcomm = new MPI_Comm;
    (*newcomm)->is_ep = false;

    ::MPI_Comm *output = new ::MPI_Comm;
      
    ::MPI_Comm_dup(to_mpi_comm(comm->mpi_comm), output);

    (*newcomm)->mpi_comm = output;
  }

  int MPI_Comm_dup_intracomm(MPI_Comm comm, MPI_Comm *newcomm)
  {
    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;
    int num_ep      = comm->ep_comm_ptr->size_rank_info[1].second;


    if(0 == ep_rank_loc)
    {
      MPI_Info info;
      MPI_Comm *out_comm;

      MPI_Comm_create_endpoints(comm->mpi_comm, num_ep, info, out_comm);
      comm->ep_comm_ptr->comm_list[0]->mem_bridge = out_comm;
    }

    MPI_Barrier_local(comm);

    *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[ep_rank_loc]);
    
  }

  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
  {
    
    if(!comm->is_ep)
    {
      Debug("MPI_Comm_dup with MPI\n");
      return MPI_Comm_dup_mpi(comm, newcomm); 
    }

    if(comm->is_intercomm) return MPI_Comm_dup_intercomm(comm, newcomm);

    
    return MPI_Comm_dup_intracomm(comm, newcomm); 

   
  }

  int MPI_Comm_dup_intercomm(MPI_Comm comm, MPI_Comm *newcomm)
  {
    
    int newcomm_ep_rank =comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 
    int newcomm_ep_rank_loc = comm->ep_comm_ptr->intercomm->size_rank_info[1].first;
    int newcomm_num_ep = comm->ep_comm_ptr->intercomm->size_rank_info[1].second;


    if(0 == newcomm_ep_rank_loc)
    {
      //printf("in dup , newcomm_ep_rank_loc = 0 :  ep %d\n", comm->ep_comm_ptr->intercomm->size_rank_info[0].first);

      MPI_Info info;
      MPI_Comm *out_comm;

      MPI_Comm_create_endpoints(comm->mpi_comm, newcomm_num_ep, info, out_comm);

      ::MPI_Comm *mpi_inter_comm = new ::MPI_Comm;
      ::MPI_Comm_dup(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), mpi_inter_comm);

      for(int i=0; i<newcomm_num_ep; i++)
      {
        out_comm[i]->is_intercomm = true;
        out_comm[i]->ep_comm_ptr->comm_label = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->comm_label;
        out_comm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
#ifdef _showinfo
        printf("new out_comm[%d]->ep_comm_ptr->intercomm = %p\n", i, out_comm[i]->ep_comm_ptr->intercomm);
#endif
        out_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter_comm;
      }


      comm->ep_comm_ptr->comm_list[0]->mem_bridge = out_comm;
    }

    MPI_Barrier_local(comm);

    *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[newcomm_ep_rank_loc]);

    (*newcomm)->ep_comm_ptr->size_rank_info[0] = comm->ep_comm_ptr->size_rank_info[0];
    (*newcomm)->ep_comm_ptr->size_rank_info[1] = comm->ep_comm_ptr->size_rank_info[1];
    (*newcomm)->ep_comm_ptr->size_rank_info[2] = comm->ep_comm_ptr->size_rank_info[2];

    (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[0] = comm->ep_comm_ptr->intercomm->size_rank_info[0];
    (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[1] = comm->ep_comm_ptr->intercomm->size_rank_info[1];
    (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[2] = comm->ep_comm_ptr->intercomm->size_rank_info[2];

    (*newcomm)->ep_comm_ptr->intercomm->intercomm_tag = comm->ep_comm_ptr->intercomm->intercomm_tag;


    int ep_rank_loc = (*newcomm)->ep_comm_ptr->size_rank_info[1].first;
    
    if(ep_rank_loc == 0)
    {
      int world_rank;
      MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);

      (*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new INTERCOMM_RANK_MAP;
      (*newcomm)->ep_comm_ptr->intercomm->local_rank_map = new EP_RANK_MAP;

      *(*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = *comm->ep_comm_ptr->intercomm->intercomm_rank_map;
      *(*newcomm)->ep_comm_ptr->intercomm->local_rank_map     = *comm->ep_comm_ptr->intercomm->local_rank_map;
    }

    MPI_Barrier_local(comm);

    if(ep_rank_loc !=0 )
    {
      int target = (*newcomm)->ep_comm_ptr->intercomm->intercomm_tag;
      (*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = (*newcomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->intercomm_rank_map; 
      (*newcomm)->ep_comm_ptr->intercomm->local_rank_map     = (*newcomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->local_rank_map;
    }

    



    
  }



}