source: XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp @ 1527

Last change on this file since 1527 was 1527, checked in by yushan, 6 years ago

save dev

File size: 4.0 KB
Line 
1/*!
2   \file ep_create.cpp
3   \since 2 may 2016
4
5   \brief Definitions of MPI endpoint function: MPI_Comm_create_endpoints
6 */
7
8#include "ep_lib.hpp"
9#include <mpi.h>
10#include "ep_declaration.hpp"
11#include "ep_mpi.hpp"
12
13using namespace std;
14
15namespace ep_lib {
16
17  /*!
18    Dynamic creation of endpoints for each MPI process.
19    The output of this function is an array of communicator handles of length num_ep, where each handle
20    corresponds to a nez local randk in the output communicator.
21    Once created, endpoints behave as MPI processes.
22    \param [in] mpi_comm Parent MPI communicator.
23    \param [in] num_ep Number of endpoints per process.
24    \param [out] info Information of the EP creation.
25    \param [out] out_comm_hdls Handles of EP communicators.
26  */
27 
28  int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls)
29  {
30
31    int base_rank;
32    int base_size;
33   
34    ::MPI_Comm mpi_base_comm = to_mpi_comm(base_comm_ptr);
35
36    ::MPI_Comm_size(mpi_base_comm, &base_size);  // ep_lib::mpi_comm_size
37    ::MPI_Comm_rank(mpi_base_comm, &base_rank);  // ep_lib::mpi_comm_rank
38                    // parent_comm can also be endpoints communicators
39
40    std::vector<int> recv_num_ep(base_size);
41
42    out_comm_hdls = new MPI_Comm[num_ep];
43#ifdef _showinfo
44    printf("new out_comm_hdls = %p\n", out_comm_hdls);
45#endif
46
47    ::MPI_Comm *parent_comm = new ::MPI_Comm;
48    ::MPI_Comm_dup(to_mpi_comm(base_comm_ptr), parent_comm);
49
50#ifdef _showinfo
51    printf("new out_comm_hdls->mpi_comm = %p\n", parent_comm);
52#endif
53   
54    for (int idx = 0; idx < num_ep; ++idx)
55    {
56
57      out_comm_hdls[idx] = new ep_comm;
58#ifdef _showinfo
59      printf("new out_comm_hdls[%d] = %p\n", idx, out_comm_hdls[idx]);
60#endif
61
62      out_comm_hdls[idx]->is_ep = true;
63      out_comm_hdls[idx]->is_intercomm = false;
64     
65      out_comm_hdls[idx]->ep_comm_ptr = new ep_communicator;     
66#ifdef _showinfo
67      printf("new out_comm_hdls[%d]->ep_comm_ptr = %p\n", idx, out_comm_hdls[idx]->ep_comm_ptr);
68#endif
69
70
71      out_comm_hdls[idx]->mpi_comm = parent_comm;
72      out_comm_hdls[idx]->ep_comm_ptr->comm_list = out_comm_hdls;
73    }
74
75    ::MPI_Allgather(&num_ep, 1, to_mpi_type(MPI_INT), &recv_num_ep[0], 1, to_mpi_type(MPI_INT), mpi_base_comm);
76
77
78    int sum = 0;  // representing total ep number of process with smaller rank
79    for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; }
80
81    int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0);
82
83    out_comm_hdls[0]->ep_barrier = new ep_barrier(num_ep);
84#ifdef _showinfo
85    printf("new out_comm_hdls[0]->ep_barrier = %p\n", out_comm_hdls[0]->ep_barrier);
86#endif
87
88    out_comm_hdls[0]->my_buffer = new BUFFER;
89#ifdef _showinfo
90    printf("new out_comm_hdls[0]->my_buffer = %p\n", out_comm_hdls[0]->my_buffer);
91#endif
92
93    out_comm_hdls[0]->ep_rank_map = new EP_RANK_MAP;
94#ifdef _showinfo
95    printf("new out_comm_hdls[0]->ep_rank_map = %p\n", out_comm_hdls[0]->ep_rank_map);
96#endif
97
98    for (int i = 1; i < num_ep; i++)
99    {
100      out_comm_hdls[i]->ep_barrier = out_comm_hdls[0]->ep_barrier;
101      out_comm_hdls[i]->my_buffer  = out_comm_hdls[0]->my_buffer;
102      out_comm_hdls[i]->ep_rank_map= out_comm_hdls[0]->ep_rank_map;
103    }
104
105
106    for (int i = 0; i < num_ep; i++)
107    {
108      out_comm_hdls[i]->ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size);
109      out_comm_hdls[i]->ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep);
110      out_comm_hdls[i]->ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size);
111
112      out_comm_hdls[i]->ep_comm_ptr->message_queue = new Message_list;
113#ifdef _showinfo
114      printf("new out_comm_hdls[%d]->ep_comm_ptr->message_queue = %p\n", i, out_comm_hdls[i]->ep_comm_ptr->message_queue);
115#endif
116
117    }
118
119
120    int ind = 0;
121
122    for(int i=0; i<base_size; i++)
123    {
124      for(int j=0; j<recv_num_ep[i]; j++)
125      {
126        out_comm_hdls[0]->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(ind, j, i));
127        ind++;
128      }
129    }
130
131  } //MPI_Comm_create_endpoints
132
133 
134
135} //namespace ep_lib
Note: See TracBrowser for help on using the repository browser.