[1134] | 1 | #include "ep_lib.hpp" |
---|
| 2 | #include <mpi.h> |
---|
| 3 | #include "ep_declaration.hpp" |
---|
[1354] | 4 | #include "ep_mpi.hpp" |
---|
[1134] | 5 | |
---|
| 6 | using namespace std; |
---|
| 7 | |
---|
| 8 | namespace ep_lib |
---|
| 9 | { |
---|
| 10 | |
---|
[1354] | 11 | // #ifdef _openmpi |
---|
[1134] | 12 | |
---|
[1354] | 13 | // int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm) |
---|
| 14 | // { |
---|
[1134] | 15 | |
---|
[1354] | 16 | // int ep_rank, ep_rank_loc, mpi_rank; |
---|
| 17 | // int ep_size, num_ep, mpi_size; |
---|
[1134] | 18 | |
---|
[1354] | 19 | // ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first; |
---|
| 20 | // ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first; |
---|
| 21 | // mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first; |
---|
| 22 | // ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second; |
---|
| 23 | // num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second; |
---|
| 24 | // mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second; |
---|
[1134] | 25 | |
---|
| 26 | |
---|
[1354] | 27 | // std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm |
---|
| 28 | // //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm |
---|
[1134] | 29 | |
---|
[1354] | 30 | // int rank_in_world; |
---|
| 31 | // int rank_in_local_parent; |
---|
[1134] | 32 | |
---|
[1354] | 33 | // int local_ep_size = ep_size; |
---|
| 34 | // int remote_ep_size; |
---|
[1134] | 35 | |
---|
[1354] | 36 | // ::MPI_Comm peer_comm = to_mpi_comm(peer_comm_ptr); |
---|
| 37 | // ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm.mpi_comm); |
---|
[1134] | 38 | |
---|
[1354] | 39 | // ::MPI_Comm_rank(peer_comm, &rank_in_world); |
---|
[1134] | 40 | |
---|
[1354] | 41 | // ::MPI_Comm_rank(local_mpi_comm, &rank_in_local_parent); |
---|
[1134] | 42 | |
---|
[1354] | 43 | // bool is_proc_master = false; |
---|
| 44 | // bool is_local_leader = false; |
---|
| 45 | // bool is_final_master = false; |
---|
[1134] | 46 | |
---|
| 47 | |
---|
[1354] | 48 | // if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;} |
---|
| 49 | // if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true; |
---|
[1134] | 50 | |
---|
| 51 | |
---|
[1354] | 52 | // int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm |
---|
[1134] | 53 | |
---|
[1354] | 54 | // int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader |
---|
[1134] | 55 | |
---|
| 56 | |
---|
[1354] | 57 | // std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm |
---|
[1134] | 58 | |
---|
[1354] | 59 | // std::vector<int> new_rank_info[4]; |
---|
| 60 | // std::vector<int> new_ep_info[2]; |
---|
[1134] | 61 | |
---|
[1354] | 62 | // std::vector<int> offset; |
---|
[1134] | 63 | |
---|
[1354] | 64 | // if(is_proc_master) |
---|
| 65 | // { |
---|
[1134] | 66 | |
---|
[1354] | 67 | // size_info[0] = mpi_size; |
---|
[1134] | 68 | |
---|
[1354] | 69 | // rank_info[0].resize(size_info[0]); |
---|
| 70 | // rank_info[1].resize(size_info[0]); |
---|
[1134] | 71 | |
---|
| 72 | |
---|
| 73 | |
---|
[1354] | 74 | // ep_info[0].resize(size_info[0]); |
---|
[1134] | 75 | |
---|
[1354] | 76 | // vector<int> send_buf(6); |
---|
| 77 | // vector<int> recv_buf(3*size_info[0]); |
---|
[1134] | 78 | |
---|
[1354] | 79 | // send_buf[0] = rank_in_world; |
---|
| 80 | // send_buf[1] = rank_in_local_parent; |
---|
| 81 | // send_buf[2] = num_ep; |
---|
[1134] | 82 | |
---|
[1354] | 83 | // ::MPI_Allgather(send_buf.data(), 3, to_mpi_type(MPI_INT), recv_buf.data(), 3, to_mpi_type(MPI_INT), local_mpi_comm); |
---|
[1134] | 84 | |
---|
[1354] | 85 | // for(int i=0; i<size_info[0]; i++) |
---|
| 86 | // { |
---|
| 87 | // rank_info[0][i] = recv_buf[3*i]; |
---|
| 88 | // rank_info[1][i] = recv_buf[3*i+1]; |
---|
| 89 | // ep_info[0][i] = recv_buf[3*i+2]; |
---|
| 90 | // } |
---|
[1134] | 91 | |
---|
[1354] | 92 | // if(is_local_leader) |
---|
| 93 | // { |
---|
| 94 | // leader_info[0] = rank_in_world; |
---|
| 95 | // leader_info[1] = mpi_remote_leader; |
---|
[1134] | 96 | |
---|
[1354] | 97 | // ::MPI_Status mpi_status; |
---|
[1134] | 98 | |
---|
[1354] | 99 | // send_buf[0] = size_info[0]; |
---|
| 100 | // send_buf[1] = local_ep_size; |
---|
[1134] | 101 | |
---|
[1354] | 102 | // ::MPI_Send(send_buf.data(), 2, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 103 | |
---|
[1354] | 104 | // ::MPI_Recv(recv_buf.data(), 2, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
[1134] | 105 | |
---|
[1354] | 106 | // recv_buf[2] = rank_in_world; |
---|
| 107 | // recv_buf[3] = mpi_remote_leader; |
---|
[1134] | 108 | |
---|
[1354] | 109 | // } |
---|
[1134] | 110 | |
---|
[1354] | 111 | // ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 112 | |
---|
[1354] | 113 | // size_info[1] = recv_buf[0]; |
---|
| 114 | // remote_ep_size = recv_buf[1]; |
---|
| 115 | // leader_info[0] = recv_buf[2]; |
---|
| 116 | // leader_info[1] = recv_buf[3]; |
---|
[1134] | 117 | |
---|
[1354] | 118 | // rank_info[2].resize(size_info[1]); |
---|
| 119 | // rank_info[3].resize(size_info[1]); |
---|
[1134] | 120 | |
---|
[1354] | 121 | // ep_info[1].resize(size_info[1]); |
---|
[1134] | 122 | |
---|
[1354] | 123 | // send_buf.resize(3*size_info[0]); |
---|
| 124 | // recv_buf.resize(3*size_info[1]); |
---|
[1134] | 125 | |
---|
[1354] | 126 | // if(is_local_leader) |
---|
| 127 | // { |
---|
| 128 | // ::MPI_Status mpi_status; |
---|
[1134] | 129 | |
---|
| 130 | |
---|
[1354] | 131 | // std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() ); |
---|
| 132 | // std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] ); |
---|
| 133 | // std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] ); |
---|
[1134] | 134 | |
---|
[1354] | 135 | // ::MPI_Send(send_buf.data(), 3*size_info[0], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 136 | |
---|
[1354] | 137 | // ::MPI_Recv(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
[1134] | 138 | |
---|
[1354] | 139 | // } |
---|
[1134] | 140 | |
---|
[1354] | 141 | // ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 142 | |
---|
[1354] | 143 | // std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() ); |
---|
| 144 | // std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() ); |
---|
| 145 | // std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() ); |
---|
[1134] | 146 | |
---|
[1354] | 147 | // offset.resize(size_info[0]); |
---|
[1134] | 148 | |
---|
[1354] | 149 | // if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader |
---|
| 150 | // { |
---|
[1134] | 151 | |
---|
[1354] | 152 | // bool found = false; |
---|
| 153 | // int ep_tmp; |
---|
| 154 | // int ep_local; |
---|
| 155 | // int ep_remote; |
---|
| 156 | // for(int i=0; i<size_info[0]; i++) |
---|
| 157 | // { |
---|
| 158 | // int target = rank_info[0][i]; |
---|
| 159 | // found = false; |
---|
| 160 | // for(int j=0; j<size_info[1]; j++) |
---|
| 161 | // { |
---|
| 162 | // if(target == rank_info[2][j]) |
---|
| 163 | // { |
---|
| 164 | // found = true; |
---|
| 165 | // ep_tmp = ep_info[1][j]; |
---|
| 166 | // ep_local = ep_info[0][j]; |
---|
| 167 | // ep_remote = ep_info[1][j]; |
---|
| 168 | // break; |
---|
| 169 | // } |
---|
| 170 | // } |
---|
| 171 | // if(found) |
---|
| 172 | // { |
---|
[1134] | 173 | |
---|
[1354] | 174 | // if(target == leader_info[0]) // the leader is doubled in remote |
---|
| 175 | // { |
---|
| 176 | // new_rank_info[0].push_back(target); |
---|
| 177 | // new_rank_info[1].push_back(rank_info[1][i]); |
---|
[1134] | 178 | |
---|
[1354] | 179 | // new_ep_info[0].push_back(ep_local + ep_remote); |
---|
| 180 | // offset[i] = 0; |
---|
| 181 | // } |
---|
| 182 | // else |
---|
| 183 | // { |
---|
| 184 | // offset[i] = ep_local; |
---|
| 185 | // } |
---|
| 186 | // } |
---|
| 187 | // else |
---|
| 188 | // { |
---|
| 189 | // new_rank_info[0].push_back(target); |
---|
| 190 | // new_rank_info[1].push_back(rank_info[1][i]); |
---|
[1134] | 191 | |
---|
[1354] | 192 | // new_ep_info[0].push_back(ep_info[0][i]); |
---|
[1134] | 193 | |
---|
[1354] | 194 | // offset[i] = 0; |
---|
| 195 | // } |
---|
[1134] | 196 | |
---|
[1354] | 197 | // } |
---|
| 198 | // } |
---|
[1134] | 199 | |
---|
[1354] | 200 | // else // erase rank doubled with remote leader |
---|
| 201 | // { |
---|
[1134] | 202 | |
---|
[1354] | 203 | // bool found = false; |
---|
| 204 | // int ep_tmp; |
---|
| 205 | // int ep_local; |
---|
| 206 | // int ep_remote; |
---|
| 207 | // for(int i=0; i<size_info[0]; i++) |
---|
| 208 | // { |
---|
| 209 | // int target = rank_info[0][i]; |
---|
| 210 | // found = false; |
---|
| 211 | // for(int j=0; j<size_info[1]; j++) |
---|
| 212 | // { |
---|
[1134] | 213 | |
---|
[1354] | 214 | // if(target == rank_info[2][j]) |
---|
| 215 | // { |
---|
| 216 | // found = true; |
---|
| 217 | // ep_tmp = ep_info[1][j]; |
---|
| 218 | // ep_local = ep_info[0][j]; |
---|
| 219 | // ep_remote = ep_info[1][j]; |
---|
| 220 | // break; |
---|
| 221 | // } |
---|
| 222 | // } |
---|
| 223 | // if(found) |
---|
| 224 | // { |
---|
| 225 | // if(target != leader_info[1]) |
---|
| 226 | // { |
---|
| 227 | // new_rank_info[0].push_back(target); |
---|
| 228 | // new_rank_info[1].push_back(rank_info[1][i]); |
---|
[1134] | 229 | |
---|
[1354] | 230 | // new_ep_info[0].push_back(ep_local + ep_remote); |
---|
| 231 | // offset[i] = 0; |
---|
| 232 | // } |
---|
| 233 | // else // found remote leader |
---|
| 234 | // { |
---|
| 235 | // offset[i] = ep_remote; |
---|
| 236 | // } |
---|
| 237 | // } |
---|
| 238 | // else |
---|
| 239 | // { |
---|
| 240 | // new_rank_info[0].push_back(target); |
---|
| 241 | // new_rank_info[1].push_back(rank_info[1][i]); |
---|
[1134] | 242 | |
---|
[1354] | 243 | // new_ep_info[0].push_back(ep_info[0][i]); |
---|
| 244 | // offset[i] = 0; |
---|
| 245 | // } |
---|
| 246 | // } |
---|
| 247 | // } |
---|
[1134] | 248 | |
---|
[1354] | 249 | // if(offset[mpi_rank] == 0) |
---|
| 250 | // { |
---|
| 251 | // is_final_master = true; |
---|
| 252 | // } |
---|
[1134] | 253 | |
---|
| 254 | |
---|
[1354] | 255 | // // size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote |
---|
[1134] | 256 | |
---|
[1354] | 257 | // if(is_local_leader) |
---|
| 258 | // { |
---|
| 259 | // size_info[2] = new_ep_info[0].size(); |
---|
| 260 | // ::MPI_Status mpi_status; |
---|
[1134] | 261 | |
---|
[1354] | 262 | // ::MPI_Send(&size_info[2], 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 263 | |
---|
[1354] | 264 | // ::MPI_Recv(&size_info[3], 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
| 265 | // } |
---|
[1134] | 266 | |
---|
[1354] | 267 | // ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 268 | |
---|
[1354] | 269 | // new_rank_info[2].resize(size_info[3]); |
---|
| 270 | // new_rank_info[3].resize(size_info[3]); |
---|
| 271 | // new_ep_info[1].resize(size_info[3]); |
---|
[1134] | 272 | |
---|
[1354] | 273 | // send_buf.resize(size_info[2]); |
---|
| 274 | // recv_buf.resize(size_info[3]); |
---|
[1134] | 275 | |
---|
[1354] | 276 | // if(is_local_leader) |
---|
| 277 | // { |
---|
| 278 | // ::MPI_Status mpi_status; |
---|
[1134] | 279 | |
---|
[1354] | 280 | // std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() ); |
---|
| 281 | // std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] ); |
---|
| 282 | // std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] ); |
---|
[1134] | 283 | |
---|
[1354] | 284 | // ::MPI_Send(send_buf.data(), 3*size_info[2], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
| 285 | // ::MPI_Recv(recv_buf.data(), 3*size_info[3], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
[1134] | 286 | |
---|
[1354] | 287 | // } |
---|
[1134] | 288 | |
---|
[1354] | 289 | // ::MPI_Bcast(recv_buf.data(), 3*size_info[3], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 290 | |
---|
[1354] | 291 | // std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() ); |
---|
| 292 | // std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() ); |
---|
| 293 | // std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() ); |
---|
[1134] | 294 | |
---|
[1354] | 295 | // } |
---|
[1134] | 296 | |
---|
| 297 | |
---|
| 298 | |
---|
[1354] | 299 | // if(is_proc_master) |
---|
| 300 | // { |
---|
| 301 | // // leader_info[4]: 2-> rank of local leader in new_group generated comm; |
---|
| 302 | // // 3-> rank of remote leader in new_group generated comm; |
---|
| 303 | // ::MPI_Group local_group; |
---|
| 304 | // ::MPI_Group new_group; |
---|
| 305 | // ::MPI_Comm *new_comm = new ::MPI_Comm; |
---|
| 306 | // ::MPI_Comm *intercomm = new ::MPI_Comm; |
---|
[1134] | 307 | |
---|
[1354] | 308 | // ::MPI_Comm_group(local_mpi_comm, &local_group); |
---|
[1134] | 309 | |
---|
[1354] | 310 | // ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group); |
---|
[1134] | 311 | |
---|
[1354] | 312 | // ::MPI_Comm_create(local_mpi_comm, new_group, new_comm); |
---|
[1134] | 313 | |
---|
| 314 | |
---|
| 315 | |
---|
[1354] | 316 | // if(is_local_leader) |
---|
| 317 | // { |
---|
| 318 | // ::MPI_Comm_rank(*new_comm, &leader_info[2]); |
---|
| 319 | // } |
---|
[1134] | 320 | |
---|
[1354] | 321 | // ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 322 | |
---|
[1354] | 323 | // if(new_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) |
---|
| 324 | // { |
---|
| 325 | // ::MPI_Barrier(*new_comm); |
---|
[1134] | 326 | |
---|
[1354] | 327 | // ::MPI_Intercomm_create(*new_comm, leader_info[2], peer_comm, leader_info[1], tag, intercomm); |
---|
[1134] | 328 | |
---|
[1354] | 329 | // int id; |
---|
| 330 | // ::MPI_Comm_rank(*new_comm, &id); |
---|
| 331 | // int my_num_ep = new_ep_info[0][id]; |
---|
[1134] | 332 | |
---|
[1354] | 333 | // MPI_Comm *ep_intercomm; |
---|
| 334 | // MPI_Info info; |
---|
| 335 | // MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm); |
---|
[1134] | 336 | |
---|
| 337 | |
---|
| 338 | |
---|
[1354] | 339 | // for(int i= 0; i<my_num_ep; i++) |
---|
| 340 | // { |
---|
| 341 | // ep_intercomm[i].is_intercomm = true; |
---|
[1134] | 342 | |
---|
[1354] | 343 | // ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; |
---|
| 344 | // ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm; |
---|
| 345 | // ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0]; |
---|
| 346 | // } |
---|
[1134] | 347 | |
---|
[1354] | 348 | // #pragma omp critical (write_to_tag_list) |
---|
| 349 | // tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm)); |
---|
| 350 | // } |
---|
[1134] | 351 | |
---|
| 352 | |
---|
[1354] | 353 | // } |
---|
[1134] | 354 | |
---|
| 355 | |
---|
[1354] | 356 | // MPI_Barrier_local(local_comm); |
---|
[1134] | 357 | |
---|
[1354] | 358 | // vector<int> bcast_buf(8); |
---|
| 359 | // if(is_local_leader) |
---|
| 360 | // { |
---|
| 361 | // std::copy(size_info, size_info+4, bcast_buf.begin()); |
---|
| 362 | // std::copy(leader_info, leader_info+4, bcast_buf.begin()+4); |
---|
| 363 | // } |
---|
[1134] | 364 | |
---|
[1354] | 365 | // MPI_Bcast(bcast_buf.data(), 8, MPI_INT, local_leader, local_comm); |
---|
[1134] | 366 | |
---|
[1354] | 367 | // if(!is_local_leader) |
---|
| 368 | // { |
---|
| 369 | // std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info); |
---|
| 370 | // std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info); |
---|
| 371 | // } |
---|
[1134] | 372 | |
---|
[1354] | 373 | // if(!is_local_leader) |
---|
| 374 | // { |
---|
| 375 | // new_rank_info[1].resize(size_info[2]); |
---|
| 376 | // ep_info[1].resize(size_info[1]); |
---|
| 377 | // offset.resize(size_info[0]); |
---|
| 378 | // } |
---|
[1134] | 379 | |
---|
[1354] | 380 | // bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1); |
---|
[1134] | 381 | |
---|
[1354] | 382 | // if(is_local_leader) |
---|
| 383 | // { |
---|
| 384 | // bcast_buf[0] = remote_ep_size; |
---|
| 385 | // std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1); |
---|
| 386 | // std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1); |
---|
| 387 | // std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1); |
---|
| 388 | // } |
---|
[1134] | 389 | |
---|
[1354] | 390 | // MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT, local_leader, local_comm); |
---|
[1134] | 391 | |
---|
[1354] | 392 | // if(!is_local_leader) |
---|
| 393 | // { |
---|
| 394 | // remote_ep_size = bcast_buf[0]; |
---|
| 395 | // std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin()); |
---|
| 396 | // std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin()); |
---|
| 397 | // std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin()); |
---|
| 398 | // } |
---|
[1134] | 399 | |
---|
| 400 | |
---|
[1354] | 401 | // int my_position = offset[rank_in_local_parent]+ep_rank_loc; |
---|
[1134] | 402 | |
---|
| 403 | |
---|
[1354] | 404 | // MPI_Barrier_local(local_comm); |
---|
| 405 | // #pragma omp flush |
---|
[1134] | 406 | |
---|
| 407 | |
---|
[1354] | 408 | // #pragma omp critical (read_from_tag_list) |
---|
| 409 | // { |
---|
| 410 | // bool found = false; |
---|
| 411 | // while(!found) |
---|
| 412 | // { |
---|
| 413 | // for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++) |
---|
| 414 | // { |
---|
| 415 | // if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1]))) |
---|
| 416 | // { |
---|
| 417 | // *newintercomm = iter->second[my_position]; |
---|
[1134] | 418 | |
---|
[1354] | 419 | // found = true; |
---|
| 420 | // //tag_list.erase(iter); |
---|
| 421 | // break; |
---|
| 422 | // } |
---|
| 423 | // } |
---|
| 424 | // } |
---|
| 425 | // } |
---|
[1134] | 426 | |
---|
[1354] | 427 | // MPI_Barrier(local_comm); |
---|
[1134] | 428 | |
---|
[1354] | 429 | // if(is_local_leader) |
---|
| 430 | // { |
---|
| 431 | // int local_flag = true; |
---|
| 432 | // int remote_flag = false; |
---|
| 433 | // ::MPI_Status mpi_status; |
---|
[1287] | 434 | |
---|
[1354] | 435 | // ::MPI_Send(&local_flag, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1287] | 436 | |
---|
[1354] | 437 | // ::MPI_Recv(&remote_flag, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
| 438 | // } |
---|
[1287] | 439 | |
---|
[1354] | 440 | // MPI_Barrier(local_comm); |
---|
[1287] | 441 | |
---|
[1354] | 442 | // if(is_proc_master) |
---|
| 443 | // { |
---|
| 444 | // for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++) |
---|
| 445 | // { |
---|
| 446 | // if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1]))) |
---|
| 447 | // { |
---|
| 448 | // tag_list.erase(iter); |
---|
| 449 | // break; |
---|
| 450 | // } |
---|
| 451 | // } |
---|
| 452 | // } |
---|
[1134] | 453 | |
---|
| 454 | |
---|
| 455 | |
---|
[1354] | 456 | // int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank; |
---|
| 457 | // int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size; |
---|
[1134] | 458 | |
---|
[1354] | 459 | // intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first; |
---|
| 460 | // intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first; |
---|
| 461 | // intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first; |
---|
| 462 | // intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second; |
---|
| 463 | // intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second; |
---|
| 464 | // intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second; |
---|
[1134] | 465 | |
---|
[1354] | 466 | // MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm); |
---|
[1134] | 467 | |
---|
[1354] | 468 | // int my_rank_map_elem[2]; |
---|
[1134] | 469 | |
---|
| 470 | |
---|
[1354] | 471 | // my_rank_map_elem[0] = intercomm_ep_rank; |
---|
[1134] | 472 | |
---|
[1354] | 473 | // my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label; |
---|
[1134] | 474 | |
---|
[1354] | 475 | // vector<pair<int, int> > local_rank_map_array; |
---|
| 476 | // vector<pair<int, int> > remote_rank_map_array; |
---|
[1134] | 477 | |
---|
| 478 | |
---|
[1354] | 479 | // (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; |
---|
| 480 | // (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size); |
---|
[1134] | 481 | |
---|
[1354] | 482 | // MPI_Allgather2(my_rank_map_elem, 2, MPI_INT, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); |
---|
[1134] | 483 | |
---|
[1354] | 484 | // (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; |
---|
| 485 | // (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size); |
---|
[1134] | 486 | |
---|
[1354] | 487 | // int local_intercomm_size = intercomm_ep_size; |
---|
| 488 | // int remote_intercomm_size; |
---|
[1134] | 489 | |
---|
| 490 | |
---|
| 491 | |
---|
| 492 | |
---|
[1354] | 493 | // if(is_local_leader) |
---|
| 494 | // { |
---|
| 495 | // ::MPI_Status status; |
---|
[1134] | 496 | |
---|
[1354] | 497 | // ::MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 498 | |
---|
[1354] | 499 | // ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); |
---|
[1134] | 500 | |
---|
[1354] | 501 | // ::MPI_Send(&local_intercomm_size, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 502 | |
---|
[1354] | 503 | // ::MPI_Recv(&remote_intercomm_size, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); |
---|
| 504 | // } |
---|
[1134] | 505 | |
---|
[1354] | 506 | // MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm); |
---|
| 507 | // MPI_Bcast(&remote_intercomm_size, 1, MPI_INT, 0, *newintercomm); |
---|
[1134] | 508 | |
---|
| 509 | |
---|
[1354] | 510 | // (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; |
---|
| 511 | // (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); |
---|
[1134] | 512 | |
---|
[1354] | 513 | // (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0]; |
---|
| 514 | // (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1]; |
---|
| 515 | // (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2]; |
---|
[1134] | 516 | |
---|
| 517 | |
---|
[1354] | 518 | // if(is_local_leader) |
---|
| 519 | // { |
---|
| 520 | // ::MPI_Status status; |
---|
[1134] | 521 | |
---|
[1354] | 522 | // ::MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 523 | |
---|
[1354] | 524 | // ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); |
---|
| 525 | // } |
---|
[1134] | 526 | |
---|
[1354] | 527 | // MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), 0, *newintercomm); |
---|
[1134] | 528 | |
---|
[1354] | 529 | // (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]); |
---|
| 530 | // (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label; |
---|
[1134] | 531 | |
---|
| 532 | |
---|
[1354] | 533 | // return MPI_SUCCESS; |
---|
[1134] | 534 | |
---|
[1354] | 535 | // } |
---|
[1134] | 536 | |
---|
| 537 | |
---|
| 538 | |
---|
[1354] | 539 | // #elif _intelmpi |
---|
[1356] | 540 | int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm) |
---|
[1134] | 541 | { |
---|
| 542 | int ep_rank, ep_rank_loc, mpi_rank; |
---|
| 543 | int ep_size, num_ep, mpi_size; |
---|
| 544 | |
---|
| 545 | ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first; |
---|
| 546 | ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first; |
---|
| 547 | mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first; |
---|
| 548 | ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second; |
---|
| 549 | num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second; |
---|
| 550 | mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second; |
---|
| 551 | |
---|
| 552 | std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm |
---|
| 553 | //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm |
---|
| 554 | |
---|
| 555 | int rank_in_world; |
---|
| 556 | int rank_in_local_parent; |
---|
| 557 | |
---|
| 558 | int local_ep_size = ep_size; |
---|
| 559 | int remote_ep_size; |
---|
| 560 | |
---|
[1354] | 561 | ::MPI_Comm peer_comm = to_mpi_comm(peer_comm_ptr); |
---|
| 562 | ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm.mpi_comm); |
---|
[1134] | 563 | |
---|
| 564 | ::MPI_Comm_rank(peer_comm, &rank_in_world); |
---|
| 565 | |
---|
[1354] | 566 | ::MPI_Comm_rank(local_mpi_comm, &rank_in_local_parent); |
---|
[1134] | 567 | |
---|
| 568 | bool is_proc_master = false; |
---|
| 569 | bool is_local_leader = false; |
---|
| 570 | bool is_final_master = false; |
---|
| 571 | |
---|
| 572 | |
---|
| 573 | if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;} |
---|
| 574 | if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true; |
---|
| 575 | |
---|
| 576 | |
---|
| 577 | int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm |
---|
| 578 | |
---|
| 579 | int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader |
---|
| 580 | |
---|
| 581 | |
---|
| 582 | std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm |
---|
| 583 | |
---|
| 584 | std::vector<int> new_rank_info[4]; |
---|
| 585 | std::vector<int> new_ep_info[2]; |
---|
| 586 | |
---|
| 587 | std::vector<int> offset; |
---|
| 588 | |
---|
| 589 | if(is_proc_master) |
---|
| 590 | { |
---|
| 591 | |
---|
| 592 | size_info[0] = mpi_size; |
---|
| 593 | |
---|
| 594 | rank_info[0].resize(size_info[0]); |
---|
| 595 | rank_info[1].resize(size_info[0]); |
---|
| 596 | |
---|
| 597 | ep_info[0].resize(size_info[0]); |
---|
| 598 | |
---|
| 599 | vector<int> send_buf(6); |
---|
| 600 | vector<int> recv_buf(3*size_info[0]); |
---|
| 601 | |
---|
| 602 | send_buf[0] = rank_in_world; |
---|
| 603 | send_buf[1] = rank_in_local_parent; |
---|
| 604 | send_buf[2] = num_ep; |
---|
| 605 | |
---|
[1354] | 606 | ::MPI_Allgather(send_buf.data(), 3, to_mpi_type(MPI_INT), recv_buf.data(), 3, to_mpi_type(MPI_INT), local_mpi_comm); |
---|
[1134] | 607 | |
---|
| 608 | for(int i=0; i<size_info[0]; i++) |
---|
| 609 | { |
---|
| 610 | rank_info[0][i] = recv_buf[3*i]; |
---|
| 611 | rank_info[1][i] = recv_buf[3*i+1]; |
---|
| 612 | ep_info[0][i] = recv_buf[3*i+2]; |
---|
| 613 | } |
---|
| 614 | |
---|
| 615 | |
---|
| 616 | if(is_local_leader) |
---|
| 617 | { |
---|
| 618 | leader_info[0] = rank_in_world; |
---|
| 619 | leader_info[1] = mpi_remote_leader; |
---|
| 620 | |
---|
| 621 | ::MPI_Status mpi_status; |
---|
| 622 | |
---|
| 623 | send_buf[0] = size_info[0]; |
---|
| 624 | send_buf[1] = local_ep_size; |
---|
| 625 | |
---|
[1354] | 626 | ::MPI_Send(send_buf.data(), 2, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 627 | |
---|
[1354] | 628 | ::MPI_Recv(recv_buf.data(), 2, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
[1134] | 629 | |
---|
| 630 | recv_buf[2] = rank_in_world; |
---|
| 631 | recv_buf[3] = mpi_remote_leader; |
---|
| 632 | |
---|
| 633 | } |
---|
| 634 | |
---|
[1354] | 635 | ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 636 | |
---|
| 637 | size_info[1] = recv_buf[0]; |
---|
| 638 | remote_ep_size = recv_buf[1]; |
---|
| 639 | leader_info[0] = recv_buf[2]; |
---|
| 640 | leader_info[1] = recv_buf[3]; |
---|
| 641 | |
---|
| 642 | rank_info[2].resize(size_info[1]); |
---|
| 643 | rank_info[3].resize(size_info[1]); |
---|
| 644 | |
---|
| 645 | ep_info[1].resize(size_info[1]); |
---|
| 646 | |
---|
| 647 | send_buf.resize(3*size_info[0]); |
---|
| 648 | recv_buf.resize(3*size_info[1]); |
---|
| 649 | |
---|
| 650 | if(is_local_leader) |
---|
| 651 | { |
---|
| 652 | ::MPI_Status mpi_status; |
---|
| 653 | |
---|
| 654 | |
---|
| 655 | std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() ); |
---|
| 656 | std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] ); |
---|
| 657 | std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] ); |
---|
| 658 | |
---|
[1354] | 659 | ::MPI_Send(send_buf.data(), 3*size_info[0], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 660 | |
---|
[1354] | 661 | ::MPI_Recv(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
[1134] | 662 | |
---|
| 663 | } |
---|
| 664 | |
---|
[1354] | 665 | ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 666 | |
---|
| 667 | std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() ); |
---|
| 668 | std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() ); |
---|
| 669 | std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() ); |
---|
| 670 | |
---|
| 671 | offset.resize(size_info[0]); |
---|
| 672 | |
---|
| 673 | if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader |
---|
| 674 | { |
---|
| 675 | |
---|
| 676 | bool found = false; |
---|
| 677 | int ep_tmp; |
---|
| 678 | int ep_local; |
---|
| 679 | int ep_remote; |
---|
| 680 | for(int i=0; i<size_info[0]; i++) |
---|
| 681 | { |
---|
| 682 | int target = rank_info[0][i]; |
---|
| 683 | found = false; |
---|
| 684 | for(int j=0; j<size_info[1]; j++) |
---|
| 685 | { |
---|
| 686 | if(target == rank_info[2][j]) |
---|
| 687 | { |
---|
| 688 | found = true; |
---|
| 689 | ep_tmp = ep_info[1][j]; |
---|
| 690 | ep_local = ep_info[0][j]; |
---|
| 691 | ep_remote = ep_info[1][j]; |
---|
| 692 | break; |
---|
| 693 | } |
---|
| 694 | } |
---|
| 695 | if(found) |
---|
| 696 | { |
---|
| 697 | |
---|
| 698 | if(target == leader_info[0]) // the leader is doubled in remote |
---|
| 699 | { |
---|
| 700 | new_rank_info[0].push_back(target); |
---|
| 701 | new_rank_info[1].push_back(rank_info[1][i]); |
---|
| 702 | |
---|
| 703 | new_ep_info[0].push_back(ep_local + ep_remote); |
---|
| 704 | offset[i] = 0; |
---|
| 705 | } |
---|
| 706 | else |
---|
| 707 | { |
---|
| 708 | offset[i] = ep_local; |
---|
| 709 | } |
---|
| 710 | } |
---|
| 711 | else |
---|
| 712 | { |
---|
| 713 | new_rank_info[0].push_back(target); |
---|
| 714 | new_rank_info[1].push_back(rank_info[1][i]); |
---|
| 715 | |
---|
| 716 | new_ep_info[0].push_back(ep_info[0][i]); |
---|
| 717 | |
---|
| 718 | offset[i] = 0; |
---|
| 719 | } |
---|
| 720 | |
---|
| 721 | } |
---|
| 722 | } |
---|
| 723 | |
---|
| 724 | else // erase rank doubled with remote leader |
---|
| 725 | { |
---|
| 726 | |
---|
| 727 | bool found = false; |
---|
| 728 | int ep_tmp; |
---|
| 729 | int ep_local; |
---|
| 730 | int ep_remote; |
---|
| 731 | for(int i=0; i<size_info[0]; i++) |
---|
| 732 | { |
---|
| 733 | int target = rank_info[0][i]; |
---|
| 734 | found = false; |
---|
| 735 | for(int j=0; j<size_info[1]; j++) |
---|
| 736 | { |
---|
| 737 | |
---|
| 738 | if(target == rank_info[2][j]) |
---|
| 739 | { |
---|
| 740 | found = true; |
---|
| 741 | ep_tmp = ep_info[1][j]; |
---|
| 742 | ep_local = ep_info[0][j]; |
---|
| 743 | ep_remote = ep_info[1][j]; |
---|
| 744 | break; |
---|
| 745 | } |
---|
| 746 | } |
---|
| 747 | if(found) |
---|
| 748 | { |
---|
| 749 | if(target != leader_info[1]) |
---|
| 750 | { |
---|
| 751 | new_rank_info[0].push_back(target); |
---|
| 752 | new_rank_info[1].push_back(rank_info[1][i]); |
---|
| 753 | |
---|
| 754 | new_ep_info[0].push_back(ep_local + ep_remote); |
---|
| 755 | offset[i] = 0; |
---|
| 756 | } |
---|
| 757 | else // found remote leader |
---|
| 758 | { |
---|
| 759 | offset[i] = ep_remote; |
---|
| 760 | } |
---|
| 761 | } |
---|
| 762 | else |
---|
| 763 | { |
---|
| 764 | new_rank_info[0].push_back(target); |
---|
| 765 | new_rank_info[1].push_back(rank_info[1][i]); |
---|
| 766 | |
---|
| 767 | new_ep_info[0].push_back(ep_info[0][i]); |
---|
| 768 | offset[i] = 0; |
---|
| 769 | } |
---|
| 770 | } |
---|
| 771 | } |
---|
| 772 | |
---|
| 773 | if(offset[mpi_rank] == 0) |
---|
| 774 | { |
---|
| 775 | is_final_master = true; |
---|
| 776 | } |
---|
| 777 | |
---|
| 778 | |
---|
| 779 | // size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote |
---|
| 780 | |
---|
| 781 | if(is_local_leader) |
---|
| 782 | { |
---|
| 783 | size_info[2] = new_ep_info[0].size(); |
---|
| 784 | ::MPI_Status mpi_status; |
---|
| 785 | |
---|
[1354] | 786 | ::MPI_Send(&size_info[2], 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 787 | |
---|
[1354] | 788 | ::MPI_Recv(&size_info[3], 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
[1134] | 789 | } |
---|
| 790 | |
---|
[1354] | 791 | ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 792 | |
---|
| 793 | new_rank_info[2].resize(size_info[3]); |
---|
| 794 | new_rank_info[3].resize(size_info[3]); |
---|
| 795 | new_ep_info[1].resize(size_info[3]); |
---|
| 796 | |
---|
| 797 | send_buf.resize(size_info[2]); |
---|
| 798 | recv_buf.resize(size_info[3]); |
---|
| 799 | |
---|
| 800 | if(is_local_leader) |
---|
| 801 | { |
---|
| 802 | ::MPI_Status mpi_status; |
---|
| 803 | |
---|
| 804 | std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() ); |
---|
| 805 | std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] ); |
---|
| 806 | std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] ); |
---|
| 807 | |
---|
[1354] | 808 | ::MPI_Send(send_buf.data(), 3*size_info[2], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 809 | |
---|
[1354] | 810 | ::MPI_Recv(recv_buf.data(), 3*size_info[3], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
[1134] | 811 | } |
---|
| 812 | |
---|
[1354] | 813 | ::MPI_Bcast(recv_buf.data(), 3*size_info[3], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 814 | |
---|
| 815 | std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() ); |
---|
| 816 | std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() ); |
---|
| 817 | std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() ); |
---|
| 818 | |
---|
| 819 | } |
---|
| 820 | |
---|
| 821 | |
---|
| 822 | |
---|
| 823 | if(is_proc_master) |
---|
| 824 | { |
---|
| 825 | // leader_info[4]: 2-> rank of local leader in new_group generated comm; |
---|
| 826 | // 3-> rank of remote leader in new_group generated comm; |
---|
| 827 | ::MPI_Group local_group; |
---|
| 828 | ::MPI_Group new_group; |
---|
[1354] | 829 | ::MPI_Comm *new_comm = new ::MPI_Comm; |
---|
| 830 | ::MPI_Comm *intercomm = new ::MPI_Comm; |
---|
[1134] | 831 | |
---|
| 832 | ::MPI_Comm_group(local_mpi_comm, &local_group); |
---|
| 833 | |
---|
| 834 | ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group); |
---|
| 835 | |
---|
[1354] | 836 | ::MPI_Comm_create(local_mpi_comm, new_group, new_comm); |
---|
[1134] | 837 | |
---|
| 838 | |
---|
| 839 | |
---|
| 840 | if(is_local_leader) |
---|
| 841 | { |
---|
[1354] | 842 | ::MPI_Comm_rank(*new_comm, &leader_info[2]); |
---|
[1134] | 843 | } |
---|
| 844 | |
---|
[1354] | 845 | ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); |
---|
[1134] | 846 | |
---|
[1354] | 847 | if(new_comm != static_cast< ::MPI_Comm* >(MPI_COMM_NULL.mpi_comm)) |
---|
[1134] | 848 | { |
---|
[1354] | 849 | ::MPI_Barrier(*new_comm); |
---|
[1134] | 850 | |
---|
[1354] | 851 | ::MPI_Intercomm_create(*new_comm, leader_info[2], peer_comm, leader_info[1], tag, intercomm); |
---|
[1134] | 852 | |
---|
| 853 | int id; |
---|
[1354] | 854 | ::MPI_Comm_rank(*new_comm, &id); |
---|
[1134] | 855 | int my_num_ep = new_ep_info[0][id]; |
---|
| 856 | |
---|
| 857 | MPI_Comm *ep_intercomm; |
---|
| 858 | MPI_Info info; |
---|
| 859 | MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm); |
---|
| 860 | |
---|
| 861 | for(int i= 0; i<my_num_ep; i++) |
---|
| 862 | { |
---|
| 863 | ep_intercomm[i].is_intercomm = true; |
---|
| 864 | |
---|
| 865 | ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; |
---|
| 866 | ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm; |
---|
| 867 | ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0]; |
---|
| 868 | } |
---|
| 869 | |
---|
| 870 | |
---|
| 871 | #pragma omp critical (write_to_tag_list) |
---|
| 872 | tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm)); |
---|
[1287] | 873 | |
---|
[1134] | 874 | |
---|
| 875 | } |
---|
| 876 | |
---|
| 877 | |
---|
| 878 | } |
---|
| 879 | |
---|
| 880 | |
---|
| 881 | vector<int> bcast_buf(8); |
---|
| 882 | if(is_local_leader) |
---|
| 883 | { |
---|
| 884 | std::copy(size_info, size_info+4, bcast_buf.begin()); |
---|
| 885 | std::copy(leader_info, leader_info+4, bcast_buf.begin()+4); |
---|
| 886 | } |
---|
| 887 | |
---|
| 888 | |
---|
| 889 | |
---|
[1354] | 890 | MPI_Bcast(bcast_buf.data(), 8, MPI_INT, local_leader, local_comm); |
---|
[1134] | 891 | |
---|
| 892 | |
---|
| 893 | if(!is_local_leader) |
---|
| 894 | { |
---|
| 895 | std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info); |
---|
| 896 | std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info); |
---|
| 897 | } |
---|
| 898 | |
---|
| 899 | |
---|
| 900 | |
---|
| 901 | if(!is_local_leader) |
---|
| 902 | { |
---|
| 903 | new_rank_info[1].resize(size_info[2]); |
---|
| 904 | ep_info[1].resize(size_info[1]); |
---|
| 905 | offset.resize(size_info[0]); |
---|
| 906 | } |
---|
| 907 | |
---|
| 908 | bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1); |
---|
| 909 | |
---|
| 910 | if(is_local_leader) |
---|
| 911 | { |
---|
| 912 | bcast_buf[0] = remote_ep_size; |
---|
| 913 | std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1); |
---|
| 914 | std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1); |
---|
| 915 | std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1); |
---|
| 916 | } |
---|
| 917 | |
---|
[1354] | 918 | MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT, local_leader, local_comm); |
---|
[1134] | 919 | |
---|
| 920 | |
---|
| 921 | if(!is_local_leader) |
---|
| 922 | { |
---|
| 923 | remote_ep_size = bcast_buf[0]; |
---|
| 924 | std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin()); |
---|
| 925 | std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin()); |
---|
| 926 | std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin()); |
---|
| 927 | } |
---|
| 928 | |
---|
| 929 | |
---|
| 930 | int my_position = offset[rank_in_local_parent]+ep_rank_loc; |
---|
| 931 | |
---|
| 932 | |
---|
| 933 | #pragma omp critical (read_from_tag_list) |
---|
| 934 | { |
---|
| 935 | bool found = false; |
---|
| 936 | while(!found) |
---|
| 937 | { |
---|
| 938 | for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++) |
---|
| 939 | { |
---|
| 940 | if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1]))) |
---|
| 941 | { |
---|
| 942 | *newintercomm = iter->second[my_position]; |
---|
| 943 | |
---|
| 944 | found = true; |
---|
| 945 | break; |
---|
| 946 | } |
---|
| 947 | } |
---|
| 948 | } |
---|
| 949 | } |
---|
| 950 | |
---|
[1287] | 951 | MPI_Barrier(local_comm); |
---|
[1134] | 952 | |
---|
[1287] | 953 | if(is_local_leader) |
---|
| 954 | { |
---|
| 955 | int local_flag = true; |
---|
| 956 | int remote_flag = false; |
---|
| 957 | ::MPI_Status mpi_status; |
---|
| 958 | |
---|
[1354] | 959 | ::MPI_Send(&local_flag, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1287] | 960 | |
---|
[1354] | 961 | ::MPI_Recv(&remote_flag, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); |
---|
[1287] | 962 | } |
---|
| 963 | |
---|
| 964 | MPI_Barrier(local_comm); |
---|
| 965 | |
---|
[1134] | 966 | if(is_proc_master) |
---|
| 967 | { |
---|
| 968 | for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++) |
---|
| 969 | { |
---|
| 970 | if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1]))) |
---|
| 971 | { |
---|
| 972 | tag_list.erase(iter); |
---|
| 973 | break; |
---|
| 974 | } |
---|
| 975 | } |
---|
| 976 | } |
---|
| 977 | |
---|
| 978 | int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank; |
---|
| 979 | int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size; |
---|
| 980 | |
---|
| 981 | intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first; |
---|
| 982 | intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first; |
---|
| 983 | intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first; |
---|
| 984 | intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second; |
---|
| 985 | intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second; |
---|
| 986 | intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second; |
---|
| 987 | |
---|
| 988 | |
---|
[1354] | 989 | MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm); |
---|
[1134] | 990 | |
---|
| 991 | int my_rank_map_elem[2]; |
---|
| 992 | |
---|
| 993 | |
---|
| 994 | my_rank_map_elem[0] = intercomm_ep_rank; |
---|
| 995 | |
---|
| 996 | my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label; |
---|
| 997 | |
---|
| 998 | vector<pair<int, int> > local_rank_map_array; |
---|
| 999 | vector<pair<int, int> > remote_rank_map_array; |
---|
| 1000 | |
---|
| 1001 | |
---|
| 1002 | (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; |
---|
| 1003 | (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size); |
---|
| 1004 | |
---|
[1287] | 1005 | MPI_Allgather(my_rank_map_elem, 2, MPI_INT, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); |
---|
| 1006 | |
---|
[1134] | 1007 | (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; |
---|
| 1008 | (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size); |
---|
| 1009 | |
---|
| 1010 | int local_intercomm_size = intercomm_ep_size; |
---|
| 1011 | int remote_intercomm_size; |
---|
| 1012 | |
---|
| 1013 | |
---|
| 1014 | if(is_local_leader) |
---|
| 1015 | { |
---|
| 1016 | ::MPI_Status status; |
---|
| 1017 | |
---|
[1354] | 1018 | ::MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 1019 | |
---|
[1354] | 1020 | ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); |
---|
[1134] | 1021 | |
---|
[1354] | 1022 | ::MPI_Send(&local_intercomm_size, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 1023 | |
---|
[1354] | 1024 | ::MPI_Recv(&remote_intercomm_size, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); |
---|
[1134] | 1025 | } |
---|
| 1026 | |
---|
[1354] | 1027 | MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm); |
---|
| 1028 | MPI_Bcast(&remote_intercomm_size, 1, MPI_INT, 0, *newintercomm); |
---|
[1134] | 1029 | |
---|
| 1030 | |
---|
| 1031 | (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; |
---|
| 1032 | (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); |
---|
| 1033 | |
---|
| 1034 | (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0]; |
---|
| 1035 | (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1]; |
---|
| 1036 | (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2]; |
---|
| 1037 | |
---|
| 1038 | |
---|
| 1039 | if(is_local_leader) |
---|
| 1040 | { |
---|
| 1041 | ::MPI_Status status; |
---|
| 1042 | |
---|
[1354] | 1043 | ::MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); |
---|
[1134] | 1044 | |
---|
[1354] | 1045 | ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); |
---|
[1134] | 1046 | } |
---|
| 1047 | |
---|
[1354] | 1048 | MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, 0, *newintercomm); |
---|
[1134] | 1049 | |
---|
| 1050 | (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]); |
---|
| 1051 | (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label; |
---|
| 1052 | |
---|
| 1053 | return MPI_SUCCESS; |
---|
| 1054 | |
---|
| 1055 | } |
---|
| 1056 | |
---|
[1354] | 1057 | // #endif |
---|
[1134] | 1058 | |
---|
| 1059 | } |
---|
| 1060 | |
---|