Ignore:
Timestamp:
12/04/17 17:12:00 (4 years ago)
Author:
yushan
Message:

omp dev : unify MPI_Comm type

File:
1 edited

Legend:

Unmodified
Added
Removed
  • XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm_kernel.cpp

    r1304 r1354  
    22#include <mpi.h> 
    33#include "ep_declaration.hpp" 
     4#include "ep_mpi.hpp" 
    45 
    56using namespace std; 
     
    3233 
    3334 
    34     ::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm); 
     35    ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm.mpi_comm); 
    3536 
    3637     
    37     ::MPI_Comm_rank(static_cast< ::MPI_Comm>(MPI_COMM_WORLD.mpi_comm), &rank_in_world); 
    38     ::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent); 
     38    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD.mpi_comm), &rank_in_world); 
     39    ::MPI_Comm_rank(local_mpi_comm, &rank_in_local_parent); 
    3940     
    4041 
     
    7980      send_buf[2] = num_ep; 
    8081 
    81       ::MPI_Allgather(send_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), recv_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), local_mpi_comm); 
     82      ::MPI_Allgather(send_buf.data(), 3, to_mpi_type(MPI_INT), recv_buf.data(), 3, to_mpi_type(MPI_INT), local_mpi_comm); 
    8283 
    8384      for(int i=0; i<size_info[0]; i++) 
     
    9394        leader_info[1] = remote_leader; 
    9495 
    95         ::MPI_Comm_rank(static_cast< ::MPI_Comm>(peer_comm.mpi_comm), &rank_in_peer_mpi[0]); 
    96  
    97          
     96        ::MPI_Comm_rank(to_mpi_comm(peer_comm.mpi_comm), &rank_in_peer_mpi[0]); 
    9897 
    9998        send_buf[0] = size_info[0]; 
     
    106105        MPI_Status statuses[2]; 
    107106         
    108         MPI_Isend(send_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &requests[0]); 
    109         MPI_Irecv(recv_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &requests[1]); 
     107        MPI_Isend(send_buf.data(), 3, MPI_INT, remote_leader, tag, peer_comm, &requests[0]); 
     108        MPI_Irecv(recv_buf.data(), 3, MPI_INT, remote_leader, tag, peer_comm, &requests[1]); 
    110109 
    111110 
     
    126125      send_buf[4] = rank_in_peer_mpi[1]; 
    127126 
    128       ::MPI_Bcast(send_buf.data(), 5, static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     127      ::MPI_Bcast(send_buf.data(), 5, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    129128 
    130129      size_info[1] = send_buf[0]; 
     
    152151        std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] ); 
    153152 
    154         MPI_Isend(send_buf.data(), 3*size_info[0], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+1, peer_comm, &requests[0]); 
    155         MPI_Irecv(recv_buf.data(), 3*size_info[1], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+1, peer_comm, &requests[1]); 
     153        MPI_Isend(send_buf.data(), 3*size_info[0], MPI_INT, remote_leader, tag+1, peer_comm, &requests[0]); 
     154        MPI_Irecv(recv_buf.data(), 3*size_info[1], MPI_INT, remote_leader, tag+1, peer_comm, &requests[1]); 
    156155         
    157156        MPI_Waitall(2, requests, statuses); 
    158157      } 
    159158 
    160       ::MPI_Bcast(recv_buf.data(), 3*size_info[1], static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     159      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT, local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    161160 
    162161      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() ); 
     
    276275        MPI_Request requests[2]; 
    277276        MPI_Status statuses[2]; 
    278         MPI_Isend(&size_info[2], 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+2, peer_comm, &requests[0]); 
    279         MPI_Irecv(&size_info[3], 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+2, peer_comm, &requests[1]); 
     277        MPI_Isend(&size_info[2], 1, MPI_INT, remote_leader, tag+2, peer_comm, &requests[0]); 
     278        MPI_Irecv(&size_info[3], 1, MPI_INT, remote_leader, tag+2, peer_comm, &requests[1]); 
    280279          
    281280        MPI_Waitall(2, requests, statuses); 
    282281      } 
    283282 
    284       ::MPI_Bcast(&size_info[2], 2, static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     283      ::MPI_Bcast(&size_info[2], 2, MPI_INT, local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    285284 
    286285      new_rank_info[2].resize(size_info[3]); 
     
    300299        std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] ); 
    301300 
    302         MPI_Isend(send_buf.data(), 3*size_info[2], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+3, peer_comm, &requests[0]); 
    303         MPI_Irecv(recv_buf.data(), 3*size_info[3], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+3, peer_comm, &requests[1]); 
     301        MPI_Isend(send_buf.data(), 3*size_info[2], MPI_INT, remote_leader, tag+3, peer_comm, &requests[0]); 
     302        MPI_Irecv(recv_buf.data(), 3*size_info[3], MPI_INT, remote_leader, tag+3, peer_comm, &requests[1]); 
    304303         
    305304        MPI_Waitall(2, requests, statuses); 
    306305      } 
    307306 
    308       ::MPI_Bcast(recv_buf.data(),   3*size_info[3], static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     307      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    309308 
    310309      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() ); 
     
    322321      ::MPI_Group local_group; 
    323322      ::MPI_Group new_group; 
    324       ::MPI_Comm new_comm; 
    325       ::MPI_Comm intercomm; 
     323      ::MPI_Comm *new_comm = new ::MPI_Comm; 
     324      ::MPI_Comm *intercomm = new ::MPI_Comm; 
    326325 
    327326      ::MPI_Comm_group(local_mpi_comm, &local_group); 
     
    329328      ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group); 
    330329 
    331       ::MPI_Comm_create(local_mpi_comm, new_group, &new_comm); 
     330      ::MPI_Comm_create(local_mpi_comm, new_group, new_comm); 
    332331 
    333332 
     
    335334      if(is_local_leader) 
    336335      { 
    337         ::MPI_Comm_rank(new_comm, &leader_info[2]); 
    338       } 
    339  
    340       ::MPI_Bcast(&leader_info[2], 1, static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    341  
    342       if(new_comm != static_cast< ::MPI_Comm>(MPI_COMM_NULL.mpi_comm)) 
    343       { 
    344  
    345         ::MPI_Barrier(new_comm); 
    346  
    347         ::MPI_Intercomm_create(new_comm, leader_info[2], static_cast< ::MPI_Comm>(peer_comm.mpi_comm), rank_in_peer_mpi[1], tag, &intercomm); 
     336        ::MPI_Comm_rank(*new_comm, &leader_info[2]); 
     337      } 
     338 
     339      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     340 
     341      if(new_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 
     342      { 
     343 
     344        ::MPI_Barrier(*new_comm); 
     345 
     346        ::MPI_Intercomm_create(*new_comm, leader_info[2], to_mpi_comm(peer_comm.mpi_comm), rank_in_peer_mpi[1], tag, intercomm); 
    348347 
    349348        int id; 
    350349 
    351         ::MPI_Comm_rank(new_comm, &id); 
     350        ::MPI_Comm_rank(*new_comm, &id); 
    352351        int my_num_ep = new_ep_info[0][id]; 
    353352 
     
    380379    } 
    381380 
    382     MPI_Bcast(bcast_buf.data(), 8, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm); 
     381    MPI_Bcast(bcast_buf.data(), 8, MPI_INT, local_leader, local_comm); 
    383382 
    384383    if(!is_local_leader) 
     
    405404    } 
    406405 
    407     MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm); 
     406    MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT, local_leader, local_comm); 
    408407 
    409408    if(!is_local_leader) 
     
    478477    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second; 
    479478 
    480     MPI_Bcast(&remote_ep_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm); 
     479    MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm); 
    481480 
    482481    int my_rank_map_elem[2]; 
     
    492491    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size); 
    493492 
    494     MPI_Allgather(my_rank_map_elem, 2, static_cast< ::MPI_Datatype> (MPI_INT),  
    495       (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, static_cast< ::MPI_Datatype> (MPI_INT), local_comm); 
     493    MPI_Allgather(my_rank_map_elem, 2, MPI_INT,  
     494      (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 
    496495 
    497496    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
     
    514513      MPI_Status statuses[4]; 
    515514       
    516       MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+4, peer_comm, &requests[0]); 
    517       MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+4, peer_comm, &requests[1]); 
    518  
    519       MPI_Isend(&local_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+5, peer_comm, &requests[2]); 
    520       MPI_Irecv(&remote_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+5, peer_comm, &requests[3]); 
     515      MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT, remote_leader, tag+4, peer_comm, &requests[0]); 
     516      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, remote_leader, tag+4, peer_comm, &requests[1]); 
     517 
     518      MPI_Isend(&local_intercomm_size, 1, MPI_INT, remote_leader, tag+5, peer_comm, &requests[2]); 
     519      MPI_Irecv(&remote_intercomm_size, 1, MPI_INT, remote_leader, tag+5, peer_comm, &requests[3]); 
    521520       
    522521      MPI_Waitall(4, requests, statuses); 
     
    525524    } 
    526525 
    527     MPI_Allreduce(&new_bcast_root_0, &new_bcast_root, 1, static_cast< ::MPI_Datatype> (MPI_INT), static_cast< ::MPI_Op>(MPI_SUM), *newintercomm); 
    528  
    529  
    530     MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm); 
    531     MPI_Bcast(&remote_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), new_bcast_root, *newintercomm); 
     526    MPI_Allreduce(&new_bcast_root_0, &new_bcast_root, 1, MPI_INT, static_cast< ::MPI_Op>(MPI_SUM), *newintercomm); 
     527 
     528 
     529    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm); 
     530    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT, new_bcast_root, *newintercomm); 
    532531 
    533532 
     
    543542      MPI_Status statuses[2]; 
    544543       
    545       MPI_Isend((*newintercomm).rank_map->data(), 2*local_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+6, peer_comm, &requests[0]); 
    546       MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+6, peer_comm, &requests[1]); 
     544      MPI_Isend((*newintercomm).rank_map->data(), 2*local_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[0]); 
     545      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[1]); 
    547546       
    548547      MPI_Waitall(2, requests, statuses); 
    549548    } 
    550549 
    551     MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), new_bcast_root, *newintercomm); 
     550    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, new_bcast_root, *newintercomm); 
    552551 
    553552    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]); 
     
    610609    int rank_in_peer_mpi[2]; 
    611610 
    612     ::MPI_Comm_rank(static_cast< ::MPI_Comm >(MPI_COMM_WORLD.mpi_comm), &rank_in_world); 
     611    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD.mpi_comm), &rank_in_world); 
    613612 
    614613 
     
    639638      MPI_Request req_s, req_r; 
    640639 
    641       MPI_Isend(send_buf.data(), 2, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &req_s); 
    642       MPI_Irecv(recv_buf.data(), 2, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &req_r); 
     640      MPI_Isend(send_buf.data(), 2, MPI_INT, remote_leader, tag, peer_comm, &req_s); 
     641      MPI_Irecv(recv_buf.data(), 2, MPI_INT, remote_leader, tag, peer_comm, &req_r); 
    643642 
    644643 
     
    650649    } 
    651650 
    652     MPI_Bcast(recv_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm); 
     651    MPI_Bcast(recv_buf.data(), 3, MPI_INT, local_leader, local_comm); 
    653652 
    654653    remote_num_ep = recv_buf[0]; 
     
    665664      if(ep_rank == local_leader) 
    666665      { 
    667         ::MPI_Comm mpi_dup; 
     666        ::MPI_Comm *mpi_dup = new ::MPI_Comm; 
    668667         
    669         ::MPI_Comm_dup(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &mpi_dup); 
     668        ::MPI_Comm_dup(to_mpi_comm(local_comm.mpi_comm), mpi_dup); 
    670669 
    671670        MPI_Comm *ep_intercomm; 
     
    691690        MPI_Request req_s; 
    692691        MPI_Status sta_s; 
    693         MPI_Isend(tag_label, 2, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &req_s); 
     692        MPI_Isend(tag_label, 2, MPI_INT, remote_leader, tag, peer_comm, &req_s); 
    694693 
    695694        MPI_Wait(&req_s, &sta_s); 
     
    705704        MPI_Status status; 
    706705        MPI_Request req_r; 
    707         MPI_Irecv(tag_label, 2, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &req_r); 
     706        MPI_Irecv(tag_label, 2, MPI_INT, remote_leader, tag, peer_comm, &req_r); 
    708707        MPI_Wait(&req_r, &status); 
    709708      } 
    710709    } 
    711710 
    712     MPI_Bcast(tag_label, 2, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm); 
     711    MPI_Bcast(tag_label, 2, MPI_INT, local_leader, local_comm); 
    713712 
    714713 
     
    776775    local_rank_map_ele[1] = (*newintercomm).ep_comm_ptr->comm_label; 
    777776 
    778     MPI_Allgather(local_rank_map_ele, 2, static_cast< ::MPI_Datatype> (MPI_INT),  
    779       (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, static_cast< ::MPI_Datatype> (MPI_INT), local_comm); 
     777    MPI_Allgather(local_rank_map_ele, 2, MPI_INT,  
     778      (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 
    780779 
    781780    if(ep_rank == local_leader) 
     
    784783      MPI_Request req_s, req_r; 
    785784 
    786       MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_num_ep, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &req_s); 
    787       MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &req_r); 
     785      MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_num_ep, MPI_INT, remote_leader, tag, peer_comm, &req_s); 
     786      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT, remote_leader, tag, peer_comm, &req_r); 
    788787 
    789788 
     
    793792    } 
    794793 
    795     MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm); 
     794    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT, local_leader, local_comm); 
    796795    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]); 
    797796    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = tag; 
Note: See TracChangeset for help on using the changeset viewer.