Changeset 1304 for XIOS/dev


Ignore:
Timestamp:
10/11/17 16:19:45 (7 years ago)
Author:
yushan
Message:

change blocking calls to non-blocking calls for intercomm creation. Tested with test_client and test_complete for upto 100 clients

File:
1 edited

Legend:

Unmodified
Added
Removed

  • XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm_kernel.cpp

    r1287 r1304  
    101101        send_buf[2] = rank_in_peer_mpi[0]; 
    102102 
    103         MPI_Request req_send, req_recv; 
    104         MPI_Status sta_send, sta_recv; 
    105103         
    106         MPI_Isend(send_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &req_send); 
    107         MPI_Irecv(recv_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &req_recv); 
    108  
    109  
    110         MPI_Wait(&req_send, &sta_send); 
    111         MPI_Wait(&req_recv, &sta_recv); 
     104         
     105        MPI_Request requests[2]; 
     106        MPI_Status statuses[2]; 
     107         
     108        MPI_Isend(send_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &requests[0]); 
     109        MPI_Irecv(recv_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag, peer_comm, &requests[1]); 
     110 
     111 
     112        MPI_Waitall(2, requests, statuses); 
    112113         
    113114        size_info[1] = recv_buf[0]; 
    … …  
    144145      if(is_local_leader) 
    145146      { 
    146         MPI_Status status; 
     147        MPI_Request requests[2]; 
     148        MPI_Status statuses[2]; 
    147149 
    148150        std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() ); 
    … …  
    150152        std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] ); 
    151153 
    152         MPI_Send(send_buf.data(), 3*size_info[0], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+1, peer_comm); 
    153         MPI_Recv(recv_buf.data(), 3*size_info[1], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+1, peer_comm, &status); 
     154        MPI_Isend(send_buf.data(), 3*size_info[0], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+1, peer_comm, &requests[0]); 
     155        MPI_Irecv(recv_buf.data(), 3*size_info[1], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+1, peer_comm, &requests[1]); 
     156         
     157        MPI_Waitall(2, requests, statuses); 
    154158      } 
    155159 
    … …  
    270274      { 
    271275        size_info[2] = new_ep_info[0].size(); 
    272         MPI_Status status; 
    273         MPI_Send(&size_info[2], 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+2, peer_comm); 
    274         MPI_Recv(&size_info[3], 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+2, peer_comm, &status); 
     276        MPI_Request requests[2]; 
     277        MPI_Status statuses[2]; 
     278        MPI_Isend(&size_info[2], 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+2, peer_comm, &requests[0]); 
     279        MPI_Irecv(&size_info[3], 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+2, peer_comm, &requests[1]); 
     280          
     281        MPI_Waitall(2, requests, statuses); 
    275282      } 
    276283 
    … …  
    286293      if(is_local_leader) 
    287294      { 
    288         MPI_Status status; 
     295        MPI_Request requests[2]; 
     296        MPI_Status statuses[2]; 
    289297 
    290298        std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() ); 
    … …  
    292300        std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] ); 
    293301 
    294         MPI_Send(send_buf.data(), 3*size_info[2], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+3, peer_comm); 
    295         MPI_Recv(recv_buf.data(), 3*size_info[3], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+3, peer_comm, &status); 
     302        MPI_Isend(send_buf.data(), 3*size_info[2], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+3, peer_comm, &requests[0]); 
     303        MPI_Irecv(recv_buf.data(), 3*size_info[3], static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+3, peer_comm, &requests[1]); 
     304         
     305        MPI_Waitall(2, requests, statuses); 
    296306      } 
    297307 
    … …  
    434444      int local_flag = true; 
    435445      int remote_flag = false; 
    436       MPI_Status mpi_status; 
     446      MPI_Request mpi_requests[2]; 
     447      MPI_Status mpi_statuses[2]; 
    437448       
    438       MPI_Send(&local_flag, 1, MPI_INT, remote_leader, tag, peer_comm); 
    439  
    440       MPI_Recv(&remote_flag, 1, MPI_INT, remote_leader, tag, peer_comm, &mpi_status); 
     449      MPI_Isend(&local_flag, 1, MPI_INT, remote_leader, tag, peer_comm, &mpi_requests[0]); 
     450      MPI_Irecv(&remote_flag, 1, MPI_INT, remote_leader, tag, peer_comm, &mpi_requests[1]); 
     451       
     452      MPI_Waitall(2, mpi_requests, mpi_statuses); 
    441453    } 
    442454 
    … …  
    499511    if(is_local_leader) 
    500512    { 
    501       MPI_Status status; 
    502       MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+4, peer_comm); 
    503       MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+4, peer_comm, &status); 
    504  
    505       MPI_Send(&local_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+5, peer_comm); 
    506       MPI_Recv(&remote_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+5, peer_comm, &status); 
     513      MPI_Request requests[4]; 
     514      MPI_Status statuses[4]; 
     515       
     516      MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+4, peer_comm, &requests[0]); 
     517      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+4, peer_comm, &requests[1]); 
     518 
     519      MPI_Isend(&local_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+5, peer_comm, &requests[2]); 
     520      MPI_Irecv(&remote_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+5, peer_comm, &requests[3]); 
     521       
     522      MPI_Waitall(4, requests, statuses); 
    507523 
    508524      new_bcast_root_0 = intercomm_ep_rank; 
    … …  
    524540    if(is_local_leader) 
    525541    { 
    526       MPI_Status status; 
    527       MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+6, peer_comm); 
    528       MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+6, peer_comm, &status); 
     542      MPI_Request requests[2]; 
     543      MPI_Status statuses[2]; 
     544       
     545      MPI_Isend((*newintercomm).rank_map->data(), 2*local_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+6, peer_comm, &requests[0]); 
     546      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), remote_leader, tag+6, peer_comm, &requests[1]); 
     547       
     548      MPI_Waitall(2, requests, statuses); 
    529549    } 
    530550 
Note: See TracChangeset for help on using the changeset viewer.