Ignore:
Timestamp:
05/28/18 09:54:32 (6 years ago)
Author:
yushan
Message:

save dev

File:
1 edited

Legend:

Unmodified
Added
Removed

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_send.cpp

    r1381 r1500  
    1717  int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
    1818  { 
    19     if(!comm.is_ep) 
    20       return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm.mpi_comm)); 
    21     if(comm.is_intercomm) 
     19    if(!comm->is_ep) 
     20      return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 
     21    if(comm->is_intercomm) 
    2222    { 
    2323      MPI_Request request; 
    … …  
    2828    else 
    2929    { 
    30       int ep_src_loc  = comm.ep_comm_ptr->size_rank_info[1].first; 
    31       int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first; 
     30      int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     31      int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first; 
    3232      int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    33       int mpi_dest    = comm.ep_comm_ptr->comm_list->rank_map->at(dest).second; 
    34  
    35       ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm)); 
    36       //printf("call mpi_send for intracomm, dest = %d, tag = %d\n", dest, tag); 
     33      int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second; 
     34 
     35      ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm)); 
    3736    } 
    3837    //check_sum_send(buf, count, datatype, dest, tag, comm); 
    … …  
    4443  int MPI_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
    4544  { 
    46     if(!comm.is_ep) 
    47     { 
    48       return ::MPI_Ssend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm.mpi_comm)); 
     45    if(!comm->is_ep) 
     46    { 
     47      return ::MPI_Ssend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 
    4948    } 
    5049 
    … …  
    6968     
    7069 
    71     if(!comm.is_ep) 
    72     { 
    73       ::MPI_Request mpi_request; 
    74       ::MPI_Isend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    75  
    76       request->mpi_request = new ::MPI_Request(mpi_request); 
    77  
    78       request->ep_src = src_rank; 
    79       request->ep_tag = tag; 
    80       request->ep_datatype = datatype; 
    81       request->type = 1; 
    82       request->comm = comm; 
     70    if(!comm->is_ep) 
     71    { 
     72      ::MPI_Request mpi_request; 
     73      ::MPI_Isend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
     74 
     75      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     76 
     77      (*request)->ep_src = src_rank; 
     78      (*request)->ep_tag = tag; 
     79      (*request)->ep_datatype = datatype; 
     80      (*request)->type = 1; 
     81      (*request)->comm = comm; 
    8382 
    8483      return 0; 
    8584    } 
    8685 
    87     if(comm.is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request); 
     86    if(comm->is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request); 
    8887 
    8988    // EP intracomm 
    9089 
    9190    //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    92  
    93     int ep_src_loc  = comm.ep_comm_ptr->size_rank_info[1].first; 
    94     int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first; 
     91     
     92    *request = new ep_request; 
     93    printf("new %p : in ep_lib::MPI_Isend, *request = new ep_request\n", (*request)); 
     94 
     95    (*request)->mpi_request = new ::MPI_Request; 
     96    printf("new %p : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request\n", (*request)->mpi_request); 
     97     
     98 
     99    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     100    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first; 
    95101    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    96     int mpi_dest    = comm.ep_comm_ptr->comm_list->rank_map->at(dest).second; 
    97  
    98     request->ep_src  = src_rank; 
    99     request->ep_tag  = tag; 
    100     request->ep_datatype = datatype; 
    101  
    102     ::MPI_Request mpi_request; 
    103  
    104     ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    105  
    106     request->mpi_request = new ::MPI_Request(mpi_request); 
    107     request->type = 1;    // used in wait 
    108     request->comm = comm; 
    109     request->buf = const_cast<void*>(buf); 
    110  
    111     //Message_Check(comm); 
     102    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second; 
     103 
     104    (*request)->ep_src  = src_rank; 
     105    (*request)->ep_tag  = tag; 
     106    (*request)->ep_datatype = datatype; 
     107 
     108    ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
     109 
     110    (*request)->type = 1;    // used in wait 
     111    (*request)->comm = comm; 
     112    (*request)->buf = const_cast<void*>(buf); 
     113 
    112114 
    113115    return 0; 
    … …  
    126128     
    127129 
    128     if(!comm.is_ep) 
    129     { 
    130       ::MPI_Request mpi_request; 
    131       ::MPI_Issend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    132  
    133       request->mpi_request = new ::MPI_Request(mpi_request); 
    134       request->ep_src = src_rank; 
    135       request->ep_tag = tag; 
    136       request->ep_datatype = datatype; 
    137       request->type = 1; 
    138       request->comm = comm; 
     130    if(!comm->is_ep) 
     131    { 
     132      ::MPI_Request mpi_request; 
     133      ::MPI_Issend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
     134 
     135      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     136      (*request)->ep_src = src_rank; 
     137      (*request)->ep_tag = tag; 
     138      (*request)->ep_datatype = datatype; 
     139      (*request)->type = 1; 
     140      (*request)->comm = comm; 
    139141 
    140142      return 0; 
    141143    } 
    142144 
    143     if(comm.is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request); 
     145    if(comm->is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request); 
    144146 
    145147    // EP intracomm 
    … …  
    147149    //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    148150 
    149     int ep_src_loc  = comm.ep_comm_ptr->size_rank_info[1].first; 
    150     int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first; 
     151    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     152    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first; 
    151153    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    152     int mpi_dest    = comm.ep_comm_ptr->comm_list->rank_map->at(dest).second; 
    153      
    154     request->ep_src = src_rank; 
    155     request->ep_tag = tag; 
    156     request->ep_datatype = datatype; 
     154    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second; 
     155     
     156    (*request)->ep_src = src_rank; 
     157    (*request)->ep_tag = tag; 
     158    (*request)->ep_datatype = datatype; 
    157159 
    158160    ::MPI_Request mpi_request; 
    159161 
    160     ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    161  
    162     request->mpi_request = new ::MPI_Request(mpi_request); 
    163     request->type = 1;    // used in wait 
    164     request->comm = comm; 
    165     request->buf = NULL; 
     162    ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
     163 
     164    (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     165    (*request)->type = 1;    // used in wait 
     166    (*request)->comm = comm; 
     167    (*request)->buf = NULL; 
    166168     
    167169 
    … …  
    180182    //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    181183 
    182     int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
    183     int dest_remote_comm_label = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; 
    184  
    185     int src_ep_rank    = comm.ep_comm_ptr->intercomm->size_rank_info[0].first; 
     184    int dest_remote_ep_rank    = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
     185    int dest_remote_comm_label = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; 
     186 
     187    int src_ep_rank    = comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 
    186188    int src_comm_label; 
    187189 
    188     src_comm_label = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second; 
     190    src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second; 
    189191 
    190192 
    … …  
    196198    if(dest_remote_comm_label == src_comm_label)       // mpi_dest differs 
    197199    { 
    198       int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    199       int ep_src_loc = comm.rank_map->at(inter_src).first; 
    200       int ep_dest_loc = comm.rank_map->at(dest_remote_ep_rank).first; 
    201       int mpi_dest    = comm.rank_map->at(dest_remote_ep_rank).second; 
     200      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
     201      int ep_src_loc = comm->rank_map->at(inter_src).first; 
     202      int ep_dest_loc = comm->rank_map->at(dest_remote_ep_rank).first; 
     203      int mpi_dest    = comm->rank_map->at(dest_remote_ep_rank).second; 
    202204      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    203205 
    204206      ::MPI_Request mpi_request; 
    205207   
    206       ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    207  
    208       request->mpi_request = new ::MPI_Request(mpi_request); 
    209       request->type = 1;    // used in wait 
    210       request->comm = comm; 
    211  
    212       request->ep_src = src_ep_rank; 
    213       request->ep_tag = tag; 
    214       request->ep_datatype = datatype; 
     208      ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
     209 
     210      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     211      (*request)->type = 1;    // used in wait 
     212      (*request)->comm = comm; 
     213 
     214      (*request)->ep_src = src_ep_rank; 
     215      (*request)->ep_tag = tag; 
     216      (*request)->ep_datatype = datatype; 
    215217    } 
    216218 
    217219    else   // dest_remote_comm_label != src_comm_label 
    218220    {  
    219       int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    220       int ep_src_loc = comm.rank_map->at(inter_src).first; 
    221       int ep_dest_loc = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first; 
    222       int mpi_dest    = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second; 
     221      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
     222      int ep_src_loc = comm->rank_map->at(inter_src).first; 
     223      int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first; 
     224      int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second; 
    223225      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    224226 
    225227      ::MPI_Request mpi_request; 
    226228 
    227       ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request); 
    228  
    229       request->mpi_request = new ::MPI_Request(mpi_request); 
    230       request->type = 1;    // used in wait 
    231       request->comm = comm; 
     229      ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request); 
     230 
     231      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     232      (*request)->type = 1;    // used in wait 
     233      (*request)->comm = comm; 
    232234    
    233       request->ep_src = src_ep_rank; 
    234       request->ep_tag = tag; 
    235       request->ep_datatype = datatype; 
     235      (*request)->ep_src = src_ep_rank; 
     236      (*request)->ep_tag = tag; 
     237      (*request)->ep_datatype = datatype; 
    236238    } 
    237239 
    … …  
    247249    //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    248250 
    249     int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
    250     int dest_remote_comm_label = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; 
    251  
    252     int src_ep_rank    = comm.ep_comm_ptr->intercomm->size_rank_info[0].first; 
     251    int dest_remote_ep_rank    = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
     252    int dest_remote_comm_label = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; 
     253 
     254    int src_ep_rank    = comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 
    253255    int src_comm_label; 
    254256 
    255     for(int i=0; i<comm.ep_comm_ptr->intercomm->local_rank_map->size(); i++) 
    256     { 
    257       if(comm.ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank) 
     257    for(int i=0; i<comm->ep_comm_ptr->intercomm->local_rank_map->size(); i++) 
     258    { 
     259      if(comm->ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank) 
    258260      { 
    259         src_comm_label = comm.ep_comm_ptr->intercomm->local_rank_map->at(i).second; 
     261        src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(i).second; 
    260262        break; 
    261263      } 
    … …  
    267269    if(dest_remote_comm_label == src_comm_label)       // dest rank (loc, mpi) differs 
    268270    { 
    269       int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    270       int ep_src_loc = comm.rank_map->at(inter_src).first; 
    271       int ep_dest_loc = comm.rank_map->at(dest_remote_ep_rank).first; 
    272       int mpi_dest    = comm.rank_map->at(dest_remote_ep_rank).second; 
     271      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
     272      int ep_src_loc = comm->rank_map->at(inter_src).first; 
     273      int ep_dest_loc = comm->rank_map->at(dest_remote_ep_rank).first; 
     274      int mpi_dest    = comm->rank_map->at(dest_remote_ep_rank).second; 
    273275      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    274276 
    275277      ::MPI_Request mpi_request; 
    276278   
    277       ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    278  
    279       request->mpi_request = new ::MPI_Request(mpi_request); 
    280       request->type = 1;    // used in wait 
    281       request->comm = comm; 
    282  
    283       request->ep_src = src_ep_rank; 
    284       request->ep_tag = tag; 
    285       request->ep_datatype = datatype; 
     279      ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
     280 
     281      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     282      (*request)->type = 1;    // used in wait 
     283      (*request)->comm = comm; 
     284 
     285      (*request)->ep_src = src_ep_rank; 
     286      (*request)->ep_tag = tag; 
     287      (*request)->ep_datatype = datatype; 
    286288    } 
    287289 
    288290    else   // dest_remote_comm_label != src_comm_label 
    289291    {  
    290       int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    291       int ep_src_loc = comm.rank_map->at(inter_src).first; 
    292       int ep_dest_loc = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first; 
    293       int mpi_dest    = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second; 
     292      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
     293      int ep_src_loc = comm->rank_map->at(inter_src).first; 
     294      int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first; 
     295      int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second; 
    294296      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    295297 
    296298      ::MPI_Request mpi_request; 
    297299 
    298       ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request); 
    299  
    300       request->mpi_request = new ::MPI_Request(mpi_request); 
    301       request->type = 1;    // used in wait 
    302       request->comm = comm; 
     300      ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request); 
     301 
     302      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     303      (*request)->type = 1;    // used in wait 
     304      (*request)->comm = comm; 
    303305    
    304       request->ep_src = src_ep_rank; 
    305       request->ep_tag = tag; 
    306       request->ep_datatype = datatype; 
     306      (*request)->ep_src = src_ep_rank; 
     307      (*request)->ep_tag = tag; 
     308      (*request)->ep_datatype = datatype; 
    307309    } 
    308310 
Note: See TracChangeset for help on using the changeset viewer.