Changeset 1503


Ignore:
Timestamp:
05/28/18 17:28:07 (2 years ago)
Author:
yushan
Message:

rank_map is passed from vector to map, in order to have more flexibility in comm_split

Location:
XIOS/dev/branch_openmp/extern/ep_dev
Files:
36 edited

Legend:

Unmodified
Added
Removed
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_accumulate.cpp

    r1500 r1503  
    1212                     int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win) 
    1313  { 
    14     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    15     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     14    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     15    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    1616    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1717    if(num_ep==1) 
     
    2727                     int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request) 
    2828  { 
    29     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    30     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     29    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     30    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    3131    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    3232    ::MPI_Request mpi_request; 
     
    6060                         int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win) 
    6161  { 
    62     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    63     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     62    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     63    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    6464    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    6565    if(num_ep==1) 
     
    7575                         int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request) 
    7676  { 
    77     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    78     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     77    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     78    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    7979    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    8080    ::MPI_Request mpi_request; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allgather.cpp

    r1500 r1503  
    8686      for(int i=0; i<ep_size; i++) 
    8787      { 
    88         offset = mpi_displs[comm->rank_map->at(i).second] + comm->rank_map->at(i).first * sendcount;  
     88        offset = mpi_displs[comm->ep_rank_map->at(i).second] + comm->ep_rank_map->at(i).first * sendcount;  
    8989        memcpy(recvbuf + i*sendcount*datasize, tmp_recvbuf+offset*datasize, sendcount*datasize); 
    9090      } 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allgatherv.cpp

    r1500 r1503  
    9292      { 
    9393        int extra = 0; 
    94         for(int j=0, k=0; j<ep_size, k<comm->rank_map->at(i).first; j++) 
    95           if(comm->rank_map->at(i).second == comm->rank_map->at(j).second) 
     94        for(int j=0, k=0; j<ep_size, k<comm->ep_rank_map->at(i).first; j++) 
     95          if(comm->ep_rank_map->at(i).second == comm->ep_rank_map->at(j).second) 
    9696          { 
    9797            extra += recvcounts[j]; 
     
    9999          }   
    100100 
    101         offset = mpi_displs[comm->rank_map->at(i).second] +  extra; 
     101        offset = mpi_displs[comm->ep_rank_map->at(i).second] +  extra; 
    102102 
    103103        memcpy(recvbuf+displs[i]*datasize, tmp_recvbuf+offset*datasize, recvcounts[i]*datasize); 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_bcast.cpp

    r1500 r1503  
    5757    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
    5858 
    59     int root_mpi_rank = comm->rank_map->at(root).second; 
    60     int root_ep_rank_loc = comm->rank_map->at(root).first; 
     59    int root_mpi_rank = comm->ep_rank_map->at(root).second; 
     60    int root_ep_rank_loc = comm->ep_rank_map->at(root).first; 
    6161 
    6262 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_comm.hpp

    r1502 r1503  
    1010typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank> 
    1111typedef std::map<int, std::pair<int, int> > EP_RANK_MAP;  // key(ep_rank) = <ep_rank_local, mpi_rank> 
    12 typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP; 
     12//typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP; 
    1313typedef std::list<ep_lib::MPI_Message > Message_list; 
    1414 
     
    5252    BUFFER     *my_buffer; 
    5353    ep_barrier *ep_barrier; 
    54     RANK_MAP   *rank_map; 
     54    //RANK_MAP   *rank_map; 
     55    EP_RANK_MAP   *ep_rank_map; 
    5556    void* mpi_comm; 
    5657    ep_communicator *ep_comm_ptr; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_compare.cpp

    r1500 r1503  
    1111        int MPI_Compare_and_swap(const void *origin_addr, const void *compare_addr, void *result_addr, MPI_Datatype datatype, int target_rank, MPI_Aint target_disp, MPI_Win win) 
    1212  { 
    13     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    14     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     13    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     14    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    1515    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1616    if(num_ep==1) 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp

    r1500 r1503  
    6464    out_comm_hdls[0]->my_buffer = new BUFFER; 
    6565 
    66     out_comm_hdls[0]->rank_map = new RANK_MAP; 
    67     out_comm_hdls[0]->rank_map->resize(ep_size); 
     66    //out_comm_hdls[0]->rank_map = new RANK_MAP; 
     67    //out_comm_hdls[0]->rank_map->resize(ep_size); 
     68 
     69    out_comm_hdls[0]->ep_rank_map = new EP_RANK_MAP; 
     70    //out_comm_hdls[0]->ep_rank_map->resize(ep_size); 
    6871 
    6972 
     
    7275      out_comm_hdls[i]->ep_barrier = out_comm_hdls[0]->ep_barrier; 
    7376      out_comm_hdls[i]->my_buffer  = out_comm_hdls[0]->my_buffer; 
    74       out_comm_hdls[i]->rank_map   = out_comm_hdls[0]->rank_map; 
     77      //out_comm_hdls[i]->rank_map   = out_comm_hdls[0]->rank_map; 
     78      out_comm_hdls[i]->ep_rank_map= out_comm_hdls[0]->ep_rank_map; 
    7579    } 
    7680 
     
    9296      for(int j=0; j<recv_num_ep[i]; j++) 
    9397      { 
    94         out_comm_hdls[0]->rank_map->at(ind) = make_pair(j, i); 
     98        //out_comm_hdls[0]->rank_map->at(ind) = make_pair(j, i); 
     99        out_comm_hdls[0]->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(ind, j, i)); 
    95100        ind++; 
    96101      } 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_exscan.cpp

    r1500 r1503  
    253253    std::vector<int> my_map(mpi_size, 0); 
    254254 
    255     for(int i=0; i<comm->rank_map->size(); i++) my_map[comm->rank_map->at(i).second]++; 
     255    for(int i=0; i<comm->ep_rank_map->size(); i++) my_map[comm->ep_rank_map->at(i).second]++; 
    256256 
    257257    for(int i=0; i<mpi_rank; i++) my_src += my_map[i]; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_fetch.cpp

    r1500 r1503  
    1111        int MPI_Fetch_and_op(const void *origin_addr, void *result_addr, MPI_Datatype datatype, int target_rank, MPI_Aint target_disp, MPI_Op op, MPI_Win win) 
    1212  { 
    13     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    14     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     13    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     14    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    1515    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1616    if(num_ep==1) 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_free.cpp

    r1500 r1503  
    4949 
    5050 
    51         if( ! (*comm)->rank_map->empty() ) 
     51        if( ! (*comm)->ep_rank_map->empty() ) 
    5252        { 
    53           (*comm)->rank_map->clear(); 
    54           delete (*comm)->rank_map; 
    55           Debug("rank_map emptied\n"); 
     53          (*comm)->ep_rank_map->clear(); 
     54          delete (*comm)->ep_rank_map; 
     55          Debug("ep_rank_map emptied and freed\n"); 
    5656        } 
    5757 
     
    110110 
    111111 
    112       if( ! (*comm)->rank_map->empty() ) 
     112      if( ! (*comm)->ep_rank_map->empty() ) 
    113113      { 
    114         (*comm)->rank_map->clear(); 
    115         Debug("rank_map emptied\n"); 
     114        (*comm)->ep_rank_map->clear(); 
     115        delete (*comm)->ep_rank_map; 
     116        Debug("ep_rank_map emptied and freed\n"); 
    116117      } 
    117118 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_gather.cpp

    r1500 r1503  
    5959    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    6060 
    61     int root_mpi_rank = comm->rank_map->at(root).second; 
    62     int root_ep_loc = comm->rank_map->at(root).first; 
     61    int root_mpi_rank = comm->ep_rank_map->at(root).second; 
     62    int root_ep_loc = comm->ep_rank_map->at(root).first; 
    6363 
    6464    ::MPI_Aint datasize, lb; 
     
    9090      for(int i=0; i<ep_size; i++) 
    9191      { 
    92         recvcounts[comm->rank_map->at(i).second]+=sendcount; 
     92        recvcounts[comm->ep_rank_map->at(i).second]+=sendcount; 
    9393      } 
    9494 
     
    111111      for(int i=0; i<ep_size; i++) 
    112112      { 
    113         offset = displs[comm->rank_map->at(i).second] + comm->rank_map->at(i).first * sendcount;  
     113        offset = displs[comm->ep_rank_map->at(i).second] + comm->ep_rank_map->at(i).first * sendcount;  
    114114        memcpy(recvbuf + i*sendcount*datasize, tmp_recvbuf+offset*datasize, sendcount*datasize); 
    115115 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_gatherv.cpp

    r1500 r1503  
    6363    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    6464 
    65     int root_mpi_rank = comm->rank_map->at(root).second; 
    66     int root_ep_loc = comm->rank_map->at(root).first; 
     65    int root_mpi_rank = comm->ep_rank_map->at(root).second; 
     66    int root_ep_loc = comm->ep_rank_map->at(root).first; 
    6767 
    6868    ::MPI_Aint datasize, lb; 
     
    126126      for(int i=0; i<ep_size; i++) 
    127127      { 
    128         mpi_recvcounts[comm->rank_map->at(i).second]+=recvcounts[i]; 
     128        mpi_recvcounts[comm->ep_rank_map->at(i).second]+=recvcounts[i]; 
    129129      } 
    130130 
     
    144144      { 
    145145        int extra = 0; 
    146         for(int j=0, k=0; j<ep_size, k<comm->rank_map->at(i).first; j++) 
    147           if(comm->rank_map->at(i).second == comm->rank_map->at(j).second) 
     146        for(int j=0, k=0; j<ep_size, k<comm->ep_rank_map->at(i).first; j++) 
     147          if(comm->ep_rank_map->at(i).second == comm->ep_rank_map->at(j).second) 
    148148          { 
    149149            extra += recvcounts[j]; 
     
    151151          }   
    152152 
    153         offset = mpi_displs[comm->rank_map->at(i).second] +  extra; 
     153        offset = mpi_displs[comm->ep_rank_map->at(i).second] +  extra; 
    154154 
    155155        memcpy(recvbuf+displs[i]*datasize, tmp_recvbuf+offset*datasize, recvcounts[i]*datasize); 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_get.cpp

    r1500 r1503  
    1212                  int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win) 
    1313  { 
    14     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    15     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     14    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     15    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    1616    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1717    if(num_ep==1) 
     
    2727              int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request) 
    2828  { 
    29     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    30     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     29    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     30    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    3131    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    3232 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp

    r1500 r1503  
    7777            // change leader 
    7878            is_decider = true; 
    79             int target = local_comm->rank_map->at(local_leader).second; 
     79            int target = local_comm->ep_rank_map->at(local_leader).second; 
    8080            { 
    8181              for(int i=0; i<ep_size; i++) 
    8282              { 
    83                 if(local_comm->rank_map->at(i).second != target && local_comm->rank_map->at(i).first == 0) 
     83                if(local_comm->ep_rank_map->at(i).second != target && local_comm->ep_rank_map->at(i).first == 0) 
    8484                { 
    8585                  new_local_leader = i; 
     
    100100            // change leader 
    101101            is_decider = true; 
    102             int target = local_comm->rank_map->at(local_leader).second; 
     102            int target = local_comm->ep_rank_map->at(local_leader).second; 
    103103            { 
    104104              for(int i=0; i<ep_size; i++) 
    105105              { 
    106                 if(local_comm->rank_map->at(i).second != target && local_comm->rank_map->at(i).first == 0) 
     106                if(local_comm->ep_rank_map->at(i).second != target && local_comm->ep_rank_map->at(i).first == 0) 
    107107                { 
    108108                  new_local_leader = i; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.hpp

    r1502 r1503  
    55typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank> 
    66typedef std::map<int, std::pair<int, int> > EP_RANK_MAP;  // key(ep_rank) = <ep_rank_local, mpi_rank> 
    7 typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP; 
     7//typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP; 
    88 
    99namespace ep_lib 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp

    r1500 r1503  
    4646 
    4747    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;} 
    48     if(ep_rank_loc == 0 && mpi_rank != local_comm->rank_map->at(local_leader).second) is_proc_master = true; 
     48    if(ep_rank_loc == 0 && mpi_rank != local_comm->ep_rank_map->at(local_leader).second) is_proc_master = true; 
    4949 
    5050 
     
    125125      send_buf[4] = rank_in_peer_mpi[1]; 
    126126 
    127       ::MPI_Bcast(send_buf.data(), 5, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     127      ::MPI_Bcast(send_buf.data(), 5, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    128128 
    129129      size_info[1] = send_buf[0]; 
     
    157157      } 
    158158 
    159       ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     159      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    160160 
    161161      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() ); 
     
    281281      } 
    282282 
    283       ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     283      ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    284284 
    285285      new_rank_info[2].resize(size_info[3]); 
     
    305305      } 
    306306 
    307       ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     307      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    308308 
    309309      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() ); 
     
    337337      } 
    338338 
    339       ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     339      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    340340 
    341341      if(new_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
     
    542542      MPI_Status statuses[2]; 
    543543       
    544       MPI_Isend((*newintercomm)->rank_map->data(), 2*local_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[0]); 
    545       MPI_Irecv((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[1]); 
     544      std::vector<std::pair<int, std::pair<int, int> > > map2vec((*newintercomm)->ep_rank_map->size()); 
     545      std::vector<std::pair<int, std::pair<int, int> > > vec2map((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->size()); 
     546       
     547      int ii=0; 
     548      for(std::map<int, std::pair<int, int> >::iterator it = (*newintercomm)->ep_rank_map->begin(); it != (*newintercomm)->ep_rank_map->end(); it++) 
     549      { 
     550        map2vec[ii++] = make_pair(it->first, make_pair(it->second.first, it->second.second)); 
     551      } 
     552       
     553       
     554      MPI_Isend(map2vec.data(), 3*local_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[0]); 
     555      MPI_Irecv(vec2map.data(), 3*remote_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[1]); 
     556       
     557      
     558      for(ii=0; ii<vec2map.size(); ii++) 
     559      { 
     560        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(vec2map[ii].first) = make_pair(vec2map[ii].second.first, vec2map[ii].second.second); 
     561      } 
    546562       
    547563      MPI_Waitall(2, requests, statuses); 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_world.cpp

    r1500 r1503  
    4444 
    4545    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;} 
    46     if(ep_rank_loc == 0 && mpi_rank != local_comm->rank_map->at(local_leader).second) is_proc_master = true; 
     46    if(ep_rank_loc == 0 && mpi_rank != local_comm->ep_rank_map->at(local_leader).second) is_proc_master = true; 
    4747 
    4848 
     
    105105      } 
    106106 
    107       ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     107      ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    108108 
    109109      size_info[1] = recv_buf[0]; 
     
    135135      } 
    136136 
    137       ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     137      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    138138 
    139139      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() ); 
     
    261261      } 
    262262 
    263       ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     263      ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    264264 
    265265      new_rank_info[2].resize(size_info[3]); 
     
    283283      } 
    284284 
    285       ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     285      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    286286 
    287287      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() ); 
     
    315315      } 
    316316 
    317       ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     317      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm); 
    318318 
    319319      if(new_comm != static_cast< ::MPI_Comm* >(MPI_COMM_NULL->mpi_comm)) 
     
    512512    { 
    513513      ::MPI_Status status; 
    514  
    515       ::MPI_Send((*newintercomm)->rank_map->data(), 2*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    516  
    517       ::MPI_Recv((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 
     514       
     515      std::vector<std::pair<int, std::pair<int, int> > > map2vec((*newintercomm)->ep_rank_map->size()); 
     516      std::vector<std::pair<int, std::pair<int, int> > > vec2map((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->size()); 
     517       
     518      int ii=0; 
     519      for(std::map<int, std::pair<int, int> >::iterator it = (*newintercomm)->ep_rank_map->begin(); it != (*newintercomm)->ep_rank_map->end(); it++) 
     520      { 
     521        map2vec[ii++] = make_pair(it->first, make_pair(it->second.first, it->second.second)); 
     522      } 
     523       
     524       
     525      ::MPI_Send(map2vec.data(), 3*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
     526      ::MPI_Recv(vec2map.data(), 3*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 
     527       
     528      
     529      for(ii=0; ii<vec2map.size(); ii++) 
     530      { 
     531        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(vec2map[ii].first) = make_pair(vec2map[ii].second.first, vec2map[ii].second.second); 
     532      } 
    518533    } 
    519534 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.cpp

    r1500 r1503  
    3333  int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank) 
    3434  { 
    35     for(int i=0; i<comm->rank_map->size(); i++) 
    36     { 
    37       if(   ( comm->rank_map->at(i).first  == ep_rank_loc ) 
    38          && ( comm->rank_map->at(i).second == mpi_rank ) ) 
     35    for(int i=0; i<comm->ep_rank_map->size(); i++) 
     36    { 
     37      if(   ( comm->ep_rank_map->at(i).first  == ep_rank_loc ) 
     38         && ( comm->ep_rank_map->at(i).second == mpi_rank ) ) 
    3939      { 
    4040        return i; 
     
    8989  void check_sum_send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, int type) 
    9090  { 
     91 
    9192    int src_rank; 
    9293    int int_count; 
     
    121122  void check_sum_recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, int type) 
    122123  { 
     124 
    123125    int dest_rank; 
    124126    int int_count; 
     
    147149    } 
    148150    else printf("Unable to open file\n"); 
     151 
    149152 
    150153  } 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.hpp

    r1500 r1503  
    99#include "ep_lib_fortran.hpp" 
    1010#include "ep_lib_win.hpp" 
     11#include "ep_lib_mpi.hpp" 
     12 
    1113 
    1214namespace ep_lib 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp

    r1500 r1503  
    9292      for(int i=0; i<intra_ep_size; i++) 
    9393      { 
    94         tmp_rank_map[reorder[i]] = (*newintracomm)->rank_map->at(i) ; 
     94        tmp_rank_map[reorder[i]] = (*newintracomm)->ep_rank_map->at(i) ; 
    9595      } 
    9696 
    97       (*newintracomm)->rank_map->swap(tmp_rank_map); 
     97      //(*newintracomm)->rank_map->swap(tmp_rank_map); 
     98      (*newintracomm)->ep_rank_map->clear(); 
     99      for(int i=0; i<tmp_rank_map.size(); i++) 
     100      { 
     101        (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(i, tmp_rank_map[i].first, tmp_rank_map[i].second)); 
     102      } 
     103       
    98104 
    99105      tmp_rank_map.clear(); 
     
    206212    for(int i=0; i<newintracomm_ep_size; i++) 
    207213    { 
    208       (*newintracomm)->rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1]; 
    209       (*newintracomm)->rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2]; 
     214      (*newintracomm)->ep_rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1]; 
     215      (*newintracomm)->ep_rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2]; 
    210216    } 
    211217 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_message.cpp

    r1500 r1503  
    2020  int Request_Check() 
    2121  { 
     22    if(EP_PendingRequests == 0 ) EP_PendingRequests = new std::list< MPI_Request* >; 
     23     
     24    if(EP_PendingRequests->size() == 0) return 0; 
     25     
    2226    MPI_Status status; 
    2327    MPI_Message *message; 
     
    2630    std::list<MPI_Request* >::iterator it; 
    2731     
    28     if(EP_PendingRequests == 0 ) return 0; 
    2932     
    3033    for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); it++) 
     
    3841      if((*(*it))->state == 0) 
    3942      { 
    40            
    41         message = new MPI_Message; 
    42         *message = new ep_message; 
    43          
    44         printf("new %p : in ep_lib::Request_Check, message = new MPI_Message\n", message); 
    45         printf("new %p : in ep_lib::Request_Check, *message = new ep_message\n", *message); 
     43        #pragma omp critical (_query0) 
     44        { 
     45          MPI_Iprobe((*(*it))->ep_src, (*(*it))->ep_tag, ((*(*it))->comm), &probed, &status); 
     46          if(probed) 
     47          { 
     48            message = new MPI_Message; 
     49            *message = new ep_message; 
     50         
     51            memcheck("new "<< message <<" : in ep_lib::Request_Check, message = new MPI_Message"); 
     52            memcheck("new "<< *message <<" : in ep_lib::Request_Check, *message = new ep_message"); 
    4653           
    4754           
    48         MPI_Improbe((*(*it))->ep_src, (*(*it))->ep_tag, ((*(*it))->comm), &probed, message, &status); 
    49         printf("in Request_Check, after improbe, mpi_status = %p\n", to_mpi_status_ptr(status)); 
     55            MPI_Improbe((*(*it))->ep_src, (*(*it))->ep_tag, (*(*it))->comm, &probed, message, &status); 
     56         
     57          } 
     58        } 
    5059       
    5160         
     
    5463          ::MPI_Get_count(to_mpi_status_ptr(status), to_mpi_type((*(*it))->ep_datatype), &recv_count); 
    5564           
    56           printf("in Request_Check, imrecv, buf = %p, recv_count = %d, status = %p\n", (*(*it))->buf, recv_count, to_mpi_status_ptr(status)); 
    5765          MPI_Imrecv((*(*it))->buf, recv_count, (*(*it))->ep_datatype, message, *it); 
    5866          (*(*it))->type = 3; 
    5967          (*(*it))->state = 1; 
    60            
    61  
    62           printf("delete %p : in ep_lib::Request_Check, delete *message\n", *message); 
    63           printf("delete %p : in ep_lib::Request_Check, delete message\n", message); 
     68 
     69          memcheck("delete "<< status.mpi_status <<" : in ep_lib::Request_Check, delete status.mpi_status"); 
     70          delete status.mpi_status;           
     71 
     72          memcheck("delete "<< *message <<" : in ep_lib::Request_Check, delete *message"); 
     73          memcheck("delete "<< message <<" : in ep_lib::Request_Check, delete message"); 
    6474 
    6575          delete *message; 
     
    6878          it++; 
    6979          continue;       
    70         }       
     80        }               
    7181      } 
    7282       
    7383      if((*(*it))->state == 2) 
    7484      { 
    75         printf("delete %p : in ep_lib::Request_Check, delete (*(*it))\n", (*(*it))); 
     85        int ep_rank = ((*(*it))->comm)->ep_comm_ptr->size_rank_info[0].first; 
     86        memcheck("delete "<< (*(*it)) <<" : in ep_lib::Request_Check, delete (*(*it))"); 
    7687        delete (*(*it)); 
    7788         
    7889        EP_PendingRequests->erase(it); 
    79         printf("EP_PendingRequests->size() = %lu\n", EP_PendingRequests->size()); 
     90        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size()); 
    8091        it = EP_PendingRequests->begin(); 
    8192        continue; 
     
    128139        msg->mpi_message = new ::MPI_Message(message); 
    129140 
    130         printf("new %p : in ep_lib::Message_Check, msg = new ep_message\n", msg); 
    131         printf("new %p : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message\n", msg->mpi_message); 
     141        memcheck("new "<< msg <<" : in ep_lib::Message_Check, msg = new ep_message"); 
     142        memcheck("new "<< msg->mpi_message <<" : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message"); 
    132143               
    133144 
     
    139150        msg->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);        
    140151        msg->mpi_status = new ::MPI_Status(status);   
    141         printf("new %p : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status\n", msg->mpi_status); 
     152        memcheck("new "<< msg->mpi_status <<" : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status"); 
    142153 
    143154        #pragma omp critical (_query) 
     
    145156          #pragma omp flush 
    146157          comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->push_back(msg);   
    147           int test_count; 
    148           ::MPI_Get_count(static_cast< ::MPI_Status* >(msg->mpi_status), 1275070475, &test_count); 
    149           printf("status1 = %p, test_count2 = %d\n", static_cast< ::MPI_Status* >(msg->mpi_status), test_count); 
    150            
    151           ::MPI_Get_count(static_cast< ::MPI_Status* >(comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->back()->mpi_status), 1275070475, &test_count); 
    152           printf("status2 = %p, test_count2 = %d\n", static_cast< ::MPI_Status* >(comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->back()->mpi_status), test_count); 
    153                     
     158          int dest_mpi = comm->ep_comm_ptr->size_rank_info[2].first; 
     159          memcheck("message_queue["<<dest_mpi<<","<<dest_loc<<"]->size = "<<comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->size()); 
    154160          #pragma omp flush 
    155161        } 
     
    199205        msg->mpi_message = new ::MPI_Message(message); 
    200206 
    201         printf("new %p : in ep_lib::Message_Check, msg = new ep_message\n", msg); 
    202         printf("new %p : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message\n", msg->mpi_message); 
     207        memcheck("new "<< msg <<" : in ep_lib::Message_Check, msg = new ep_message"); 
     208        memcheck("new "<< msg->mpi_message <<" : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message"); 
    203209               
    204210 
     
    210216              
    211217        msg->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);        
    212         msg->mpi_status = new ::MPI_Status(status);   
    213         printf("new %p : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status\n", msg->mpi_status); 
     218        msg->mpi_status = new ::MPI_Status(status);  
     219        memcheck("new "<< msg->mpi_status <<" : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status"); 
    214220 
    215221        #pragma omp critical (_query) 
     
    217223          #pragma omp flush 
    218224          comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->push_back(msg); 
     225          memcheck("comm->ep_comm_ptr->comm_list["<<dest_loc<<"]->ep_comm_ptr->message_queue->size = "<<comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->size()); 
    219226          #pragma omp flush 
    220227        } 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_probe.cpp

    r1500 r1503  
    1313    if(!comm->is_ep) 
    1414    { 
    15       ::MPI_Status *mpi_status = static_cast< ::MPI_Status* >(status->mpi_status); 
    16       ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, mpi_status); 
     15      Debug("calling MPI_Iprobe MPI\n"); 
     16      ::MPI_Status mpi_status; 
     17      ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status); 
    1718 
    18       status->mpi_status = mpi_status; 
     19      status->mpi_status = &mpi_status; 
    1920      status->ep_src = src; 
    2021      status->ep_tag = tag; 
     
    2324 
    2425    Debug("calling MPI_Iprobe EP\n"); 
    25  
    2626    Message_Check(comm); 
    2727 
     
    4343          ::MPI_Status mpi_status = *(static_cast< ::MPI_Status *>((*it)->mpi_status)); 
    4444 
    45           status->mpi_status = new ::MPI_Status(mpi_status); 
     45          status->mpi_status = (*it)->mpi_status; 
    4646          status->ep_src = (*it)->ep_src; 
    4747          status->ep_tag = (*it)->ep_tag; 
     
    4949          break; 
    5050        } 
    51  
    5251      } 
    5352    } 
     
    6059  int MPI_Improbe(int src, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status) 
    6160  { 
     61    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     62    int mpi_rank    = comm->ep_comm_ptr->size_rank_info[2].first; 
    6263    *flag = false; 
    6364    if(!comm->is_ep) 
     
    102103      for(Message_list::iterator it = comm->ep_comm_ptr->message_queue->begin(); it!= comm->ep_comm_ptr->message_queue->end(); ++it) 
    103104      { 
    104  
    105         //printf("in ep_lib::Improbe, it->mpi_message = %p, it->mpi_status = %p, it->ep_src = %d, it->ep_tag = %d\n",  
    106         //                            (*(*it))->mpi_message,(*(*it))->mpi_status,(*(*it))->ep_src,(*(*it))->ep_tag); 
    107105                                           
    108106        bool src_matched = src<0? true: (*it)->ep_src == src; 
     
    113111          *flag = true; 
    114112 
    115           status->mpi_status = (*it)->mpi_status; 
     113          status->mpi_status = new ::MPI_Status(*static_cast< ::MPI_Status*>((*it)->mpi_status)); 
     114          memcheck("new "<< status->mpi_status << " : in ep_lib::MPI_Improbe, status->mpi_status = new ::MPI_Status"); 
    116115          status->ep_src = (*it)->ep_src; 
    117116          status->ep_tag = (*it)->ep_tag; 
    118117 
    119           (*message)->mpi_message = (*it)->mpi_message; 
     118          (*message)->mpi_message = new ::MPI_Message(*static_cast< ::MPI_Message*>((*it)->mpi_message)); 
     119          memcheck("new "<< (*message)->mpi_message <<" : in ep_lib::MPI_Improbe, (*message)->mpi_message = new ::MPI_Message"); 
    120120          (*message)->ep_src = (*it)->ep_src; 
    121121          (*message)->ep_tag = (*it)->ep_tag; 
    122            
    123           int test_count; 
    124           ::MPI_Get_count(to_mpi_status_ptr(status), 1275070475, &test_count); 
    125           printf("status = %p, test_count = %d\n", to_mpi_status_ptr(status), test_count); 
    126            
    127            
    128           ::MPI_Get_count(static_cast< ::MPI_Status* >(comm->ep_comm_ptr->message_queue->back()->mpi_status), 1275070475, &test_count); 
    129           printf("in ep_lib::Improbe, status = %p, test_count = %d\n", static_cast< ::MPI_Status* >(comm->ep_comm_ptr->message_queue->back()->mpi_status), test_count); 
    130122                                       
    131123 
    132124          #pragma omp critical (_query2) 
    133125          {               
    134             printf("delete %p : in ep_lib::Message_Check, delete (*it)->mpi_message\n", (*it)->mpi_message); 
    135             printf("delete %p : in ep_lib::Message_Check, delete (*it)->mpi_status\n", (*it)->mpi_status); 
    136             printf("delete %p : in ep_lib::Message_Check, delete (*it)\n", (*it)); 
     126            memcheck("delete "<< (*it)->mpi_message <<" : in ep_lib::Message_Check, delete (*it)->mpi_message"); 
     127            memcheck("delete "<< (*it)->mpi_status <<" : in ep_lib::Message_Check, delete (*it)->mpi_status"); 
     128            memcheck("delete "<< (*it) <<" : in ep_lib::Message_Check, delete (*it)"); 
     129             
    137130             
    138131            delete (*it)->mpi_message;      
     
    140133            delete *it; 
    141134             
     135                        
    142136            comm->ep_comm_ptr->message_queue->erase(it); 
    143             printf("message queue siez = %lu\n", comm->ep_comm_ptr->message_queue->size()); 
     137            memcheck("message_queue["<<mpi_rank<<","<<ep_rank_loc<<"]->size = "<<comm->ep_comm_ptr->message_queue->size()); 
    144138            #pragma omp flush 
    145139          } 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_put.cpp

    r1500 r1503  
    1212                  int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win) 
    1313  { 
    14     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    15     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     14    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     15    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    1616    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1717    if(num_ep==1) 
     
    2727              int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request) 
    2828  { 
    29     int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
    30     int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     29    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second; 
     30    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first; 
    3131    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    3232     
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_recv.cpp

    r1500 r1503  
    1919namespace ep_lib  
    2020{ 
    21    
    2221 
    2322  int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status) 
    2423  { 
    2524 
    26     if(!comm->is_ep) 
    27     { 
    28       ::MPI_Status mpi_status; 
    29       ::MPI_Recv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_status); 
    30  
    31       status->ep_src = src; 
    32       status->ep_tag = tag; 
    33       status->ep_datatype = datatype; 
    34  
    35       return 0;   
    36     } 
    37  
    38     //Message_Check(comm); 
     25    if(!comm->is_ep) return MPI_Recv_mpi(buf, count, datatype, src, tag, comm, status); 
     26     
     27    Debug("MPI_Recv with EP"); 
    3928 
    4029    MPI_Request request; 
     
    4534  } 
    4635 
    47  
     36   
    4837 
    4938 
    5039  int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request) 
    5140  { 
    52  
     41    if(!comm->is_ep) return MPI_Irecv_mpi(buf, count, datatype, src, tag, comm, request); 
     42     
    5343    Debug("MPI_Irecv with EP"); 
    5444    int dest_rank; 
    5545    MPI_Comm_rank(comm, &dest_rank); 
    5646     
    57  
    58     if(!comm->is_ep) 
    59     { 
    60       ::MPI_Request mpi_request; 
    61       ::MPI_Irecv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
    62  
    63       (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    64       (*request)->ep_src = src; 
    65       (*request)->ep_datatype = datatype; 
    66       (*request)->ep_tag = tag; 
    67     } 
    68      
    69     *request = new ep_request; 
    70     printf("new %p : in ep_lib::MPI_Irecv, *request = new ep_request\n", (*request)); 
     47    *request = new ep_request;    
     48    memcheck("new " << *request <<" : in ep_lib::MPI_Irecv, *request = new ep_request"); 
    7149 
    7250    (*request)->mpi_request = new ::MPI_Request; 
    73     printf("new %p : in ep_lib::MPI_Irecv, (*request)->mpi_request = new ::MPI_Request\n", (*request)->mpi_request); 
     51    memcheck("new " << (*request)->mpi_request << " : in ep_lib::MPI_Irecv, (*request)->mpi_request = new ::MPI_Request"); 
    7452     
    7553    (*request)->buf = buf; 
     
    8260    (*request)->ep_tag = tag; 
    8361    (*request)->ep_datatype = datatype; 
    84      
    85      
    86     /* With Improbe*/ 
     62   
     63    if(EP_PendingRequests == 0 ) EP_PendingRequests = new std::list< MPI_Request* >; 
    8764 
    88     if(EP_PendingRequests == 0 )  
    89     { 
    90       EP_PendingRequests = new std::list< MPI_Request* >; 
    91     } 
    92  
    93     EP_PendingRequests->push_back(request);                                                                 
     65    EP_PendingRequests->push_back(request);   
     66    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     67    memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());                                                               
    9468                                           
    95     Request_Check(); 
    96  
    97     return 0; 
     69    return Request_Check(); 
    9870  } 
    99  
    100   int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request) 
    101   { 
    102     Debug("MPI_Imrecv"); 
    103  
    104     (*request)->type = 3; 
    105          
    106     ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request));                 
    107  
    108     (*request)->ep_datatype = datatype; 
    109     (*request)->ep_tag = (*message)->ep_tag; 
    110     (*request)->ep_src = (*message)->ep_src; 
    111      
    112      
    113     return 0; 
    114   } 
    115  
    116  
     71   
    11772  int MPI_Mrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Status *status) 
    11873  { 
    119     Debug("EP Mrecv called\n"); 
     74    Debug("MPI_Mrecv with MPI/EP"); 
    12075 
    121     ::MPI_Status mpi_status; 
    122     ::MPI_Mrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >((*message)->mpi_message), &mpi_status); 
     76    status->mpi_status = new ::MPI_Status; 
     77    memcheck("new " << status->mpi_status << " : in ep_lib::MPI_Mrecv, status->mpi_status = new ::MPI_Status"); 
     78     
     79    ::MPI_Mrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >((*message)->mpi_message), to_mpi_status_ptr(*status)); 
    12380 
    124     status->mpi_status = new ::MPI_Status(mpi_status); 
     81     
    12582    status->ep_src = (*message)->ep_src; 
    12683    status->ep_datatype = datatype; 
    12784    status->ep_tag = (*message)->ep_tag; 
    12885 
     86    memcheck("delete " << (*message)->mpi_message << " : in ep_lib::MPI_Mrecv, delete (*message)->mpi_message"); 
    12987    delete (*message)->mpi_message; 
    13088 
    131     //check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag); 
     89#ifdef _check_sum 
     90    check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag); 
     91#endif 
    13292 
    133     return 0; 
     93    return Request_Check(); 
    13494  } 
    13595 
     96 
     97  int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request) 
     98  { 
     99    Debug("MPI_Imrecv with MPI/EP"); 
     100 
     101    (*request)->type = 3; 
     102    (*request)->ep_datatype = datatype; 
     103    (*request)->ep_tag = (*message)->ep_tag; 
     104    (*request)->ep_src = (*message)->ep_src; 
     105         
     106    ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request));                 
     107     
     108    memcheck("delete " << (*message)->mpi_message << " : in ep_lib::MPI_Imrecv, delete (*message)->mpi_message"); 
     109    delete (*message)->mpi_message; 
     110 
     111#ifdef _check_sum 
     112    check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag); 
     113#endif 
     114 
     115     
     116    return Request_Check(); 
     117  } 
     118 
     119 
     120   int MPI_Recv_mpi(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status) 
     121  { 
     122    Debug("MPI_Recv with MPI"); 
     123    status->ep_src = src; 
     124    status->ep_tag = tag; 
     125    status->ep_datatype = datatype; 
     126     
     127    return ::MPI_Recv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), to_mpi_status_ptr(*status));  
     128  } 
     129   
     130  int MPI_Irecv_mpi(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request) 
     131  { 
     132    Debug("MPI_Irecv with MPI"); 
     133    int dest_rank; 
     134    MPI_Comm_rank(comm, &dest_rank); 
     135     
     136    *request = new ep_request; 
     137    memcheck("new "<< *request <<" : in ep_lib::MPI_Irecv, *request = new ep_request"); 
     138 
     139    (*request)->mpi_request = new ::MPI_Request; 
     140    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Irecv, (*request)->mpi_request = new ::MPI_Request"); 
     141       
     142    (*request)->ep_src = src; 
     143    (*request)->ep_datatype = datatype; 
     144    (*request)->type = 2; 
     145    (*request)->ep_tag = tag; 
     146     
     147    return ::MPI_Irecv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
     148  } 
    136149} 
    137150 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_reduce.cpp

    r1500 r1503  
    302302    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    303303 
    304     int root_mpi_rank = comm->rank_map->at(root).second; 
    305     int root_ep_loc = comm->rank_map->at(root).first; 
     304    int root_mpi_rank = comm->ep_rank_map->at(root).second; 
     305    int root_ep_loc = comm->ep_rank_map->at(root).first; 
    306306 
    307307    ::MPI_Aint datasize, lb; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_request.hpp

    r1502 r1503  
    1414 
    1515      int type; //! type of the non-blocking communication.  
    16                 // 1: Isend  
    17                 // 2:Irecv  
    18                 // 3:Imrecv  
    19                 // 4:Issend 
     16                // 1: Isend or Issend 
     17                // 2: Irecv  
     18                // 3: Imrecv  
     19                 
    2020      int state; // 0: new  
    2121                 // 1: imrecvd 
     
    2727      int ep_tag; 
    2828      void* ep_datatype; 
     29       
     30       
    2931 
    3032      ep_comm *comm;    //! EP communicator related to the communication 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_scan.cpp

    r1500 r1503  
    375375    std::vector<int> my_map(mpi_size, 0); 
    376376 
    377     for(int i=0; i<comm->rank_map->size(); i++) my_map[comm->rank_map->at(i).second]++; 
     377    for(int i=0; i<comm->ep_rank_map->size(); i++) my_map[comm->ep_rank_map->at(i).second]++; 
    378378 
    379379    for(int i=0; i<mpi_rank; i++) my_src += my_map[i]; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_scatter.cpp

    r1500 r1503  
    5656    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    5757 
    58     int root_mpi_rank = comm->rank_map->at(root).second; 
    59     int root_ep_loc = comm->rank_map->at(root).first; 
     58    int root_mpi_rank = comm->ep_rank_map->at(root).second; 
     59    int root_ep_loc = comm->ep_rank_map->at(root).first; 
    6060 
    6161    bool is_master = (ep_rank_loc==0 && mpi_rank != root_mpi_rank ) || ep_rank == root; 
     
    8787      for(int i=0; i<ep_size; i++) 
    8888      { 
    89         recvcounts[comm->rank_map->at(i).second]++; 
     89        recvcounts[comm->ep_rank_map->at(i).second]++; 
    9090      } 
    9191 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_scatterv.cpp

    r1500 r1503  
    5959    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    6060 
    61     int root_mpi_rank = comm->rank_map->at(root).second; 
    62     int root_ep_loc = comm->rank_map->at(root).first; 
     61    int root_mpi_rank = comm->ep_rank_map->at(root).second; 
     62    int root_ep_loc = comm->ep_rank_map->at(root).first; 
    6363 
    6464    bool is_master = (ep_rank_loc==0 && mpi_rank != root_mpi_rank ) || ep_rank == root; 
     
    9090      for(int i=0; i<ep_size; i++) 
    9191      { 
    92         recvcounts[comm->rank_map->at(i).second]++; 
     92        recvcounts[comm->ep_rank_map->at(i).second]++; 
    9393      } 
    9494 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_send.cpp

    r1500 r1503  
    1212 
    1313 
    14 namespace ep_lib { 
    15  
    16  
     14namespace ep_lib  
     15{ 
     16   
    1717  int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
    1818  { 
    19     if(!comm->is_ep) 
    20       return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 
    21     if(comm->is_intercomm) 
    22     { 
    23       MPI_Request request; 
    24       MPI_Status status; 
    25       MPI_Isend(buf, count, datatype, dest, tag, comm, &request); 
    26       MPI_Wait(&request, &status); 
    27     } 
    28     else 
    29     { 
    30       int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
    31       int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first; 
    32       int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    33       int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second; 
    34  
    35       ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm)); 
    36     } 
    37     //check_sum_send(buf, count, datatype, dest, tag, comm); 
    38  
    39     return 0; 
     19    if(!comm->is_ep)       return MPI_Send_mpi(buf, count, datatype, dest, tag, comm); 
     20    if(comm->is_intercomm) return MPI_Send_intercomm(buf, count, datatype, dest, tag, comm); 
     21     
     22    Debug("\nMPI_Send with EP\n"); 
     23     
     24    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     25    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first; 
     26    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
     27    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second; 
     28 
     29#ifdef _check_sum     
     30    check_sum_send(buf, count, datatype, dest, tag, comm); 
     31#endif 
     32 
     33    return ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm)); 
    4034  } 
    4135 
     
    4337  int MPI_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
    4438  { 
    45     if(!comm->is_ep) 
    46     { 
    47       return ::MPI_Ssend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 
    48     } 
    49  
     39    if(!comm->is_ep)       return MPI_Ssend_mpi(buf, count, datatype, dest, tag, comm); 
     40    if(comm->is_intercomm) return MPI_Ssend_intercomm(buf, count, datatype, dest, tag, comm); 
     41     
     42    Debug("\nMPI_Ssend with EP\n"); 
     43 
     44    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     45    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first; 
     46    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
     47    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second; 
     48     
     49#ifdef _check_sum     
     50    check_sum_send(buf, count, datatype, dest, tag, comm); 
     51#endif 
     52     
     53    return ::MPI_Ssend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm)); 
     54  } 
     55   
     56 
     57 
     58 
     59 
     60  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
     61  { 
     62    if(!comm->is_ep)       return MPI_Isend_mpi(buf, count, datatype, dest, tag, comm, request); 
     63    if(comm->is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request); 
     64 
     65    Debug("\nMPI_Isend with EP\n"); 
     66    
     67    int src_rank; 
     68    MPI_Comm_rank(comm, &src_rank); 
     69 
     70 
     71#ifdef _check_sum     
     72    check_sum_send(buf, count, datatype, dest, tag, comm); 
     73#endif 
     74     
     75    *request = new ep_request; 
     76    memcheck("new "<< *request <<" : in ep_lib::MPI_Isend, *request = new ep_request"); 
     77 
     78    (*request)->mpi_request = new ::MPI_Request; 
     79    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request"); 
     80     
     81 
     82    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     83    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first; 
     84    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
     85    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second; 
     86 
     87    (*request)->ep_src  = src_rank; 
     88    (*request)->ep_tag  = tag; 
     89    (*request)->ep_datatype = datatype; 
     90 
     91    (*request)->type = 1;    // used in wait 
     92    (*request)->comm = comm; 
     93    (*request)->buf = const_cast<void*>(buf); 
     94 
     95 
     96    return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
     97  } 
     98 
     99 
     100 
     101 
     102  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
     103  { 
     104  
     105    if(!comm->is_ep) return MPI_Issend_mpi(buf, count, datatype, dest, tag, comm, request); 
     106    if(comm->is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request); 
     107 
     108    Debug("\nMPI_Issend with EP\n"); 
     109    
     110    int src_rank; 
     111    MPI_Comm_rank(comm, &src_rank); 
     112 
     113 
     114#ifdef _check_sum     
     115    check_sum_send(buf, count, datatype, dest, tag, comm); 
     116#endif 
     117     
     118    *request = new ep_request; 
     119    memcheck("new "<< *request <<" : in ep_lib::MPI_Issend, *request = new ep_request"); 
     120 
     121    (*request)->mpi_request = new ::MPI_Request; 
     122    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend, (*request)->mpi_request = new ::MPI_Request"); 
     123     
     124 
     125    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     126    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first; 
     127    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
     128    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second; 
     129 
     130    (*request)->ep_src  = src_rank; 
     131    (*request)->ep_tag  = tag; 
     132    (*request)->ep_datatype = datatype; 
     133 
     134    (*request)->type = 1;    // used in wait 
     135    (*request)->comm = comm; 
     136    (*request)->buf = const_cast<void*>(buf); 
     137     
     138    return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));     
     139  } 
     140   
     141  int MPI_Send_mpi(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
     142  { 
     143    Debug("\nMPI_Send with MPI\n"); 
     144    return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 
     145  } 
     146   
     147  int MPI_Send_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
     148  { 
     149    Debug("\nMPI_Send with EP_Intercomm\n"); 
     150    MPI_Request request; 
     151    MPI_Status status; 
     152    MPI_Isend(buf, count, datatype, dest, tag, comm, &request); 
     153    MPI_Wait(&request, &status); 
     154    return MPI_SUCCESS; 
     155  } 
     156   
     157   
     158  int MPI_Ssend_mpi(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
     159  { 
     160    Debug("\nMPI_Ssend with MPI\n"); 
     161    return ::MPI_Ssend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 
     162  } 
     163   
     164 
     165  int MPI_Ssend_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
     166  { 
     167    Debug("\nMPI_Ssend with EP_Intercomm\n"); 
    50168    MPI_Request request; 
    51169    MPI_Status status; 
    52170    MPI_Issend(buf, count, datatype, dest, tag, comm, &request); 
    53171    MPI_Wait(&request, &status); 
    54     //check_sum_send(buf, count, datatype, dest, tag, comm); 
    55     return 0; 
    56   } 
    57    
    58  
    59  
    60  
    61  
    62   int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
    63   { 
    64     Debug("\nMPI_Isend with EP\n"); 
     172    return MPI_SUCCESS; 
     173  } 
     174   
     175  int MPI_Isend_mpi(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
     176  { 
     177    Debug("\nMPI_Isend with MPI\n"); 
     178     
    65179    int src_rank; 
    66180    MPI_Comm_rank(comm, &src_rank); 
    67  
    68      
    69  
    70     if(!comm->is_ep) 
    71     { 
    72       ::MPI_Request mpi_request; 
    73       ::MPI_Isend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
    74  
    75       (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    76  
    77       (*request)->ep_src = src_rank; 
    78       (*request)->ep_tag = tag; 
    79       (*request)->ep_datatype = datatype; 
    80       (*request)->type = 1; 
    81       (*request)->comm = comm; 
    82  
    83       return 0; 
    84     } 
    85  
    86     if(comm->is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request); 
    87  
    88     // EP intracomm 
    89  
    90     //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    91181     
    92182    *request = new ep_request; 
    93     printf("new %p : in ep_lib::MPI_Isend, *request = new ep_request\n", (*request)); 
     183    memcheck("new "<< *request <<" : in ep_lib::MPI_Isend, *request = new ep_request"); 
    94184 
    95185    (*request)->mpi_request = new ::MPI_Request; 
    96     printf("new %p : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request\n", (*request)->mpi_request); 
    97      
    98  
    99     int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
    100     int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first; 
    101     int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    102     int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second; 
    103  
    104     (*request)->ep_src  = src_rank; 
    105     (*request)->ep_tag  = tag; 
    106     (*request)->ep_datatype = datatype; 
    107  
    108     ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
    109  
    110     (*request)->type = 1;    // used in wait 
    111     (*request)->comm = comm; 
    112     (*request)->buf = const_cast<void*>(buf); 
    113  
    114  
    115     return 0; 
    116   } 
    117  
    118  
    119  
    120  
    121   int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
    122   { 
    123     Debug("\nMPI_Issend with EP\n"); 
    124  
    125     int src_rank; 
    126     MPI_Comm_rank(comm, &src_rank); 
    127  
    128      
    129  
    130     if(!comm->is_ep) 
    131     { 
    132       ::MPI_Request mpi_request; 
    133       ::MPI_Issend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
    134  
    135       (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    136       (*request)->ep_src = src_rank; 
    137       (*request)->ep_tag = tag; 
    138       (*request)->ep_datatype = datatype; 
    139       (*request)->type = 1; 
    140       (*request)->comm = comm; 
    141  
    142       return 0; 
    143     } 
    144  
    145     if(comm->is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request); 
    146  
    147     // EP intracomm 
    148  
    149     //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    150  
    151     int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
    152     int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first; 
    153     int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    154     int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second; 
    155      
     186    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request"); 
     187       
    156188    (*request)->ep_src = src_rank; 
    157189    (*request)->ep_tag = tag; 
    158190    (*request)->ep_datatype = datatype; 
    159  
    160     ::MPI_Request mpi_request; 
    161  
    162     ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
    163  
    164     (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    165     (*request)->type = 1;    // used in wait 
     191    (*request)->type = 1; 
    166192    (*request)->comm = comm; 
    167     (*request)->buf = NULL; 
    168      
    169  
    170     //Message_Check(comm); 
    171  
    172     return 0; 
    173   } 
    174    
    175  
    176  
    177  
     193     
     194    return ::MPI_Isend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
     195  } 
     196   
    178197  int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
    179198  { 
    180     Debug("MPI_Isend with intercomm\n"); 
    181  
    182     //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
     199    Debug("MPI_Isend with EP_Intercomm\n"); 
     200 
     201#ifdef _check_sum     
     202    check_sum_send(buf, count, datatype, dest, tag, comm); 
     203#endif 
    183204 
    184205    int dest_remote_ep_rank    = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
     
    189210 
    190211    src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second; 
    191  
    192  
    193      
    194  
    195212    //Message_Check(comm); 
    196  
    197213 
    198214    if(dest_remote_comm_label == src_comm_label)       // mpi_dest differs 
    199215    { 
    200216      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    201       int ep_src_loc = comm->rank_map->at(inter_src).first; 
    202       int ep_dest_loc = comm->rank_map->at(dest_remote_ep_rank).first; 
    203       int mpi_dest    = comm->rank_map->at(dest_remote_ep_rank).second; 
     217      int ep_src_loc = comm->ep_rank_map->at(inter_src).first; 
     218      int ep_dest_loc = comm->ep_rank_map->at(dest_remote_ep_rank).first; 
     219      int mpi_dest    = comm->ep_rank_map->at(dest_remote_ep_rank).second; 
    204220      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    205221 
    206       ::MPI_Request mpi_request; 
    207    
    208       ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
    209  
    210       (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     222      *request = new ep_request; 
     223      memcheck("new "<< *request <<" : in ep_lib::MPI_Isend_intercomm, *request = new ep_request"); 
     224 
     225      (*request)->mpi_request = new ::MPI_Request; 
     226      memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend_intercomm, (*request)->mpi_request = new ::MPI_Request"); 
     227       
    211228      (*request)->type = 1;    // used in wait 
    212229      (*request)->comm = comm; 
    213  
    214230      (*request)->ep_src = src_ep_rank; 
    215231      (*request)->ep_tag = tag; 
    216232      (*request)->ep_datatype = datatype; 
     233 
     234      return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
     235 
    217236    } 
    218237 
     
    220239    {  
    221240      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    222       int ep_src_loc = comm->rank_map->at(inter_src).first; 
     241      int ep_src_loc = comm->ep_rank_map->at(inter_src).first; 
    223242      int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first; 
    224243      int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second; 
    225244      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    226245 
    227       ::MPI_Request mpi_request; 
    228  
    229       ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request); 
    230  
    231       (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     246      *request = new ep_request; 
     247      memcheck("new "<< *request <<" : in ep_lib::MPI_Isend_intercomm, *request = new ep_request"); 
     248 
     249      (*request)->mpi_request = new ::MPI_Request; 
     250      memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend_intercomm, (*request)->mpi_request = new ::MPI_Request"); 
     251 
    232252      (*request)->type = 1;    // used in wait 
    233253      (*request)->comm = comm; 
    234     
    235254      (*request)->ep_src = src_ep_rank; 
    236255      (*request)->ep_tag = tag; 
    237256      (*request)->ep_datatype = datatype; 
     257       
     258      return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request));  
    238259    } 
    239  
    240     return 0; 
    241  
     260  } 
     261 
     262  int MPI_Issend_mpi(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
     263  { 
     264    Debug("\nMPI_Issend with MPI\n"); 
     265     
     266    int src_rank; 
     267    MPI_Comm_rank(comm, &src_rank); 
     268     
     269    *request = new ep_request; 
     270    memcheck("new "<< *request <<" : in ep_lib::MPI_Issend, *request = new ep_request"); 
     271 
     272    (*request)->mpi_request = new ::MPI_Request; 
     273    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend, (*request)->mpi_request = new ::MPI_Request"); 
     274       
     275    (*request)->ep_src = src_rank; 
     276    (*request)->ep_tag = tag; 
     277    (*request)->ep_datatype = datatype; 
     278    (*request)->type = 1; 
     279    (*request)->comm = comm; 
     280     
     281    return ::MPI_Issend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
    242282  } 
    243283 
     
    245285  int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
    246286  { 
    247     Debug("MPI_Issend with intercomm\n"); 
    248  
    249     //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
     287    Debug("MPI_Issend with EP_Intercomm\n"); 
     288 
     289#ifdef _check_sum     
     290    check_sum_send(buf, count, datatype, dest, tag, comm); 
     291#endif 
    250292 
    251293    int dest_remote_ep_rank    = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
     
    255297    int src_comm_label; 
    256298 
    257     for(int i=0; i<comm->ep_comm_ptr->intercomm->local_rank_map->size(); i++) 
    258     { 
    259       if(comm->ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank) 
    260       { 
    261         src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(i).second; 
    262         break; 
    263       } 
    264     } 
    265  
     299    src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second; 
    266300    //Message_Check(comm); 
    267301 
    268  
    269     if(dest_remote_comm_label == src_comm_label)       // dest rank (loc, mpi) differs 
     302    if(dest_remote_comm_label == src_comm_label)       // mpi_dest differs 
    270303    { 
    271304      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    272       int ep_src_loc = comm->rank_map->at(inter_src).first; 
    273       int ep_dest_loc = comm->rank_map->at(dest_remote_ep_rank).first; 
    274       int mpi_dest    = comm->rank_map->at(dest_remote_ep_rank).second; 
     305      int ep_src_loc = comm->ep_rank_map->at(inter_src).first; 
     306      int ep_dest_loc = comm->ep_rank_map->at(dest_remote_ep_rank).first; 
     307      int mpi_dest    = comm->ep_rank_map->at(dest_remote_ep_rank).second; 
    275308      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    276309 
    277       ::MPI_Request mpi_request; 
    278    
    279       ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
    280  
    281       (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     310      *request = new ep_request; 
     311      memcheck("new "<< *request <<" : in ep_lib::MPI_Issend_intercomm, *request = new ep_request"); 
     312 
     313      (*request)->mpi_request = new ::MPI_Request; 
     314      memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend_intercomm, (*request)->mpi_request = new ::MPI_Request"); 
     315       
    282316      (*request)->type = 1;    // used in wait 
    283317      (*request)->comm = comm; 
    284  
    285318      (*request)->ep_src = src_ep_rank; 
    286319      (*request)->ep_tag = tag; 
    287320      (*request)->ep_datatype = datatype; 
     321 
     322      return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
     323 
    288324    } 
    289325 
     
    291327    {  
    292328      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    293       int ep_src_loc = comm->rank_map->at(inter_src).first; 
     329      int ep_src_loc = comm->ep_rank_map->at(inter_src).first; 
    294330      int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first; 
    295331      int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second; 
    296332      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    297333 
    298       ::MPI_Request mpi_request; 
    299  
    300       ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request); 
    301  
    302       (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     334      *request = new ep_request; 
     335      memcheck("new "<< *request <<" : in ep_lib::MPI_Issend_intercomm, *request = new ep_request"); 
     336 
     337      (*request)->mpi_request = new ::MPI_Request; 
     338      memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend_intercomm, (*request)->mpi_request = new ::MPI_Request"); 
     339 
    303340      (*request)->type = 1;    // used in wait 
    304341      (*request)->comm = comm; 
    305     
    306342      (*request)->ep_src = src_ep_rank; 
    307343      (*request)->ep_tag = tag; 
    308344      (*request)->ep_datatype = datatype; 
     345       
     346      return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request));  
    309347    } 
    310  
    311     return 0; 
    312  
    313348  } 
    314349   
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_size.cpp

    r1500 r1503  
    3838 
    3939      ::MPI_Comm_size(mpi_comm, &mpi_size); 
    40       printf("============ ep comm size called for non ep comm, %d\n", mpi_size); 
    4140      *size = mpi_size; 
    4241      return 0; 
     
    4645    { 
    4746      *size = MPI_UNDEFINED; 
    48       printf("============ ep comm size called for non ep comm not defined\n"); 
    4947      return 0; 
    5048    } 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_type.cpp

    r1500 r1503  
    125125    my_buffer = NULL; 
    126126    ep_barrier = NULL; 
    127     rank_map = NULL; 
     127    ep_rank_map = NULL; 
    128128    ep_comm_ptr = NULL; 
    129129    mem_bridge = NULL; 
     
    140140    my_buffer = NULL; 
    141141    ep_barrier = NULL; 
    142     rank_map = NULL; 
     142    ep_rank_map = NULL; 
    143143    ep_comm_ptr = NULL; 
    144144    mem_bridge = NULL; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_type.hpp

    r1500 r1503  
    2323#endif 
    2424 
     25#ifdef _Memory_check 
     26#define memcheck(x) std::cout << x << std::endl 
     27#else 
     28#define memcheck(x) 
     29#endif 
     30 
     31 
    2532#define BUFFER_SIZE 10000 
    2633 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_wait.cpp

    r1500 r1503  
    4141    (*request)->state = 2; 
    4242 
    43     printf("delete %p : in ep_lib::MPI_Wait, delete (*request)->mpi_request\n", (*request)->mpi_request); 
     43    memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Wait, delete (*request)->mpi_request"); 
    4444    delete (*request)->mpi_request; 
    4545    Request_Check(); 
     
    9191      array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype; 
    9292      array_of_requests[i]->state = 2; 
    93       printf("delete %p : in ep_lib::MPI_Wait, array_of_requests[i]->mpi_request\n", array_of_requests[i]->mpi_request); 
     93      memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]->mpi_request"); 
    9494      delete array_of_requests[i]->mpi_request; 
    9595      if(array_of_requests[i]->type == 1)  
    9696      { 
    97         printf("delete %p : in ep_lib::MPI_Wait, array_of_requests[i]\n", array_of_requests[i]); 
     97        memcheck("delete "<< array_of_requests[i] <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]"); 
    9898        delete array_of_requests[i]; 
    9999      } 
  • XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

    r1500 r1503  
    5252 
    5353 
     54    MPI_Comm comm_for_dup; // this should act as EP_COMM_WORLD 
    5455    MPI_Comm comm; // this should act as EP_COMM_WORLD 
    55     comm = passage[omp_get_thread_num()]; 
     56         
     57    comm_for_dup = passage[omp_get_thread_num()]; 
     58    MPI_Comm_dup(comm_for_dup, &comm); 
     59     
     60    MPI_Comm_free(&comm_for_dup); 
     61     
     62    MPI_Barrier(comm); 
    5663 
    5764    int rank, size; 
    5865    MPI_Comm_rank(comm, &rank); 
    5966    MPI_Comm_size(comm, &size); 
     67     
     68    if(rank == 0) printf("           \t test MPI_Comm_dup \t OK \n"); 
    6069/* 
    6170    // TIMING SYCHRONIZATION 
     
    109118      MPI_Barrier(comm); 
    110119       
     120      MPI_Comm equal_comm = comm; 
     121       
     122       
     123       
     124       
    111125      double sendbuf[10]; 
    112126      double recvbuf[20]; 
    113127       
    114       int sender = 1; 
    115       int receiver = 2; 
     128      int sender; 
     129      if(rank == 0) sender = rand() % size; 
     130      MPI_Bcast(&sender, 1, MPI_INT, 0, comm); 
     131       
     132      int receiver = sender; 
     133      if(rank == 0) while(sender == receiver) {receiver = rand() % size;} 
     134      MPI_Bcast(&receiver, 1, MPI_INT, 0, comm); 
     135       
     136       
     137       
    116138       
    117139      if(rank == sender) 
    118140      { 
    119141        for(int i=0; i<10; i++) sendbuf[i] = 99.99; 
    120         MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 99, comm); 
     142        MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 99, equal_comm); 
    121143        for(int i=0; i<10; i++) sendbuf[i] = -99.99; 
    122         MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 11, comm); 
     144        MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 11, equal_comm); 
    123145      } 
    124146       
     
    131153         
    132154        for(int i=0; i<20; i++) std::cout << "recvbuf["<< i <<"] = "<< recvbuf[i] << std::endl; 
     155        printf("sender = %d\nreceiver = %d \tTEST of p2p blocking communication\tOK\n", sender, receiver); 
    133156      } 
    134157       
     
    145168      double recvbuf[20]; 
    146169       
    147       int sender = 3; 
    148       int receiver = 7; 
    149        
     170      int sender; 
     171      if(rank == 0) sender = rand() % size; 
     172      MPI_Bcast(&sender, 1, MPI_INT, 0, comm); 
     173       
     174      int receiver = sender; 
     175      if(rank == 0) receiver = rand() % size; 
     176      MPI_Bcast(&receiver, 1, MPI_INT, 0, comm); 
     177 
     178 
     179             
    150180      MPI_Request request[2]; 
    151181       
     
    179209      { 
    180210        for(int i=0; i<20; i++) std::cout << "recvbuf["<< i <<"] = "<< recvbuf[i] << std::endl; 
     211        printf("sender = %d\nreceiver = %d \tTEST of p2p non-blocking communication\tOK\n", sender, receiver); 
    181212      } 
    182213     
  • XIOS/dev/branch_openmp/extern/ep_dev/makefile

    r1500 r1503  
    11appname := EP_test 
    22 
    3 CXX := mpiicc -D_intelmpi -openmp 
     3CXX := mpiicc -D_intelmpi -D_noMemory_check -D_nocheck_sum -openmp 
    44CXXFLAGS := -std=c++11  -g 
    55#CXXFLAGS := -std=c++11  -g 
Note: See TracChangeset for help on using the changeset viewer.