Changeset 1500


Ignore:
Timestamp:
05/28/18 09:54:32 (6 years ago)
Author:
yushan
Message:

save dev

Location:
XIOS/dev/branch_openmp/extern/ep_dev
Files:
48 edited

Legend:

Unmodified
Added
Removed

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_accumulate.cpp

    r1398 r1500  
    1212                     int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win) 
    1313  { 
    14     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    15     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    16     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     14    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     15    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     16    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1717    if(num_ep==1) 
    1818      return  ::MPI_Accumulate(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    19                                to_mpi_op(op), to_mpi_win(win.server_win[target_local_rank])); 
     19                               to_mpi_op(op), to_mpi_win(win->server_win[target_local_rank])); 
    2020 
    2121    else 
    2222        return  ::MPI_Accumulate(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    23                                to_mpi_op(op), to_mpi_win(win.client_win)); 
     23                               to_mpi_op(op), to_mpi_win(win->client_win)); 
    2424  } 
    2525 
    … …  
    2727                     int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request) 
    2828  { 
    29     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    30     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    31     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     29    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     30    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     31    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    3232    ::MPI_Request mpi_request; 
    3333 
    … …  
    3535    { 
    3636      int return_value = ::MPI_Raccumulate(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    37                                    to_mpi_op(op), to_mpi_win(win.server_win[target_local_rank]), &mpi_request); 
    38       request->mpi_request = new ::MPI_Request(mpi_request); 
     37                                   to_mpi_op(op), to_mpi_win(win->server_win[target_local_rank]), &mpi_request); 
     38      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    3939 
    40       request->ep_datatype = origin_datatype; 
    41       request->type = 1; 
     40      (*request)->ep_datatype = origin_datatype; 
     41      (*request)->type = 1; 
    4242      return return_value; 
    4343    } 
    … …  
    4646    { 
    4747      int return_value = ::MPI_Raccumulate(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    48                                    to_mpi_op(op), to_mpi_win(win.client_win), &mpi_request); 
    49       request->mpi_request = new ::MPI_Request(mpi_request); 
     48                                   to_mpi_op(op), to_mpi_win(win->client_win), &mpi_request); 
     49      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    5050 
    51       request->ep_datatype = origin_datatype; 
    52       request->type = 1; 
     51      (*request)->ep_datatype = origin_datatype; 
     52      (*request)->type = 1; 
    5353      return return_value; 
    5454    } 
    … …  
    6060                         int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win) 
    6161  { 
    62     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    63     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    64     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     62    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     63    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     64    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    6565    if(num_ep==1) 
    6666      return ::MPI_Get_accumulate(origin_addr, origin_count, to_mpi_type(origin_datatype), result_addr, result_count, to_mpi_type(result_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, 
    67                                   to_mpi_type(target_datatype), to_mpi_op(op), to_mpi_win(win.server_win[target_local_rank])); 
     67                                  to_mpi_type(target_datatype), to_mpi_op(op), to_mpi_win(win->server_win[target_local_rank])); 
    6868    else 
    6969      return ::MPI_Get_accumulate(origin_addr, origin_count, to_mpi_type(origin_datatype), result_addr, result_count, to_mpi_type(result_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, 
    70                                   to_mpi_type(target_datatype), to_mpi_op(op), to_mpi_win(win.client_win)); 
     70                                  to_mpi_type(target_datatype), to_mpi_op(op), to_mpi_win(win->client_win)); 
    7171  } 
    7272 
    … …  
    7575                         int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request) 
    7676  { 
    77     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    78     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    79     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     77    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     78    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     79    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    8080    ::MPI_Request mpi_request; 
    8181    if(num_ep==1) 
    8282    { 
    8383      int return_value = ::MPI_Rget_accumulate(origin_addr, origin_count, to_mpi_type(origin_datatype), result_addr, result_count, to_mpi_type(result_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, 
    84                                       to_mpi_type(target_datatype), to_mpi_op(op), to_mpi_win(win.server_win[target_local_rank]), &mpi_request); 
    85       request->mpi_request = new ::MPI_Request(mpi_request); 
     84                                      to_mpi_type(target_datatype), to_mpi_op(op), to_mpi_win(win->server_win[target_local_rank]), &mpi_request); 
     85      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    8686 
    87       request->ep_datatype = origin_datatype; 
    88       request->type = 1; 
     87      (*request)->ep_datatype = origin_datatype; 
     88      (*request)->type = 1; 
    8989      return return_value; 
    9090    } 
    … …  
    9292    { 
    9393      int return_value = ::MPI_Rget_accumulate(origin_addr, origin_count, to_mpi_type(origin_datatype), result_addr, result_count, to_mpi_type(result_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, 
    94                                       to_mpi_type(target_datatype), to_mpi_op(op), to_mpi_win(win.client_win), &mpi_request); 
    95       request->mpi_request = new ::MPI_Request(mpi_request); 
     94                                      to_mpi_type(target_datatype), to_mpi_op(op), to_mpi_win(win->client_win), &mpi_request); 
     95      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    9696 
    97       request->ep_datatype = origin_datatype; 
    98       request->type = 1; 
     97      (*request)->ep_datatype = origin_datatype; 
     98      (*request)->type = 1; 
    9999      return return_value; 
    100100    } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allgather.cpp

    r1381 r1500  
    2020  { 
    2121 
    22     if(!comm.is_ep && comm.mpi_comm) 
     22    if(!comm->is_ep && comm->mpi_comm) 
    2323    { 
    24       return ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm.mpi_comm)); 
     24      return ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 
    2525    } 
    2626 
    … …  
    3737 
    3838 
    39     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    40     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    41     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    42     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    43     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    44     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     39    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     40    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     41    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     42    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     43    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     44    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    4545 
    4646    bool is_master = ep_rank_loc==0; 
    … …  
    7070      int local_sendcount = num_ep * count; 
    7171 
    72       ::MPI_Allgather(&local_sendcount, 1, to_mpi_type(MPI_INT), mpi_recvcounts, 1, to_mpi_type(MPI_INT), to_mpi_comm(comm.mpi_comm)); 
     72      ::MPI_Allgather(&local_sendcount, 1, to_mpi_type(MPI_INT), mpi_recvcounts, 1, to_mpi_type(MPI_INT), to_mpi_comm(comm->mpi_comm)); 
    7373 
    7474      mpi_displs[0] = 0; 
    … …  
    7979 
    8080     
    81       ::MPI_Allgatherv(local_recvbuf, num_ep * count, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts, mpi_displs, to_mpi_type(datatype), to_mpi_comm(comm.mpi_comm)); 
     81      ::MPI_Allgatherv(local_recvbuf, num_ep * count, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts, mpi_displs, to_mpi_type(datatype), to_mpi_comm(comm->mpi_comm)); 
    8282 
    8383 
    … …  
    8686      for(int i=0; i<ep_size; i++) 
    8787      { 
    88         offset = mpi_displs[comm.rank_map->at(i).second] + comm.rank_map->at(i).first * sendcount;  
     88        offset = mpi_displs[comm->rank_map->at(i).second] + comm->rank_map->at(i).first * sendcount;  
    8989        memcpy(recvbuf + i*sendcount*datasize, tmp_recvbuf+offset*datasize, sendcount*datasize); 
    9090      } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allgatherv.cpp

    r1381 r1500  
    2020  { 
    2121 
    22     if(!comm.is_ep && comm.mpi_comm) 
     22    if(!comm->is_ep && comm->mpi_comm) 
    2323    { 
    24       return ::MPI_Allgatherv(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcounts, displs, to_mpi_type(recvtype), to_mpi_comm(comm.mpi_comm)); 
     24      return ::MPI_Allgatherv(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcounts, displs, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 
    2525    } 
    2626 
    27     if(!comm.mpi_comm) return 0; 
     27    if(!comm->mpi_comm) return 0; 
    2828 
    2929 
    … …  
    3838 
    3939 
    40     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    41     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    42     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    43     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    44     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    45     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     40    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     41    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     42    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     43    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     44    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     45    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    4646 
    4747    assert(sendcount == recvcounts[ep_rank]); 
    … …  
    7979 
    8080      int local_sendcount = std::accumulate(local_recvcounts.begin(), local_recvcounts.end(), 0); 
    81       ::MPI_Allgather(&local_sendcount, 1, to_mpi_type(MPI_INT), mpi_recvcounts.data(), 1, to_mpi_type(MPI_INT), to_mpi_comm(comm.mpi_comm)); 
     81      ::MPI_Allgather(&local_sendcount, 1, to_mpi_type(MPI_INT), mpi_recvcounts.data(), 1, to_mpi_type(MPI_INT), to_mpi_comm(comm->mpi_comm)); 
    8282 
    8383      for(int i=1; i<mpi_size; i++) 
    … …  
    8585 
    8686 
    87       ::MPI_Allgatherv(local_recvbuf, local_sendcount, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts.data(), mpi_displs.data(), to_mpi_type(datatype), to_mpi_comm(comm.mpi_comm)); 
     87      ::MPI_Allgatherv(local_recvbuf, local_sendcount, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts.data(), mpi_displs.data(), to_mpi_type(datatype), to_mpi_comm(comm->mpi_comm)); 
    8888 
    8989      // reorder  
    … …  
    9292      { 
    9393        int extra = 0; 
    94         for(int j=0, k=0; j<ep_size, k<comm.rank_map->at(i).first; j++) 
    95           if(comm.rank_map->at(i).second == comm.rank_map->at(j).second) 
     94        for(int j=0, k=0; j<ep_size, k<comm->rank_map->at(i).first; j++) 
     95          if(comm->rank_map->at(i).second == comm->rank_map->at(j).second) 
    9696          { 
    9797            extra += recvcounts[j]; 
    … …  
    9999          }   
    100100 
    101         offset = mpi_displs[comm.rank_map->at(i).second] +  extra; 
     101        offset = mpi_displs[comm->rank_map->at(i).second] +  extra; 
    102102 
    103103        memcpy(recvbuf+displs[i]*datasize, tmp_recvbuf+offset*datasize, recvcounts[i]*datasize); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allocate.cpp

    r1388 r1500  
    1616   } 
    1717    
    18   // int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr) 
    19   // { 
    20   //   ::MPI_Alloc_mem(size, *(static_cast< ::MPI_Info*>(info.mpi_info)), baseptr); 
    21   //   //::MPI_Alloc_mem(size, MPI_INFO_NULL_STD, baseptr); 
    22   //   return 0; 
    23   // } 
     18  int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr) 
     19  { 
     20    ::MPI_Alloc_mem(size, *(static_cast< ::MPI_Info*>(info->mpi_info)), baseptr); 
     21    //::MPI_Alloc_mem(size, MPI_INFO_NULL_STD, baseptr); 
     22    return 0; 
     23  } 
    2424 
    2525  int MPI_Win_allocate (MPI_Aint size, int disp_unit, MPI_Info info, MPI_Comm comm, void *baseptr, MPI_Win *win) 
    … …  
    2727    int rank, rank_loc, num_ep; 
    2828    MPI_Comm_rank(comm, &rank); 
    29     rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    30     num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
     29    rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     30    num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
    3131     
    32     win->is_ep = comm.is_ep; 
     32    *win = new ep_win; 
     33     
     34    (*win)->is_ep = comm->is_ep; 
    3335 
    3436    int num_ep_max; 
    … …  
    3739    assert(num_ep_max > 1); 
    3840     
    39      
     41    info = new ep_info; 
    4042 
    4143    if(num_ep == 1)  // server 
    … …  
    4345      for(int i=0; i<num_ep_max; i++) 
    4446      { 
    45         win->server_win[i] = new ::MPI_Win; 
     47        (*win)->server_win[i] = new ::MPI_Win; 
    4648      } 
    4749 
    … …  
    4951      for(int i=0; i<num_ep_max; i++) 
    5052      { 
    51         ::MPI_Win_allocate(to_mpi_aint(size), disp_unit, to_mpi_info(info), to_mpi_comm(comm.mpi_comm), baseptr, static_cast< ::MPI_Win*>(win->server_win[i])); 
    52         win->comm = comm; 
     53        ::MPI_Win_allocate(to_mpi_aint(size), disp_unit, to_mpi_info(info), to_mpi_comm(comm->mpi_comm), baseptr, static_cast< ::MPI_Win*>((*win)->server_win[i])); 
     54        (*win)->comm = comm; 
    5355      } 
    5456    } 
    5557    else  // client 
    5658    { 
    57       win->client_win = new ::MPI_Win; 
     59      (*win)->client_win = new ::MPI_Win; 
    5860       
    5961      for(int i=0; i<num_ep; i++) 
    … …  
    6264        { 
    6365          printf("Calling MPI_Win_allocate from client, rank = %d\n", rank); 
    64           ::MPI_Win_allocate(to_mpi_aint(size), disp_unit, to_mpi_info(info), to_mpi_comm(comm.mpi_comm), baseptr, static_cast< ::MPI_Win*>(win->client_win));  
    65           win->comm = comm; 
     66          ::MPI_Win_allocate(to_mpi_aint(size), disp_unit, to_mpi_info(info), to_mpi_comm(comm->mpi_comm), baseptr, static_cast< ::MPI_Win*>((*win)->client_win));  
     67          (*win)->comm = comm; 
    6668        } 
    6769        MPI_Barrier_local(comm); 
    … …  
    7476          for(int i=0; i<num_ep_max-num_ep; i++) 
    7577          { 
    76             win->null_win[i] = new ::MPI_Win; 
    77             ::MPI_Win_allocate(0, disp_unit, to_mpi_info(info), to_mpi_comm(comm.mpi_comm), baseptr, static_cast< ::MPI_Win*>(win->null_win[i]));  
     78            (*win)->null_win[i] = new ::MPI_Win; 
     79            ::MPI_Win_allocate(0, disp_unit, to_mpi_info(info), to_mpi_comm(comm->mpi_comm), baseptr, static_cast< ::MPI_Win*>((*win)->null_win[i]));  
    7880          }   
    7981        }   

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allreduce.cpp

    r1381 r1500  
    2121  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
    2222  { 
    23     if(!comm.is_ep && comm.mpi_comm) 
     23    if(!comm->is_ep && comm->mpi_comm) 
    2424    { 
    25       return ::MPI_Allreduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm)); 
     25      return ::MPI_Allreduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    2626    } 
    2727 
    2828 
    2929 
    30     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    31     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    32     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    33     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    34     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    35     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     30    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     31    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     32    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     33    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     34    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     35    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    3636 
    3737 
    … …  
    5555    if(is_master) 
    5656    { 
    57       ::MPI_Allreduce(local_recvbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm)); 
     57      ::MPI_Allreduce(local_recvbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    5858    } 
    5959 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_alltoall.cpp

    r1381 r1500  
    99  int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) 
    1010  { 
    11     if(!comm.is_ep) 
     11    if(!comm->is_ep) 
    1212    { 
    13       return ::MPI_Alltoall(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm.mpi_comm)); 
     13      return ::MPI_Alltoall(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 
    1414    } 
    1515 
    … …  
    2323    int count = sendcount; 
    2424     
    25     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    26     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    27     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    28     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    29     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    30     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     25    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     26    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     27    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     28    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     29    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     30    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    3131 
    3232    void* tmp_recvbuf; 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_barrier.cpp

    r1499 r1500  
    99  int MPI_Barrier(MPI_Comm comm) 
    1010  { 
    11     if(comm.is_intercomm) return MPI_Barrier_intercomm(comm); 
     11    if(comm->is_intercomm) return MPI_Barrier_intercomm(comm); 
    1212 
    13     if(comm.is_ep) 
     13    if(comm->is_ep) 
    1414    { 
    15       int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
     15      int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
    1616 
    1717      MPI_Barrier_local(comm); 
    … …  
    1919      if(ep_rank_loc == 0) 
    2020      { 
    21         ::MPI_Comm mpi_comm = to_mpi_comm(comm.mpi_comm); 
     21        ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
    2222 
    2323        ::MPI_Barrier(mpi_comm); 
    … …  
    2828      return 0; 
    2929    } 
    30     else if(comm.mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 
     30    else if(comm->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
    3131    { 
    32       ::MPI_Comm mpi_comm = to_mpi_comm(comm.mpi_comm); 
     32      ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
    3333      ::MPI_Barrier(mpi_comm); 
    3434      return 0; 
    … …  
    4242  { 
    4343 
    44     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
     44    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
    4545 
    4646    MPI_Barrier_local(comm); 
    … …  
    4848    if(ep_rank_loc == 0) 
    4949    { 
    50       ::MPI_Comm mpi_comm = to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm); 
     50      ::MPI_Comm mpi_comm = to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm); 
    5151      ::MPI_Barrier(mpi_comm); 
    5252    } 
    … …  
    6262  { 
    6363    //Message_Check(comm); 
    64     comm.ep_barrier->wait(); 
     64    comm->ep_barrier->wait(); 
    6565  } 
    6666 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_bcast.cpp

    r1381 r1500  
    2020    assert(valid_type(datatype)); 
    2121 
    22     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
     22    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
    2323 
    2424    ::MPI_Aint datasize, lb; 
    … …  
    2828    if(ep_rank_loc == local_root) 
    2929    { 
    30       comm.my_buffer->void_buffer[local_root] = buffer; 
     30      comm->my_buffer->void_buffer[local_root] = buffer; 
    3131    } 
    3232 
    … …  
    3636    { 
    3737      #pragma omp critical (_bcast)       
    38       memcpy(buffer, comm.my_buffer->void_buffer[local_root], datasize * count); 
     38      memcpy(buffer, comm->my_buffer->void_buffer[local_root], datasize * count); 
    3939    } 
    4040 
    … …  
    4545  { 
    4646 
    47     if(!comm.is_ep) 
     47    if(!comm->is_ep) 
    4848    { 
    4949      #pragma omp single nowait 
    50       ::MPI_Bcast(buffer, count, to_mpi_type(datatype), root, to_mpi_comm(comm.mpi_comm)); 
     50      ::MPI_Bcast(buffer, count, to_mpi_type(datatype), root, to_mpi_comm(comm->mpi_comm)); 
    5151      return 0; 
    5252    } 
    5353 
    5454 
    55     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    56     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    57     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
     55    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     56    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     57    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
    5858 
    59     int root_mpi_rank = comm.rank_map->at(root).second; 
    60     int root_ep_rank_loc = comm.rank_map->at(root).first; 
     59    int root_mpi_rank = comm->rank_map->at(root).second; 
     60    int root_ep_rank_loc = comm->rank_map->at(root).first; 
    6161 
    6262 
    6363    if((ep_rank_loc==0 && mpi_rank != root_mpi_rank ) || ep_rank == root) 
    6464    { 
    65       ::MPI_Bcast(buffer, count, to_mpi_type(datatype), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     65      ::MPI_Bcast(buffer, count, to_mpi_type(datatype), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    6666    } 
    6767 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_compare.cpp

    r1395 r1500  
    1111        int MPI_Compare_and_swap(const void *origin_addr, const void *compare_addr, void *result_addr, MPI_Datatype datatype, int target_rank, MPI_Aint target_disp, MPI_Win win) 
    1212  { 
    13     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    14     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    15     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     13    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     14    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     15    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1616    if(num_ep==1) 
    17       return  ::MPI_Compare_and_swap(origin_addr, compare_addr, result_addr, to_mpi_type(datatype), target_mpi_rank, to_mpi_aint(target_disp), to_mpi_win(win.server_win[target_local_rank])); 
     17      return  ::MPI_Compare_and_swap(origin_addr, compare_addr, result_addr, to_mpi_type(datatype), target_mpi_rank, to_mpi_aint(target_disp), to_mpi_win(win->server_win[target_local_rank])); 
    1818 
    1919    else 
    20         return  ::MPI_Compare_and_swap(origin_addr, compare_addr, result_addr, to_mpi_type(datatype), target_mpi_rank, to_mpi_aint(target_disp), to_mpi_win(win.client_win)); 
     20        return  ::MPI_Compare_and_swap(origin_addr, compare_addr, result_addr, to_mpi_type(datatype), target_mpi_rank, to_mpi_aint(target_disp), to_mpi_win(win->client_win)); 
    2121  } 
    2222 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp

    r1499 r1500  
    2525    \param [out] out_comm_hdls Handles of EP communicators. 
    2626  */ 
    27   // #ifdef _intelmpi 
    28   // int MPI_Comm_create_endpoints(int base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls) 
    29   // { 
    30   //   int base_rank; 
    31   //   int base_size; 
    32  
    33   //   ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm_ptr); 
    34  
    35   //   ::MPI_Comm_size(mpi_base_comm, &base_size);  // ep_lib::mpi_comm_size 
    36   //   ::MPI_Comm_rank(mpi_base_comm, &base_rank);  // ep_lib::mpi_comm_rank 
    37                 //                                                        // parent_comm can also be endpoints communicators 
    38  
    39   //   std::vector<int> recv_num_ep(base_size); 
    40  
    41   //   out_comm_hdls = new MPI_Comm[num_ep]; 
    42  
    43   //   for (int idx = 0; idx < num_ep; ++idx) 
    44   //   { 
    45   //     out_comm_hdls[idx].is_ep = true; 
    46   //     out_comm_hdls[idx].is_intercomm = false; 
    47   //     out_comm_hdls[idx].ep_comm_ptr = new ep_communicator; 
    48   //     out_comm_hdls[idx].mpi_comm = base_comm_ptr; 
    49   //     out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls; 
    50   //     out_comm_hdls[idx].ep_comm_ptr->comm_label = 0; 
    51   //   } 
    52  
    53   //   ::MPI_Allgather(&num_ep, 1, static_cast< ::MPI_Datatype>(MPI_INT), &recv_num_ep[0], 1, static_cast< ::MPI_Datatype>(MPI_INT), mpi_base_comm); 
    54  
    55  
    56   //   int sum = 0;  // representing total ep number of process with smaller rank 
    57   //   for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; } 
    58  
    59   //   int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0); 
    60  
    61   //   out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep); 
    62  
    63   //   out_comm_hdls[0].my_buffer = new BUFFER; 
    64  
    65   //   out_comm_hdls[0].rank_map = new RANK_MAP; 
    66   //   out_comm_hdls[0].rank_map->resize(ep_size); 
    67  
    68  
    69   //   for (int i = 1; i < num_ep; i++) 
    70   //   { 
    71   //     out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier; 
    72   //     out_comm_hdls[i].my_buffer  = out_comm_hdls[0].my_buffer; 
    73   //     out_comm_hdls[i].rank_map   = out_comm_hdls[0].rank_map; 
    74   //   } 
    75  
    76  
    77   //   for (int i = 0; i < num_ep; i++) 
    78   //   { 
    79   //     out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size); 
    80   //     out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep); 
    81   //     out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size); 
    82  
    83   //     out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list; 
    84   //   } 
    85  
    86  
    87   //   int ind = 0; 
    88  
    89   //   for(int i=0; i<base_size; i++) 
    90   //   { 
    91   //     for(int j=0; j<recv_num_ep[i]; j++) 
    92   //     { 
    93   //       out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i); 
    94   //       ind++; 
    95   //     } 
    96   //   } 
    97  
    98  
    99  
    100   //   return 0; 
    101  
    102   // } //MPI_Comm_create_endpoints 
    103  
    104   // #elif _openmpi 
    105   // int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls) 
    106   // { 
    107  
    108   //   int base_rank; 
    109   //   int base_size; 
    110  
    111   //   ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm_ptr); 
    112  
    113   //   ::MPI_Comm_size(mpi_base_comm, &base_size);  // ep_lib::mpi_comm_size 
    114   //   ::MPI_Comm_rank(mpi_base_comm, &base_rank);  // ep_lib::mpi_comm_rank 
    115   //                                                // parent_comm can also be endpoints communicators ? 
    116   //   std::vector<int> recv_num_ep(base_size); 
    117  
    118   //   out_comm_hdls = new MPI_Comm[num_ep]; 
    119  
    120   //   for (int idx = 0; idx < num_ep; ++idx) 
    121   //   { 
    122   //     out_comm_hdls[idx].is_ep = true; 
    123   //     out_comm_hdls[idx].is_intercomm = false; 
    124   //     out_comm_hdls[idx].ep_comm_ptr = new ep_communicator; 
    125   //     out_comm_hdls[idx].mpi_comm = base_comm_ptr; 
    126   //     out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls; 
    127   //     out_comm_hdls[idx].ep_comm_ptr->comm_label = 0; 
    128   //   } 
    129  
    130   //   ::MPI_Allgather(&num_ep, 1, static_cast< ::MPI_Datatype> (MPI_INT),  
    131   //                  &recv_num_ep[0], 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_base_comm); 
    132  
    133   //   int sum = 0;  // representing total ep number of process with smaller rank 
    134   //   for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; } 
    135  
    136   //   int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0); 
    137  
    138   //   out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep); 
    139   //   out_comm_hdls[0].my_buffer = new BUFFER; 
    140  
    141   //   out_comm_hdls[0].rank_map = new RANK_MAP; 
    142   //   out_comm_hdls[0].rank_map->resize(ep_size); 
    143  
    144  
    145   //   for (int i = 1; i < num_ep; i++) 
    146   //   { 
    147   //     out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier; 
    148   //     out_comm_hdls[i].my_buffer  = out_comm_hdls[0].my_buffer; 
    149   //     out_comm_hdls[i].rank_map   = out_comm_hdls[0].rank_map; 
    150   //   } 
    151  
    152  
    153   //   for (int i = 0; i < num_ep; i++) 
    154   //   { 
    155   //     out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size); 
    156   //     out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep); 
    157   //     out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size); 
    158  
    159   //     out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list; 
    160   //   } 
    161  
    162  
    163   //   int ind = 0; 
    164  
    165   //   for(int i=0; i<base_size; i++) 
    166   //   { 
    167   //     for(int j=0; j<recv_num_ep[i]; j++) 
    168   //     { 
    169   //       out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i); 
    170   //       ind++; 
    171   //     } 
    172   //   } 
    173  
    174   //   return 0; 
    175  
    176   // } //MPI_Comm_create_endpoints 
    177  
    178   // #endif 
    179    
    18027   
    18128  int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls) 
    … …  
    19744    for (int idx = 0; idx < num_ep; ++idx) 
    19845    { 
    199       out_comm_hdls[idx].is_ep = true; 
    200       out_comm_hdls[idx].is_intercomm = false; 
    201       out_comm_hdls[idx].ep_comm_ptr = new ep_communicator;      
    202       *(static_cast< ::MPI_Comm*>(out_comm_hdls[idx].mpi_comm)) = *(static_cast< ::MPI_Comm*>(base_comm_ptr)); 
    203       out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls; 
    204       out_comm_hdls[idx].ep_comm_ptr->comm_label = 0; 
     46      out_comm_hdls[idx] = new ep_comm; 
     47      out_comm_hdls[idx]->is_ep = true; 
     48      out_comm_hdls[idx]->is_intercomm = false; 
     49      out_comm_hdls[idx]->ep_comm_ptr = new ep_communicator;      
     50      *(static_cast< ::MPI_Comm*>(out_comm_hdls[idx]->mpi_comm)) = *(static_cast< ::MPI_Comm*>(base_comm_ptr)); 
     51      out_comm_hdls[idx]->ep_comm_ptr->comm_list = out_comm_hdls; 
     52      out_comm_hdls[idx]->ep_comm_ptr->comm_label = 0; 
    20553    } 
    20654 
    … …  
    21361    int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0); 
    21462 
    215     out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep); 
    216     out_comm_hdls[0].my_buffer = new BUFFER; 
     63    out_comm_hdls[0]->ep_barrier = new ep_barrier(num_ep); 
     64    out_comm_hdls[0]->my_buffer = new BUFFER; 
    21765 
    218     out_comm_hdls[0].rank_map = new RANK_MAP; 
    219     out_comm_hdls[0].rank_map->resize(ep_size); 
     66    out_comm_hdls[0]->rank_map = new RANK_MAP; 
     67    out_comm_hdls[0]->rank_map->resize(ep_size); 
    22068 
    22169 
    22270    for (int i = 1; i < num_ep; i++) 
    22371    { 
    224       out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier; 
    225       out_comm_hdls[i].my_buffer  = out_comm_hdls[0].my_buffer; 
    226       out_comm_hdls[i].rank_map   = out_comm_hdls[0].rank_map; 
     72      out_comm_hdls[i]->ep_barrier = out_comm_hdls[0]->ep_barrier; 
     73      out_comm_hdls[i]->my_buffer  = out_comm_hdls[0]->my_buffer; 
     74      out_comm_hdls[i]->rank_map   = out_comm_hdls[0]->rank_map; 
    22775    } 
    22876 
    … …  
    23078    for (int i = 0; i < num_ep; i++) 
    23179    { 
    232       out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size); 
    233       out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep); 
    234       out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size); 
     80      out_comm_hdls[i]->ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size); 
     81      out_comm_hdls[i]->ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep); 
     82      out_comm_hdls[i]->ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size); 
    23583 
    236       out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list; 
     84      out_comm_hdls[i]->ep_comm_ptr->message_queue = new Message_list; 
    23785    } 
    23886 
    … …  
    24492      for(int j=0; j<recv_num_ep[i]; j++) 
    24593      { 
    246         out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i); 
     94        out_comm_hdls[0]->rank_map->at(ind) = make_pair(j, i); 
    24795        ind++; 
    24896      } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_declaration.cpp

    r1499 r1500  
    9494ep_lib::MPI_Op MPI_REPLACE = &MPI_REPLACE_STD; 
    9595 
    96 ep_lib::MPI_Comm MPI_COMM_WORLD(&MPI_COMM_WORLD_STD); 
    97 ep_lib::MPI_Comm MPI_COMM_NULL(&MPI_COMM_NULL_STD); 
     96ep_lib::ep_comm EP_COMM_WORLD(&MPI_COMM_WORLD_STD); 
     97ep_lib::ep_comm EP_COMM_NULL(&MPI_COMM_NULL_STD); 
    9898 
    99 ep_lib::MPI_Request MPI_REQUEST_NULL(&MPI_REQUEST_NULL_STD); 
    100 ep_lib::MPI_Info MPI_INFO_NULL(&MPI_INFO_NULL_STD); 
     99ep_lib::MPI_Comm MPI_COMM_WORLD = &EP_COMM_WORLD; 
     100ep_lib::MPI_Comm MPI_COMM_NULL = &EP_COMM_NULL; 
     101 
     102//ep_lib::ep_status EP_STATUS_IGNORE(&MPI_STATUS_IGNORE_STD); 
     103ep_lib::ep_request EP_REQUEST_NULL(&MPI_REQUEST_NULL_STD); 
     104ep_lib::ep_info EP_INFO_NULL(&MPI_INFO_NULL_STD); 
     105 
     106//ep_lib::MPI_Status MPI_STATUS_IGNORE = &EP_STATUS_IGNORE; 
     107ep_lib::MPI_Request MPI_REQUEST_NULL = &EP_REQUEST_NULL; 
     108ep_lib::MPI_Info MPI_INFO_NULL = &EP_INFO_NULL; 
    101109 
    102110 
    103111 
    104  

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_dup.cpp

    r1499 r1500  
    99  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm) 
    1010  { 
    11     if(!comm.is_ep) 
     11    if(!comm->is_ep) 
    1212    { 
    1313      Debug("Comm_dup MPI\n"); 
    1414      newcomm = new MPI_Comm; 
    15       newcomm->is_ep = comm.is_ep; 
     15      (*newcomm)->is_ep = comm->is_ep; 
    1616 
    17       ::MPI_Comm input = to_mpi_comm(comm.mpi_comm); 
     17      ::MPI_Comm input = to_mpi_comm(comm->mpi_comm); 
    1818      ::MPI_Comm *output = new ::MPI_Comm; 
    1919 
    … …  
    2121      ::MPI_Comm_dup(input, output); 
    2222 
    23       newcomm->mpi_comm = output; 
     23      (*newcomm)->mpi_comm = output; 
    2424 
    2525      return 0; 
    2626    } 
    2727 
    28     if(comm.is_intercomm) return MPI_Comm_dup_intercomm(comm, newcomm); 
     28    if(comm->is_intercomm) return MPI_Comm_dup_intercomm(comm, newcomm); 
    2929 
    3030    // for intracomm 
    31     if(comm.mpi_comm == static_cast< ::MPI_Comm* >(MPI_COMM_NULL.mpi_comm)) return 0; 
     31    if(comm->mpi_comm == static_cast< ::MPI_Comm* >(MPI_COMM_NULL->mpi_comm)) return 0; 
    3232 
    3333 
    34     int my_rank = comm.ep_comm_ptr->size_rank_info[1].first; 
    35     int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second; 
     34    int my_rank = comm->ep_comm_ptr->size_rank_info[1].first; 
     35    int num_ep  = comm->ep_comm_ptr->size_rank_info[1].second; 
    3636 
    3737 
    … …  
    4242      ::MPI_Comm *mpi_dup = new ::MPI_Comm; 
    4343 
    44       ::MPI_Comm in_comm = to_mpi_comm(comm.mpi_comm); 
     44      ::MPI_Comm in_comm = to_mpi_comm(comm->mpi_comm); 
    4545 
    4646      ::MPI_Comm_dup(in_comm, mpi_dup); 
    4747 
    4848      MPI_Comm_create_endpoints(mpi_dup, num_ep, info, out_comm); 
    49       comm.ep_comm_ptr->comm_list->mem_bridge = out_comm; 
     49      comm->ep_comm_ptr->comm_list[0]->mem_bridge = out_comm; 
    5050    } 
    5151 
    5252    MPI_Barrier(comm); 
    5353 
    54     *newcomm = (comm.ep_comm_ptr->comm_list->mem_bridge[my_rank]); 
     54    *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[my_rank]); 
    5555 
    5656    return MPI_SUCCESS; 
    … …  
    6060  { 
    6161     
    62     if(comm.mpi_comm == static_cast< ::MPI_Comm* >(MPI_COMM_NULL.mpi_comm)) return 0; 
     62    if(comm->mpi_comm == static_cast< ::MPI_Comm* >(MPI_COMM_NULL->mpi_comm)) return 0; 
    6363 
    64     int my_rank = comm.ep_comm_ptr->size_rank_info[1].first; 
    65     int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second; 
     64    int my_rank = comm->ep_comm_ptr->size_rank_info[1].first; 
     65    int num_ep  = comm->ep_comm_ptr->size_rank_info[1].second; 
    6666 
    6767 
    … …  
    7272      ::MPI_Comm *mpi_dup = new ::MPI_Comm; 
    7373 
    74       ::MPI_Comm in_comm = to_mpi_comm(comm.mpi_comm); 
     74      ::MPI_Comm in_comm = to_mpi_comm(comm->mpi_comm); 
    7575 
    7676      ::MPI_Comm_dup(in_comm, mpi_dup); 
    … …  
    8080      ::MPI_Comm *mpi_inter = new ::MPI_Comm; 
    8181 
    82       ::MPI_Comm_dup(to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm), mpi_inter); 
     82      ::MPI_Comm_dup(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), mpi_inter); 
    8383       
    8484      for(int i=0; i<num_ep; i++) 
    8585      { 
    86         out_comm[i].ep_comm_ptr->comm_label = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->comm_label; 
    87         out_comm[i].ep_comm_ptr->intercomm = new ep_intercomm; 
     86        out_comm[i]->ep_comm_ptr->comm_label = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->comm_label; 
     87        out_comm[i]->ep_comm_ptr->intercomm = new ep_intercomm; 
    8888 
    89         out_comm[i].ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter; 
    90         out_comm[i].is_intercomm = true; 
     89        out_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter; 
     90        out_comm[i]->is_intercomm = true; 
    9191 
    92         out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 
    93         out_comm[i].ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; 
    94         out_comm[i].ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
     92        out_comm[i]->ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 
     93        out_comm[i]->ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; 
     94        out_comm[i]->ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
    9595 
    9696        int map_size = 0; 
    97         map_size = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map->size(); 
    98         out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map->resize(map_size); 
     97        map_size = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_rank_map->size(); 
     98        out_comm[i]->ep_comm_ptr->intercomm->intercomm_rank_map->resize(map_size); 
    9999        for(int ii=0; ii<map_size; ii++) 
    100           out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map->at(ii) = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map->at(ii); 
     100          out_comm[i]->ep_comm_ptr->intercomm->intercomm_rank_map->at(ii) = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_rank_map->at(ii); 
    101101 
    102         map_size = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map->size(); 
    103         out_comm[i].ep_comm_ptr->intercomm->local_rank_map->resize(map_size); 
     102        map_size = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_rank_map->size(); 
     103        out_comm[i]->ep_comm_ptr->intercomm->local_rank_map->resize(map_size); 
    104104        for(int ii=0; ii<map_size; ii++) 
    105           out_comm[i].ep_comm_ptr->intercomm->local_rank_map->at(ii) = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map->at(ii); 
     105          out_comm[i]->ep_comm_ptr->intercomm->local_rank_map->at(ii) = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_rank_map->at(ii); 
    106106 
    107         map_size = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map->size(); 
    108         out_comm[i].ep_comm_ptr->intercomm->remote_rank_map->resize(map_size); 
     107        map_size = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->remote_rank_map->size(); 
     108        out_comm[i]->ep_comm_ptr->intercomm->remote_rank_map->resize(map_size); 
    109109        for(int ii=0; ii<map_size; ii++) 
    110           out_comm[i].ep_comm_ptr->intercomm->remote_rank_map->at(ii) = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map->at(ii); 
     110          out_comm[i]->ep_comm_ptr->intercomm->remote_rank_map->at(ii) = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->remote_rank_map->at(ii); 
    111111 
    112112 
    113         //out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map; 
    114         //out_comm[i].ep_comm_ptr->intercomm->local_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map; 
    115         //out_comm[i].ep_comm_ptr->intercomm->remote_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map; 
     113        //out_comm[i]->ep_comm_ptr->intercomm->intercomm_rank_map = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_rank_map; 
     114        //out_comm[i]->ep_comm_ptr->intercomm->local_rank_map = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_rank_map; 
     115        //out_comm[i]->ep_comm_ptr->intercomm->remote_rank_map = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->remote_rank_map; 
    116116 
    117         out_comm[i].ep_comm_ptr->intercomm->local_comm = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_comm;         
    118         out_comm[i].ep_comm_ptr->intercomm->intercomm_tag = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_tag; 
     117        out_comm[i]->ep_comm_ptr->intercomm->local_comm = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_comm;         
     118        out_comm[i]->ep_comm_ptr->intercomm->intercomm_tag = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_tag; 
    119119 
    120120        for(int j =0; j<3; j++) 
    121121        { 
    122           out_comm[i].ep_comm_ptr->intercomm->size_rank_info[j] = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->size_rank_info[j]; 
     122          out_comm[i]->ep_comm_ptr->intercomm->size_rank_info[j] = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->size_rank_info[j]; 
    123123        } 
    124124 
    125125      } 
    126126 
    127       comm.ep_comm_ptr->comm_list->mem_bridge = out_comm; 
     127      comm->ep_comm_ptr->comm_list[0]->mem_bridge = out_comm; 
    128128    } 
    129129 
    130130    MPI_Barrier(comm); 
    131131 
    132     *newcomm = comm.ep_comm_ptr->comm_list->mem_bridge[my_rank]; 
     132    *newcomm = comm->ep_comm_ptr->comm_list[0]->mem_bridge[my_rank]; 
    133133     
    134134 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_exscan.cpp

    r1381 r1500  
    5050    valid_op(op); 
    5151 
    52     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    53     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    54     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
     52    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     53    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     54    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
    5555     
    5656 
    … …  
    6060    if(ep_rank_loc == 0 && mpi_rank != 0) 
    6161    { 
    62       comm.my_buffer->void_buffer[0] = recvbuf; 
     62      comm->my_buffer->void_buffer[0] = recvbuf; 
    6363    } 
    6464    if(ep_rank_loc == 0 && mpi_rank == 0) 
    6565    { 
    66       comm.my_buffer->void_buffer[0] = const_cast<void*>(sendbuf);   
     66      comm->my_buffer->void_buffer[0] = const_cast<void*>(sendbuf);   
    6767    }  
    6868       
    … …  
    7070    MPI_Barrier_local(comm); 
    7171 
    72     memcpy(recvbuf, comm.my_buffer->void_buffer[0], datasize*count); 
     72    memcpy(recvbuf, comm->my_buffer->void_buffer[0], datasize*count); 
    7373 
    7474    MPI_Barrier_local(comm); 
    7575 
    76     comm.my_buffer->void_buffer[ep_rank_loc] = const_cast<void*>(sendbuf);   
     76    comm->my_buffer->void_buffer[ep_rank_loc] = const_cast<void*>(sendbuf);   
    7777     
    7878    MPI_Barrier_local(comm); 
    … …  
    8484        assert(datasize == sizeof(int)); 
    8585        for(int i=0; i<ep_rank_loc; i++) 
    86           reduce_sum<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
     86          reduce_sum<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
    8787      } 
    8888      
    … …  
    9191        assert(datasize == sizeof(float)); 
    9292        for(int i=0; i<ep_rank_loc; i++) 
    93           reduce_sum<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
     93          reduce_sum<float>(static_cast<float*>(comm->my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
    9494      } 
    9595       
    … …  
    9999        assert(datasize == sizeof(double)); 
    100100        for(int i=0; i<ep_rank_loc; i++) 
    101           reduce_sum<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
     101          reduce_sum<double>(static_cast<double*>(comm->my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
    102102      } 
    103103 
    … …  
    106106        assert(datasize == sizeof(char)); 
    107107        for(int i=0; i<ep_rank_loc; i++) 
    108           reduce_sum<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
     108          reduce_sum<char>(static_cast<char*>(comm->my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
    109109      } 
    110110 
    … …  
    113113        assert(datasize == sizeof(long)); 
    114114        for(int i=0; i<ep_rank_loc; i++) 
    115           reduce_sum<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
     115          reduce_sum<long>(static_cast<long*>(comm->my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
    116116      } 
    117117 
    … …  
    120120        assert(datasize == sizeof(unsigned long)); 
    121121        for(int i=0; i<ep_rank_loc; i++) 
    122           reduce_sum<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
     122          reduce_sum<unsigned long>(static_cast<unsigned long*>(comm->my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
    123123      } 
    124124 
    … …  
    134134        assert(datasize == sizeof(int)); 
    135135        for(int i=0; i<ep_rank_loc; i++) 
    136           reduce_max<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
     136          reduce_max<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
    137137      } 
    138138 
    … …  
    141141        assert(datasize == sizeof(float)); 
    142142        for(int i=0; i<ep_rank_loc; i++) 
    143           reduce_max<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
     143          reduce_max<float>(static_cast<float*>(comm->my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
    144144      } 
    145145 
    … …  
    148148        assert(datasize == sizeof(double)); 
    149149        for(int i=0; i<ep_rank_loc; i++) 
    150           reduce_max<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
     150          reduce_max<double>(static_cast<double*>(comm->my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
    151151      } 
    152152 
    … …  
    155155        assert(datasize == sizeof(char)); 
    156156        for(int i=0; i<ep_rank_loc; i++) 
    157           reduce_max<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
     157          reduce_max<char>(static_cast<char*>(comm->my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
    158158      } 
    159159 
    … …  
    162162        assert(datasize == sizeof(long)); 
    163163        for(int i=0; i<ep_rank_loc; i++) 
    164           reduce_max<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
     164          reduce_max<long>(static_cast<long*>(comm->my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
    165165      } 
    166166 
    … …  
    169169        assert(datasize == sizeof(unsigned long)); 
    170170        for(int i=0; i<ep_rank_loc; i++) 
    171           reduce_max<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
     171          reduce_max<unsigned long>(static_cast<unsigned long*>(comm->my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
    172172      } 
    173173      
    … …  
    181181        assert(datasize == sizeof(int)); 
    182182        for(int i=0; i<ep_rank_loc; i++) 
    183           reduce_min<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
     183          reduce_min<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
    184184      } 
    185185 
    … …  
    188188        assert(datasize == sizeof(float)); 
    189189        for(int i=0; i<ep_rank_loc; i++) 
    190           reduce_min<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
     190          reduce_min<float>(static_cast<float*>(comm->my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
    191191      } 
    192192 
    … …  
    195195        assert(datasize == sizeof(double)); 
    196196        for(int i=0; i<ep_rank_loc; i++) 
    197           reduce_min<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
     197          reduce_min<double>(static_cast<double*>(comm->my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
    198198      } 
    199199 
    … …  
    202202        assert(datasize == sizeof(char)); 
    203203        for(int i=0; i<ep_rank_loc; i++) 
    204           reduce_min<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
     204          reduce_min<char>(static_cast<char*>(comm->my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
    205205      } 
    206206 
    … …  
    209209        assert(datasize == sizeof(long)); 
    210210        for(int i=0; i<ep_rank_loc; i++) 
    211           reduce_min<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
     211          reduce_min<long>(static_cast<long*>(comm->my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
    212212      } 
    213213 
    … …  
    216216        assert(datasize == sizeof(unsigned long)); 
    217217        for(int i=0; i<ep_rank_loc; i++) 
    218           reduce_min<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
     218          reduce_min<unsigned long>(static_cast<unsigned long*>(comm->my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
    219219      } 
    220220 
    … …  
    228228  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
    229229  { 
    230     if(!comm.is_ep) 
    231     { 
    232       return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm)); 
     230    if(!comm->is_ep) 
     231    { 
     232      return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    233233    } 
    234234     
    235235    valid_type(datatype); 
    236236 
    237     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    238     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    239     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    240     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    241     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    242     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     237    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     238    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     239    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     240    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     241    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     242    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    243243 
    244244    ::MPI_Aint datasize, lb; 
    … …  
    253253    std::vector<int> my_map(mpi_size, 0); 
    254254 
    255     for(int i=0; i<comm.rank_map->size(); i++) my_map[comm.rank_map->at(i).second]++; 
     255    for(int i=0; i<comm->rank_map->size(); i++) my_map[comm->rank_map->at(i).second]++; 
    256256 
    257257    for(int i=0; i<mpi_rank; i++) my_src += my_map[i]; 
    … …  
    291291 
    292292    if(ep_rank_loc == 0) 
    293       ::MPI_Exscan(MPI_IN_PLACE, tmp_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm)); 
     293      ::MPI_Exscan(MPI_IN_PLACE, tmp_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    294294 
    295295    // printf(" ID=%d : %d  %d \n", ep_rank, static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_fetch.cpp

    r1394 r1500  
    1111        int MPI_Fetch_and_op(const void *origin_addr, void *result_addr, MPI_Datatype datatype, int target_rank, MPI_Aint target_disp, MPI_Op op, MPI_Win win) 
    1212  { 
    13     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    14     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    15     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     13    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     14    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     15    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1616    if(num_ep==1) 
    17       return  ::MPI_Fetch_and_op(origin_addr, result_addr, to_mpi_type(datatype), target_mpi_rank, to_mpi_aint(target_disp), to_mpi_op(op), to_mpi_win(win.server_win[target_local_rank])); 
     17      return  ::MPI_Fetch_and_op(origin_addr, result_addr, to_mpi_type(datatype), target_mpi_rank, to_mpi_aint(target_disp), to_mpi_op(op), to_mpi_win(win->server_win[target_local_rank])); 
    1818 
    1919    else 
    20         return  ::MPI_Fetch_and_op(origin_addr, result_addr, to_mpi_type(datatype), target_mpi_rank, to_mpi_aint(target_disp), to_mpi_op(op), to_mpi_win(win.client_win)); 
     20        return  ::MPI_Fetch_and_op(origin_addr, result_addr, to_mpi_type(datatype), target_mpi_rank, to_mpi_aint(target_disp), to_mpi_op(op), to_mpi_win(win->client_win)); 
    2121  } 
    2222 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_finalize.cpp

    r1381 r1500  
    2626    if(id == 0) 
    2727    { 
    28       ::MPI_Comm mpi_comm = to_mpi_comm(comm.mpi_comm); 
     28      ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
    2929      ::MPI_Abort(mpi_comm, errorcode); 
    3030    } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_fortran.cpp

    r1381 r1500  
    1616    void* fint = new ::MPI_Fint; 
    1717    #ifdef _intelmpi 
    18     *static_cast< ::MPI_Fint*>(fint) = (::MPI_Fint)(to_mpi_comm(comm.mpi_comm)); 
     18    *static_cast< ::MPI_Fint*>(fint) = (::MPI_Fint)(to_mpi_comm(comm->mpi_comm)); 
    1919    #elif _openmpi 
    20     *static_cast< ::MPI_Fint*>(fint) = MPI_Comm_c2f(to_mpi_comm(comm.mpi_comm)); 
     20    *static_cast< ::MPI_Fint*>(fint) = MPI_Comm_c2f(to_mpi_comm(comm->mpi_comm)); 
    2121    #endif 
    2222     
    … …  
    2929      { 
    3030        fc_comm_map.insert(std::make_pair( std::make_pair( *static_cast< ::MPI_Fint*>(fint), omp_get_thread_num()) , comm)); 
    31         printf("EP_Comm_c2f : MAP %p insert: %d, %d, %p\n", &fc_comm_map, *static_cast< ::MPI_Fint*>(fint), omp_get_thread_num(), comm.ep_comm_ptr); 
     31        printf("EP_Comm_c2f : MAP %p insert: %d, %d, %p\n", &fc_comm_map, *static_cast< ::MPI_Fint*>(fint), omp_get_thread_num(), comm->ep_comm_ptr); 
    3232      } 
    3333    } 
    … …  
    5252      MPI_Comm comm_ptr; 
    5353      comm_ptr = it->second; 
    54       printf("EP_Comm_f2c : MAP %p find: %d, %d, %p\n", &fc_comm_map, it->first.first, it->first.second, comm_ptr.ep_comm_ptr); 
     54      printf("EP_Comm_f2c : MAP %p find: %d, %d, %p\n", &fc_comm_map, it->first.first, it->first.second, comm_ptr->ep_comm_ptr); 
    5555      return  comm_ptr; 
    5656    } 
    … …  
    6565    #endif 
    6666 
    67     if(*base_comm != to_mpi_comm(MPI_COMM_NULL.mpi_comm)) 
     67    if(*base_comm != to_mpi_comm(MPI_COMM_NULL->mpi_comm)) 
    6868    { 
    6969      if(omp_get_thread_num() == 0) 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_free.cpp

    r1499 r1500  
    99  int MPI_Comm_free(MPI_Comm *comm) 
    1010  { 
    11     if(! comm->is_ep) 
     11    if(! (*comm)->is_ep) 
    1212    { 
    13       if(comm->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 
     13      if((*comm)->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
    1414      { 
    15         ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
     15        ::MPI_Comm mpi_comm = to_mpi_comm((*comm)->mpi_comm); 
    1616 
    1717        ::MPI_Comm_free(&mpi_comm); 
    … …  
    2121    } 
    2222 
    23     else if(comm->is_intercomm) 
     23    else if((*comm)->is_intercomm) 
    2424    { 
    2525      return MPI_Comm_free_intercomm(comm); 
    … …  
    3030      int ep_rank_loc, num_ep; 
    3131 
    32       ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
    33       num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     32      ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first; 
     33      num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second; 
    3434 
    3535      MPI_Barrier(*comm); 
    … …  
    3939        Debug("comm is EP, mpi_comm_ptr != NULL\n"); 
    4040 
    41         delete comm->my_buffer; 
     41        delete (*comm)->my_buffer; 
    4242 
    4343 
    44         if(comm->ep_barrier != NULL) 
     44        if((*comm)->ep_barrier != NULL) 
    4545        { 
    46           comm->ep_barrier->~OMPbarrier(); 
     46          (*comm)->ep_barrier->~ep_barrier(); 
    4747          Debug("ep_barrier freed\n"); 
    4848        } 
    4949 
    5050 
    51         if( ! comm->rank_map->empty() ) 
     51        if( ! (*comm)->rank_map->empty() ) 
    5252        { 
    53           comm->rank_map->clear(); 
    54           delete comm->rank_map; 
     53          (*comm)->rank_map->clear(); 
     54          delete (*comm)->rank_map; 
    5555          Debug("rank_map emptied\n"); 
    5656        } 
    … …  
    5858        for(int i=0; i<num_ep; i++) 
    5959        { 
    60           comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue->clear(); 
    61           delete comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue; 
     60          (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear(); 
     61          delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue; 
    6262          Debug("message queue freed\n"); 
    6363           
    6464 
    65           if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr != NULL) 
     65          if((*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr != NULL) 
    6666          { 
    67             delete comm->ep_comm_ptr->comm_list[i].ep_comm_ptr; 
     67            delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr; 
    6868            Debug("ep_comm_ptr freed\n"); 
    6969          } 
    7070        } 
    7171 
    72         if( to_mpi_comm(comm->mpi_comm) != *static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)  
    73           && to_mpi_comm(comm->mpi_comm) != *static_cast< ::MPI_Comm*>(MPI_COMM_WORLD.mpi_comm)) 
     72        if( to_mpi_comm((*comm)->mpi_comm) != *static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)  
     73          && to_mpi_comm((*comm)->mpi_comm) != *static_cast< ::MPI_Comm*>(MPI_COMM_WORLD->mpi_comm)) 
    7474        { 
    75           ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
     75          ::MPI_Comm mpi_comm = to_mpi_comm((*comm)->mpi_comm); 
    7676          ::MPI_Comm_free(&mpi_comm); 
    7777          Debug("mpi_comm freed\n"); 
    7878        } 
    7979 
    80        if(comm != NULL) {delete[] comm->ep_comm_ptr->comm_list; Debug("comm freed\n");} 
     80       //if(comm != NULL) {delete[] comm->ep_comm_ptr->comm_list; Debug("comm freed\n");} 
    8181 
    8282      } 
    … …  
    9393    int ep_rank_loc, num_ep; 
    9494 
    95     ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
    96     num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     95    ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first; 
     96    num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second; 
    9797 
    9898    //MPI_Barrier(*comm); 
    … …  
    103103 
    104104 
    105       if(comm->ep_barrier != NULL) 
     105      if((*comm)->ep_barrier != NULL) 
    106106      { 
    107         comm->ep_barrier->~OMPbarrier(); 
     107        (*comm)->ep_barrier->~ep_barrier(); 
    108108        Debug("ep_barrier freed\n"); 
    109109      } 
    110110 
    111111 
    112       if( ! comm->rank_map->empty() ) 
     112      if( ! (*comm)->rank_map->empty() ) 
    113113      { 
    114         comm->rank_map->clear(); 
     114        (*comm)->rank_map->clear(); 
    115115        Debug("rank_map emptied\n"); 
    116116      } 
    … …  
    118118      for(int i=0; i<num_ep; i++) 
    119119      { 
    120         comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue->clear(); 
     120        (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear(); 
    121121        Debug("message queue freed\n"); 
    122122 
    123123        #pragma omp critical (memory_free) 
    124         if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm != NULL) 
     124        if((*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm != NULL) 
    125125        { 
    126           comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map->clear(); 
    127           comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map->clear(); 
    128           comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map->clear(); 
     126          (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_rank_map->clear(); 
     127          (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->remote_rank_map->clear(); 
     128          (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_rank_map->clear(); 
    129129          Debug("intercomm local/remote/intercomm_rank_map emptied\n"); 
    130130        }   
    131131 
    132         if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr != NULL) 
     132        if((*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr != NULL) 
    133133        { 
    134           delete comm->ep_comm_ptr->comm_list[i].ep_comm_ptr; 
     134          delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr; 
    135135          Debug("ep_comm_ptr freed\n"); 
    136136        } 
    137137      } 
    138138 
    139       if(comm->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 
     139      if((*comm)->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
    140140      { 
    141         ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
     141        ::MPI_Comm mpi_comm = to_mpi_comm((*comm)->mpi_comm); 
    142142        ::MPI_Comm_free(&mpi_comm); 
    143143        Debug("mpi_comm freed\n"); 
    144144      } 
    145145 
    146       if(comm->ep_comm_ptr->intercomm->mpi_inter_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 
     146      if((*comm)->ep_comm_ptr->intercomm->mpi_inter_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
    147147      { 
    148         ::MPI_Comm mpi_comm = to_mpi_comm(comm->ep_comm_ptr->comm_list->ep_comm_ptr->intercomm->mpi_inter_comm); 
     148        ::MPI_Comm mpi_comm = to_mpi_comm((*comm)->ep_comm_ptr->comm_list[0]->ep_comm_ptr->intercomm->mpi_inter_comm); 
    149149        ::MPI_Comm_free(&mpi_comm); 
    150150        Debug("mpi_intercomm freed\n"); 
    151151      } 
    152152 
    153      if(comm != NULL) {delete[] comm->ep_comm_ptr->comm_list; Debug("comm freed\n");} 
     153     if(comm != NULL) {delete[] (*comm)->ep_comm_ptr->comm_list; Debug("comm freed\n");} 
    154154 
    155155    } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_gather.cpp

    r1499 r1500  
    2323    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize); 
    2424 
    25     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    26     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
     25    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     26    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
    2727 
    2828    #pragma omp critical (_gather) 
    29     comm.my_buffer->void_buffer[ep_rank_loc] = const_cast< void* >(sendbuf); 
     29    comm->my_buffer->void_buffer[ep_rank_loc] = const_cast< void* >(sendbuf); 
    3030 
    3131    MPI_Barrier_local(comm); 
    … …  
    3434    { 
    3535      for(int i=0; i<num_ep; i++) 
    36         memcpy(recvbuf + datasize * i * count, comm.my_buffer->void_buffer[i], datasize * count); 
     36        memcpy(recvbuf + datasize * i * count, comm->my_buffer->void_buffer[i], datasize * count); 
    3737 
    3838      //printf("local_recvbuf = %d %d \n", static_cast<int*>(recvbuf)[0], static_cast<int*>(recvbuf)[1] ); 
    … …  
    4444  int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 
    4545  { 
    46     if(!comm.is_ep) 
     46    if(!comm->is_ep) 
    4747    { 
    4848      return ::MPI_Gather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), 
    49                    root, to_mpi_comm(comm.mpi_comm)); 
     49                   root, to_mpi_comm(comm->mpi_comm)); 
    5050    } 
    5151 
    5252    assert(sendcount == recvcount && sendtype == recvtype); 
    5353 
    54     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    55     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    56     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    57     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    58     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    59     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     54    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     55    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     56    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     57    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     58    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     59    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    6060 
    61     int root_mpi_rank = comm.rank_map->at(root).second; 
    62     int root_ep_loc = comm.rank_map->at(root).first; 
     61    int root_mpi_rank = comm->rank_map->at(root).second; 
     62    int root_ep_loc = comm->rank_map->at(root).first; 
    6363 
    6464    ::MPI_Aint datasize, lb; 
    … …  
    9090      for(int i=0; i<ep_size; i++) 
    9191      { 
    92         recvcounts[comm.rank_map->at(i).second]+=sendcount; 
     92        recvcounts[comm->rank_map->at(i).second]+=sendcount; 
    9393      } 
    9494 
    … …  
    9696        displs[i] = displs[i-1] + recvcounts[i-1]; 
    9797 
    98       ::MPI_Gatherv(local_recvbuf, sendcount*num_ep, to_mpi_type(sendtype), tmp_recvbuf, recvcounts.data(), displs.data(), to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     98      ::MPI_Gatherv(local_recvbuf, sendcount*num_ep, to_mpi_type(sendtype), tmp_recvbuf, recvcounts.data(), displs.data(), to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    9999    }    
    100100 
    … …  
    111111      for(int i=0; i<ep_size; i++) 
    112112      { 
    113         offset = displs[comm.rank_map->at(i).second] + comm.rank_map->at(i).first * sendcount;  
     113        offset = displs[comm->rank_map->at(i).second] + comm->rank_map->at(i).first * sendcount;  
    114114        memcpy(recvbuf + i*sendcount*datasize, tmp_recvbuf+offset*datasize, sendcount*datasize); 
    115115 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_gatherv.cpp

    r1381 r1500  
    2323    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize); 
    2424 
    25     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    26     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
     25    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     26    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
    2727 
    2828 
    2929    #pragma omp critical (_gatherv) 
    30     comm.my_buffer->void_buffer[ep_rank_loc] = const_cast< void* >(sendbuf); 
     30    comm->my_buffer->void_buffer[ep_rank_loc] = const_cast< void* >(sendbuf); 
    3131 
    3232    MPI_Barrier_local(comm); 
    … …  
    3535    { 
    3636      for(int i=0; i<num_ep; i++) 
    37         memcpy(recvbuf + datasize*displs[i], comm.my_buffer->void_buffer[i], datasize*recvcounts[i]); 
     37        memcpy(recvbuf + datasize*displs[i], comm->my_buffer->void_buffer[i], datasize*recvcounts[i]); 
    3838 
    3939    } 
    … …  
    4646  { 
    4747   
    48     if(!comm.is_ep) 
     48    if(!comm->is_ep) 
    4949    { 
    5050      return ::MPI_Gatherv(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, const_cast<int*>(input_recvcounts), const_cast<int*>(input_displs), 
    51                     to_mpi_type(recvtype), root, to_mpi_comm(comm.mpi_comm)); 
     51                    to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 
    5252    } 
    5353 
    … …  
    5656 
    5757     
    58     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    59     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    60     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    61     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    62     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    63     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     58    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     59    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     60    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     61    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     62    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     63    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    6464 
    65     int root_mpi_rank = comm.rank_map->at(root).second; 
    66     int root_ep_loc = comm.rank_map->at(root).first; 
     65    int root_mpi_rank = comm->rank_map->at(root).second; 
     66    int root_ep_loc = comm->rank_map->at(root).first; 
    6767 
    6868    ::MPI_Aint datasize, lb; 
    … …  
    126126      for(int i=0; i<ep_size; i++) 
    127127      { 
    128         mpi_recvcounts[comm.rank_map->at(i).second]+=recvcounts[i]; 
     128        mpi_recvcounts[comm->rank_map->at(i).second]+=recvcounts[i]; 
    129129      } 
    130130 
    … …  
    133133 
    134134 
    135       ::MPI_Gatherv(local_recvbuf, sendcount*num_ep, to_mpi_type(sendtype), tmp_recvbuf, mpi_recvcounts.data(), mpi_displs.data(), to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     135      ::MPI_Gatherv(local_recvbuf, sendcount*num_ep, to_mpi_type(sendtype), tmp_recvbuf, mpi_recvcounts.data(), mpi_displs.data(), to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    136136    }    
    137137 
    … …  
    144144      { 
    145145        int extra = 0; 
    146         for(int j=0, k=0; j<ep_size, k<comm.rank_map->at(i).first; j++) 
    147           if(comm.rank_map->at(i).second == comm.rank_map->at(j).second) 
     146        for(int j=0, k=0; j<ep_size, k<comm->rank_map->at(i).first; j++) 
     147          if(comm->rank_map->at(i).second == comm->rank_map->at(j).second) 
    148148          { 
    149149            extra += recvcounts[j]; 
    … …  
    151151          }   
    152152 
    153         offset = mpi_displs[comm.rank_map->at(i).second] +  extra; 
     153        offset = mpi_displs[comm->rank_map->at(i).second] +  extra; 
    154154 
    155155        memcpy(recvbuf+displs[i]*datasize, tmp_recvbuf+offset*datasize, recvcounts[i]*datasize); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_get.cpp

    r1398 r1500  
    1212                  int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win) 
    1313  { 
    14     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    15     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    16     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     14    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     15    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     16    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1717    if(num_ep==1) 
    1818      return  ::MPI_Get(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    19                        to_mpi_win(win.server_win[target_local_rank])); 
     19                       to_mpi_win(win->server_win[target_local_rank])); 
    2020 
    2121    else 
    2222        return  ::MPI_Get(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    23                        to_mpi_win(win.client_win)); 
     23                       to_mpi_win(win->client_win)); 
    2424  } 
    2525 
    … …  
    2727              int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request) 
    2828  { 
    29     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    30     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    31     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     29    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     30    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     31    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    3232 
    3333    ::MPI_Request mpi_request; 
    … …  
    3535    { 
    3636      int return_value = ::MPI_Rget(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    37                          to_mpi_win(win.server_win[target_local_rank]), &mpi_request); 
     37                         to_mpi_win(win->server_win[target_local_rank]), &mpi_request); 
    3838 
    39       request->mpi_request = new ::MPI_Request(mpi_request); 
     39      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    4040 
    41       request->ep_datatype = origin_datatype; 
    42       request->type = 1; 
     41      (*request)->ep_datatype = origin_datatype; 
     42      (*request)->type = 1; 
    4343      return return_value; 
    4444    } 
    … …  
    4646    { 
    4747      int return_value = ::MPI_Rget(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    48                          to_mpi_win(win.client_win), &mpi_request); 
     48                         to_mpi_win(win->client_win), &mpi_request); 
    4949 
    50       request->mpi_request = new ::MPI_Request(mpi_request); 
     50      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    5151 
    52       request->ep_datatype = origin_datatype; 
    53       request->type = 1; 
     52      (*request)->ep_datatype = origin_datatype; 
     53      (*request)->type = 1; 
    5454      return return_value; 
    5555    } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp

    r1499 r1500  
    1010  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 
    1111  { 
    12     assert(local_comm.is_ep); 
     12    assert(local_comm->is_ep); 
    1313 
    1414    int ep_rank, ep_rank_loc, mpi_rank; 
    1515    int ep_size, num_ep, mpi_size; 
    1616 
    17     ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first; 
    18     ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first; 
    19     mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first; 
    20     ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second; 
    21     num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second; 
    22     mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second; 
     17    ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first; 
     18    ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first; 
     19    mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first; 
     20    ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second; 
     21    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second; 
     22    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second; 
    2323 
    2424 
    … …  
    6262      { 
    6363        if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_unique_leader\n"); 
    64         local_comm.ep_comm_ptr->comm_label = -99; 
     64        local_comm->ep_comm_ptr->comm_label = -99; 
    6565 
    6666        return MPI_Intercomm_create_unique_leader(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm); 
    … …  
    7777            // change leader 
    7878            is_decider = true; 
    79             int target = local_comm.rank_map->at(local_leader).second; 
     79            int target = local_comm->rank_map->at(local_leader).second; 
    8080            { 
    8181              for(int i=0; i<ep_size; i++) 
    8282              { 
    83                 if(local_comm.rank_map->at(i).second != target && local_comm.rank_map->at(i).first == 0) 
     83                if(local_comm->rank_map->at(i).second != target && local_comm->rank_map->at(i).first == 0) 
    8484                { 
    8585                  new_local_leader = i; 
    … …  
    100100            // change leader 
    101101            is_decider = true; 
    102             int target = local_comm.rank_map->at(local_leader).second; 
     102            int target = local_comm->rank_map->at(local_leader).second; 
    103103            { 
    104104              for(int i=0; i<ep_size; i++) 
    105105              { 
    106                 if(local_comm.rank_map->at(i).second != target && local_comm.rank_map->at(i).first == 0) 
     106                if(local_comm->rank_map->at(i).second != target && local_comm->rank_map->at(i).first == 0) 
    107107                { 
    108108                  new_local_leader = i; 
    … …  
    146146        if(ep_rank == new_local_leader) 
    147147        { 
    148           ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD.mpi_comm), &leader_in_world[0]); 
     148          ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &leader_in_world[0]); 
    149149        } 
    150150 
    … …  
    165165        MPI_Bcast(&leader_in_world[1], 1, MPI_INT, local_leader, local_comm); 
    166166 
    167         local_comm.ep_comm_ptr->comm_label = tag; 
     167        local_comm->ep_comm_ptr->comm_label = tag; 
    168168 
    169169        if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_from_world\n"); 
    170170 
    171         return MPI_Intercomm_create_from_world(local_comm, new_local_leader, MPI_COMM_WORLD.mpi_comm, leader_in_world[1], new_tag_in_world, newintercomm); 
     171        return MPI_Intercomm_create_from_world(local_comm, new_local_leader, MPI_COMM_WORLD->mpi_comm, leader_in_world[1], new_tag_in_world, newintercomm); 
    172172         
    173173      } 
    … …  
    183183  { 
    184184    *flag = false; 
    185     if(comm.is_ep) 
    186     { 
    187       *flag = comm.is_intercomm; 
     185    if(comm->is_ep) 
     186    { 
     187      *flag = comm->is_intercomm; 
    188188      return 0; 
    189189    }  
    190     else if(comm.mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 
    191     { 
    192       ::MPI_Comm mpi_comm = to_mpi_comm(comm.mpi_comm); 
     190    else if(comm->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
     191    { 
     192      ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
    193193       
    194194      ::MPI_Comm_test_inter(mpi_comm, flag); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp

    r1499 r1500  
    1414    int ep_size, num_ep, mpi_size; 
    1515 
    16     ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first; 
    17     ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first; 
    18     mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first; 
    19     ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second; 
    20     num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second; 
    21     mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second; 
     16    ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first; 
     17    ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first; 
     18    mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first; 
     19    ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second; 
     20    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second; 
     21    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second; 
    2222 
    2323    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm 
    … …  
    3333 
    3434 
    35     ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm.mpi_comm); 
     35    ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm->mpi_comm); 
    3636 
    3737     
    38     ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD.mpi_comm), &rank_in_world); 
     38    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &rank_in_world); 
    3939    ::MPI_Comm_rank(local_mpi_comm, &rank_in_local_parent); 
    4040     
    … …  
    4646 
    4747    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;} 
    48     if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true; 
     48    if(ep_rank_loc == 0 && mpi_rank != local_comm->rank_map->at(local_leader).second) is_proc_master = true; 
    4949 
    5050 
    … …  
    9494        leader_info[1] = remote_leader; 
    9595 
    96         ::MPI_Comm_rank(to_mpi_comm(peer_comm.mpi_comm), &rank_in_peer_mpi[0]); 
     96        ::MPI_Comm_rank(to_mpi_comm(peer_comm->mpi_comm), &rank_in_peer_mpi[0]); 
    9797 
    9898        send_buf[0] = size_info[0]; 
    … …  
    125125      send_buf[4] = rank_in_peer_mpi[1]; 
    126126 
    127       ::MPI_Bcast(send_buf.data(), 5, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     127      ::MPI_Bcast(send_buf.data(), 5, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
    128128 
    129129      size_info[1] = send_buf[0]; 
    … …  
    157157      } 
    158158 
    159       ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     159      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
    160160 
    161161      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() ); 
    … …  
    281281      } 
    282282 
    283       ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     283      ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
    284284 
    285285      new_rank_info[2].resize(size_info[3]); 
    … …  
    305305      } 
    306306 
    307       ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     307      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
    308308 
    309309      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() ); 
    … …  
    337337      } 
    338338 
    339       ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    340  
    341       if(new_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 
     339      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     340 
     341      if(new_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
    342342      { 
    343343 
    344344        ::MPI_Barrier(*new_comm); 
    345345 
    346         ::MPI_Intercomm_create(*new_comm, leader_info[2], to_mpi_comm(peer_comm.mpi_comm), rank_in_peer_mpi[1], tag, intercomm); 
     346        ::MPI_Intercomm_create(*new_comm, leader_info[2], to_mpi_comm(peer_comm->mpi_comm), rank_in_peer_mpi[1], tag, intercomm); 
    347347 
    348348        int id; 
    … …  
    358358        for(int i= 0; i<my_num_ep; i++) 
    359359        { 
    360           ep_intercomm[i].is_intercomm = true; 
    361  
    362           ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
    363           ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm; 
    364           ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0]; 
     360          ep_intercomm[i]->is_intercomm = true; 
     361 
     362          ep_intercomm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
     363          ep_intercomm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = intercomm; 
     364          ep_intercomm[i]->ep_comm_ptr->comm_label = leader_info[0]; 
    365365        } 
    366366 
    … …  
    470470    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size; 
    471471 
    472     intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first; 
    473     intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first; 
    474     intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first; 
    475     intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second; 
    476     intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second; 
    477     intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second; 
     472    intercomm_ep_rank = (*newintercomm)->ep_comm_ptr->size_rank_info[0].first; 
     473    intercomm_ep_rank_loc = (*newintercomm)->ep_comm_ptr->size_rank_info[1].first; 
     474    intercomm_mpi_rank = (*newintercomm)->ep_comm_ptr->size_rank_info[2].first; 
     475    intercomm_ep_size = (*newintercomm)->ep_comm_ptr->size_rank_info[0].second; 
     476    intercomm_num_ep = (*newintercomm)->ep_comm_ptr->size_rank_info[1].second; 
     477    intercomm_mpi_size = (*newintercomm)->ep_comm_ptr->size_rank_info[2].second; 
    478478 
    479479    MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm); 
    … …  
    482482 
    483483    my_rank_map_elem[0] = intercomm_ep_rank; 
    484     my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label; 
     484    my_rank_map_elem[1] = (*newintercomm)->ep_comm_ptr->comm_label; 
    485485 
    486486    vector<pair<int, int> > local_rank_map_array; 
    … …  
    488488 
    489489 
    490     (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; 
    491     (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size); 
     490    (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; 
     491    (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size); 
    492492 
    493493    MPI_Allgather(my_rank_map_elem, 2, MPI_INT,  
    494       (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 
    495  
    496     (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
    497     (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size); 
    498  
    499     (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0]; 
    500     (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1]; 
    501     (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2]; 
     494      (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 
     495 
     496    (*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
     497    (*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size); 
     498 
     499    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[0] = local_comm->ep_comm_ptr->size_rank_info[0]; 
     500    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[1] = local_comm->ep_comm_ptr->size_rank_info[1]; 
     501    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[2] = local_comm->ep_comm_ptr->size_rank_info[2]; 
    502502 
    503503    int local_intercomm_size = intercomm_ep_size; 
    … …  
    513513      MPI_Status statuses[4]; 
    514514       
    515       MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT, remote_leader, tag+4, peer_comm, &requests[0]); 
    516       MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, remote_leader, tag+4, peer_comm, &requests[1]); 
     515      MPI_Isend((*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT, remote_leader, tag+4, peer_comm, &requests[0]); 
     516      MPI_Irecv((*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, remote_leader, tag+4, peer_comm, &requests[1]); 
    517517 
    518518      MPI_Isend(&local_intercomm_size, 1, MPI_INT, remote_leader, tag+5, peer_comm, &requests[2]); 
    … …  
    527527 
    528528 
    529     MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm); 
     529    MPI_Bcast((*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm); 
    530530    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT, new_bcast_root, *newintercomm); 
    531531 
    532532 
    533     (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 
    534     (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); 
     533    (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 
     534    (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); 
    535535 
    536536 
    … …  
    542542      MPI_Status statuses[2]; 
    543543       
    544       MPI_Isend((*newintercomm).rank_map->data(), 2*local_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[0]); 
    545       MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[1]); 
     544      MPI_Isend((*newintercomm)->rank_map->data(), 2*local_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[0]); 
     545      MPI_Irecv((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[1]); 
    546546       
    547547      MPI_Waitall(2, requests, statuses); 
    548548    } 
    549549 
    550     MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, new_bcast_root, *newintercomm); 
    551  
    552     (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]); 
    553     (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = tag; 
     550    MPI_Bcast((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, new_bcast_root, *newintercomm); 
     551 
     552    (*newintercomm)->ep_comm_ptr->intercomm->local_comm = (local_comm->ep_comm_ptr->comm_list[ep_rank_loc]); 
     553    (*newintercomm)->ep_comm_ptr->intercomm->intercomm_tag = tag; 
    554554 
    555555/* 
    556556    for(int i=0; i<local_ep_size; i++) 
    557     if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, local_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i, 
    558           (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).second); 
     557    if(local_comm->ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, local_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i, 
     558          (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->at(i).first, (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->at(i).second); 
    559559 
    560560    for(int i=0; i<remote_ep_size; i++) 
    561     if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i, 
    562           (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->at(i).second); 
     561    if(local_comm->ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i, 
     562          (*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->at(i).first, (*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->at(i).second); 
    563563 
    564564    for(int i=0; i<remote_intercomm_size; i++) 
    565     if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i, 
    566           (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second); 
     565    if(local_comm->ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i, 
     566          (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second); 
    567567*/ 
    568568 
    569 //    for(int i=0; i<(*newintercomm).rank_map->size(); i++) 
    570 //    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i, 
    571 //          (*newintercomm).rank_map->at(i).first, (*newintercomm).rank_map->at(i).second); 
     569//    for(int i=0; i<(*newintercomm)->rank_map->size(); i++) 
     570//    if(local_comm->ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i, 
     571//          (*newintercomm)->rank_map->at(i).first, (*newintercomm)->rank_map->at(i).second); 
    572572 
    573573//    MPI_Comm *test_comm = newintercomm->ep_comm_ptr->intercomm->local_comm; 
    … …  
    593593    int ep_size, num_ep, mpi_size; 
    594594 
    595     ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first; 
    596     ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first; 
    597     mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first; 
    598     ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second; 
    599     num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second; 
    600     mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second; 
     595    ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first; 
     596    ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first; 
     597    mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first; 
     598    ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second; 
     599    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second; 
     600    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second; 
    601601 
    602602 
    … …  
    609609    int rank_in_peer_mpi[2]; 
    610610 
    611     ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD.mpi_comm), &rank_in_world); 
     611    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &rank_in_world); 
    612612 
    613613 
    … …  
    666666        ::MPI_Comm *mpi_dup = new ::MPI_Comm; 
    667667         
    668         ::MPI_Comm_dup(to_mpi_comm(local_comm.mpi_comm), mpi_dup); 
     668        ::MPI_Comm_dup(to_mpi_comm(local_comm->mpi_comm), mpi_dup); 
    669669 
    670670        MPI_Comm *ep_intercomm; 
    … …  
    675675        for(int i=0; i<total_num_ep; i++) 
    676676        { 
    677           ep_intercomm[i].is_intercomm = true; 
    678           ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
    679           ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = 0; 
    680  
    681           ep_intercomm[i].ep_comm_ptr->comm_label = leader_rank_in_peer[0]; 
     677          ep_intercomm[i]->is_intercomm = true; 
     678          ep_intercomm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
     679          ep_intercomm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = 0; 
     680 
     681          ep_intercomm[i]->ep_comm_ptr->comm_label = leader_rank_in_peer[0]; 
    682682        } 
    683683 
    … …  
    751751    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size; 
    752752 
    753     intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first; 
    754     intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first; 
    755     intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first; 
    756     intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second; 
    757     intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second; 
    758     intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second; 
    759  
    760  
    761  
    762     (*newintercomm).ep_comm_ptr->intercomm->local_rank_map  = new RANK_MAP; 
    763     (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
    764     (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_num_ep); 
    765     (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_num_ep); 
    766  
    767     (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0]; 
    768     (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1]; 
    769     (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2]; 
     753    intercomm_ep_rank = (*newintercomm)->ep_comm_ptr->size_rank_info[0].first; 
     754    intercomm_ep_rank_loc = (*newintercomm)->ep_comm_ptr->size_rank_info[1].first; 
     755    intercomm_mpi_rank = (*newintercomm)->ep_comm_ptr->size_rank_info[2].first; 
     756    intercomm_ep_size = (*newintercomm)->ep_comm_ptr->size_rank_info[0].second; 
     757    intercomm_num_ep = (*newintercomm)->ep_comm_ptr->size_rank_info[1].second; 
     758    intercomm_mpi_size = (*newintercomm)->ep_comm_ptr->size_rank_info[2].second; 
     759 
     760 
     761 
     762    (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map  = new RANK_MAP; 
     763    (*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
     764    (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->resize(local_num_ep); 
     765    (*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->resize(remote_num_ep); 
     766 
     767    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[0] = local_comm->ep_comm_ptr->size_rank_info[0]; 
     768    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[1] = local_comm->ep_comm_ptr->size_rank_info[1]; 
     769    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[2] = local_comm->ep_comm_ptr->size_rank_info[2]; 
    770770 
    771771 
    … …  
    773773    int local_rank_map_ele[2]; 
    774774    local_rank_map_ele[0] = intercomm_ep_rank; 
    775     local_rank_map_ele[1] = (*newintercomm).ep_comm_ptr->comm_label; 
     775    local_rank_map_ele[1] = (*newintercomm)->ep_comm_ptr->comm_label; 
    776776 
    777777    MPI_Allgather(local_rank_map_ele, 2, MPI_INT,  
    778       (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 
     778      (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 
    779779 
    780780    if(ep_rank == local_leader) 
    … …  
    783783      MPI_Request req_s, req_r; 
    784784 
    785       MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_num_ep, MPI_INT, remote_leader, tag, peer_comm, &req_s); 
    786       MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT, remote_leader, tag, peer_comm, &req_r); 
     785      MPI_Isend((*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_num_ep, MPI_INT, remote_leader, tag, peer_comm, &req_s); 
     786      MPI_Irecv((*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT, remote_leader, tag, peer_comm, &req_r); 
    787787 
    788788 
    … …  
    792792    } 
    793793 
    794     MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT, local_leader, local_comm); 
    795     (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]); 
    796     (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = tag; 
     794    MPI_Bcast((*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT, local_leader, local_comm); 
     795    (*newintercomm)->ep_comm_ptr->intercomm->local_comm = (local_comm->ep_comm_ptr->comm_list[ep_rank_loc]); 
     796    (*newintercomm)->ep_comm_ptr->intercomm->intercomm_tag = tag; 
    797797 
    798798 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_world.cpp

    r1499 r1500  
    99{ 
    1010 
    11   // #ifdef _openmpi 
    12  
    13   // int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm) 
    14   // { 
    15  
    16   //   int ep_rank, ep_rank_loc, mpi_rank; 
    17   //   int ep_size, num_ep, mpi_size; 
    18  
    19   //   ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first; 
    20   //   ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first; 
    21   //   mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first; 
    22   //   ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second; 
    23   //   num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second; 
    24   //   mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second; 
    25  
    26  
    27   //   std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm 
    28   //                                   //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm 
    29  
    30   //   int rank_in_world; 
    31   //   int rank_in_local_parent; 
    32  
    33   //   int local_ep_size = ep_size; 
    34   //   int remote_ep_size; 
    35  
    36   //   ::MPI_Comm peer_comm = to_mpi_comm(peer_comm_ptr); 
    37   //   ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm.mpi_comm); 
    38  
    39   //   ::MPI_Comm_rank(peer_comm, &rank_in_world); 
    40  
    41   //   ::MPI_Comm_rank(local_mpi_comm, &rank_in_local_parent); 
    42  
    43   //   bool is_proc_master = false; 
    44   //   bool is_local_leader = false; 
    45   //   bool is_final_master = false; 
    46  
    47  
    48   //   if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;} 
    49   //   if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true; 
    50  
    51  
    52   //   int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm 
    53  
    54   //   int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader 
    55  
    56  
    57   //   std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm 
    58  
    59   //   std::vector<int> new_rank_info[4]; 
    60   //   std::vector<int> new_ep_info[2]; 
    61  
    62   //   std::vector<int> offset; 
    63  
    64   //   if(is_proc_master) 
    65   //   { 
    66  
    67   //     size_info[0] = mpi_size; 
    68  
    69   //     rank_info[0].resize(size_info[0]); 
    70   //     rank_info[1].resize(size_info[0]); 
    71  
    72  
    73  
    74   //     ep_info[0].resize(size_info[0]); 
    75  
    76   //     vector<int> send_buf(6); 
    77   //     vector<int> recv_buf(3*size_info[0]); 
    78  
    79   //     send_buf[0] = rank_in_world; 
    80   //     send_buf[1] = rank_in_local_parent; 
    81   //     send_buf[2] = num_ep; 
    82  
    83   //     ::MPI_Allgather(send_buf.data(), 3, to_mpi_type(MPI_INT), recv_buf.data(), 3, to_mpi_type(MPI_INT), local_mpi_comm); 
    84  
    85   //     for(int i=0; i<size_info[0]; i++) 
    86   //     { 
    87   //       rank_info[0][i] = recv_buf[3*i]; 
    88   //       rank_info[1][i] = recv_buf[3*i+1]; 
    89   //       ep_info[0][i]   = recv_buf[3*i+2]; 
    90   //     } 
    91  
    92   //     if(is_local_leader) 
    93   //     { 
    94   //       leader_info[0] = rank_in_world; 
    95   //       leader_info[1] = mpi_remote_leader; 
    96  
    97   //       ::MPI_Status mpi_status; 
    98  
    99   //       send_buf[0] = size_info[0]; 
    100   //       send_buf[1] = local_ep_size; 
    101  
    102   //       ::MPI_Send(send_buf.data(), 2, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    103  
    104   //       ::MPI_Recv(recv_buf.data(), 2, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 
    105  
    106   //       recv_buf[2] = rank_in_world; 
    107   //       recv_buf[3] = mpi_remote_leader; 
    108  
    109   //     } 
    110  
    111   //     ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    112  
    113   //     size_info[1] = recv_buf[0]; 
    114   //     remote_ep_size = recv_buf[1]; 
    115   //     leader_info[0] = recv_buf[2]; 
    116   //     leader_info[1] = recv_buf[3]; 
    117  
    118   //     rank_info[2].resize(size_info[1]); 
    119   //     rank_info[3].resize(size_info[1]); 
    120  
    121   //     ep_info[1].resize(size_info[1]); 
    122  
    123   //     send_buf.resize(3*size_info[0]); 
    124   //     recv_buf.resize(3*size_info[1]); 
    125  
    126   //     if(is_local_leader) 
    127   //     { 
    128   //       ::MPI_Status mpi_status; 
    129  
    130  
    131   //       std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() ); 
    132   //       std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] ); 
    133   //       std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] ); 
    134  
    135   //       ::MPI_Send(send_buf.data(), 3*size_info[0], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    136  
    137   //       ::MPI_Recv(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 
    138  
    139   //     } 
    140  
    141   //     ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    142  
    143   //     std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() ); 
    144   //     std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin()  ); 
    145   //     std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() ); 
    146  
    147   //     offset.resize(size_info[0]); 
    148  
    149   //     if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader 
    150   //     { 
    151  
    152   //       bool found = false; 
    153   //       int ep_tmp; 
    154   //       int ep_local; 
    155   //       int ep_remote; 
    156   //       for(int i=0; i<size_info[0]; i++) 
    157   //       { 
    158   //         int target = rank_info[0][i]; 
    159   //         found = false; 
    160   //         for(int j=0; j<size_info[1]; j++) 
    161   //         { 
    162   //           if(target == rank_info[2][j]) 
    163   //           { 
    164   //             found = true; 
    165   //             ep_tmp = ep_info[1][j]; 
    166   //             ep_local = ep_info[0][j]; 
    167   //             ep_remote = ep_info[1][j]; 
    168   //             break; 
    169   //           } 
    170   //         } 
    171   //         if(found) 
    172   //         { 
    173  
    174   //           if(target == leader_info[0]) // the leader is doubled in remote 
    175   //           { 
    176   //             new_rank_info[0].push_back(target); 
    177   //             new_rank_info[1].push_back(rank_info[1][i]); 
    178  
    179   //             new_ep_info[0].push_back(ep_local + ep_remote); 
    180   //             offset[i] = 0; 
    181   //           } 
    182   //           else 
    183   //           { 
    184   //             offset[i] = ep_local; 
    185   //           } 
    186   //         } 
    187   //         else 
    188   //         { 
    189   //           new_rank_info[0].push_back(target); 
    190   //           new_rank_info[1].push_back(rank_info[1][i]); 
    191  
    192   //           new_ep_info[0].push_back(ep_info[0][i]); 
    193  
    194   //           offset[i] = 0; 
    195   //         } 
    196  
    197   //       } 
    198   //     } 
    199  
    200   //     else // erase rank doubled with remote leader 
    201   //     { 
    202  
    203   //       bool found = false; 
    204   //       int ep_tmp; 
    205   //       int ep_local; 
    206   //       int ep_remote; 
    207   //       for(int i=0; i<size_info[0]; i++) 
    208   //       { 
    209   //         int target = rank_info[0][i]; 
    210   //         found = false; 
    211   //         for(int j=0; j<size_info[1]; j++) 
    212   //         { 
    213  
    214   //           if(target == rank_info[2][j]) 
    215   //           { 
    216   //             found = true; 
    217   //             ep_tmp = ep_info[1][j]; 
    218   //             ep_local = ep_info[0][j]; 
    219   //             ep_remote = ep_info[1][j]; 
    220   //             break; 
    221   //           } 
    222   //         } 
    223   //         if(found) 
    224   //         { 
    225   //           if(target != leader_info[1]) 
    226   //           { 
    227   //             new_rank_info[0].push_back(target); 
    228   //             new_rank_info[1].push_back(rank_info[1][i]); 
    229  
    230   //             new_ep_info[0].push_back(ep_local + ep_remote); 
    231   //             offset[i] = 0; 
    232   //           } 
    233   //           else // found remote leader 
    234   //           { 
    235   //             offset[i] = ep_remote; 
    236   //           } 
    237   //         } 
    238   //         else 
    239   //         { 
    240   //           new_rank_info[0].push_back(target); 
    241   //           new_rank_info[1].push_back(rank_info[1][i]); 
    242  
    243   //           new_ep_info[0].push_back(ep_info[0][i]); 
    244   //           offset[i] = 0; 
    245   //         } 
    246   //       } 
    247   //     } 
    248  
    249   //     if(offset[mpi_rank] == 0) 
    250   //     { 
    251   //       is_final_master = true; 
    252   //     } 
    253  
    254  
    255   //     // size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote 
    256  
    257   //     if(is_local_leader) 
    258   //     { 
    259   //       size_info[2] = new_ep_info[0].size(); 
    260   //       ::MPI_Status mpi_status; 
    261  
    262   //       ::MPI_Send(&size_info[2], 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    263  
    264   //       ::MPI_Recv(&size_info[3], 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 
    265   //     } 
    266  
    267   //     ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    268  
    269   //     new_rank_info[2].resize(size_info[3]); 
    270   //     new_rank_info[3].resize(size_info[3]); 
    271   //     new_ep_info[1].resize(size_info[3]); 
    272  
    273   //     send_buf.resize(size_info[2]); 
    274   //     recv_buf.resize(size_info[3]); 
    275  
    276   //     if(is_local_leader) 
    277   //     { 
    278   //       ::MPI_Status mpi_status; 
    279  
    280   //       std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() ); 
    281   //       std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] ); 
    282   //       std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] ); 
    283  
    284   //       ::MPI_Send(send_buf.data(), 3*size_info[2], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    285   //       ::MPI_Recv(recv_buf.data(), 3*size_info[3], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 
    286  
    287   //     } 
    288  
    289   //     ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    290  
    291   //     std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() ); 
    292   //     std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin()  ); 
    293   //     std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() ); 
    294  
    295   //   } 
    296  
    297  
    298  
    299   //   if(is_proc_master) 
    300   //   { 
    301   //     // leader_info[4]: 2-> rank of local leader in new_group generated comm; 
    302   //                     // 3-> rank of remote leader in new_group generated comm; 
    303   //     ::MPI_Group local_group; 
    304   //     ::MPI_Group new_group; 
    305   //     ::MPI_Comm *new_comm = new ::MPI_Comm; 
    306   //     ::MPI_Comm *intercomm = new ::MPI_Comm; 
    307  
    308   //     ::MPI_Comm_group(local_mpi_comm, &local_group); 
    309  
    310   //     ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group); 
    311  
    312   //     ::MPI_Comm_create(local_mpi_comm, new_group, new_comm); 
    313  
    314  
    315  
    316   //     if(is_local_leader) 
    317   //     { 
    318   //       ::MPI_Comm_rank(*new_comm, &leader_info[2]); 
    319   //     } 
    320  
    321   //     ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    322  
    323   //     if(new_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 
    324   //     { 
    325   //       ::MPI_Barrier(*new_comm); 
    326  
    327   //       ::MPI_Intercomm_create(*new_comm, leader_info[2], peer_comm, leader_info[1], tag, intercomm); 
    328  
    329   //       int id; 
    330   //       ::MPI_Comm_rank(*new_comm, &id); 
    331   //       int my_num_ep = new_ep_info[0][id]; 
    332  
    333   //       MPI_Comm *ep_intercomm; 
    334   //       MPI_Info info; 
    335   //       MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm); 
    336  
    337  
    338  
    339   //       for(int i= 0; i<my_num_ep; i++) 
    340   //       { 
    341   //         ep_intercomm[i].is_intercomm = true; 
    342  
    343   //         ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
    344   //         ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm; 
    345   //         ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0]; 
    346   //       } 
    347  
    348   //       #pragma omp critical (write_to_tag_list) 
    349   //       tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm)); 
    350   //     } 
    351  
    352  
    353   //   } 
    354  
    355  
    356   //   MPI_Barrier_local(local_comm); 
    357  
    358   //   vector<int> bcast_buf(8); 
    359   //   if(is_local_leader) 
    360   //   { 
    361   //     std::copy(size_info, size_info+4, bcast_buf.begin()); 
    362   //     std::copy(leader_info, leader_info+4, bcast_buf.begin()+4); 
    363   //   } 
    364  
    365   //   MPI_Bcast(bcast_buf.data(), 8, MPI_INT, local_leader, local_comm); 
    366  
    367   //   if(!is_local_leader) 
    368   //   { 
    369   //     std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info); 
    370   //     std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info); 
    371   //   } 
    372  
    373   //   if(!is_local_leader) 
    374   //   { 
    375   //     new_rank_info[1].resize(size_info[2]); 
    376   //     ep_info[1].resize(size_info[1]); 
    377   //     offset.resize(size_info[0]); 
    378   //   } 
    379  
    380   //   bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1); 
    381  
    382   //   if(is_local_leader) 
    383   //   { 
    384   //     bcast_buf[0] = remote_ep_size; 
    385   //     std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1); 
    386   //     std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1); 
    387   //     std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1); 
    388   //   } 
    389  
    390   //   MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT, local_leader, local_comm); 
    391  
    392   //   if(!is_local_leader) 
    393   //   { 
    394   //     remote_ep_size = bcast_buf[0]; 
    395   //     std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin()); 
    396   //     std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin()); 
    397   //     std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin()); 
    398   //   } 
    399  
    400  
    401   //   int my_position = offset[rank_in_local_parent]+ep_rank_loc; 
    402  
    403  
    404   //   MPI_Barrier_local(local_comm); 
    405   //   #pragma omp flush 
    406  
    407  
    408   //   #pragma omp critical (read_from_tag_list) 
    409   //   { 
    410   //     bool found = false; 
    411   //     while(!found) 
    412   //     { 
    413   //       for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++) 
    414   //       { 
    415   //         if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1]))) 
    416   //         { 
    417   //           *newintercomm =  iter->second[my_position]; 
    418  
    419   //           found = true; 
    420   //           //tag_list.erase(iter); 
    421   //           break; 
    422   //         } 
    423   //       } 
    424   //     } 
    425   //   } 
    426  
    427   //   MPI_Barrier(local_comm); 
    428  
    429   //   if(is_local_leader) 
    430   //   { 
    431   //     int local_flag = true; 
    432   //     int remote_flag = false; 
    433   //     ::MPI_Status mpi_status; 
    434        
    435   //     ::MPI_Send(&local_flag, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    436  
    437   //     ::MPI_Recv(&remote_flag, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 
    438   //   } 
    439  
    440   //   MPI_Barrier(local_comm); 
    441  
    442   //   if(is_proc_master) 
    443   //   { 
    444   //     for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++) 
    445   //     { 
    446   //       if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1]))) 
    447   //       { 
    448   //         tag_list.erase(iter); 
    449   //         break; 
    450   //       } 
    451   //     } 
    452   //   } 
    453  
    454      
    455  
    456   //   int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank; 
    457   //   int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size; 
    458  
    459   //   intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first; 
    460   //   intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first; 
    461   //   intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first; 
    462   //   intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second; 
    463   //   intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second; 
    464   //   intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second; 
    465  
    466   //   MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm); 
    467  
    468   //   int my_rank_map_elem[2]; 
    469  
    470  
    471   //   my_rank_map_elem[0] = intercomm_ep_rank; 
    472  
    473   //   my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label; 
    474  
    475   //   vector<pair<int, int> > local_rank_map_array; 
    476   //   vector<pair<int, int> > remote_rank_map_array; 
    477  
    478  
    479   //   (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; 
    480   //   (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size); 
    481  
    482   //   MPI_Allgather2(my_rank_map_elem, 2, MPI_INT, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 
    483  
    484   //   (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
    485   //   (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size); 
    486  
    487   //   int local_intercomm_size = intercomm_ep_size; 
    488   //   int remote_intercomm_size; 
    489  
    490  
    491  
    492  
    493   //   if(is_local_leader) 
    494   //   { 
    495   //     ::MPI_Status status; 
    496  
    497   //     ::MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    498  
    499   //     ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 
    500  
    501   //     ::MPI_Send(&local_intercomm_size, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    502  
    503   //     ::MPI_Recv(&remote_intercomm_size, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 
    504   //   } 
    505  
    506   //   MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm); 
    507   //   MPI_Bcast(&remote_intercomm_size, 1, MPI_INT, 0, *newintercomm); 
    508  
    509  
    510   //   (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 
    511   //   (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); 
    512  
    513   //   (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0]; 
    514   //   (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1]; 
    515   //   (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2]; 
    516  
    517  
    518   //   if(is_local_leader) 
    519   //   { 
    520   //     ::MPI_Status status; 
    521  
    522   //     ::MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    523  
    524   //     ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 
    525   //   } 
    526  
    527   //   MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), 0, *newintercomm); 
    528  
    529   //   (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]); 
    530   //   (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label; 
    531  
    532  
    533   //   return MPI_SUCCESS; 
    534  
    535   // } 
    536  
    537  
    538  
    539   // #elif _intelmpi 
     11 
    54012  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm) 
    54113  { 
    … …  
    54315    int ep_size, num_ep, mpi_size; 
    54416 
    545     ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first; 
    546     ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first; 
    547     mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first; 
    548     ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second; 
    549     num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second; 
    550     mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second; 
     17    ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first; 
     18    ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first; 
     19    mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first; 
     20    ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second; 
     21    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second; 
     22    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second; 
    55123 
    55224    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm 
    … …  
    56032 
    56133    ::MPI_Comm peer_comm = to_mpi_comm(peer_comm_ptr); 
    562     ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm.mpi_comm); 
     34    ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm->mpi_comm); 
    56335 
    56436    ::MPI_Comm_rank(peer_comm, &rank_in_world); 
    … …  
    57244 
    57345    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;} 
    574     if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true; 
     46    if(ep_rank_loc == 0 && mpi_rank != local_comm->rank_map->at(local_leader).second) is_proc_master = true; 
    57547 
    57648 
    … …  
    633105      } 
    634106 
    635       ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     107      ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
    636108 
    637109      size_info[1] = recv_buf[0]; 
    … …  
    663135      } 
    664136 
    665       ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     137      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
    666138 
    667139      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() ); 
    … …  
    789261      } 
    790262 
    791       ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     263      ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
    792264 
    793265      new_rank_info[2].resize(size_info[3]); 
    … …  
    811283      } 
    812284 
    813       ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
     285      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
    814286 
    815287      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() ); 
    … …  
    843315      } 
    844316 
    845       ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 
    846  
    847       if(new_comm != static_cast< ::MPI_Comm* >(MPI_COMM_NULL.mpi_comm)) 
     317      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm); 
     318 
     319      if(new_comm != static_cast< ::MPI_Comm* >(MPI_COMM_NULL->mpi_comm)) 
    848320      { 
    849321        ::MPI_Barrier(*new_comm); 
    … …  
    861333        for(int i= 0; i<my_num_ep; i++) 
    862334        { 
    863           ep_intercomm[i].is_intercomm = true; 
    864  
    865           ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
    866           ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm; 
    867           ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0]; 
     335          ep_intercomm[i]->is_intercomm = true; 
     336 
     337          ep_intercomm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
     338          ep_intercomm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = intercomm; 
     339          ep_intercomm[i]->ep_comm_ptr->comm_label = leader_info[0]; 
    868340        } 
    869341 
    … …  
    979451    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size; 
    980452 
    981     intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first; 
    982     intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first; 
    983     intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first; 
    984     intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second; 
    985     intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second; 
    986     intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second; 
     453    intercomm_ep_rank = (*newintercomm)->ep_comm_ptr->size_rank_info[0].first; 
     454    intercomm_ep_rank_loc = (*newintercomm)->ep_comm_ptr->size_rank_info[1].first; 
     455    intercomm_mpi_rank = (*newintercomm)->ep_comm_ptr->size_rank_info[2].first; 
     456    intercomm_ep_size = (*newintercomm)->ep_comm_ptr->size_rank_info[0].second; 
     457    intercomm_num_ep = (*newintercomm)->ep_comm_ptr->size_rank_info[1].second; 
     458    intercomm_mpi_size = (*newintercomm)->ep_comm_ptr->size_rank_info[2].second; 
    987459 
    988460 
    … …  
    994466    my_rank_map_elem[0] = intercomm_ep_rank; 
    995467 
    996     my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label; 
     468    my_rank_map_elem[1] = (*newintercomm)->ep_comm_ptr->comm_label; 
    997469 
    998470    vector<pair<int, int> > local_rank_map_array; 
    … …  
    1000472 
    1001473 
    1002     (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; 
    1003     (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size); 
    1004  
    1005     MPI_Allgather(my_rank_map_elem, 2, MPI_INT, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 
     474    (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; 
     475    (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size); 
     476 
     477    MPI_Allgather(my_rank_map_elem, 2, MPI_INT, (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 
    1006478    
    1007     (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
    1008     (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size); 
     479    (*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
     480    (*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size); 
    1009481 
    1010482    int local_intercomm_size = intercomm_ep_size; 
    … …  
    1016488      ::MPI_Status status; 
    1017489 
    1018       ::MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    1019  
    1020       ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 
     490      ::MPI_Send((*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
     491 
     492      ::MPI_Recv((*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 
    1021493 
    1022494      ::MPI_Send(&local_intercomm_size, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    … …  
    1025497    } 
    1026498 
    1027     MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm); 
     499    MPI_Bcast((*newintercomm)->ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm); 
    1028500    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT, 0, *newintercomm); 
    1029501 
    1030502 
    1031     (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 
    1032     (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); 
    1033  
    1034     (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0]; 
    1035     (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1]; 
    1036     (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2]; 
     503    (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 
     504    (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); 
     505 
     506    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[0] = local_comm->ep_comm_ptr->size_rank_info[0]; 
     507    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[1] = local_comm->ep_comm_ptr->size_rank_info[1]; 
     508    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[2] = local_comm->ep_comm_ptr->size_rank_info[2]; 
    1037509 
    1038510 
    … …  
    1041513      ::MPI_Status status; 
    1042514 
    1043       ::MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
    1044  
    1045       ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 
    1046     } 
    1047  
    1048     MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, 0, *newintercomm); 
    1049  
    1050     (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]); 
    1051     (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label; 
     515      ::MPI_Send((*newintercomm)->rank_map->data(), 2*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 
     516 
     517      ::MPI_Recv((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 
     518    } 
     519 
     520    MPI_Bcast((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, 0, *newintercomm); 
     521 
     522    (*newintercomm)->ep_comm_ptr->intercomm->local_comm = (local_comm->ep_comm_ptr->comm_list[ep_rank_loc]); 
     523    (*newintercomm)->ep_comm_ptr->intercomm->intercomm_tag = local_comm->ep_comm_ptr->comm_label; 
    1052524 
    1053525    return MPI_SUCCESS; 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.cpp

    r1499 r1500  
    1414namespace ep_lib 
    1515{  
    16   bool MPI_Comm::is_null() 
     16  bool ep_comm::is_null() 
    1717  { 
    1818    if(!this->is_intercomm) 
    19       return this->mpi_comm == MPI_COMM_NULL.mpi_comm; 
     19      return this->mpi_comm == MPI_COMM_NULL->mpi_comm; 
    2020    else 
    21       return this->ep_comm_ptr->intercomm->mpi_inter_comm == MPI_COMM_NULL.mpi_comm; 
     21      return this->ep_comm_ptr->intercomm->mpi_inter_comm == MPI_COMM_NULL->mpi_comm; 
    2222  } 
    2323 
    … …  
    3333  int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank) 
    3434  { 
    35     for(int i=0; i<comm.rank_map->size(); i++) 
    36     { 
    37       if(   ( comm.rank_map->at(i).first  == ep_rank_loc ) 
    38          && ( comm.rank_map->at(i).second == mpi_rank ) ) 
     35    for(int i=0; i<comm->rank_map->size(); i++) 
     36    { 
     37      if(   ( comm->rank_map->at(i).first  == ep_rank_loc ) 
     38         && ( comm->rank_map->at(i).second == mpi_rank ) ) 
    3939      { 
    4040        return i; 
    … …  
    4848    // intercomm 
    4949    int inter_rank; 
    50     for(int i=0; i<comm.ep_comm_ptr->intercomm->intercomm_rank_map->size(); i++) 
    51     { 
    52       if(   ( comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first  == ep_rank_loc ) 
    53          && ( comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second == mpi_rank ) ) 
     50    for(int i=0; i<comm->ep_comm_ptr->intercomm->intercomm_rank_map->size(); i++) 
     51    { 
     52      if(   ( comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first  == ep_rank_loc ) 
     53         && ( comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second == mpi_rank ) ) 
    5454      { 
    5555        inter_rank =  i; 
    … …  
    5858    } 
    5959 
    60     for(int i=0; i<comm.ep_comm_ptr->intercomm->remote_rank_map->size(); i++) 
    61     { 
    62       if(  comm.ep_comm_ptr->intercomm->remote_rank_map->at(i).first  == inter_rank  ) 
     60    for(int i=0; i<comm->ep_comm_ptr->intercomm->remote_rank_map->size(); i++) 
     61    { 
     62      if(  comm->ep_comm_ptr->intercomm->remote_rank_map->at(i).first  == inter_rank  ) 
    6363      { 
    6464        //printf("get_ep_rank for intercomm, ep_rank_loc = %d, mpi_rank = %d => ep_src = %d\n", ep_rank_loc, mpi_rank, i); 
    … …  
    258258} 
    259259 
     260MPI_Message* to_mpi_message_ptr(ep_lib::MPI_Message message) 
     261{ 
     262  return static_cast< MPI_Message* >(message->mpi_message); 
     263} 
     264 
    260265MPI_Info to_mpi_info(ep_lib::MPI_Info info) 
    261266{ 
    262   return *(static_cast< MPI_Info* >(info.mpi_info)); 
     267  return *(static_cast< MPI_Info* >(info->mpi_info)); 
    263268} 
    264269 
    … …  
    273278} 
    274279 
    275  
    276  
     280MPI_Status* to_mpi_status_ptr(ep_lib::MPI_Status status) 
     281{ 
     282  return static_cast< MPI_Status* >(status.mpi_status); 
     283} 
     284 
     285MPI_Request* to_mpi_request_ptr(ep_lib::MPI_Request request) 
     286{ 
     287  return static_cast< MPI_Request* >(request->mpi_request); 
     288} 
     289 
     290 
     291 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.hpp

    r1499 r1500  
    5353 
    5454  int Message_Check(MPI_Comm comm); 
     55  int Message_Check_intracomm(MPI_Comm comm); 
    5556  int Request_Check(); 
    5657 
    … …  
    7374  int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr); 
    7475   
    75   //int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr); 
     76  int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr); 
    7677 
    7778 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_win.hpp

    r1398 r1500  
    1010 
    1111  int MPI_Win_create(void *base, MPI_Aint size, int disp_unit, MPI_Info info, MPI_Comm comm, MPI_Win *win); 
     12   
    1213  int MPI_Win_free(MPI_Win *win); 
    1314  int MPI_Win_fence(int assert, MPI_Win win); 
    … …  
    4647  int MPI_Compare_and_swap(const void *origin_addr, const void *compare_addr, void *result_addr, MPI_Datatype datatype,  
    4748                           int target_rank, MPI_Aint target_disp, MPI_Win win); 
     49                            
     50  // MPI_lock & MPI_Unlock 
    4851 
    4952} 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp

    r1381 r1500  
    1818    int ep_size, num_ep, mpi_size; 
    1919 
    20     ep_rank = inter_comm.ep_comm_ptr->size_rank_info[0].first; 
    21     ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first; 
    22     mpi_rank = inter_comm.ep_comm_ptr->size_rank_info[2].first; 
    23     ep_size = inter_comm.ep_comm_ptr->size_rank_info[0].second; 
    24     num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second; 
    25     mpi_size = inter_comm.ep_comm_ptr->size_rank_info[2].second; 
     20    ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 
     21    ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
     22    mpi_rank = inter_comm->ep_comm_ptr->size_rank_info[2].first; 
     23    ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 
     24    num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
     25    mpi_size = inter_comm->ep_comm_ptr->size_rank_info[2].second; 
    2626 
    2727    int local_high = high; 
    2828    int remote_high; 
    2929 
    30     int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size(); 
     30    int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size(); 
    3131 
    3232    int local_ep_rank, local_ep_rank_loc, local_mpi_rank; 
    3333    int local_ep_size, local_num_ep, local_mpi_size; 
    3434 
    35     local_ep_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first; 
    36     local_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
    37     local_mpi_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first; 
    38     local_ep_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second; 
    39     local_num_ep = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
    40     local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second; 
     35    local_ep_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first; 
     36    local_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
     37    local_mpi_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first; 
     38    local_ep_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second; 
     39    local_num_ep = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
     40    local_mpi_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second; 
    4141 
    4242 
    … …  
    4545      MPI_Status status[2]; 
    4646      MPI_Request request[2]; 
    47       MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[0]); 
    48       MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[1]); 
     47      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[0]); 
     48      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[1]); 
    4949 
    5050      MPI_Waitall(2, request, status); 
    … …  
    5252 
    5353 
    54     MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm)); 
     54    MPI_Bcast(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->local_comm); 
    5555 
    5656 
    … …  
    6464    int intra_ep_size, intra_num_ep, intra_mpi_size; 
    6565 
    66     intra_ep_rank = newintracomm->ep_comm_ptr->size_rank_info[0].first; 
    67     intra_ep_rank_loc = newintracomm->ep_comm_ptr->size_rank_info[1].first; 
    68     intra_mpi_rank = newintracomm->ep_comm_ptr->size_rank_info[2].first; 
    69     intra_ep_size = newintracomm->ep_comm_ptr->size_rank_info[0].second; 
    70     intra_num_ep = newintracomm->ep_comm_ptr->size_rank_info[1].second; 
    71     intra_mpi_size = newintracomm->ep_comm_ptr->size_rank_info[2].second; 
     66    intra_ep_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 
     67    intra_ep_rank_loc = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 
     68    intra_mpi_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
     69    intra_ep_size = (*newintracomm)->ep_comm_ptr->size_rank_info[0].second; 
     70    intra_num_ep = (*newintracomm)->ep_comm_ptr->size_rank_info[1].second; 
     71    intra_mpi_size = (*newintracomm)->ep_comm_ptr->size_rank_info[2].second; 
    7272 
    7373 
    … …  
    8585    if(intra_ep_rank_loc == 0) 
    8686    { 
    87       ::MPI_Bcast(reorder, intra_ep_size, to_mpi_type(MPI_INT), 0, to_mpi_comm(newintracomm->mpi_comm)); 
     87      ::MPI_Bcast(reorder, intra_ep_size, to_mpi_type(MPI_INT), 0, to_mpi_comm((*newintracomm)->mpi_comm)); 
    8888 
    8989      vector< pair<int, int> > tmp_rank_map(intra_ep_size); 
    … …  
    9292      for(int i=0; i<intra_ep_size; i++) 
    9393      { 
    94         tmp_rank_map[reorder[i]] = newintracomm->rank_map->at(i) ; 
     94        tmp_rank_map[reorder[i]] = (*newintracomm)->rank_map->at(i) ; 
    9595      } 
    9696 
    97       newintracomm->rank_map->swap(tmp_rank_map); 
     97      (*newintracomm)->rank_map->swap(tmp_rank_map); 
    9898 
    9999      tmp_rank_map.clear(); 
    … …  
    102102    MPI_Barrier_local(*newintracomm); 
    103103 
    104     (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
     104    (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
    105105 
    106106    if(intra_ep_rank_loc == 0) 
    … …  
    119119  { 
    120120 
    121     assert(inter_comm.is_intercomm); 
    122  
    123     if(inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99) 
     121    assert(inter_comm->is_intercomm); 
     122 
     123    if(inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99) 
    124124    { 
    125125        return MPI_Intercomm_merge_unique_leader(inter_comm, high, newintracomm); 
    … …  
    132132    int num_ep; 
    133133 
    134     ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first; 
    135     num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second; 
    136  
    137  
    138  
    139     int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size(); 
     134    ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
     135    num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
     136 
     137 
     138 
     139    int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size(); 
    140140 
    141141 
    … …  
    149149    { 
    150150 
    151       ::MPI_Comm mpi_comm = to_mpi_comm(inter_comm.ep_comm_ptr->intercomm->mpi_inter_comm); 
     151      ::MPI_Comm mpi_comm = to_mpi_comm(inter_comm->ep_comm_ptr->intercomm->mpi_inter_comm); 
    152152 
    153153      ::MPI_Intercomm_merge(mpi_comm, high, mpi_intracomm); 
    … …  
    155155      MPI_Comm_create_endpoints(mpi_intracomm, num_ep, info, ep_intracomm); 
    156156 
    157       inter_comm.ep_comm_ptr->comm_list->mem_bridge = ep_intracomm; 
     157      inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_intracomm; 
    158158 
    159159    } 
    … …  
    165165     
    166166    int my_ep_rank = high? inter_rank+remote_ep_size : inter_rank; 
    167     int my_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
    168     int my_num_ep_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
    169     int my_num_ep_total = inter_comm.ep_comm_ptr->comm_list->mem_bridge[0].ep_comm_ptr->size_rank_info[1].second; 
    170     int my_ep_size = inter_comm.ep_comm_ptr->comm_list->mem_bridge[0].ep_comm_ptr->size_rank_info[0].second; 
     167    int my_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
     168    int my_num_ep_loc = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
     169    int my_num_ep_total = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[0]->ep_comm_ptr->size_rank_info[1].second; 
     170    int my_ep_size = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[0]->ep_comm_ptr->size_rank_info[0].second; 
    171171 
    172172    int tmp_intra_ep_rank_loc = high?my_ep_rank_loc+my_num_ep_total-my_num_ep_loc: my_ep_rank_loc; 
    173173 
    174174 
    175     *newintracomm = inter_comm.ep_comm_ptr->comm_list->mem_bridge[tmp_intra_ep_rank_loc]; 
    176  
    177     int newintracomm_ep_rank = (*newintracomm).ep_comm_ptr->size_rank_info[0].first; 
    178     int newintracomm_ep_rank_loc = (*newintracomm).ep_comm_ptr->size_rank_info[1].first; 
    179     int newintracomm_mpi_rank = (*newintracomm).ep_comm_ptr->size_rank_info[2].first; 
    180     int newintracomm_ep_size = (*newintracomm).ep_comm_ptr->size_rank_info[0].second; 
    181     int newintracomm_num_ep = (*newintracomm).ep_comm_ptr->size_rank_info[1].second; 
    182     int newintracomm_mpi_size = (*newintracomm).ep_comm_ptr->size_rank_info[2].second; 
     175    *newintracomm = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[tmp_intra_ep_rank_loc]; 
     176 
     177    int newintracomm_ep_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 
     178    int newintracomm_ep_rank_loc = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 
     179    int newintracomm_mpi_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
     180    int newintracomm_ep_size = (*newintracomm)->ep_comm_ptr->size_rank_info[0].second; 
     181    int newintracomm_num_ep = (*newintracomm)->ep_comm_ptr->size_rank_info[1].second; 
     182    int newintracomm_mpi_size = (*newintracomm)->ep_comm_ptr->size_rank_info[2].second; 
    183183 
    184184 
    … …  
    186186    buf[0] = my_ep_rank; 
    187187    buf[1] = tmp_intra_ep_rank_loc; 
    188     buf[2] = newintracomm->ep_comm_ptr->size_rank_info[2].first; 
     188    buf[2] = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
    189189 
    190190    // printf("my_ep_rank = %d, tmp_intra_ep_rank_loc = %d, mpi_rank = %d\n", my_ep_rank, tmp_intra_ep_rank_loc, newintracomm->ep_comm_ptr->size_rank_info[2].first); 
    … …  
    206206    for(int i=0; i<newintracomm_ep_size; i++) 
    207207    { 
    208       (*newintracomm).rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1]; 
    209       (*newintracomm).rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2]; 
    210     } 
    211  
    212  
    213     (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
    214     (*newintracomm).ep_comm_ptr->size_rank_info[1].first = tmp_intra_ep_rank_loc; 
     208      (*newintracomm)->rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1]; 
     209      (*newintracomm)->rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2]; 
     210    } 
     211 
     212 
     213    (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
     214    (*newintracomm)->ep_comm_ptr->size_rank_info[1].first = tmp_intra_ep_rank_loc; 
    215215     
    216216 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_message.cpp

    r1381 r1500  
    1818namespace ep_lib 
    1919{ 
    20  
     20  int Request_Check() 
     21  { 
     22    MPI_Status status; 
     23    MPI_Message *message; 
     24    int probed = false; 
     25    int recv_count = 0; 
     26    std::list<MPI_Request* >::iterator it; 
     27     
     28    if(EP_PendingRequests == 0 ) return 0; 
     29     
     30    for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); it++) 
     31    {  
     32      Message_Check(((*(*it))->comm)); 
     33    } 
     34 
     35 
     36    for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); ) 
     37    { 
     38      if((*(*it))->state == 0) 
     39      { 
     40           
     41        message = new MPI_Message; 
     42        *message = new ep_message; 
     43         
     44        printf("new %p : in ep_lib::Request_Check, message = new MPI_Message\n", message); 
     45        printf("new %p : in ep_lib::Request_Check, *message = new ep_message\n", *message); 
     46           
     47           
     48        MPI_Improbe((*(*it))->ep_src, (*(*it))->ep_tag, ((*(*it))->comm), &probed, message, &status); 
     49        printf("in Request_Check, after improbe, mpi_status = %p\n", to_mpi_status_ptr(status)); 
     50       
     51         
     52        if(probed) 
     53        { 
     54          ::MPI_Get_count(to_mpi_status_ptr(status), to_mpi_type((*(*it))->ep_datatype), &recv_count); 
     55           
     56          printf("in Request_Check, imrecv, buf = %p, recv_count = %d, status = %p\n", (*(*it))->buf, recv_count, to_mpi_status_ptr(status)); 
     57          MPI_Imrecv((*(*it))->buf, recv_count, (*(*it))->ep_datatype, message, *it); 
     58          (*(*it))->type = 3; 
     59          (*(*it))->state = 1; 
     60           
     61 
     62          printf("delete %p : in ep_lib::Request_Check, delete *message\n", *message); 
     63          printf("delete %p : in ep_lib::Request_Check, delete message\n", message); 
     64 
     65          delete *message; 
     66          delete message; 
     67         
     68          it++; 
     69          continue;       
     70        }       
     71      } 
     72       
     73      if((*(*it))->state == 2) 
     74      { 
     75        printf("delete %p : in ep_lib::Request_Check, delete (*(*it))\n", (*(*it))); 
     76        delete (*(*it)); 
     77         
     78        EP_PendingRequests->erase(it); 
     79        printf("EP_PendingRequests->size() = %lu\n", EP_PendingRequests->size()); 
     80        it = EP_PendingRequests->begin(); 
     81        continue; 
     82      } 
     83      else it++; 
     84    } 
     85  } 
     86   
     87   
     88   
    2189  int Message_Check(MPI_Comm comm) 
    2290  { 
    23     if(!comm.is_ep) return 0; 
    24  
    25     if(comm.is_intercomm) 
     91    if(!comm->is_ep) return MPI_SUCCESS; 
     92 
     93    if(comm->is_intercomm) 
    2694    { 
    2795      return  Message_Check_intercomm(comm); 
    2896    } 
     97     
     98    return Message_Check_intracomm(comm); 
     99 
     100  } 
     101   
     102   
     103  int Message_Check_intracomm(MPI_Comm comm) 
     104  { 
     105     
     106    int flag = true; 
     107    ::MPI_Status status; 
     108    ::MPI_Message message; 
     109 
     110    while(flag) // loop until the end of global queue 
     111    { 
     112      Debug("Message probing for intracomm\n"); 
     113      
     114      #pragma omp critical (_mpi_call) 
     115      { 
     116        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm->mpi_comm), &flag, &status); 
     117        if(flag) 
     118        { 
     119          Debug("find message in mpi comm \n"); 
     120          ::MPI_Mprobe(status.MPI_SOURCE, status.MPI_TAG, to_mpi_comm(comm->mpi_comm), &message, &status); 
     121        } 
     122      } 
     123 
     124       
     125      if(flag) 
     126      { 
     127        MPI_Message msg = new ep_message;  
     128        msg->mpi_message = new ::MPI_Message(message); 
     129 
     130        printf("new %p : in ep_lib::Message_Check, msg = new ep_message\n", msg); 
     131        printf("new %p : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message\n", msg->mpi_message); 
     132               
     133 
     134        msg->ep_tag  = bitset<15>(status.MPI_TAG >> 16).to_ulong();  
     135        int src_loc  = bitset<8> (status.MPI_TAG >> 8) .to_ulong();  
     136        int dest_loc = bitset<8> (status.MPI_TAG)           .to_ulong(); 
     137        int src_mpi  = status.MPI_SOURCE; 
     138              
     139        msg->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);        
     140        msg->mpi_status = new ::MPI_Status(status);   
     141        printf("new %p : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status\n", msg->mpi_status); 
     142 
     143        #pragma omp critical (_query) 
     144        { 
     145          #pragma omp flush 
     146          comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->push_back(msg);   
     147          int test_count; 
     148          ::MPI_Get_count(static_cast< ::MPI_Status* >(msg->mpi_status), 1275070475, &test_count); 
     149          printf("status1 = %p, test_count2 = %d\n", static_cast< ::MPI_Status* >(msg->mpi_status), test_count); 
     150           
     151          ::MPI_Get_count(static_cast< ::MPI_Status* >(comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->back()->mpi_status), 1275070475, &test_count); 
     152          printf("status2 = %p, test_count2 = %d\n", static_cast< ::MPI_Status* >(comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->back()->mpi_status), test_count); 
     153                    
     154          #pragma omp flush 
     155        } 
     156      } 
     157    } 
     158 
     159    return MPI_SUCCESS; 
     160  } 
     161 
     162 
     163   
     164   
     165   
     166 
     167 
     168  int Message_Check_intercomm(MPI_Comm comm) 
     169  { 
     170    if(!comm->ep_comm_ptr->intercomm->mpi_inter_comm) return 0; 
     171 
     172    Debug("Message probing for intercomm\n"); 
    29173 
    30174    int flag = true; 
    31175    ::MPI_Message message; 
    32176    ::MPI_Status status; 
    33     int mpi_source; 
    34  
    35     while(flag) // loop until the end of global queue 
    36     { 
    37       Debug("Message probing for intracomm\n"); 
    38        
    39       #pragma omp critical (_mpi_call) 
    40       { 
    41         ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm.mpi_comm), &flag, &status); 
    42         if(flag) 
    43         { 
    44           Debug("find message in mpi comm \n"); 
    45           mpi_source = status.MPI_SOURCE; 
    46           int tag = status.MPI_TAG; 
    47           ::MPI_Mprobe(mpi_source, tag, to_mpi_comm(comm.mpi_comm), &message, &status); 
    48  
    49         } 
    50       } 
    51  
    52        
    53       if(flag) 
    54       { 
    55  
    56         MPI_Message *msg_block = new MPI_Message;  
    57         msg_block->mpi_message = new ::MPI_Message; 
    58         *(static_cast< ::MPI_Message*>(msg_block->mpi_message)) = message;   
    59         msg_block->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong();  
    60         int src_loc       = bitset<8> (status.MPI_TAG >> 8) .to_ulong();  
    61         int dest_loc      = bitset<8> (status.MPI_TAG)      .to_ulong(); 
    62         int src_mpi       = status.MPI_SOURCE; 
    63               
    64         msg_block->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);        
    65         msg_block->mpi_status = new ::MPI_Status(status); 
    66  
    67         MPI_Comm* ptr_comm_list = comm.ep_comm_ptr->comm_list; 
    68         MPI_Comm* ptr_comm_target = &ptr_comm_list[dest_loc]; 
    69  
    70  
    71         #pragma omp critical (_query) 
    72         { 
    73           #pragma omp flush 
    74           comm.ep_comm_ptr->comm_list[dest_loc].ep_comm_ptr->message_queue->push_back(*msg_block);       
    75           #pragma omp flush 
    76         } 
    77          
    78         delete msg_block; 
    79       } 
    80  
    81     } 
    82  
    83     return MPI_SUCCESS; 
    84   } 
    85  
    86  
    87  
    88   int Message_Check_intercomm(MPI_Comm comm) 
    89   { 
    90     if(!comm.ep_comm_ptr->intercomm->mpi_inter_comm) return 0; 
    91  
    92     Debug("Message probing for intercomm\n"); 
    93  
    94     int flag = true; 
    95     ::MPI_Message message; 
    96     ::MPI_Status status; 
    97     int mpi_source; 
    98177    int current_ep_rank; 
    99178    MPI_Comm_rank(comm, &current_ep_rank); 
    100179 
    101     while(flag) // loop until the end of global queue "comm.ep_comm_ptr->intercomm->mpi_inter_comm" 
     180    while(flag) // loop until the end of global queue "comm->ep_comm_ptr->intercomm->mpi_inter_comm" 
    102181    { 
    103182      Debug("Message probing for intracomm\n"); 
    … …  
    105184      #pragma omp critical (_mpi_call) 
    106185      { 
    107         ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm), &flag, &status); 
     186        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &flag, &status); 
    108187        if(flag) 
    109188        { 
    110189          Debug("find message in mpi comm \n"); 
    111           mpi_source = status.MPI_SOURCE; 
    112           int tag = status.MPI_TAG; 
    113           ::MPI_Mprobe(mpi_source, tag, to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm), &message, &status); 
    114  
     190          ::MPI_Mprobe(status.MPI_SOURCE, status.MPI_TAG, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &message, &status); 
    115191        } 
    116192      } 
    … …  
    120196      { 
    121197 
    122         MPI_Message *msg_block = new MPI_Message; 
    123         msg_block->mpi_message = new ::MPI_Message; 
    124         *(static_cast< ::MPI_Message*>(msg_block->mpi_message)) = message; 
    125         msg_block->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong(); 
    126         int src_loc       = bitset<8> (status.MPI_TAG >> 8) .to_ulong(); 
    127         int dest_loc      = bitset<8> (status.MPI_TAG)      .to_ulong(); 
    128         int src_mpi       = status.MPI_SOURCE; 
    129         int current_inter = comm.ep_comm_ptr->intercomm->local_rank_map->at(current_ep_rank).first; 
     198        MPI_Message msg = new ep_message;  
     199        msg->mpi_message = new ::MPI_Message(message); 
     200 
     201        printf("new %p : in ep_lib::Message_Check, msg = new ep_message\n", msg); 
     202        printf("new %p : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message\n", msg->mpi_message); 
     203               
     204 
     205        msg->ep_tag  = bitset<15>(status.MPI_TAG >> 16).to_ulong();  
     206        int src_loc  = bitset<8> (status.MPI_TAG >> 8) .to_ulong();  
     207        int dest_loc = bitset<8> (status.MPI_TAG)           .to_ulong(); 
     208        int src_mpi  = status.MPI_SOURCE; 
     209        int current_inter = comm->ep_comm_ptr->intercomm->local_rank_map->at(current_ep_rank).first; 
    130210              
    131         msg_block->ep_src  = get_ep_rank_intercomm(comm, src_loc,  src_mpi); 
    132         msg_block->mpi_status = new ::MPI_Status(status); 
    133  
    134  
    135         MPI_Comm* ptr_comm_list = comm.ep_comm_ptr->comm_list; 
    136         MPI_Comm* ptr_comm_target = &ptr_comm_list[dest_loc]; 
    137  
     211        msg->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);        
     212        msg->mpi_status = new ::MPI_Status(status);   
     213        printf("new %p : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status\n", msg->mpi_status); 
    138214 
    139215        #pragma omp critical (_query) 
    140216        { 
    141217          #pragma omp flush 
    142           comm.ep_comm_ptr->comm_list[dest_loc].ep_comm_ptr->message_queue->push_back(*msg_block); 
    143           #pragma omp flush 
    144         } 
    145          
    146         delete msg_block; 
    147          
    148       } 
    149  
    150     } 
    151  
    152     flag = true; 
    153     while(flag) // loop until the end of global queue "comm.mpi_comm" 
    154     { 
    155       Debug("Message probing for intracomm\n"); 
    156       
    157       #pragma omp critical (_mpi_call) 
    158       { 
    159         ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm.mpi_comm), &flag, &status); 
    160         if(flag) 
    161         { 
    162           Debug("find message in mpi comm \n"); 
    163           mpi_source = status.MPI_SOURCE; 
    164           int tag = status.MPI_TAG; 
    165           ::MPI_Mprobe(mpi_source, tag, to_mpi_comm(comm.mpi_comm), &message, &status); 
    166  
    167         } 
    168       } 
    169        
    170  
    171       if(flag) 
    172       { 
    173  
    174         MPI_Message *msg_block = new MPI_Message; 
    175         msg_block->mpi_message = new ::MPI_Message; 
    176         *(static_cast< ::MPI_Message*>(msg_block->mpi_message)) = message; 
    177         msg_block->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong(); 
    178         int src_loc       = bitset<8> (status.MPI_TAG >> 8) .to_ulong(); 
    179         int dest_loc      = bitset<8> (status.MPI_TAG)      .to_ulong(); 
    180         int src_mpi       = status.MPI_SOURCE; 
    181          
    182         msg_block->ep_src  = get_ep_rank_intercomm(comm, src_loc, src_mpi); 
    183         msg_block->mpi_status = new ::MPI_Status(status); 
    184          
    185  
    186         MPI_Comm* ptr_comm_list = comm.ep_comm_ptr->comm_list; 
    187         MPI_Comm* ptr_comm_target = &ptr_comm_list[dest_loc]; 
    188  
    189  
    190         #pragma omp critical (_query) 
    191         { 
    192           #pragma omp flush 
    193           comm.ep_comm_ptr->comm_list[dest_loc].ep_comm_ptr->message_queue->push_back(*msg_block); 
    194           #pragma omp flush 
    195         } 
    196          
    197         delete msg_block; 
    198          
    199       } 
    200  
    201     } 
     218          comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->push_back(msg); 
     219          #pragma omp flush 
     220        } 
     221      } 
     222    } 
     223 
     224    Message_Check_intracomm(comm); 
    202225 
    203226    return MPI_SUCCESS; 
    204227  } 
    205228 
    206   int Request_Check() 
    207   { 
    208     MPI_Status status; 
    209     MPI_Message message; 
    210     int probed = false; 
    211     int recv_count = 0; 
    212     std::list<MPI_Request* >::iterator it; 
    213      
    214     for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); it++) 
    215     {  
    216       Message_Check((*it)->comm); 
    217     } 
    218  
    219  
    220     for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); ) 
    221     { 
    222       MPI_Improbe((*it)->ep_src, (*it)->ep_tag, (*it)->comm, &probed, &message, &status); 
    223       if(probed) 
    224       { 
    225         MPI_Get_count(&status, (*it)->ep_datatype, &recv_count); 
    226         MPI_Imrecv((*it)->buf, recv_count, (*it)->ep_datatype, &message, *it); 
    227         (*it)->type = 3; 
    228         EP_PendingRequests->erase(it); 
    229         it = EP_PendingRequests->begin(); 
    230         continue; 
    231       } 
    232       it++; 
    233     } 
    234   } 
     229   
    235230 
    236231} 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_mpi.hpp

    r1388 r1500  
    1111MPI_Win      to_mpi_win(void* win); 
    1212MPI_Aint     to_mpi_aint(ep_lib::MPI_Aint aint); 
     13MPI_Status*  to_mpi_status_ptr (ep_lib::MPI_Status status); 
     14MPI_Request* to_mpi_request_ptr(ep_lib::MPI_Request request); 
     15MPI_Message* to_mpi_message_ptr(ep_lib::MPI_Message message); 
    1316 
    1417#endif // EP_MPI_HPP_INCLUDED 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_probe.cpp

    r1381 r1500  
    1111    *flag = false; 
    1212 
    13     if(!comm.is_ep) 
     13    if(!comm->is_ep) 
    1414    { 
    1515      ::MPI_Status *mpi_status = static_cast< ::MPI_Status* >(status->mpi_status); 
    16       ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm.mpi_comm), flag, mpi_status); 
     16      ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, mpi_status); 
    1717 
    1818      status->mpi_status = mpi_status; 
    … …  
    2929 
    3030    #pragma omp critical (_query) 
    31     if(!comm.ep_comm_ptr->message_queue->empty()) 
     31    if(!comm->ep_comm_ptr->message_queue->empty()) 
    3232    { 
    33       for(Message_list::iterator it = comm.ep_comm_ptr->message_queue->begin(); it!= comm.ep_comm_ptr->message_queue->end(); ++it) 
     33      for(Message_list::iterator it = comm->ep_comm_ptr->message_queue->begin(); it!= comm->ep_comm_ptr->message_queue->end(); ++it) 
    3434      { 
    35         bool src_matched = src<0? true: it->ep_src == src; 
    36         bool tag_matched = tag<0? true: it->ep_tag == tag; 
     35        bool src_matched = src<0? true: (*it)->ep_src == src; 
     36        bool tag_matched = tag<0? true: (*it)->ep_tag == tag; 
    3737         
    3838        if(src_matched && tag_matched)         
    … …  
    4141          *flag = true; 
    4242 
    43           ::MPI_Status mpi_status = *(static_cast< ::MPI_Status *>(it->mpi_status)); 
     43          ::MPI_Status mpi_status = *(static_cast< ::MPI_Status *>((*it)->mpi_status)); 
    4444 
    4545          status->mpi_status = new ::MPI_Status(mpi_status); 
    46           status->ep_src = it->ep_src; 
    47           status->ep_tag = it->ep_tag; 
     46          status->ep_src = (*it)->ep_src; 
     47          status->ep_tag = (*it)->ep_tag; 
    4848 
    4949          break; 
    … …  
    6161  { 
    6262    *flag = false; 
    63     if(!comm.is_ep) 
     63    if(!comm->is_ep) 
    6464    { 
    6565      Debug("calling MPI_Improbe MPI\n"); 
    … …  
    7171      #pragma omp critical (_mpi_call) 
    7272      { 
    73         ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm.mpi_comm), flag, &mpi_status); 
     73        ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status); 
    7474        if(*flag) 
    7575        { 
    76           ::MPI_Mprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm.mpi_comm), &mpi_message, &mpi_status); 
     76          ::MPI_Mprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_message, &mpi_status); 
    7777        } 
    7878      } 
    7979      #elif _intelmpi 
    80         ::MPI_Improbe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm.mpi_comm), flag, &mpi_message, &mpi_status); 
     80        ::MPI_Improbe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_message, &mpi_status); 
    8181      #endif 
    8282         
    83       status->mpi_status = new ::MPI_Status(mpi_status); 
     83      status->mpi_status = &mpi_status; 
    8484      status->ep_src = src; 
    8585      status->ep_tag = tag; 
    8686 
    87       message->mpi_message = new ::MPI_Message; 
    88       *(static_cast< ::MPI_Message*>(message->mpi_message)) = mpi_message; 
    89       message->ep_src = src; 
    90       message->ep_tag = tag; 
     87      (*message)->mpi_message = &message; 
     88      (*message)->ep_src = src; 
     89      (*message)->ep_tag = tag; 
     90       
     91       
    9192      return 0; 
    9293    } 
    9394 
    94      
    95     //Message_Check(comm); 
     95    
    9696 
    9797    #pragma omp flush 
    9898 
    9999    #pragma omp critical (_query) 
    100     if(! comm.ep_comm_ptr->message_queue->empty()) 
     100    if(! comm->ep_comm_ptr->message_queue->empty()) 
    101101    { 
    102       for(Message_list::iterator it = comm.ep_comm_ptr->message_queue->begin(); it!= comm.ep_comm_ptr->message_queue->end(); ++it) 
     102      for(Message_list::iterator it = comm->ep_comm_ptr->message_queue->begin(); it!= comm->ep_comm_ptr->message_queue->end(); ++it) 
    103103      { 
    104         bool src_matched = src<0? true: it->ep_src == src; 
    105         bool tag_matched = tag<0? true: it->ep_tag == tag; 
     104 
     105        //printf("in ep_lib::Improbe, it->mpi_message = %p, it->mpi_status = %p, it->ep_src = %d, it->ep_tag = %d\n",  
     106        //                            (*(*it))->mpi_message,(*(*it))->mpi_status,(*(*it))->ep_src,(*(*it))->ep_tag); 
     107                                           
     108        bool src_matched = src<0? true: (*it)->ep_src == src; 
     109        bool tag_matched = tag<0? true: (*it)->ep_tag == tag; 
    106110         
    107111        if(src_matched && tag_matched) 
    … …  
    109113          *flag = true; 
    110114 
    111           ::MPI_Status mpi_status; 
    112           mpi_status = *(static_cast< ::MPI_Status *>(it->mpi_status)); 
     115          status->mpi_status = (*it)->mpi_status; 
     116          status->ep_src = (*it)->ep_src; 
     117          status->ep_tag = (*it)->ep_tag; 
    113118 
    114           status->mpi_status = new ::MPI_Status(mpi_status); 
    115           status->ep_src = it->ep_src; 
    116           status->ep_tag = it->ep_tag; 
    117  
    118           message->mpi_message = it->mpi_message; 
    119           message->ep_tag = it->ep_tag; 
    120           message->ep_src = it->ep_src; 
     119          (*message)->mpi_message = (*it)->mpi_message; 
     120          (*message)->ep_src = (*it)->ep_src; 
     121          (*message)->ep_tag = (*it)->ep_tag; 
     122           
     123          int test_count; 
     124          ::MPI_Get_count(to_mpi_status_ptr(status), 1275070475, &test_count); 
     125          printf("status = %p, test_count = %d\n", to_mpi_status_ptr(status), test_count); 
     126           
     127           
     128          ::MPI_Get_count(static_cast< ::MPI_Status* >(comm->ep_comm_ptr->message_queue->back()->mpi_status), 1275070475, &test_count); 
     129          printf("in ep_lib::Improbe, status = %p, test_count = %d\n", static_cast< ::MPI_Status* >(comm->ep_comm_ptr->message_queue->back()->mpi_status), test_count); 
     130                                       
    121131 
    122132          #pragma omp critical (_query2) 
    123133          {               
    124             delete it->mpi_status; 
    125             comm.ep_comm_ptr->message_queue->erase(it); 
     134            printf("delete %p : in ep_lib::Message_Check, delete (*it)->mpi_message\n", (*it)->mpi_message); 
     135            printf("delete %p : in ep_lib::Message_Check, delete (*it)->mpi_status\n", (*it)->mpi_status); 
     136            printf("delete %p : in ep_lib::Message_Check, delete (*it)\n", (*it)); 
     137             
     138            delete (*it)->mpi_message;      
     139            delete (*it)->mpi_status;  
     140            delete *it; 
     141             
     142            comm->ep_comm_ptr->message_queue->erase(it); 
     143            printf("message queue siez = %lu\n", comm->ep_comm_ptr->message_queue->size()); 
    126144            #pragma omp flush 
    127145          } 
    128  
     146           
    129147          break; 
    130148        } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_put.cpp

    r1398 r1500  
    1212                  int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win) 
    1313  { 
    14     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    15     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    16     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     14    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     15    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     16    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    1717    if(num_ep==1) 
    1818      return  ::MPI_Put(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    19                        to_mpi_win(win.server_win[target_local_rank])); 
     19                       to_mpi_win(win->server_win[target_local_rank])); 
    2020 
    2121    else 
    2222        return  ::MPI_Put(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    23                        to_mpi_win(win.client_win)); 
     23                       to_mpi_win(win->client_win)); 
    2424  } 
    2525 
    … …  
    2727              int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request) 
    2828  { 
    29     int target_mpi_rank   = win.comm.rank_map->at(target_rank).second; 
    30     int target_local_rank = win.comm.rank_map->at(target_rank).first; 
    31     int num_ep = win.comm.ep_comm_ptr->size_rank_info[1].second; 
     29    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second; 
     30    int target_local_rank = win->comm->rank_map->at(target_rank).first; 
     31    int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second; 
    3232     
    3333    ::MPI_Request mpi_request; 
    … …  
    3636    { 
    3737      int return_value = ::MPI_Rput(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    38                            to_mpi_win(win.server_win[target_local_rank]), &mpi_request); 
     38                           to_mpi_win(win->server_win[target_local_rank]), &mpi_request); 
    3939 
    40       request->mpi_request = new ::MPI_Request(mpi_request); 
     40      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    4141 
    42       request->ep_datatype = origin_datatype; 
    43       request->type = 1; 
     42      (*request)->ep_datatype = origin_datatype; 
     43      (*request)->type = 1; 
    4444      return return_value; 
    4545    } 
    … …  
    4848    { 
    4949      int return_value = ::MPI_Rput(origin_addr, origin_count, to_mpi_type(origin_datatype), target_mpi_rank, to_mpi_aint(target_disp), target_count, to_mpi_type(target_datatype),  
    50                          to_mpi_win(win.client_win), &mpi_request); 
     50                         to_mpi_win(win->client_win), &mpi_request); 
    5151 
    52       request->mpi_request = new ::MPI_Request(mpi_request); 
     52      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
    5353 
    54       request->ep_datatype = origin_datatype; 
    55       request->type = 1; 
     54      (*request)->ep_datatype = origin_datatype; 
     55      (*request)->type = 1; 
    5656      return return_value; 
    5757    } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_rank.cpp

    r1381 r1500  
    1212  { 
    1313 
    14     if(comm.is_ep) 
     14    if(comm->is_ep) 
    1515    { 
    1616      Debug("Calling EP_Comm_rank\n"); 
    1717 
    18       if(comm.is_intercomm) 
     18      if(comm->is_intercomm) 
    1919      { 
    20         *rank = comm.ep_comm_ptr->intercomm->size_rank_info[0].first; 
     20        *rank = comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 
    2121        return 0; 
    2222      } 
    2323      else 
    2424      { 
    25         *rank = comm.ep_comm_ptr->size_rank_info[0].first; 
     25        *rank = comm->ep_comm_ptr->size_rank_info[0].first; 
    2626        return 0; 
    2727      } 
    … …  
    3131    if(comm != MPI_COMM_NULL) 
    3232    { 
    33       ::MPI_Comm mpi_comm = to_mpi_comm(comm.mpi_comm); 
     33      ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
    3434      ::MPI_Comm_rank(mpi_comm, rank); 
    3535      return 0; 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_recv.cpp

    r1381 r1500  
    2424  { 
    2525 
    26     if(!comm.is_ep) 
     26    if(!comm->is_ep) 
    2727    { 
    2828      ::MPI_Status mpi_status; 
    29       ::MPI_Recv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm.mpi_comm), &mpi_status); 
     29      ::MPI_Recv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_status); 
    3030 
    3131      status->ep_src = src; 
    … …  
    3636    } 
    3737 
    38     Message_Check(comm); 
     38    //Message_Check(comm); 
    3939 
    4040    MPI_Request request; 
    … …  
    5656     
    5757 
    58     if(!comm.is_ep) 
     58    if(!comm->is_ep) 
    5959    { 
    6060      ::MPI_Request mpi_request; 
    61       ::MPI_Irecv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
     61      ::MPI_Irecv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
    6262 
    63       request->mpi_request = new ::MPI_Request(mpi_request); 
    64       request->ep_src = src; 
    65       request->ep_datatype = datatype; 
    66       request->ep_tag = tag; 
     63      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     64      (*request)->ep_src = src; 
     65      (*request)->ep_datatype = datatype; 
     66      (*request)->ep_tag = tag; 
    6767    } 
     68     
     69    *request = new ep_request; 
     70    printf("new %p : in ep_lib::MPI_Irecv, *request = new ep_request\n", (*request)); 
    6871 
    69     request->buf = buf; 
    70     request->comm = comm; 
    71     request->type = 2; 
     72    (*request)->mpi_request = new ::MPI_Request; 
     73    printf("new %p : in ep_lib::MPI_Irecv, (*request)->mpi_request = new ::MPI_Request\n", (*request)->mpi_request); 
     74     
     75    (*request)->buf = buf; 
     76    (*request)->comm = comm; 
     77    (*request)->type = 2; 
     78    (*request)->state = 0; 
     79     
    7280 
    73     request->ep_src = src; 
    74     request->ep_tag = tag; 
    75     request->ep_datatype = datatype; 
    76  
    77  
    78  
     81    (*request)->ep_src = src; 
     82    (*request)->ep_tag = tag; 
     83    (*request)->ep_datatype = datatype; 
     84     
     85     
    7986    /* With Improbe*/ 
    80     Message_Check(comm); 
    8187 
    8288    if(EP_PendingRequests == 0 )  
    … …  
    8591    } 
    8692 
    87  
    88     EP_PendingRequests->push_back(request); 
    89  
    90  
     93    EP_PendingRequests->push_back(request);                                                                 
     94                                           
    9195    Request_Check(); 
    9296 
    … …  
    98102    Debug("MPI_Imrecv"); 
    99103 
    100     request->type = 3; 
     104    (*request)->type = 3; 
     105         
     106    ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request));                 
    101107 
    102     ::MPI_Request mpi_request; 
    103     ::MPI_Imrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >(message->mpi_message), &mpi_request);          
    104  
    105     request->mpi_request = new ::MPI_Request(mpi_request); 
    106     request->ep_datatype = datatype; 
    107     request->ep_tag = message->ep_tag; 
    108     request->ep_src = message->ep_src; 
     108    (*request)->ep_datatype = datatype; 
     109    (*request)->ep_tag = (*message)->ep_tag; 
     110    (*request)->ep_src = (*message)->ep_src; 
    109111     
    110     delete message->mpi_message; 
    111  
     112     
    112113    return 0; 
    113114  } 
    … …  
    119120 
    120121    ::MPI_Status mpi_status; 
    121     ::MPI_Mrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >(message->mpi_message), &mpi_status); 
     122    ::MPI_Mrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >((*message)->mpi_message), &mpi_status); 
    122123 
    123124    status->mpi_status = new ::MPI_Status(mpi_status); 
    124     status->ep_src = message->ep_src; 
     125    status->ep_src = (*message)->ep_src; 
    125126    status->ep_datatype = datatype; 
    126     status->ep_tag = message->ep_tag; 
     127    status->ep_tag = (*message)->ep_tag; 
    127128 
    128     delete message->mpi_message; 
     129    delete (*message)->mpi_message; 
    129130 
    130131    //check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_reduce.cpp

    r1499 r1500  
    6666    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize); 
    6767 
    68     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    69     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    70     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
     68    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     69    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     70    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
    7171 
    7272    #pragma omp critical (_reduce) 
    73     comm.my_buffer->void_buffer[ep_rank_loc] = const_cast< void* >(sendbuf); 
     73    comm->my_buffer->void_buffer[ep_rank_loc] = const_cast< void* >(sendbuf); 
    7474 
    7575    MPI_Barrier_local(comm); 
    … …  
    7878    { 
    7979 
    80       memcpy(recvbuf, comm.my_buffer->void_buffer[0], datasize * count); 
     80      memcpy(recvbuf, comm->my_buffer->void_buffer[0], datasize * count); 
    8181 
    8282      if(op == MPI_MAX) 
    … …  
    8686          assert(datasize == sizeof(int)); 
    8787          for(int i=1; i<num_ep; i++) 
    88             reduce_max<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count); 
     88            reduce_max<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count); 
    8989        } 
    9090 
    … …  
    9393          assert(datasize == sizeof(float)); 
    9494          for(int i=1; i<num_ep; i++) 
    95             reduce_max<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count); 
     95            reduce_max<float>(static_cast<float*>(comm->my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count); 
    9696        } 
    9797 
    … …  
    100100          assert(datasize == sizeof(double)); 
    101101          for(int i=1; i<num_ep; i++) 
    102             reduce_max<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
     102            reduce_max<double>(static_cast<double*>(comm->my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
    103103        } 
    104104 
    … …  
    107107          assert(datasize == sizeof(char)); 
    108108          for(int i=1; i<num_ep; i++) 
    109             reduce_max<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
     109            reduce_max<char>(static_cast<char*>(comm->my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
    110110        } 
    111111 
    … …  
    114114          assert(datasize == sizeof(long)); 
    115115          for(int i=1; i<num_ep; i++) 
    116             reduce_max<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
     116            reduce_max<long>(static_cast<long*>(comm->my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
    117117        } 
    118118 
    … …  
    121121          assert(datasize == sizeof(unsigned long)); 
    122122          for(int i=1; i<num_ep; i++) 
    123             reduce_max<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count); 
     123            reduce_max<unsigned long>(static_cast<unsigned long*>(comm->my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count); 
    124124        } 
    125125 
    … …  
    128128          assert(datasize == sizeof(uint64_t)); 
    129129          for(int i=1; i<num_ep; i++) 
    130             reduce_max<uint64_t>(static_cast<uint64_t*>(comm.my_buffer->void_buffer[i]), static_cast<uint64_t*>(recvbuf), count); 
     130            reduce_max<uint64_t>(static_cast<uint64_t*>(comm->my_buffer->void_buffer[i]), static_cast<uint64_t*>(recvbuf), count); 
    131131        } 
    132132         
    … …  
    135135          assert(datasize == sizeof(long long)); 
    136136          for(int i=1; i<num_ep; i++) 
    137             reduce_max<long long>(static_cast<long long*>(comm.my_buffer->void_buffer[i]), static_cast<long long*>(recvbuf), count); 
     137            reduce_max<long long>(static_cast<long long*>(comm->my_buffer->void_buffer[i]), static_cast<long long*>(recvbuf), count); 
    138138        } 
    139139 
    … …  
    148148          assert(datasize == sizeof(int)); 
    149149          for(int i=1; i<num_ep; i++) 
    150             reduce_min<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count); 
     150            reduce_min<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count); 
    151151        } 
    152152 
    … …  
    155155          assert(datasize == sizeof(float)); 
    156156          for(int i=1; i<num_ep; i++) 
    157             reduce_min<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count); 
     157            reduce_min<float>(static_cast<float*>(comm->my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count); 
    158158        } 
    159159 
    … …  
    162162          assert(datasize == sizeof(double)); 
    163163          for(int i=1; i<num_ep; i++) 
    164             reduce_min<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
     164            reduce_min<double>(static_cast<double*>(comm->my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
    165165        } 
    166166 
    … …  
    169169          assert(datasize == sizeof(char)); 
    170170          for(int i=1; i<num_ep; i++) 
    171             reduce_min<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
     171            reduce_min<char>(static_cast<char*>(comm->my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
    172172        } 
    173173 
    … …  
    176176          assert(datasize == sizeof(long)); 
    177177          for(int i=1; i<num_ep; i++) 
    178             reduce_min<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
     178            reduce_min<long>(static_cast<long*>(comm->my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
    179179        } 
    180180 
    … …  
    183183          assert(datasize == sizeof(unsigned long)); 
    184184          for(int i=1; i<num_ep; i++) 
    185             reduce_min<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count); 
     185            reduce_min<unsigned long>(static_cast<unsigned long*>(comm->my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count); 
    186186        } 
    187187 
    … …  
    190190          assert(datasize == sizeof(uint64_t)); 
    191191          for(int i=1; i<num_ep; i++) 
    192             reduce_min<uint64_t>(static_cast<uint64_t*>(comm.my_buffer->void_buffer[i]), static_cast<uint64_t*>(recvbuf), count); 
     192            reduce_min<uint64_t>(static_cast<uint64_t*>(comm->my_buffer->void_buffer[i]), static_cast<uint64_t*>(recvbuf), count); 
    193193        } 
    194194         
    … …  
    197197          assert(datasize == sizeof(long long)); 
    198198          for(int i=1; i<num_ep; i++) 
    199             reduce_min<long long>(static_cast<long long*>(comm.my_buffer->void_buffer[i]), static_cast<long long*>(recvbuf), count); 
     199            reduce_min<long long>(static_cast<long long*>(comm->my_buffer->void_buffer[i]), static_cast<long long*>(recvbuf), count); 
    200200        } 
    201201 
    … …  
    211211          assert(datasize == sizeof(int)); 
    212212          for(int i=1; i<num_ep; i++) 
    213             reduce_sum<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count); 
     213            reduce_sum<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count); 
    214214        } 
    215215 
    … …  
    218218          assert(datasize == sizeof(float)); 
    219219          for(int i=1; i<num_ep; i++) 
    220             reduce_sum<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count); 
     220            reduce_sum<float>(static_cast<float*>(comm->my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count); 
    221221        } 
    222222 
    … …  
    225225          assert(datasize == sizeof(double)); 
    226226          for(int i=1; i<num_ep; i++) 
    227             reduce_sum<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
     227            reduce_sum<double>(static_cast<double*>(comm->my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
    228228        } 
    229229 
    … …  
    232232          assert(datasize == sizeof(char)); 
    233233          for(int i=1; i<num_ep; i++) 
    234             reduce_sum<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
     234            reduce_sum<char>(static_cast<char*>(comm->my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
    235235        } 
    236236 
    … …  
    239239          assert(datasize == sizeof(long)); 
    240240          for(int i=1; i<num_ep; i++) 
    241             reduce_sum<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
     241            reduce_sum<long>(static_cast<long*>(comm->my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
    242242        } 
    243243 
    … …  
    246246          assert(datasize == sizeof(unsigned long)); 
    247247          for(int i=1; i<num_ep; i++) 
    248             reduce_sum<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count); 
     248            reduce_sum<unsigned long>(static_cast<unsigned long*>(comm->my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count); 
    249249        } 
    250250 
    … …  
    253253          assert(datasize == sizeof(uint64_t)); 
    254254          for(int i=1; i<num_ep; i++) 
    255             reduce_sum<uint64_t>(static_cast<uint64_t*>(comm.my_buffer->void_buffer[i]), static_cast<uint64_t*>(recvbuf), count); 
     255            reduce_sum<uint64_t>(static_cast<uint64_t*>(comm->my_buffer->void_buffer[i]), static_cast<uint64_t*>(recvbuf), count); 
    256256        } 
    257257         
    … …  
    260260          assert(datasize == sizeof(long long)); 
    261261          for(int i=1; i<num_ep; i++) 
    262             reduce_sum<long long>(static_cast<long long*>(comm.my_buffer->void_buffer[i]), static_cast<long long*>(recvbuf), count); 
     262            reduce_sum<long long>(static_cast<long long*>(comm->my_buffer->void_buffer[i]), static_cast<long long*>(recvbuf), count); 
    263263        } 
    264264 
    … …  
    275275          assert(datasize == sizeof(int)); 
    276276          for(int i=1; i<num_ep; i++) 
    277             reduce_lor<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count); 
     277            reduce_lor<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count); 
    278278        } 
    279279      } 
    … …  
    288288  { 
    289289 
    290     if(!comm.is_ep && comm.mpi_comm) 
    291     { 
    292       return ::MPI_Reduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root, to_mpi_comm(comm.mpi_comm)); 
    293     } 
    294  
    295  
    296  
    297     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    298     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    299     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    300     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    301     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    302     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
    303  
    304     int root_mpi_rank = comm.rank_map->at(root).second; 
    305     int root_ep_loc = comm.rank_map->at(root).first; 
     290    if(!comm->is_ep && comm->mpi_comm) 
     291    { 
     292      return ::MPI_Reduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root, to_mpi_comm(comm->mpi_comm)); 
     293    } 
     294 
     295 
     296 
     297    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     298    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     299    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     300    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     301    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     302    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
     303 
     304    int root_mpi_rank = comm->rank_map->at(root).second; 
     305    int root_ep_loc = comm->rank_map->at(root).first; 
    306306 
    307307    ::MPI_Aint datasize, lb; 
    … …  
    326326    if(is_master) 
    327327    { 
    328       ::MPI_Reduce(local_recvbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     328      ::MPI_Reduce(local_recvbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    329329       
    330330    } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_reduce_scatter.cpp

    r1381 r1500  
    1919  int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
    2020  { 
    21     if(!comm.is_ep) 
     21    if(!comm->is_ep) 
    2222    { 
    23       return ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm)); 
     23      return ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    2424    } 
    2525 
    2626 
    27     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    28     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    29     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    30     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    31     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    32     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     27    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     28    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     29    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     30    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     31    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     32    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    3333 
    3434 
    … …  
    5252    if(is_master) 
    5353    {   
    54       ::MPI_Allreduce(MPI_IN_PLACE, local_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm)); 
     54      ::MPI_Allreduce(MPI_IN_PLACE, local_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    5555    } 
    5656 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_scan.cpp

    r1381 r1500  
    5050    valid_op(op); 
    5151 
    52     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    53     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    54     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
     52    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     53    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     54    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
    5555     
    5656 
    … …  
    184184      } 
    185185 
    186       comm.my_buffer->void_buffer[0] = recvbuf; 
     186      comm->my_buffer->void_buffer[0] = recvbuf; 
    187187    } 
    188188    else 
    189189    { 
    190       comm.my_buffer->void_buffer[ep_rank_loc] = const_cast<void*>(sendbuf);   
     190      comm->my_buffer->void_buffer[ep_rank_loc] = const_cast<void*>(sendbuf);   
    191191      memcpy(recvbuf, sendbuf, datasize*count); 
    192192    }  
    … …  
    196196    MPI_Barrier_local(comm); 
    197197 
    198     memcpy(recvbuf, comm.my_buffer->void_buffer[0], datasize*count); 
     198    memcpy(recvbuf, comm->my_buffer->void_buffer[0], datasize*count); 
    199199 
    200200 
    … …  
    205205        assert (datasize == sizeof(int)); 
    206206        for(int i=1; i<ep_rank_loc+1; i++) 
    207           reduce_sum<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
     207          reduce_sum<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
    208208      } 
    209209      
    … …  
    212212        assert(datasize == sizeof(float)); 
    213213        for(int i=1; i<ep_rank_loc+1; i++) 
    214           reduce_sum<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
     214          reduce_sum<float>(static_cast<float*>(comm->my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
    215215      } 
    216216       
    … …  
    220220        assert(datasize == sizeof(double)); 
    221221        for(int i=1; i<ep_rank_loc+1; i++) 
    222           reduce_sum<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
     222          reduce_sum<double>(static_cast<double*>(comm->my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
    223223      } 
    224224 
    … …  
    227227        assert(datasize == sizeof(char)); 
    228228        for(int i=1; i<ep_rank_loc+1; i++) 
    229           reduce_sum<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
     229          reduce_sum<char>(static_cast<char*>(comm->my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
    230230      } 
    231231 
    … …  
    234234        assert(datasize == sizeof(long)); 
    235235        for(int i=1; i<ep_rank_loc+1; i++) 
    236           reduce_sum<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
     236          reduce_sum<long>(static_cast<long*>(comm->my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
    237237      } 
    238238 
    … …  
    241241        assert(datasize == sizeof(unsigned long)); 
    242242        for(int i=1; i<ep_rank_loc+1; i++) 
    243           reduce_sum<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
     243          reduce_sum<unsigned long>(static_cast<unsigned long*>(comm->my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
    244244      } 
    245245 
    … …  
    255255        assert(datasize == sizeof(int)); 
    256256        for(int i=1; i<ep_rank_loc+1; i++) 
    257           reduce_max<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
     257          reduce_max<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
    258258      } 
    259259 
    … …  
    262262        assert(datasize == sizeof(float)); 
    263263        for(int i=1; i<ep_rank_loc+1; i++) 
    264           reduce_max<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
     264          reduce_max<float>(static_cast<float*>(comm->my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
    265265      } 
    266266 
    … …  
    269269        assert(datasize == sizeof(double)); 
    270270        for(int i=1; i<ep_rank_loc+1; i++) 
    271           reduce_max<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
     271          reduce_max<double>(static_cast<double*>(comm->my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
    272272      } 
    273273 
    … …  
    276276        assert(datasize == sizeof(char)); 
    277277        for(int i=1; i<ep_rank_loc+1; i++) 
    278           reduce_max<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
     278          reduce_max<char>(static_cast<char*>(comm->my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
    279279      } 
    280280 
    … …  
    283283        assert(datasize == sizeof(long)); 
    284284        for(int i=1; i<ep_rank_loc+1; i++) 
    285           reduce_max<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
     285          reduce_max<long>(static_cast<long*>(comm->my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
    286286      } 
    287287 
    … …  
    290290        assert(datasize == sizeof(unsigned long)); 
    291291        for(int i=1; i<ep_rank_loc+1; i++) 
    292           reduce_max<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
     292          reduce_max<unsigned long>(static_cast<unsigned long*>(comm->my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
    293293      } 
    294294      
    … …  
    302302        assert(datasize == sizeof(int)); 
    303303        for(int i=1; i<ep_rank_loc+1; i++) 
    304           reduce_min<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
     304          reduce_min<int>(static_cast<int*>(comm->my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);     
    305305      } 
    306306 
    … …  
    309309        assert(datasize == sizeof(float)); 
    310310        for(int i=1; i<ep_rank_loc+1; i++) 
    311           reduce_min<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
     311          reduce_min<float>(static_cast<float*>(comm->my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);     
    312312      } 
    313313 
    … …  
    316316        assert(datasize == sizeof(double)); 
    317317        for(int i=1; i<ep_rank_loc+1; i++) 
    318           reduce_min<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
     318          reduce_min<double>(static_cast<double*>(comm->my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count); 
    319319      } 
    320320 
    … …  
    323323        assert(datasize == sizeof(char)); 
    324324        for(int i=1; i<ep_rank_loc+1; i++) 
    325           reduce_min<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
     325          reduce_min<char>(static_cast<char*>(comm->my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count); 
    326326      } 
    327327 
    … …  
    330330        assert(datasize == sizeof(long)); 
    331331        for(int i=1; i<ep_rank_loc+1; i++) 
    332           reduce_min<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
     332          reduce_min<long>(static_cast<long*>(comm->my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count); 
    333333      } 
    334334 
    … …  
    337337        assert(datasize == sizeof(unsigned long)); 
    338338        for(int i=1; i<ep_rank_loc+1; i++) 
    339           reduce_min<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
     339          reduce_min<unsigned long>(static_cast<unsigned long*>(comm->my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);     
    340340      } 
    341341 
    … …  
    350350  int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
    351351  { 
    352     if(!comm.is_ep) 
    353     { 
    354       return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm)); 
     352    if(!comm->is_ep) 
     353    { 
     354      return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    355355    } 
    356356     
    357357    valid_type(datatype); 
    358358 
    359     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    360     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    361     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    362     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    363     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    364     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     359    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     360    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     361    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     362    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     363    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     364    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    365365 
    366366    ::MPI_Aint datasize, lb; 
    … …  
    375375    std::vector<int> my_map(mpi_size, 0); 
    376376 
    377     for(int i=0; i<comm.rank_map->size(); i++) my_map[comm.rank_map->at(i).second]++; 
     377    for(int i=0; i<comm->rank_map->size(); i++) my_map[comm->rank_map->at(i).second]++; 
    378378 
    379379    for(int i=0; i<mpi_rank; i++) my_src += my_map[i]; 
    … …  
    416416 
    417417    if(ep_rank_loc == 0) 
    418       ::MPI_Exscan(MPI_IN_PLACE, tmp_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm)); 
     418      ::MPI_Exscan(MPI_IN_PLACE, tmp_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    419419 
    420420    //printf(" ID=%d : %d  %d \n", ep_rank, static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_scatter.cpp

    r1381 r1500  
    2424    ::MPI_Type_get_extent(to_mpi_type(sendtype), &lb, &datasize); 
    2525 
    26     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    27     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
     26    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     27    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
    2828 
    2929 
    3030    if(ep_rank_loc == local_root) 
    31       comm.my_buffer->void_buffer[local_root] = const_cast<void*>(sendbuf); 
     31      comm->my_buffer->void_buffer[local_root] = const_cast<void*>(sendbuf); 
    3232 
    3333    MPI_Barrier_local(comm); 
    3434 
    3535    #pragma omp critical (_scatter)       
    36     memcpy(recvbuf, comm.my_buffer->void_buffer[local_root]+datasize*ep_rank_loc*sendcount, datasize * recvcount); 
     36    memcpy(recvbuf, comm->my_buffer->void_buffer[local_root]+datasize*ep_rank_loc*sendcount, datasize * recvcount); 
    3737     
    3838 
    … …  
    4242  int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 
    4343  { 
    44     if(!comm.is_ep) 
     44    if(!comm->is_ep) 
    4545    { 
    46       return ::MPI_Scatter(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm.mpi_comm)); 
     46      return ::MPI_Scatter(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 
    4747    } 
    4848    
    4949    assert(sendcount == recvcount); 
    5050 
    51     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    52     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    53     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    54     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    55     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    56     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     51    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     52    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     53    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     54    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     55    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     56    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    5757 
    58     int root_mpi_rank = comm.rank_map->at(root).second; 
    59     int root_ep_loc = comm.rank_map->at(root).first; 
     58    int root_mpi_rank = comm->rank_map->at(root).second; 
     59    int root_ep_loc = comm->rank_map->at(root).first; 
    6060 
    6161    bool is_master = (ep_rank_loc==0 && mpi_rank != root_mpi_rank ) || ep_rank == root; 
    … …  
    8787      for(int i=0; i<ep_size; i++) 
    8888      { 
    89         recvcounts[comm.rank_map->at(i).second]++; 
     89        recvcounts[comm->rank_map->at(i).second]++; 
    9090      } 
    9191 
    … …  
    9393        displs[i] = displs[i-1] + recvcounts[i-1]; 
    9494 
    95       ::MPI_Gatherv(local_ranks.data(), num_ep, to_mpi_type(MPI_INT), ranks.data(), recvcounts.data(), displs.data(), to_mpi_type(MPI_INT), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     95      ::MPI_Gatherv(local_ranks.data(), num_ep, to_mpi_type(MPI_INT), ranks.data(), recvcounts.data(), displs.data(), to_mpi_type(MPI_INT), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    9696    } 
    9797 
    … …  
    121121    { 
    122122      int local_sendcount = num_ep * count; 
    123       ::MPI_Gather(&local_sendcount, 1, to_mpi_type(MPI_INT), recvcounts.data(), 1, to_mpi_type(MPI_INT), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     123      ::MPI_Gather(&local_sendcount, 1, to_mpi_type(MPI_INT), recvcounts.data(), 1, to_mpi_type(MPI_INT), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    124124       
    125125      if(is_root) for(int i=1; i<mpi_size; i++) displs[i] = displs[i-1] + recvcounts[i-1]; 
    126126 
    127       ::MPI_Scatterv(tmp_sendbuf, recvcounts.data(), displs.data(), to_mpi_type(sendtype), local_recvbuf, num_ep*count, to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     127      ::MPI_Scatterv(tmp_sendbuf, recvcounts.data(), displs.data(), to_mpi_type(sendtype), local_recvbuf, num_ep*count, to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    128128 
    129129      // printf("local_recvbuf = %d %d %d %d\n", static_cast<int*>(local_recvbuf)[0], static_cast<int*>(local_recvbuf)[1], static_cast<int*>(local_recvbuf)[2], static_cast<int*>(local_recvbuf)[3]); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_scatterv.cpp

    r1381 r1500  
    2525    ::MPI_Type_get_extent(to_mpi_type(sendtype), &lb, &datasize); 
    2626 
    27     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    28     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
     27    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     28    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
    2929 
    3030    assert(recvcount == sendcounts[ep_rank_loc]); 
    3131 
    3232    if(ep_rank_loc == local_root) 
    33       comm.my_buffer->void_buffer[local_root] = const_cast<void*>(sendbuf); 
     33      comm->my_buffer->void_buffer[local_root] = const_cast<void*>(sendbuf); 
    3434 
    3535    MPI_Barrier_local(comm); 
    3636 
    3737    #pragma omp critical (_scatterv)       
    38     memcpy(recvbuf, comm.my_buffer->void_buffer[local_root]+datasize*displs[ep_rank_loc], datasize * recvcount); 
     38    memcpy(recvbuf, comm->my_buffer->void_buffer[local_root]+datasize*displs[ep_rank_loc], datasize * recvcount); 
    3939     
    4040 
    … …  
    4545                   MPI_Datatype recvtype, int root, MPI_Comm comm) 
    4646  { 
    47     if(!comm.is_ep) 
     47    if(!comm->is_ep) 
    4848    { 
    49       return ::MPI_Scatterv(sendbuf, sendcounts, displs, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm.mpi_comm)); 
     49      return ::MPI_Scatterv(sendbuf, sendcounts, displs, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 
    5050    } 
    5151    
    5252    assert(sendtype == recvtype); 
    5353 
    54     int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first; 
    55     int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first; 
    56     int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first; 
    57     int ep_size = comm.ep_comm_ptr->size_rank_info[0].second; 
    58     int num_ep = comm.ep_comm_ptr->size_rank_info[1].second; 
    59     int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second; 
     54    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 
     55    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     56    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 
     57    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 
     58    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 
     59    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 
    6060 
    61     int root_mpi_rank = comm.rank_map->at(root).second; 
    62     int root_ep_loc = comm.rank_map->at(root).first; 
     61    int root_mpi_rank = comm->rank_map->at(root).second; 
     62    int root_ep_loc = comm->rank_map->at(root).first; 
    6363 
    6464    bool is_master = (ep_rank_loc==0 && mpi_rank != root_mpi_rank ) || ep_rank == root; 
    … …  
    9090      for(int i=0; i<ep_size; i++) 
    9191      { 
    92         recvcounts[comm.rank_map->at(i).second]++; 
     92        recvcounts[comm->rank_map->at(i).second]++; 
    9393      } 
    9494 
    … …  
    9696        my_displs[i] = my_displs[i-1] + recvcounts[i-1]; 
    9797 
    98       ::MPI_Gatherv(local_ranks.data(), num_ep, to_mpi_type(MPI_INT), ranks.data(), recvcounts.data(), my_displs.data(), to_mpi_type(MPI_INT), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     98      ::MPI_Gatherv(local_ranks.data(), num_ep, to_mpi_type(MPI_INT), ranks.data(), recvcounts.data(), my_displs.data(), to_mpi_type(MPI_INT), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    9999    } 
    100100 
    … …  
    124124      local_sendbuf = new void*[datasize * local_sendcount]; 
    125125  
    126       ::MPI_Gather(&local_sendcount, 1, to_mpi_type(MPI_INT), recvcounts.data(), 1, to_mpi_type(MPI_INT), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     126      ::MPI_Gather(&local_sendcount, 1, to_mpi_type(MPI_INT), recvcounts.data(), 1, to_mpi_type(MPI_INT), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    127127 
    128128      if(is_root) for(int i=1; i<mpi_size; i++) my_displs[i] = my_displs[i-1] + recvcounts[i-1]; 
    129129 
    130       ::MPI_Scatterv(tmp_sendbuf, recvcounts.data(), my_displs.data(), to_mpi_type(sendtype), local_sendbuf, num_ep*count, to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm.mpi_comm)); 
     130      ::MPI_Scatterv(tmp_sendbuf, recvcounts.data(), my_displs.data(), to_mpi_type(sendtype), local_sendbuf, num_ep*count, to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    131131 
    132132      // printf("my_displs = %d %d %d %d\n", my_displs[0], my_displs[1], my_displs[2], my_displs[3] ); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_send.cpp

    r1381 r1500  
    1717  int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
    1818  { 
    19     if(!comm.is_ep) 
    20       return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm.mpi_comm)); 
    21     if(comm.is_intercomm) 
     19    if(!comm->is_ep) 
     20      return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 
     21    if(comm->is_intercomm) 
    2222    { 
    2323      MPI_Request request; 
    … …  
    2828    else 
    2929    { 
    30       int ep_src_loc  = comm.ep_comm_ptr->size_rank_info[1].first; 
    31       int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first; 
     30      int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     31      int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first; 
    3232      int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    33       int mpi_dest    = comm.ep_comm_ptr->comm_list->rank_map->at(dest).second; 
    34  
    35       ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm)); 
    36       //printf("call mpi_send for intracomm, dest = %d, tag = %d\n", dest, tag); 
     33      int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second; 
     34 
     35      ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm)); 
    3736    } 
    3837    //check_sum_send(buf, count, datatype, dest, tag, comm); 
    … …  
    4443  int MPI_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
    4544  { 
    46     if(!comm.is_ep) 
    47     { 
    48       return ::MPI_Ssend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm.mpi_comm)); 
     45    if(!comm->is_ep) 
     46    { 
     47      return ::MPI_Ssend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 
    4948    } 
    5049 
    … …  
    6968     
    7069 
    71     if(!comm.is_ep) 
    72     { 
    73       ::MPI_Request mpi_request; 
    74       ::MPI_Isend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    75  
    76       request->mpi_request = new ::MPI_Request(mpi_request); 
    77  
    78       request->ep_src = src_rank; 
    79       request->ep_tag = tag; 
    80       request->ep_datatype = datatype; 
    81       request->type = 1; 
    82       request->comm = comm; 
     70    if(!comm->is_ep) 
     71    { 
     72      ::MPI_Request mpi_request; 
     73      ::MPI_Isend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
     74 
     75      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     76 
     77      (*request)->ep_src = src_rank; 
     78      (*request)->ep_tag = tag; 
     79      (*request)->ep_datatype = datatype; 
     80      (*request)->type = 1; 
     81      (*request)->comm = comm; 
    8382 
    8483      return 0; 
    8584    } 
    8685 
    87     if(comm.is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request); 
     86    if(comm->is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request); 
    8887 
    8988    // EP intracomm 
    9089 
    9190    //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    92  
    93     int ep_src_loc  = comm.ep_comm_ptr->size_rank_info[1].first; 
    94     int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first; 
     91     
     92    *request = new ep_request; 
     93    printf("new %p : in ep_lib::MPI_Isend, *request = new ep_request\n", (*request)); 
     94 
     95    (*request)->mpi_request = new ::MPI_Request; 
     96    printf("new %p : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request\n", (*request)->mpi_request); 
     97     
     98 
     99    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     100    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first; 
    95101    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    96     int mpi_dest    = comm.ep_comm_ptr->comm_list->rank_map->at(dest).second; 
    97  
    98     request->ep_src  = src_rank; 
    99     request->ep_tag  = tag; 
    100     request->ep_datatype = datatype; 
    101  
    102     ::MPI_Request mpi_request; 
    103  
    104     ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    105  
    106     request->mpi_request = new ::MPI_Request(mpi_request); 
    107     request->type = 1;    // used in wait 
    108     request->comm = comm; 
    109     request->buf = const_cast<void*>(buf); 
    110  
    111     //Message_Check(comm); 
     102    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second; 
     103 
     104    (*request)->ep_src  = src_rank; 
     105    (*request)->ep_tag  = tag; 
     106    (*request)->ep_datatype = datatype; 
     107 
     108    ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
     109 
     110    (*request)->type = 1;    // used in wait 
     111    (*request)->comm = comm; 
     112    (*request)->buf = const_cast<void*>(buf); 
     113 
    112114 
    113115    return 0; 
    … …  
    126128     
    127129 
    128     if(!comm.is_ep) 
    129     { 
    130       ::MPI_Request mpi_request; 
    131       ::MPI_Issend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    132  
    133       request->mpi_request = new ::MPI_Request(mpi_request); 
    134       request->ep_src = src_rank; 
    135       request->ep_tag = tag; 
    136       request->ep_datatype = datatype; 
    137       request->type = 1; 
    138       request->comm = comm; 
     130    if(!comm->is_ep) 
     131    { 
     132      ::MPI_Request mpi_request; 
     133      ::MPI_Issend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
     134 
     135      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     136      (*request)->ep_src = src_rank; 
     137      (*request)->ep_tag = tag; 
     138      (*request)->ep_datatype = datatype; 
     139      (*request)->type = 1; 
     140      (*request)->comm = comm; 
    139141 
    140142      return 0; 
    141143    } 
    142144 
    143     if(comm.is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request); 
     145    if(comm->is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request); 
    144146 
    145147    // EP intracomm 
    … …  
    147149    //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    148150 
    149     int ep_src_loc  = comm.ep_comm_ptr->size_rank_info[1].first; 
    150     int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first; 
     151    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first; 
     152    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first; 
    151153    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    152     int mpi_dest    = comm.ep_comm_ptr->comm_list->rank_map->at(dest).second; 
    153      
    154     request->ep_src = src_rank; 
    155     request->ep_tag = tag; 
    156     request->ep_datatype = datatype; 
     154    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second; 
     155     
     156    (*request)->ep_src = src_rank; 
     157    (*request)->ep_tag = tag; 
     158    (*request)->ep_datatype = datatype; 
    157159 
    158160    ::MPI_Request mpi_request; 
    159161 
    160     ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    161  
    162     request->mpi_request = new ::MPI_Request(mpi_request); 
    163     request->type = 1;    // used in wait 
    164     request->comm = comm; 
    165     request->buf = NULL; 
     162    ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
     163 
     164    (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     165    (*request)->type = 1;    // used in wait 
     166    (*request)->comm = comm; 
     167    (*request)->buf = NULL; 
    166168     
    167169 
    … …  
    180182    //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    181183 
    182     int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
    183     int dest_remote_comm_label = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; 
    184  
    185     int src_ep_rank    = comm.ep_comm_ptr->intercomm->size_rank_info[0].first; 
     184    int dest_remote_ep_rank    = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
     185    int dest_remote_comm_label = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; 
     186 
     187    int src_ep_rank    = comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 
    186188    int src_comm_label; 
    187189 
    188     src_comm_label = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second; 
     190    src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second; 
    189191 
    190192 
    … …  
    196198    if(dest_remote_comm_label == src_comm_label)       // mpi_dest differs 
    197199    { 
    198       int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    199       int ep_src_loc = comm.rank_map->at(inter_src).first; 
    200       int ep_dest_loc = comm.rank_map->at(dest_remote_ep_rank).first; 
    201       int mpi_dest    = comm.rank_map->at(dest_remote_ep_rank).second; 
     200      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
     201      int ep_src_loc = comm->rank_map->at(inter_src).first; 
     202      int ep_dest_loc = comm->rank_map->at(dest_remote_ep_rank).first; 
     203      int mpi_dest    = comm->rank_map->at(dest_remote_ep_rank).second; 
    202204      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    203205 
    204206      ::MPI_Request mpi_request; 
    205207   
    206       ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm), &mpi_request); 
    207  
    208       request->mpi_request = new ::MPI_Request(mpi_request); 
    209       request->type = 1;    // used in wait 
    210       request->comm = comm; 
    211  
    212       request->ep_src = src_ep_rank; 
    213       request->ep_tag = tag; 
    214       request->ep_datatype = datatype; 
     208      ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request); 
     209 
     210      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     211      (*request)->type = 1;    // used in wait 
     212      (*request)->comm = comm; 
     213 
     214      (*request)->ep_src = src_ep_rank; 
     215      (*request)->ep_tag = tag; 
     216      (*request)->ep_datatype = datatype; 
    215217    } 
    216218 
    217219    else   // dest_remote_comm_label != src_comm_label 
    218220    {  
    219       int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    220       int ep_src_loc = comm.rank_map->at(inter_src).first; 
    221       int ep_dest_loc = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first; 
    222       int mpi_dest    = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second; 
     221      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
     222      int ep_src_loc = comm->rank_map->at(inter_src).first; 
     223      int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first; 
     224      int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second; 
    223225      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    224226 
    225227      ::MPI_Request mpi_request; 
    226228 
    227       ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request); 
    228  
    229       request->mpi_request = new ::MPI_Request(mpi_request); 
    230       request->type = 1;    // used in wait 
    231       request->comm = comm; 
     229      ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request); 
     230 
     231      (*request)->mpi_request = new ::MPI_Request(mpi_request); 
     232      (*request)->type = 1;    // used in wait 
     233      (*request)->comm = comm; 
    232234    
    233       request->ep_src = src_ep_rank; 
    234       request->ep_tag = tag; 
    235       request->ep_datatype = datatype; 
     235      (*request)->ep_src = src_ep_rank; 
     236      (*request)->ep_tag = tag; 
     237      (*request)->ep_datatype = datatype; 
    236238    } 
    237239 
    … …  
    247249    //check_sum_send(buf, count, datatype, dest, tag, comm, 1); 
    248250 
    249     int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
    250     int dest_remote_comm_label = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; 
    251  
    252     int src_ep_rank    = comm.ep_comm_ptr->intercomm->size_rank_info[0].first; 
     251    int dest_remote_ep_rank    = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; 
     252    int dest_remote_comm_label = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; 
     253 
     254    int src_ep_rank    = comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 
    253255    int src_comm_label; 
    254256 
    255     for(int i=0; i<comm.ep_comm_ptr->intercomm->local_rank_map->size(); i++) 
    256     { 
    257       if(comm.ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank) 
     257    for(int i=0; i<comm->ep_comm_ptr->intercomm->local_rank_map->size(); i++) 
     258    { 
     259      if(comm->ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank) 
    258260      { 
    259         src_comm_label = comm.ep_comm_ptr->intercomm->local_rank_map->at(i).second; 
     261        src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(i).second; 
    260262        break; 
    261263      } 
    … …  
    267269    if(dest_remote_comm_label == src_comm_label)       // dest rank (loc, mpi) differs 
    268270    { 
    269       int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
    270       int ep_src_loc = comm.rank_map->at(inter_src).first; 
    271       int ep_dest_loc = comm.rank_map->at(dest_remote_ep_rank).first; 
    272       int mpi_dest    = comm.rank_map->at(dest_remote_ep_rank).second; 
     271      int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; 
     272      int ep_src_loc = comm->rank_map->at(inter_src).first; 
     273      int ep_dest_loc = comm->rank_map->at(dest_remote_ep_rank).first; 
     274      int mpi_dest    = comm->rank_map->at(dest_remote_ep_rank).second; 
    273275      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    274276