Changeset 1520 for XIOS/dev/branch_openmp/extern/src_ep_dev/ep_allgather.cpp

Timestamp:

06/04/18 19:25:08 (6 years ago)

Author:

yushan

Message:

save dev. TO DO : test with xios

File:

• XIOS/dev/branch_openmp/extern/src_ep_dev/ep_allgather.cpp (modified) (5 diffs)

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_allgather.cpp

@@ -17 +17 @@
 {
 
+  int MPI_Allgather_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    assert(valid_type(datatype));
+
+    ::MPI_Aint datasize, lb;
+    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
+
+    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;
+    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second;
+
+    #pragma omp critical (write_buffer)
+    comm->my_buffer->void_buffer[ep_rank_loc] = const_cast< void* >(sendbuf);
+
+    MPI_Barrier_local(comm);
+
+    #pragma omp critical (read_buffer)
+    {
+      for(int i=0; i<num_ep; i++)
+        memcpy(recvbuf + datasize * i * count, comm->my_buffer->void_buffer[i], datasize * count);
+    }
+    MPI_Barrier_local(comm);
+  }
+
   int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)
   {
 
-    if(!comm.is_ep && comm.mpi_comm)
+    if(!comm->is_ep && comm->mpi_comm)
     {
-      return ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm.mpi_comm));
+      return ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm));
     }
 
@@ -37 +60 @@
 
 
-    int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
-    int ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first;
+    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;
+    int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first;
+    int ep_size = comm->ep_comm_ptr->size_rank_info[0].second;
+    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second;
+    int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second;
 
     bool is_master = ep_rank_loc==0;
@@ -70 +93 @@
       int local_sendcount = num_ep * count;
 
-      ::MPI_Allgather(&local_sendcount, 1, to_mpi_type(MPI_INT), mpi_recvcounts, 1, to_mpi_type(MPI_INT), to_mpi_comm(comm.mpi_comm));
+      ::MPI_Allgather(&local_sendcount, 1, to_mpi_type(MPI_INT), mpi_recvcounts, 1, to_mpi_type(MPI_INT), to_mpi_comm(comm->mpi_comm));
 
       mpi_displs[0] = 0;
@@ -79 +102 @@
 
     
-      ::MPI_Allgatherv(local_recvbuf, num_ep * count, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts, mpi_displs, to_mpi_type(datatype), to_mpi_comm(comm.mpi_comm));
+      ::MPI_Allgatherv(local_recvbuf, num_ep * count, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts, mpi_displs, to_mpi_type(datatype), to_mpi_comm(comm->mpi_comm));
 
 
@@ -86 +109 @@
       for(int i=0; i<ep_size; i++)
       {
-        offset = mpi_displs[comm.rank_map->at(i).second] + comm.rank_map->at(i).first * sendcount; 
+        offset = mpi_displs[comm->ep_rank_map->at(i).second] + comm->ep_rank_map->at(i).first * sendcount; 
         memcpy(recvbuf + i*sendcount*datasize, tmp_recvbuf+offset*datasize, sendcount*datasize);
       }