Changeset 1289

Timestamp:

10/04/17 17:02:13 (6 years ago)

Author:

yushan

Message:

EP update part 2

Location:

XIOS/dev/branch_openmp/extern

Files:

• remap/src/mpi_routing.cpp (modified) (1 diff)
• remap/src/polyg.cpp (modified) (1 diff)
• src_ep_dev/ep_allgatherv.cpp (modified) (3 diffs)
• src_ep_dev/ep_alltoall.cpp (modified) (1 diff)
• src_ep_dev/ep_bcast.cpp (modified) (2 diffs)
• src_ep_dev/ep_create.cpp (modified) (3 diffs)
• src_ep_dev/ep_exscan.cpp (modified) (2 diffs)
• src_ep_dev/ep_free.cpp (modified) (3 diffs)
• src_ep_dev/ep_gather.cpp (modified) (2 diffs)
• src_ep_dev/ep_gatherv.cpp (modified) (2 diffs)
• src_ep_dev/ep_lib.cpp (modified) (1 diff)
• src_ep_dev/ep_lib_collective.hpp (modified) (3 diffs)
• src_ep_dev/ep_lib_fortran.hpp (modified) (1 diff)
• src_ep_dev/ep_lib_local.hpp (modified) (1 diff)
• src_ep_dev/ep_merge.cpp (modified) (6 diffs)
• src_ep_dev/ep_message.cpp (modified) (13 diffs)
• src_ep_dev/ep_reduce.cpp (modified) (3 diffs)
• src_ep_dev/ep_scan.cpp (modified) (2 diffs)
• src_ep_dev/ep_scatter.cpp (modified) (4 diffs)
• src_ep_dev/ep_scatterv.cpp (modified) (3 diffs)
• src_ep_dev/ep_send.cpp (modified) (8 diffs)
• src_ep_dev/ep_split.cpp (modified) (4 diffs)
• src_ep_dev/ep_test.cpp (modified) (3 diffs)
• src_ep_dev/ep_type.hpp (modified) (1 diff)
• src_ep_dev/ep_wait.cpp (modified) (2 diffs)

XIOS/dev/branch_openmp/extern/remap/src/mpi_routing.cpp

@@ -127 +127 @@
         MPI_Info info_null;
 
-        MPI_Alloc_mem(nbTarget *sizeof(int), info_null, &targetRank);
-        MPI_Alloc_mem(nbSource *sizeof(int), info_null, &sourceRank);
+        // MPI_Alloc_mem(nbTarget *sizeof(int), info_null, &targetRank);
+        // MPI_Alloc_mem(nbSource *sizeof(int), info_null, &sourceRank);
+        MPI_Alloc_mem(nbTarget *sizeof(int), MPI_INFO_NULL, &targetRank);
+        MPI_Alloc_mem(nbSource *sizeof(int), MPI_INFO_NULL, &sourceRank);
 
         targetRankToIndex = new int[mpiSize];

XIOS/dev/branch_openmp/extern/remap/src/polyg.cpp

@@ -180 +180 @@
                 assert(sc >= -1e-10); // Error: tri a l'env (wrong orientation)
                 double area_gc = triarea(t[0], t[1], t[2]);
-                if(area_gc<=0) printf("area_gc = %e\n", area_gc);
+                //if(area_gc<=0) printf("area_gc = %e\n", area_gc);
                 double area_sc_gc_moon = 0;
                 if (d[i]) /* handle small circle case */

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_allgatherv.cpp

@@ -62 +62 @@
 
     MPI_Gather_local(&sendcount, 1, MPI_INT, local_recvcounts.data(), 0, comm);
-    for(int i=1; i<num_ep; i++) local_displs[i] = local_displs[i-1] + local_recvcounts[i]; 
+    for(int i=1; i<num_ep; i++) local_displs[i] = local_displs[i-1] + local_recvcounts[i-1]; 
 
 
     if(is_master)
     {
-      local_recvbuf = new void*[datasize * num_ep * count];
+      local_recvbuf = new void*[datasize * std::accumulate(local_recvcounts.begin(), local_recvcounts.end(), 0)];
       tmp_recvbuf = new void*[datasize * std::accumulate(recvcounts, recvcounts+ep_size, 0)];
     }
@@ -81 +81 @@
       std::vector<int>mpi_displs(mpi_size, 0);
 
-      int local_sendcount = num_ep * count;
+      int local_sendcount = std::accumulate(local_recvcounts.begin(), local_recvcounts.end(), 0);
       MPI_Allgather(&local_sendcount, 1, MPI_INT, mpi_recvcounts.data(), 1, MPI_INT, to_mpi_comm(comm.mpi_comm));
 
@@ -99 +99 @@
 
 
-      ::MPI_Allgatherv(local_recvbuf, num_ep * count, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts.data(), mpi_displs.data(), to_mpi_type(datatype), to_mpi_comm(comm.mpi_comm));
+      ::MPI_Allgatherv(local_recvbuf, local_sendcount, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts.data(), mpi_displs.data(), to_mpi_type(datatype), to_mpi_comm(comm.mpi_comm));
 
       // if(ep_rank == 0)

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_alltoall.cpp

@@ -1 +1 @@
 #include "ep_lib.hpp"
 #include <mpi.h>
-#include "ep_mpi.hpp"
-
+//#include "ep_declaration.hpp"
 
 namespace ep_lib
 {
 
+
+
   int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)
   {
-    if(!comm.is_ep)
-    {
-      return ::MPI_Alltoall(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm.mpi_comm));
-    }
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype));
+    ::MPI_Aint typesize, llb;
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(sendtype), &llb, &typesize);
+    
+    int ep_size;
+    MPI_Comm_size(comm, &ep_size);
+    
 
-
-    assert(valid_type(sendtype) && valid_type(recvtype));
-    assert(sendcount == recvcount);
-
-    ::MPI_Aint datasize, llb;
-    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(sendtype), &llb, &datasize);
-
-    int count = sendcount;
-    
-    int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
-    int ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
-
-    void* tmp_recvbuf;
-    if(ep_rank == 0) tmp_recvbuf = new void*[count * ep_size * ep_size * datasize];
-
-    MPI_Gather(sendbuf, count*ep_size, sendtype, tmp_recvbuf, count*ep_size, recvtype, 0, comm);
-
-    
-    
-    // reorder tmp_buf
-    void* tmp_sendbuf;
-    if(ep_rank == 0) tmp_sendbuf = new void*[count * ep_size * ep_size * datasize];
-
-    if(ep_rank == 0)
     for(int i=0; i<ep_size; i++)
     {
-      for(int j=0; j<ep_size; j++)
-      {
-        //printf("tmp_recv[%d] = tmp_send[%d]\n", i*ep_size*count + j*count, j*ep_size*count + i*count);
-
-        memcpy(tmp_sendbuf + j*ep_size*count*datasize + i*count*datasize, tmp_recvbuf + i*ep_size*count*datasize + j*count*datasize, count*datasize);
-      }
+      ep_lib::MPI_Gather(sendbuf+i*sendcount*typesize, sendcount, sendtype, recvbuf, recvcount, recvtype, i, comm);
     }
-
-    MPI_Scatter(tmp_sendbuf, ep_size*count, sendtype, recvbuf, ep_size*recvcount, recvtype, 0, comm);
-
-    if(ep_rank == 0)
-    {
-      delete[] tmp_recvbuf;
-      delete[] tmp_sendbuf;
-    }
+    
 
     return 0;

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_bcast.cpp

@@ -11 +11 @@
 #include "ep_mpi.hpp"
 
+
 using namespace std;
+
 
 namespace ep_lib
 {
+
+  int MPI_Bcast_local(void *buffer, int count, MPI_Datatype datatype, int local_root, MPI_Comm comm)
+  {
+    assert(valid_type(datatype));
+
+    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+
+    ::MPI_Aint datasize, lb;
+    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
+    
+
+    if(ep_rank_loc == local_root)
+    {
+      //comm.ep_comm_ptr->comm_list->collective_buffer[local_root] = buffer;
+      comm.my_buffer->void_buffer[local_root] = buffer;
+    }
+
+//    #pragma omp flush
+    MPI_Barrier_local(comm);
+//    #pragma omp flush
+
+    if(ep_rank_loc != local_root)
+    {
+      #pragma omp critical (_bcast)      
+      memcpy(buffer, comm.my_buffer->void_buffer[local_root], datasize * count);
+      //memcpy(buffer, comm.ep_comm_ptr->comm_list->collective_buffer[local_root], datasize * count);
+    }
+
+    MPI_Barrier_local(comm);
+  }
 
   int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm)
@@ -49 +81 @@
 
 
-  int MPI_Bcast_local(void *buffer, int count, MPI_Datatype datatype, int local_root, MPI_Comm comm)
-  {
-    assert(valid_type(datatype));
-
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-
-    ::MPI_Aint datasize, lb;
-    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
-    
-
-    if(ep_rank_loc == local_root)
-    {
-      comm.my_buffer->void_buffer[local_root] = buffer;
-    }
-
-//    #pragma omp flush
-    MPI_Barrier_local(comm);
-//    #pragma omp flush
-
-    if(ep_rank_loc != local_root)
-    {
-      #pragma omp critical (_bcast)      
-      memcpy(buffer, comm.my_buffer->void_buffer[local_root], datasize * count);
-    }
-
-    MPI_Barrier_local(comm);
-  }
+
+
+
+  int MPI_Bcast_local2(void *buffer, int count, MPI_Datatype datatype, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Bcast_local_int(buffer, count, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Bcast_local_float(buffer, count, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Bcast_local_double(buffer, count, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Bcast_local_char(buffer, count, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Bcast_local_long(buffer, count, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Bcast_local_char(buffer, count, comm);
+    }
+    else
+    {
+      printf("MPI_Bcast Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Bcast_local_int(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *tmp = static_cast<int*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_float(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *tmp = static_cast<float*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_double(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *tmp = static_cast<double*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Bcast_local_char(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *tmp = static_cast<char*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_long(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *tmp = static_cast<long*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_ulong(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *tmp = static_cast<unsigned long*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Bcast2(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Bcast(buffer, count, static_cast< ::MPI_Datatype>(datatype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_rank_loc = comm.rank_map->at(root).first;
+
+
+    // if root is not master thread, send first to master
+    if(root_ep_rank_loc != 0 && mpi_rank == root_mpi_rank)
+    {
+      innode_memcpy(root_ep_rank_loc, buffer, 0, buffer, count, datatype, comm);
+    }
+
+
+    if(ep_rank_loc==0)
+    {
+      ::MPI_Bcast(buffer, count, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    MPI_Bcast_local2(buffer, count, datatype, comm);
+
+    return 0;
+  }
+
 
 }

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_create.cpp

@@ -63 +63 @@
 
     out_comm_hdls[0].my_buffer = new BUFFER;
-    // out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
 
     out_comm_hdls[0].rank_map = new RANK_MAP;
@@ -145 +145 @@
 
     out_comm_hdls[0].my_buffer = new BUFFER;
-    // out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
 
     out_comm_hdls[0].rank_map = new RANK_MAP;
@@ -233 +233 @@
 
     out_comm_hdls[0].my_buffer = new BUFFER;
-    // out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
-    // out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
 
     out_comm_hdls[0].rank_map = new RANK_MAP;

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_exscan.cpp

@@ -9 +9 @@
 #include <mpi.h>
 #include "ep_declaration.hpp"
-#include "ep_mpi.hpp"
 
 using namespace std;
@@ -27 +26 @@
   }
 
-  template<typename T>
-  void reduce_max(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, max_op<T>);
-  }
-
-  template<typename T>
-  void reduce_min(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, min_op<T>);
-  }
-
-  template<typename T>
-  void reduce_sum(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, std::plus<T>());
-  }
-
-
-  int MPI_Exscan_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-  {
-    valid_op(op);
-
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+  int MPI_Exscan_local2(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Exscan_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Exscan_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Exscan_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Exscan_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Exscan_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Exscan_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Exscan Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+
+
+
+  int MPI_Exscan_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+
+  }
+
+  int MPI_Exscan_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Exscan(const_cast<void*>(sendbuf), recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
+                   static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    ::MPI_Aint datasize, lb;
     
-
-    ::MPI_Aint datasize, lb;
-    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
-
-    if(ep_rank_loc == 0 && mpi_rank != 0)
-    {
-      comm.my_buffer->void_buffer[0] = recvbuf;
-    }
-    if(ep_rank_loc == 0 && mpi_rank == 0)
-    {
-      comm.my_buffer->void_buffer[0] = const_cast<void*>(sendbuf);  
-    } 
-      
-
-    MPI_Barrier_local(comm);
-
-    memcpy(recvbuf, comm.my_buffer->void_buffer[0], datasize*count);
-
-    MPI_Barrier_local(comm);
-
-    comm.my_buffer->void_buffer[ep_rank_loc] = const_cast<void*>(sendbuf);  
-    
-    MPI_Barrier_local(comm);
-
-    if(op == MPI_SUM)
-    {
-      if(datatype == MPI_INT && datasize == sizeof(int))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);    
-      }
-     
-      else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);    
-      }
-      
-
-      else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-      }
-
-      else if(datatype == MPI_CHAR && datasize == sizeof(char))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-      }
-
-      else if(datatype == MPI_LONG && datasize == sizeof(long))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-      }
-
-      else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);    
-      }
-
-      else printf("datatype Error\n");
-
-      
-    }
-
-    else if(op == MPI_MAX)
-    {
-      if(datatype == MPI_INT && datasize == sizeof(int))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);    
-
-      else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);    
-
-      else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-
-      else if(datatype == MPI_CHAR && datasize == sizeof(char))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-
-      else if(datatype == MPI_LONG && datasize == sizeof(long))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-
-      else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);    
-     
-      else printf("datatype Error\n");
-    }
-
-    else //if(op == MPI_MIN)
-    {
-      if(datatype == MPI_INT && datasize == sizeof(int))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);    
-
-      else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);    
-
-      else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-
-      else if(datatype == MPI_CHAR && datasize == sizeof(char))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-
-      else if(datatype == MPI_LONG && datasize == sizeof(long))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-
-      else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);    
-
-      else printf("datatype Error\n");
-    }
-
-    MPI_Barrier_local(comm);
-
-  }
-
-  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-  {
-    if(!comm.is_ep)
-    {
-      return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm));
-    }
-    
-    valid_type(datatype);
-
-    int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
-    int ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
-
-    ::MPI_Aint datasize, lb;
-    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
-    
-    void* tmp_sendbuf;
-    tmp_sendbuf = new void*[datasize * count];
-
-    int my_src = 0;
-    int my_dst = ep_rank;
-
-    std::vector<int> my_map(mpi_size, 0);
-
-    for(int i=0; i<comm.rank_map->size(); i++) my_map[comm.rank_map->at(i).second]++;
-
-    for(int i=0; i<mpi_rank; i++) my_src += my_map[i];
-    my_src += ep_rank_loc;
-
-     
-    for(int i=0; i<mpi_size; i++)
-    {
-      if(my_dst < my_map[i])
-      {
-        my_dst = get_ep_rank(comm, my_dst, i); 
-        break;
-      }
-      else
-        my_dst -= my_map[i];
-    }
-
-    if(ep_rank != my_dst) 
-    {
-      MPI_Request request[2];
-      MPI_Status status[2];
-
-      MPI_Isend(sendbuf,     count, datatype, my_dst, my_dst,  comm, &request[0]);
-    
-      MPI_Irecv(tmp_sendbuf, count, datatype, my_src, ep_rank, comm, &request[1]);
-    
-      MPI_Waitall(2, request, status);
-    }
-
-    else memcpy(tmp_sendbuf, sendbuf, datasize*count);
-    
-
-    void* tmp_recvbuf;
-    tmp_recvbuf = new void*[datasize * count];    
-
-    MPI_Reduce_local(tmp_sendbuf, tmp_recvbuf, count, datatype, op, 0, comm);
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void* local_scan_recvbuf;
+    local_scan_recvbuf = new void*[datasize * count];
+
+
+    // local scan
+    MPI_Exscan_local2(sendbuf, recvbuf, count, datatype, op, comm);
+
+//     MPI_scan
+    void* local_sum;
+    void* mpi_scan_recvbuf;
+
+
+    mpi_scan_recvbuf = new void*[datasize*count];
 
     if(ep_rank_loc == 0)
-      ::MPI_Exscan(MPI_IN_PLACE, tmp_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm));
-
-    // printf(" ID=%d : %d  %d \n", ep_rank, static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]);
-    
-    MPI_Exscan_local(tmp_sendbuf, tmp_recvbuf, count, datatype, op, comm);
-
-     // printf(" ID=%d : after local tmp_sendbuf = %d %d ; tmp_recvbuf = %d  %d \n", ep_rank, static_cast<int*>(tmp_sendbuf)[0], static_cast<int*>(tmp_sendbuf)[1], static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]);
-
-
-
-    if(ep_rank != my_src) 
-    {
-      MPI_Request request[2];
-      MPI_Status status[2];
-
-      MPI_Isend(tmp_recvbuf, count, datatype, my_src, my_src,  comm, &request[0]);
-    
-      MPI_Irecv(recvbuf,     count, datatype, my_dst, ep_rank, comm, &request[1]);
-    
-      MPI_Waitall(2, request, status);
-    }
-
-    else memcpy(recvbuf, tmp_recvbuf, datasize*count);
-    
-
-
-
-    delete[] tmp_sendbuf;
-    delete[] tmp_recvbuf;
-
-  }
+    {
+      local_sum = new void*[datasize*count];
+    }
+
+
+    MPI_Reduce_local2(sendbuf, local_sum, count, datatype, op, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      ::MPI_Exscan(local_sum, mpi_scan_recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    if(mpi_rank > 0)
+    {
+      MPI_Bcast_local2(mpi_scan_recvbuf, count, datatype, comm);
+    }
+
+
+    if(datatype == MPI_DOUBLE)
+    {
+      double* sum_buf = static_cast<double*>(mpi_scan_recvbuf);
+      double* recv_buf = static_cast<double*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<double*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<double*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_FLOAT)
+    {
+      float* sum_buf = static_cast<float*>(mpi_scan_recvbuf);
+      float* recv_buf = static_cast<float*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<float*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<float*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_INT)
+    {
+      int* sum_buf = static_cast<int*>(mpi_scan_recvbuf);
+      int* recv_buf = static_cast<int*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<int*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<int*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_CHAR)
+    {
+      char* sum_buf = static_cast<char*>(mpi_scan_recvbuf);
+      char* recv_buf = static_cast<char*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<char*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<char*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_LONG)
+    {
+      long* sum_buf = static_cast<long*>(mpi_scan_recvbuf);
+      long* recv_buf = static_cast<long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* sum_buf = static_cast<unsigned long*>(mpi_scan_recvbuf);
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<unsigned long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<unsigned long*>(local_sum);
+      }
+    }
+
+
+  }
+
+
 
 }

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_free.cpp

@@ -5 +5 @@
 namespace ep_lib
 {
+
+
 
   int MPI_Comm_free(MPI_Comm *comm)
@@ -39 +41 @@
         Debug("comm is EP, mpi_comm_ptr != NULL\n");
 
+        if(comm->my_buffer != NULL)
+        {
+          if(comm->my_buffer->buf_int != NULL) delete[] comm->my_buffer->buf_int; Debug("buf_int freed\n");
+          if(comm->my_buffer->buf_float != NULL) delete[] comm->my_buffer->buf_float; Debug("buf_float freed\n");
+          if(comm->my_buffer->buf_double != NULL) delete[] comm->my_buffer->buf_double; Debug("buf_double freed\n");
+          if(comm->my_buffer->buf_long != NULL) delete[] comm->my_buffer->buf_long; Debug("buf_long freed\n");
+          if(comm->my_buffer->buf_ulong != NULL) delete[] comm->my_buffer->buf_ulong; Debug("buf_ulong freed\n");
+          if(comm->my_buffer->buf_char != NULL) delete[] comm->my_buffer->buf_char; Debug("buf_char freed\n");
+        }
 
         if(comm->ep_barrier != NULL)
@@ -97 +108 @@
       Debug("comm is EP, mpi_comm_ptr != NULL\n");
 
+      if(comm->my_buffer != NULL)
+      {
+        if(comm->my_buffer->buf_int != NULL) delete[] comm->my_buffer->buf_int; Debug("buf_int freed\n");
+        if(comm->my_buffer->buf_float != NULL) delete[] comm->my_buffer->buf_float; Debug("buf_float freed\n");
+        if(comm->my_buffer->buf_double != NULL) delete[] comm->my_buffer->buf_double; Debug("buf_double freed\n");
+        if(comm->my_buffer->buf_long != NULL) delete[] comm->my_buffer->buf_long; Debug("buf_long freed\n");
+        if(comm->my_buffer->buf_ulong != NULL) delete[] comm->my_buffer->buf_ulong; Debug("buf_ulong freed\n");
+        if(comm->my_buffer->buf_char != NULL) delete[] comm->my_buffer->buf_char; Debug("buf_char freed\n");
+      }
 
       if(comm->ep_barrier != NULL)

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_gather.cpp

@@ -15 +15 @@
 namespace ep_lib
 {
-
   int MPI_Gather_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, int local_root, MPI_Comm comm)
   {
@@ -128 +127 @@
   }
 
+  // int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)
+  // {
+
+  //   if(!comm.is_ep && comm.mpi_comm)
+  //   {
+  //     ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+  //                     static_cast< ::MPI_Comm>(comm.mpi_comm));
+  //     return 0;
+  //   }
+
+  //   if(!comm.mpi_comm) return 0;
+
+  //   assert(sendcount == recvcount);
+
+  //   assert(valid_type(sendtype) && valid_type(recvtype));
+
+  //   MPI_Datatype datatype = sendtype;
+  //   int count = sendcount;
+
+  //   ::MPI_Aint datasize, lb;
+
+  //   ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
+
+
+  //   int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+  //   int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+  //   int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+  //   int ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+  //   int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+  //   int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+  //   bool is_master = ep_rank_loc==0;
+
+  //   void* local_recvbuf;
+  //   void* tmp_recvbuf;
+
+
+  //   if(is_master)
+  //   {
+  //     local_recvbuf = new void*[datasize * num_ep * count];
+  //     tmp_recvbuf = new void*[datasize * count * ep_size];
+  //   }
+
+  //   MPI_Gather_local(sendbuf, count, datatype, local_recvbuf, 0, comm);
+
+
+  //   int* mpi_recvcounts;
+  //   int *mpi_displs;
+    
+  //   if(is_master)
+  //   {
+      
+  //     mpi_recvcounts = new int[mpi_size];
+  //     mpi_displs = new int[mpi_size];
+
+  //     int local_sendcount = num_ep * count;
+
+  //     ::MPI_Allgather(&local_sendcount, 1, to_mpi_type(MPI_INT), mpi_recvcounts, 1, to_mpi_type(MPI_INT), to_mpi_comm(comm.mpi_comm));
+
+  //     mpi_displs[0] = 0;
+  //     for(int i=1; i<mpi_size; i++)
+  //     {
+  //       mpi_displs[i] = mpi_displs[i-1] + mpi_recvcounts[i-1];
+  //     }
+
+    
+  //     ::MPI_Allgatherv(local_recvbuf, num_ep * count, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts, mpi_displs, to_mpi_type(datatype), to_mpi_comm(comm.mpi_comm));
+
+
+  //     // reorder 
+  //     int offset;
+  //     for(int i=0; i<ep_size; i++)
+  //     {
+  //       offset = mpi_displs[comm.rank_map->at(i).second] + comm.rank_map->at(i).first * sendcount; 
+  //       memcpy(recvbuf + i*sendcount*datasize, tmp_recvbuf+offset*datasize, sendcount*datasize);
+  //     }
+
+  //     delete[] mpi_recvcounts;
+  //     delete[] mpi_displs;
+  //   }
+
+  //   MPI_Bcast_local(recvbuf, count*ep_size, datatype, 0, comm);
+
+  //   MPI_Barrier(comm);
+
+
+  //   if(is_master)
+  //   {
+  //     delete[] local_recvbuf;
+  //     delete[] tmp_recvbuf;
+
+  //   }
+
+  // }
+
+  int MPI_Gather_local2(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Gather_local_int(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Gather_local_float(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Gather_local_double(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Gather_local_long(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Gather_local_ulong(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Gather_local_char(sendbuf, count, recvbuf, comm);
+    }
+    else
+    {
+      printf("MPI_Gather Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Gather_local_int(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_float(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_long(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_ulong(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Gather_local_char(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+
+  int MPI_Gather2(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Gather(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+                   root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+    
+    MPI_Bcast(&recvcount, 1, MPI_INT, root, comm);
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype) && sendcount == recvcount);
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+    void *master_recvbuf;
+    if(ep_rank_loc == 0 && mpi_rank == root_mpi_rank && root_ep_loc != 0) 
+    {
+      master_recvbuf = new void*[datasize*ep_size*count];
+    }
+
+    if(ep_rank_loc==0)
+    {
+      local_gather_recvbuf = new void*[datasize*num_ep*count];
+    }
+
+    // local gather to master
+    MPI_Gather_local2(sendbuf, count, datatype, local_gather_recvbuf, comm);
+
+    //MPI_Gather
+
+    if(ep_rank_loc == 0)
+    {
+      int *gatherv_recvcnt;
+      int *gatherv_displs;
+      int gatherv_cnt = count*num_ep;
+
+      gatherv_recvcnt = new int[mpi_size];
+      gatherv_displs = new int[mpi_size];
+
+
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT, gatherv_recvcnt, 1, MPI_INT, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      gatherv_displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+
+      if(root_ep_loc != 0) // gather to root_master
+      {
+        ::MPI_Gatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), master_recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+      else
+      {
+        ::MPI_Gatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+
+      delete[] gatherv_recvcnt;
+      delete[] gatherv_displs;
+    }
+
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      innode_memcpy(0, master_recvbuf, root_ep_loc, recvbuf, count*ep_size, datatype, comm);
+    }
+
+
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else// if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+      
+      if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) delete[] master_recvbuf;
+    }
+  }
+
+
+  int MPI_Allgather2(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+                      static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype) && sendcount == recvcount);
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+
+    if(ep_rank_loc==0)
+    {
+      local_gather_recvbuf = new void*[datasize*num_ep*count];
+    }
+
+    // local gather to master
+    MPI_Gather_local2(sendbuf, count, datatype, local_gather_recvbuf, comm);
+
+    //MPI_Gather
+
+    if(ep_rank_loc == 0)
+    {
+      int *gatherv_recvcnt;
+      int *gatherv_displs;
+      int gatherv_cnt = count*num_ep;
+
+      gatherv_recvcnt = new int[mpi_size];
+      gatherv_displs = new int[mpi_size];
+
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT, gatherv_recvcnt, 1, MPI_INT, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      gatherv_displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+
+      ::MPI_Allgatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      delete[] gatherv_recvcnt;
+      delete[] gatherv_displs;
+    }
+
+    MPI_Bcast_local2(recvbuf, count*ep_size, datatype, comm);
+
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else// if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+    }
+  }
+
+
 }

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_gatherv.cpp

@@ -15 +15 @@
 namespace ep_lib
 {
-
-  int MPI_Gatherv_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, const int recvcounts[], const int displs[], int local_root, MPI_Comm comm)
+   int MPI_Gatherv_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, const int recvcounts[], const int displs[], int local_root, MPI_Comm comm)
   {
     assert(valid_type(datatype));
@@ -186 +185 @@
   }
 
+  // int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm)
+  // {
+
+  //   if(!comm.is_ep && comm.mpi_comm)
+  //   {
+  //     ::MPI_Allgatherv(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcounts, displs, to_mpi_type(recvtype), to_mpi_comm(comm.mpi_comm));
+  //     return 0;
+  //   }
+
+  //   if(!comm.mpi_comm) return 0;
+
+
+
+
+  //   assert(valid_type(sendtype) && valid_type(recvtype));
+
+  //   MPI_Datatype datatype = sendtype;
+  //   int count = sendcount;
+
+  //   ::MPI_Aint datasize, lb;
+
+  //   ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
+
+
+  //   int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+  //   int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+  //   int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+  //   int ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+  //   int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+  //   int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+  //   assert(sendcount == recvcounts[ep_rank]);
+
+  //   bool is_master = ep_rank_loc==0;
+
+  //   void* local_recvbuf;
+  //   void* tmp_recvbuf;
+
+  //   int recvbuf_size = 0;
+  //   for(int i=0; i<ep_size; i++)
+  //     recvbuf_size = max(recvbuf_size, displs[i]+recvcounts[i]);
+
+
+  //   vector<int>local_recvcounts(num_ep, 0);
+  //   vector<int>local_displs(num_ep, 0);
+
+  //   MPI_Gather_local(&sendcount, 1, MPI_INT, local_recvcounts.data(), 0, comm);
+  //   for(int i=1; i<num_ep; i++) local_displs[i] = local_displs[i-1] + local_recvcounts[i-1]; 
+
+
+  //   if(is_master)
+  //   {
+  //     local_recvbuf = new void*[datasize * std::accumulate(local_recvcounts.begin(), local_recvcounts.begin()+num_ep, 0)];
+  //     tmp_recvbuf = new void*[datasize * std::accumulate(recvcounts, recvcounts+ep_size, 0)];
+  //   }
+
+  //   MPI_Gatherv_local(sendbuf, count, datatype, local_recvbuf, local_recvcounts.data(), local_displs.data(), 0, comm);
+
+
+  //   if(is_master)
+  //   {
+  //     std::vector<int>mpi_recvcounts(mpi_size, 0);
+  //     std::vector<int>mpi_displs(mpi_size, 0);
+
+  //     int local_sendcount = std::accumulate(local_recvcounts.begin(), local_recvcounts.begin()+num_ep, 0);
+  //     MPI_Allgather(&local_sendcount, 1, MPI_INT, mpi_recvcounts.data(), 1, MPI_INT, to_mpi_comm(comm.mpi_comm));
+
+  //     for(int i=1; i<mpi_size; i++)
+  //       mpi_displs[i] = mpi_displs[i-1] + mpi_recvcounts[i-1];
+
+
+  //     ::MPI_Allgatherv(local_recvbuf, local_sendcount, to_mpi_type(datatype), tmp_recvbuf, mpi_recvcounts.data(), mpi_displs.data(), to_mpi_type(datatype), to_mpi_comm(comm.mpi_comm));
+
+
+
+  //     // reorder 
+  //     int offset;
+  //     for(int i=0; i<ep_size; i++)
+  //     {
+  //       int extra = 0;
+  //       for(int j=0, k=0; j<ep_size, k<comm.rank_map->at(i).first; j++)
+  //         if(comm.rank_map->at(i).second == comm.rank_map->at(j).second)
+  //         {
+  //           extra += recvcounts[j];
+  //           k++;
+  //         }  
+
+  //       offset = mpi_displs[comm.rank_map->at(i).second] +  extra;
+
+  //       memcpy(recvbuf+displs[i]*datasize, tmp_recvbuf+offset*datasize, recvcounts[i]*datasize);
+        
+  //     }
+
+  //   }
+
+  //   MPI_Bcast_local(recvbuf, recvbuf_size, datatype, 0, comm);
+
+  //   if(is_master)
+  //   {
+  //     delete[] local_recvbuf;
+  //     delete[] tmp_recvbuf;
+  //   }
+
+  // }
+
+
+  int MPI_Gatherv_local2(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Gatherv_local_int(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Gatherv_local_float(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Gatherv_local_double(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Gatherv_local_long(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Gatherv_local_ulong(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Gatherv_local_char(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else
+    {
+      printf("MPI_Gatherv Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Gatherv_local_int(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; count!=0? j<count: j<count+1; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            if(count!=0) copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_float(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; count!=0? j<count: j<count+1; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            if(count!=0) copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_double(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; count!=0? j<count: j<count+1; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            if(count!=0) copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_long(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; count!=0? j<count: j<count+1; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            if(count!=0)copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_ulong(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; count!=0? j<count: j<count+1; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            if(count!=0) copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_char(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; count!=0? j<count: j<count+1; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            if(count!=0) copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Gatherv2(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+  
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Gatherv(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, const_cast<int*>(recvcounts), const_cast<int*>(displs),
+                    static_cast< ::MPI_Datatype>(recvtype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype));
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+    
+    
+    
+    if(ep_size == mpi_size) 
+      return ::MPI_Gatherv(sendbuf, sendcount, static_cast< ::MPI_Datatype>(datatype), recvbuf, recvcounts, displs,
+                              static_cast< ::MPI_Datatype>(datatype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+    if(ep_rank != root)
+    {
+      recvcounts = new int[ep_size];
+      displs = new int[ep_size];
+    }
+    
+    MPI_Bcast(const_cast< int* >(displs),     ep_size, MPI_INT, root, comm);
+    MPI_Bcast(const_cast< int* >(recvcounts), ep_size, MPI_INT, root, comm);
+                              
+
+    int recv_plus_displs[ep_size];
+    for(int i=0; i<ep_size; i++) recv_plus_displs[i] = recvcounts[i] + displs[i];
+
+    for(int j=0; j<mpi_size; j++)
+    {
+      if(recv_plus_displs[j*num_ep] < displs[j*num_ep+1] ||
+         recv_plus_displs[j*num_ep + num_ep -1] < displs[j*num_ep + num_ep -2])  
+      {  
+        Debug("Call special implementation of mpi_gatherv. 1st condition not OK\n");
+        return MPI_Allgatherv_special(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm);
+      }
+
+      for(int i=1; i<num_ep-1; i++)
+      {
+        if(recv_plus_displs[j*num_ep+i] < displs[j*num_ep+i+1] || 
+           recv_plus_displs[j*num_ep+i] < displs[j*num_ep+i-1])
+        {
+          Debug("Call special implementation of mpi_gatherv. 2nd condition not OK\n");
+          return MPI_Allgatherv_special(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm);
+        }
+      }
+    }
+
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+    int buffer_size;
+    void *master_recvbuf;
+
+    if(ep_rank_loc == 0 && mpi_rank == root_mpi_rank && root_ep_loc != 0) 
+    {
+      master_recvbuf = new void*[sizeof(recvbuf)];
+      assert(root_ep_loc == 0);
+    }
+
+    if(ep_rank_loc==0)
+    {
+      buffer_size = *std::max_element(recv_plus_displs+ep_rank, recv_plus_displs+ep_rank+num_ep);
+
+      local_gather_recvbuf = new void*[datasize*buffer_size];
+    }
+
+    MPI_Gatherv_local2(sendbuf, count, datatype, local_gather_recvbuf, recvcounts+ep_rank-ep_rank_loc, displs+ep_rank-ep_rank_loc, comm);
+
+    //MPI_Gather
+    if(ep_rank_loc == 0)
+    {
+      int *mpi_recvcnt= new int[mpi_size];
+      int *mpi_displs= new int[mpi_size];
+
+      int buff_start = *std::min_element(displs+ep_rank, displs+ep_rank+num_ep);;
+      int buff_end = buffer_size;
+
+      int mpi_sendcnt = buff_end - buff_start;
+
+
+      ::MPI_Gather(&mpi_sendcnt, 1, MPI_INT, mpi_recvcnt, 1, MPI_INT, root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      ::MPI_Gather(&buff_start,  1, MPI_INT, mpi_displs,  1, MPI_INT, root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      if(root_ep_loc == 0)
+      {  ::MPI_Gatherv(local_gather_recvbuf + datasize*buff_start, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, mpi_recvcnt,
+                       mpi_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+      else  // gatherv to master_recvbuf
+      {  ::MPI_Gatherv(local_gather_recvbuf + datasize*buff_start, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), master_recvbuf, mpi_recvcnt,
+                       mpi_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+
+      delete[] mpi_recvcnt;
+      delete[] mpi_displs;
+    }
+
+    int global_min_displs = *std::min_element(displs, displs+ep_size);
+    int global_recvcnt = *std::max_element(recv_plus_displs, recv_plus_displs+ep_size);
+
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      innode_memcpy(0, master_recvbuf+datasize*global_min_displs, root_ep_loc, recvbuf+datasize*global_min_displs, global_recvcnt, datatype, comm);
+      if(ep_rank_loc == 0) delete[] master_recvbuf;
+    }
+
+
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+      else // if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+    }
+    else
+    {
+      delete[] recvcounts;
+      delete[] displs;
+    }
+    return 0;
+  }
+
+
+
+  int MPI_Allgatherv2(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Allgatherv(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcounts, displs,
+                       static_cast< ::MPI_Datatype>(recvtype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype));
+
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+    
+    if(ep_size == mpi_size)  // needed by servers
+      return ::MPI_Allgatherv(sendbuf, sendcount, static_cast< ::MPI_Datatype>(datatype), recvbuf, recvcounts, displs,
+                              static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+    int recv_plus_displs[ep_size];
+    for(int i=0; i<ep_size; i++) recv_plus_displs[i] = recvcounts[i] + displs[i];
+
+
+    for(int j=0; j<mpi_size; j++)
+    {
+      if(recv_plus_displs[j*num_ep] < displs[j*num_ep+1] ||
+         recv_plus_displs[j*num_ep + num_ep -1] < displs[j*num_ep + num_ep -2])  
+      {  
+        printf("proc %d/%d Call special implementation of mpi_allgatherv.\n", ep_rank, ep_size);
+        for(int k=0; k<ep_size; k++)
+          printf("recv_plus_displs[%d] = %d\t displs[%d] = %d\n", k, recv_plus_displs[k], k, displs[k]);
+
+        return MPI_Allgatherv_special(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm);
+      }
+
+      for(int i=1; i<num_ep-1; i++)
+      {
+        if(recv_plus_displs[j*num_ep+i] < displs[j*num_ep+i+1] || 
+           recv_plus_displs[j*num_ep+i] < displs[j*num_ep+i-1])
+        {
+          printf("proc %d/%d Call special implementation of mpi_allgatherv.\n", ep_rank, ep_size);
+          return MPI_Allgatherv_special(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm);
+        }
+      }
+    }
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+    int buffer_size;
+
+    if(ep_rank_loc==0)
+    {
+      buffer_size = *std::max_element(recv_plus_displs+ep_rank, recv_plus_displs+ep_rank+num_ep);
+
+      local_gather_recvbuf = new void*[datasize*buffer_size];
+    }
+
+    // local gather to master
+    MPI_Gatherv_local2(sendbuf, count, datatype, local_gather_recvbuf, recvcounts+ep_rank-ep_rank_loc, displs+ep_rank-ep_rank_loc, comm);
+
+    //MPI_Gather
+    if(ep_rank_loc == 0)
+    {
+      int *mpi_recvcnt= new int[mpi_size];
+      int *mpi_displs= new int[mpi_size];
+
+      int buff_start = *std::min_element(displs+ep_rank, displs+ep_rank+num_ep);;
+      int buff_end = buffer_size;
+
+      int mpi_sendcnt = buff_end - buff_start;
+
+
+      ::MPI_Allgather(&mpi_sendcnt, 1, MPI_INT, mpi_recvcnt, 1, MPI_INT, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      ::MPI_Allgather(&buff_start,  1, MPI_INT, mpi_displs,  1, MPI_INT, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+
+      ::MPI_Allgatherv((char*)local_gather_recvbuf + datasize*buff_start, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, mpi_recvcnt,
+                       mpi_displs, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      delete[] mpi_recvcnt;
+      delete[] mpi_displs;
+    }
+
+    int global_min_displs = *std::min_element(displs, displs+ep_size);
+    int global_recvcnt = *std::max_element(recv_plus_displs, recv_plus_displs+ep_size);
+
+    MPI_Bcast_local2(recvbuf+datasize*global_min_displs, global_recvcnt, datatype, comm);
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+      else // if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+    }
+  }
+
+  int MPI_Gatherv_special(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                          MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+    ::MPI_Aint datasize, lb;
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(sendtype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+    int buffer_size;
+
+    int *local_displs = new int[num_ep];
+    int *local_rvcnts = new int[num_ep];
+    for(int i=0; i<num_ep; i++) local_rvcnts[i] = recvcounts[ep_rank-ep_rank_loc + i];
+    local_displs[0] = 0;
+    for(int i=1; i<num_ep; i++) local_displs[i] = local_displs[i-1] + local_rvcnts[i-1];
+
+    if(ep_rank_loc==0)
+    {
+      buffer_size = local_displs[num_ep-1] + recvcounts[ep_rank+num_ep-1];
+      local_gather_recvbuf = new void*[datasize*buffer_size];
+    }
+
+    // local gather to master
+    MPI_Gatherv_local2(sendbuf, sendcount, sendtype, local_gather_recvbuf, local_rvcnts, local_displs, comm); // all sendbuf gathered to master
+
+    int **mpi_recvcnts = new int*[num_ep];
+    int **mpi_displs   = new int*[num_ep];
+    for(int i=0; i<num_ep; i++) 
+    {
+      mpi_recvcnts[i] = new int[mpi_size];
+      mpi_displs[i]   = new int[mpi_size];
+      for(int j=0; j<mpi_size; j++)
+      {
+        mpi_recvcnts[i][j] = recvcounts[j*num_ep + i];
+        mpi_displs[i][j]   = displs[j*num_ep + i];
+      }
+    } 
+
+    void *master_recvbuf;
+    if(ep_rank_loc == 0 && mpi_rank == root_mpi_rank && root_ep_loc != 0) master_recvbuf = new void*[sizeof(recvbuf)];
+
+    if(ep_rank_loc == 0 && root_ep_loc == 0) // master in MPI_Allgatherv loop
+      for(int i=0; i<num_ep; i++)
+      {
+        ::MPI_Gatherv(local_gather_recvbuf + datasize*local_displs[i], recvcounts[ep_rank+i], static_cast< ::MPI_Datatype>(sendtype), recvbuf, mpi_recvcnts[i], mpi_displs[i],
+                    static_cast< ::MPI_Datatype>(recvtype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+    if(ep_rank_loc == 0 && root_ep_loc != 0)
+      for(int i=0; i<num_ep; i++)
+      {
+        ::MPI_Gatherv(local_gather_recvbuf + datasize*local_displs[i], recvcounts[ep_rank+i], static_cast< ::MPI_Datatype>(sendtype), master_recvbuf, mpi_recvcnts[i], mpi_displs[i],
+                    static_cast< ::MPI_Datatype>(recvtype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      for(int i=0; i<ep_size; i++)
+        innode_memcpy(0, master_recvbuf + datasize*displs[i], root_ep_loc, recvbuf + datasize*displs[i], recvcounts[i], sendtype, comm);
+
+      if(ep_rank_loc == 0) delete[] master_recvbuf;
+    }
+
+    
+    delete[] local_displs;
+    delete[] local_rvcnts;
+    for(int i=0; i<num_ep; i++) { delete[] mpi_recvcnts[i]; 
+                                  delete[] mpi_displs[i]; }
+    delete[] mpi_recvcnts;
+    delete[] mpi_displs;
+    if(ep_rank_loc==0)
+    {
+      if(sendtype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(sendtype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(sendtype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(sendtype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else if(sendtype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+      else // if(sendtype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+    }
+  }
+
+  int MPI_Allgatherv_special(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                             MPI_Datatype recvtype, MPI_Comm comm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint datasize, lb;
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(sendtype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+    int buffer_size;
+
+    int *local_displs = new int[num_ep];
+    int *local_rvcnts = new int[num_ep];
+    for(int i=0; i<num_ep; i++) local_rvcnts[i] = recvcounts[ep_rank-ep_rank_loc + i];
+    local_displs[0] = 0;
+    for(int i=1; i<num_ep; i++) local_displs[i] = local_displs[i-1] + local_rvcnts[i-1];
+
+    if(ep_rank_loc==0)
+    {
+      buffer_size = local_displs[num_ep-1] + recvcounts[ep_rank+num_ep-1];
+      local_gather_recvbuf = new void*[datasize*buffer_size];
+    }
+
+    // local gather to master
+    MPI_Gatherv_local2(sendbuf, sendcount, sendtype, local_gather_recvbuf, local_rvcnts, local_displs, comm); // all sendbuf gathered to master
+
+    int **mpi_recvcnts = new int*[num_ep];
+    int **mpi_displs   = new int*[num_ep];
+    for(int i=0; i<num_ep; i++) 
+    {
+      mpi_recvcnts[i] = new int[mpi_size];
+      mpi_displs[i]   = new int[mpi_size];
+      for(int j=0; j<mpi_size; j++)
+      {
+        mpi_recvcnts[i][j] = recvcounts[j*num_ep + i];
+        mpi_displs[i][j]   = displs[j*num_ep + i];
+      }
+    } 
+
+    if(ep_rank_loc == 0) // master in MPI_Allgatherv loop
+    for(int i=0; i<num_ep; i++)
+    {
+      ::MPI_Allgatherv(local_gather_recvbuf + datasize*local_displs[i], recvcounts[ep_rank+i], static_cast< ::MPI_Datatype>(sendtype), recvbuf, mpi_recvcnts[i], mpi_displs[i],
+                  static_cast< ::MPI_Datatype>(recvtype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    for(int i=0; i<ep_size; i++)
+      MPI_Bcast_local2(recvbuf + datasize*displs[i], recvcounts[i], recvtype, comm);
+
+    
+    delete[] local_displs;
+    delete[] local_rvcnts;
+    for(int i=0; i<num_ep; i++) { delete[] mpi_recvcnts[i]; 
+                                  delete[] mpi_displs[i]; }
+    delete[] mpi_recvcnts;
+    delete[] mpi_displs;
+    if(ep_rank_loc==0)
+    {
+      if(sendtype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(sendtype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(sendtype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(sendtype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else if(sendtype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+      else // if(sendtype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+    }
+  }
+
+
 }

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_lib.cpp

@@ -63 +63 @@
 
 
+  int innode_memcpy(int sender, const void* sendbuf, int receiver, void* recvbuf, int count, MPI_Datatype datatype, MPI_Comm comm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    if(datatype == MPI_INT)
+    {
+
+      int* send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+      int* recv_buf = static_cast<int*>(recvbuf);
+      int* buffer = comm.my_buffer->buf_int;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+
+      float* send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+      float* recv_buf = static_cast<float*>(recvbuf);
+      float* buffer = comm.my_buffer->buf_float;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+
+
+      double* send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+      double* recv_buf = static_cast<double*>(recvbuf);
+      double* buffer = comm.my_buffer->buf_double;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_LONG)
+    {
+      long* send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+      long* recv_buf = static_cast<long*>(recvbuf);
+      long* buffer = comm.my_buffer->buf_long;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+      unsigned long* buffer = comm.my_buffer->buf_ulong;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      char* send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+      char* recv_buf = static_cast<char*>(recvbuf);
+      char* buffer = comm.my_buffer->buf_char;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else
+    {
+      printf("datatype not supported!!\n");
+      exit(1);
+    }
+    return 0;
+  }
+
+
   int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count)
   {
-
+/*
+    ::MPI_Aint datasize, char_size, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+    ::MPI_Type_get_extent(MPI_CHAR, &lb, &char_size);
+
+    *count = status->char_count / ( datasize/ char_size);
+
+    //printf("MPI_Get_count, status_count  = %d\n", *count);
+    return 0;
+*/
     ::MPI_Status *mpi_status = static_cast< ::MPI_Status* >(status->mpi_status);
     ::MPI_Datatype mpi_datatype = static_cast< ::MPI_Datatype >(datatype);

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_lib_collective.hpp

@@ -17 +17 @@
 
   int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm);
-
   int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
 
@@ -23 +22 @@
 
   int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
-
   int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
 
@@ -33 +31 @@
   int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
                   MPI_Datatype recvtype, int root, MPI_Comm comm);
+  int MPI_Gatherv_special(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                          MPI_Datatype recvtype, int root, MPI_Comm comm);
   int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
                      MPI_Datatype recvtype, MPI_Comm comm);
 
+  int MPI_Allgatherv_special(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                             MPI_Datatype recvtype, MPI_Comm comm);
+
 
   int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm);
-
   int MPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount,
                    MPI_Datatype recvtype, int root, MPI_Comm comm);

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_lib_fortran.hpp

@@ -6 +6 @@
 namespace ep_lib
 {
+  
+  // #ifdef _intelmpi
+  
+  // MPI_Fint MPI_Comm_c2f(MPI_Comm comm);
+  // MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
+  
+  // #elif _openmpi
+  
+  // int MPI_Comm_c2f(MPI_Comm comm);
+  // ep_lib::MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
+  
+  // #endif
 
   int EP_Comm_c2f(MPI_Comm comm);

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_lib_local.hpp

@@ -30 +30 @@
                    MPI_Datatype recvtype, int local_root, MPI_Comm comm);
 
+
+
+  int MPI_Reduce_local2       (const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_int   (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_float (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_long  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_ulong (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_char  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+
+
+  int MPI_Scan_local2       (const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_int   (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_float (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_long  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_ulong (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_char  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Exscan_local2       (const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_int   (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_float (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_long  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_ulong (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_char  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Bcast_local2       (void *buffer, int count, MPI_Datatype datatype, MPI_Comm comm);
+  int MPI_Bcast_local_int   (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_float (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_double(void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_long  (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_ulong (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_char  (void *buffer, int count, MPI_Comm comm);
+
+  int MPI_Gather_local2       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_int   (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_float (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_long  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_ulong (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_char  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+
+
+  int MPI_Gatherv_local2       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf,
+                               const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_int   (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_float (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_double(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_long  (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_ulong (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_char  (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+
+  int MPI_Scatter_local2       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_int   (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_float (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_long  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_ulong (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_char  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+
+  int MPI_Scatterv_local2       (const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_int   (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_float (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_double(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_long  (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_ulong (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_char  (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+
+  int innode_memcpy(int sender, const void* sendbuf, int receiver, void* recvbuf, int count, MPI_Datatype datatype, MPI_Comm comm);
+
   int MPI_Barrier_local(MPI_Comm comm);
 

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_merge.cpp

@@ -42 +42 @@
     if(local_ep_rank == 0)
     {
-      MPI_Status status[2];
-      MPI_Request request[2];
-      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[0]);
-      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[1]);
-
-      MPI_Waitall(2, request, status);
+      MPI_Status status;
+      MPI_Request req_s, req_r;
+      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_s);
+      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_r);
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
     }
 
@@ -53 +54 @@
     MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm));
 
+//    printf("%d, %d, %d, %d\n", local_ep_size, remote_ep_size, local_high, remote_high);
 
 
@@ -84 +86 @@
     if(intra_ep_rank_loc == 0)
     {
-      ::MPI_Bcast(reorder, intra_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
+      ::MPI_Bcast(reorder, intra_ep_size, MPI_INT, 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
 
       vector< pair<int, int> > tmp_rank_map(intra_ep_size);
@@ -128 +130 @@
     Debug("intercomm_merge kernel\n");
 
-    int ep_rank_loc;
-    int num_ep;
-
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = inter_comm.ep_comm_ptr->size_rank_info[0].first;
     ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = inter_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = inter_comm.ep_comm_ptr->size_rank_info[0].second;
     num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second;
-
-
+    mpi_size = inter_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    int local_ep_rank, local_ep_rank_loc, local_mpi_rank;
+    int local_ep_size, local_num_ep, local_mpi_size;
+
+
+    local_ep_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first;
+    local_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first;
+    local_mpi_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first;
+    local_ep_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second;
+    local_num_ep = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second;
+    local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second;
 
     int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size();
 
+    int local_high = high;
+    int remote_high;
 
     MPI_Barrier(inter_comm);
+
+//    if(local_ep_rank == 0 && high == false)
+//    {
+//      MPI_Status status;
+//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
+//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
+//    }
+//
+//    if(local_ep_rank == 0 && high == true)
+//    {
+//      MPI_Status status;
+//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
+//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
+//    }
+
+    if(local_ep_rank == 0)
+    {
+      MPI_Status status;
+      MPI_Request req_s, req_r;
+      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_s);
+      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_r);
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+    }
+
+    MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm));
+
+    int intercomm_high;
+    if(ep_rank == 0) intercomm_high = local_high;
+    MPI_Bcast(&intercomm_high, 1, MPI_INT, 0, inter_comm);
+
+    //printf("remote_ep_size = %d, local_high = %d, remote_high = %d, intercomm_high = %d\n", remote_ep_size, local_high, remote_high, intercomm_high);
 
 
@@ -150 +201 @@
       ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(inter_comm.ep_comm_ptr->intercomm->mpi_inter_comm);
 
-      ::MPI_Intercomm_merge(mpi_comm, high, &mpi_intracomm);
+      ::MPI_Intercomm_merge(mpi_comm, intercomm_high, &mpi_intracomm);
       MPI_Info info;
       MPI_Comm_create_endpoints(mpi_intracomm, num_ep, info, ep_intracomm);
@@ -158 +209 @@
     }
 
+
+
     MPI_Barrier_local(inter_comm);
 
-    int inter_rank;
-    MPI_Comm_rank(inter_comm, &inter_rank);
-    
-    int my_ep_rank = high? inter_rank+remote_ep_size : inter_rank;
-    int my_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first;
-    int my_num_ep_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second;
-    int my_num_ep_total = inter_comm.ep_comm_ptr->comm_list->mem_bridge[0].ep_comm_ptr->size_rank_info[1].second;
-    int my_ep_size = inter_comm.ep_comm_ptr->comm_list->mem_bridge[0].ep_comm_ptr->size_rank_info[0].second;
-
-    int tmp_intra_ep_rank_loc = high?my_ep_rank_loc+my_num_ep_total-my_num_ep_loc: my_ep_rank_loc;
-
-
-    *newintracomm = inter_comm.ep_comm_ptr->comm_list->mem_bridge[tmp_intra_ep_rank_loc];
-
-    int newintracomm_ep_rank = (*newintracomm).ep_comm_ptr->size_rank_info[0].first;
-    int newintracomm_ep_rank_loc = (*newintracomm).ep_comm_ptr->size_rank_info[1].first;
-    int newintracomm_mpi_rank = (*newintracomm).ep_comm_ptr->size_rank_info[2].first;
-    int newintracomm_ep_size = (*newintracomm).ep_comm_ptr->size_rank_info[0].second;
-    int newintracomm_num_ep = (*newintracomm).ep_comm_ptr->size_rank_info[1].second;
-    int newintracomm_mpi_size = (*newintracomm).ep_comm_ptr->size_rank_info[2].second;
-
-
-    int buf[3];
-    buf[0] = my_ep_rank;
-    buf[1] = tmp_intra_ep_rank_loc;
-    buf[2] = newintracomm->ep_comm_ptr->size_rank_info[2].first;
-
-    // printf("my_ep_rank = %d, tmp_intra_ep_rank_loc = %d, mpi_rank = %d\n", my_ep_rank, tmp_intra_ep_rank_loc, newintracomm->ep_comm_ptr->size_rank_info[2].first);
-
-    int *rankmap_buf;
-    rankmap_buf = new int [3*my_ep_size];
-
-    MPI_Allgather(buf, 3, MPI_INT, rankmap_buf, 3, MPI_INT, *newintracomm);
-
-    
-    // printf(" ID = %d : rankmap_buf = (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d)\n", newintracomm_ep_rank,
-    //                     rankmap_buf[0], rankmap_buf[1], rankmap_buf[2], rankmap_buf[3], rankmap_buf[4], rankmap_buf[5], rankmap_buf[6], rankmap_buf[7], rankmap_buf[8], rankmap_buf[9],
-    //                     rankmap_buf[10], rankmap_buf[11], rankmap_buf[12], rankmap_buf[13], rankmap_buf[14], rankmap_buf[15], rankmap_buf[16], rankmap_buf[17], rankmap_buf[18], rankmap_buf[19], 
-    //                     rankmap_buf[20], rankmap_buf[21], rankmap_buf[22], rankmap_buf[23], rankmap_buf[24], rankmap_buf[25], rankmap_buf[26], rankmap_buf[27], rankmap_buf[28], rankmap_buf[29],
-    //                     rankmap_buf[30], rankmap_buf[31], rankmap_buf[32], rankmap_buf[33], rankmap_buf[34], rankmap_buf[35], rankmap_buf[36], rankmap_buf[37], rankmap_buf[38], rankmap_buf[39],
-    //                     rankmap_buf[40], rankmap_buf[41], rankmap_buf[42], rankmap_buf[43], rankmap_buf[44], rankmap_buf[45], rankmap_buf[46], rankmap_buf[47]);
-
-
-    for(int i=0; i<newintracomm_ep_size; i++)
-    {
-      (*newintracomm).rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1];
-      (*newintracomm).rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2];
-    }
-
+    *newintracomm = inter_comm.ep_comm_ptr->comm_list->mem_bridge[ep_rank_loc];
+
+    int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size;
+
+    int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank;
+    int intra_ep_size, intra_num_ep, intra_mpi_size;
+
+    intra_ep_rank = newintracomm->ep_comm_ptr->size_rank_info[0].first;
+    intra_ep_rank_loc = newintracomm->ep_comm_ptr->size_rank_info[1].first;
+    intra_mpi_rank = newintracomm->ep_comm_ptr->size_rank_info[2].first;
+    intra_ep_size = newintracomm->ep_comm_ptr->size_rank_info[0].second;
+    intra_num_ep = newintracomm->ep_comm_ptr->size_rank_info[1].second;
+    intra_mpi_size = newintracomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    MPI_Barrier_local(*newintracomm);
+
+
+    int *reorder;
+    if(intra_ep_rank_loc == 0)
+    {
+      reorder = new int[intra_ep_size];
+    }
+
+
+
+    MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm);
+    if(intra_ep_rank_loc == 0)
+    {
+
+      ::MPI_Bcast(reorder, intra_ep_size, MPI_INT, 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
+
+      vector< pair<int, int> > tmp_rank_map(intra_ep_size);
+
+
+      for(int i=0; i<intra_ep_size; i++)
+      {
+        tmp_rank_map[reorder[i]] = newintracomm->rank_map->at(i) ;
+      }
+
+      newintracomm->rank_map->swap(tmp_rank_map);
+
+      tmp_rank_map.clear();
+    }
+
+    MPI_Barrier_local(*newintracomm);
 
     (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank;
-    (*newintracomm).ep_comm_ptr->size_rank_info[1].first = tmp_intra_ep_rank_loc;
-    
-
-    delete[] rankmap_buf;
-
-    
+
+
+    if(intra_ep_rank_loc == 0)
+    {
+      delete[] reorder;
+
+    }
+
+    /*
+    if(intra_ep_rank == 0)
+    {
+      for(int i=0; i<intra_ep_size; i++)
+      {
+        printf("intra rank_map[%d] = (%d, %d)\n", i, newintracomm->rank_map->at(i).first, newintracomm->rank_map->at(i).second);
+      }
+    }
+*/
     return MPI_SUCCESS;
 

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_message.cpp

@@ -9 +9 @@
 #include <mpi.h>
 #include "ep_declaration.hpp"
-#include "ep_mpi.hpp"
 
 using namespace std;
@@ -21 +20 @@
   int Message_Check(MPI_Comm comm)
   {
+    int myRank;
+    MPI_Comm_rank(comm, &myRank);
+
     if(!comm.is_ep) return 0;
 
@@ -36 +38 @@
     {
       Debug("Message probing for intracomm\n");
-      
-
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
       #ifdef _openmpi
       #pragma omp critical (_mpi_call)
       {
-        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm.mpi_comm), &flag, &status);
+        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &status);
         if(flag)
         {
@@ -47 +48 @@
           mpi_source = status.MPI_SOURCE;
           int tag = status.MPI_TAG;
-          ::MPI_Mprobe(mpi_source, tag, to_mpi_comm(comm.mpi_comm), &message, &status);
+          ::MPI_Mprobe(mpi_source, tag, mpi_comm, &message, &status);
 
         }
       }
       #elif _intelmpi
-      ::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm.mpi_comm), &flag, &message, &status); 
+      #pragma omp critical (_mpi_call)
+      ::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &message, &status); 
       #endif
       
@@ -66 +68 @@
              
         msg_block->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);       
+        int dest_mpi = comm.ep_comm_ptr->size_rank_info[2].first;
+        int ep_dest = get_ep_rank(comm, dest_loc, dest_mpi);
         msg_block->mpi_status = new ::MPI_Status(status);
 
@@ -75 +79 @@
         {
           #pragma omp flush
-          comm.ep_comm_ptr->comm_list[dest_loc].ep_comm_ptr->message_queue->push_back(*msg_block);      
+          ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);  
+    
           #pragma omp flush
         }
@@ -105 +110 @@
     {
       Debug("Message probing for intracomm\n");
-
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.ep_comm_ptr->intercomm->mpi_inter_comm);  // => mpi_intercomm
+      
       #ifdef _openmpi
       #pragma omp critical (_mpi_call)
       {
-        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm), &flag, &status);
+        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &status);
         if(flag)
         {
@@ -115 +121 @@
           mpi_source = status.MPI_SOURCE;
           int tag = status.MPI_TAG;
-          ::MPI_Mprobe(mpi_source, tag, to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm), &message, &status);
+          ::MPI_Mprobe(mpi_source, tag, mpi_comm, &message, &status);
 
         }
       }
       #elif _intelmpi
-      ::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm.ep_comm_ptr->intercomm->mpi_inter_comm), &flag, &message, &status);       
+      #pragma omp critical (_mpi_call)
+      ::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &message, &status);       
       #endif
 
@@ -146 +153 @@
         {
           #pragma omp flush
-          comm.ep_comm_ptr->comm_list[dest_loc].ep_comm_ptr->message_queue->push_back(*msg_block);
+          ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);
           #pragma omp flush
         }
@@ -160 +167 @@
     {
       Debug("Message probing for intracomm\n");
-     
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
       #ifdef _openmpi
       #pragma omp critical (_mpi_call)
       {
-        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm.mpi_comm), &flag, &status);
+        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &status);
         if(flag)
         {
@@ -170 +177 @@
           mpi_source = status.MPI_SOURCE;
           int tag = status.MPI_TAG;
-          ::MPI_Mprobe(mpi_source, tag, to_mpi_comm(comm.mpi_comm), &message, &status);
+          ::MPI_Mprobe(mpi_source, tag, mpi_comm, &message, &status);
 
         }
       }
       #elif _intelmpi
-      ::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm.mpi_comm), &flag, &message, &status);       
+      #pragma omp critical (_mpi_call)
+      ::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &message, &status);       
       #endif
 
@@ -188 +196 @@
         int dest_loc      = bitset<8> (status.MPI_TAG)      .to_ulong();
         int src_mpi       = status.MPI_SOURCE;
+        int current_inter = comm.ep_comm_ptr->intercomm->local_rank_map->at(current_ep_rank).first;
         
         msg_block->ep_src  = get_ep_rank_intercomm(comm, src_loc, src_mpi);
@@ -200 +209 @@
         {
           #pragma omp flush
-          comm.ep_comm_ptr->comm_list[dest_loc].ep_comm_ptr->message_queue->push_back(*msg_block);
+          ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);
           #pragma omp flush
         }

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_reduce.cpp

@@ -9 +9 @@
 #include <mpi.h>
 #include "ep_declaration.hpp"
-#include "ep_mpi.hpp"
 
 using namespace std;
@@ -28 +27 @@
   }
 
-  template<typename T>
-  void reduce_max(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, max_op<T>);
-  }
-
-  template<typename T>
-  void reduce_min(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, min_op<T>);
-  }
-
-  template<typename T>
-  void reduce_sum(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, std::plus<T>());
-  }
-
-  int MPI_Reduce_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int local_root, MPI_Comm comm)
-  {
-    assert(valid_type(datatype));
-    assert(valid_op(op));
-
-    ::MPI_Aint datasize, lb;
-    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
-
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-
-    #pragma omp critical (_reduce)
-    comm.my_buffer->void_buffer[ep_rank_loc] = const_cast< void* >(sendbuf);
-
-    MPI_Barrier_local(comm);
-
-    if(ep_rank_loc == local_root)
-    {
-      memcpy(recvbuf, comm.my_buffer->void_buffer[0], datasize * count);
-
-      if(op == MPI_MAX)
-      {
-        if(datasize == sizeof(int))
+
+  int MPI_Reduce_local2(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Reduce_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Reduce_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Reduce Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+
+  int MPI_Reduce_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
         {
-          for(int i=1; i<num_ep; i++)
-            reduce_max<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);
+          #pragma omp flush
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+          }
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
         }
-
-        else if(datasize == sizeof(float))
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Reduce_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
         {
-          for(int i=1; i<num_ep; i++)
-            reduce_max<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);
+          #pragma omp flush
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+          }
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
         }
-
-        else if(datasize == sizeof(double))
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
         {
-          for(int i=1; i<num_ep; i++)
-            reduce_max<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
         }
-
-        else if(datasize == sizeof(char))
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
         {
-          for(int i=1; i<num_ep; i++)
-            reduce_max<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
         }
-
-        else if(datasize == sizeof(long))
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
         {
-          for(int i=1; i<num_ep; i++)
-            reduce_max<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
         }
-
-        else if(datasize == sizeof(unsigned long))
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
         {
-          for(int i=1; i<num_ep; i++)
-            reduce_max<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
         }
-
-        else printf("datatype Error\n");
-
-      }
-
-      if(op == MPI_MIN)
-      {
-        if(datasize == sizeof(int))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_min<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(float))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_min<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(double))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_min<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(char))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_min<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(long))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_min<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(unsigned long))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_min<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);
-        }
-
-        else printf("datatype Error\n");
-
-      }
-
-
-      if(op == MPI_SUM)
-      {
-        if(datasize == sizeof(int))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_sum<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(float))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_sum<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(double))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_sum<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(char))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_sum<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(long))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_sum<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-        }
-
-        else if(datasize == sizeof(unsigned long))
-        {
-          for(int i=1; i<num_ep; i++)
-            reduce_sum<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);
-        }
-
-        else printf("datatype Error\n");
-
-      }
-    }
-
-    MPI_Barrier_local(comm);
-
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
   }
 
@@ -201 +445 @@
   int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)
   {
-
     if(!comm.is_ep && comm.mpi_comm)
     {
-      return ::MPI_Reduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root, to_mpi_comm(comm.mpi_comm));
-    }
-
-
-
-    int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
-    int ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+      ::MPI_Reduce(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), root,
+                   static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+
+    if(!comm.mpi_comm) return 0;
 
     int root_mpi_rank = comm.rank_map->at(root).second;
     int root_ep_loc = comm.rank_map->at(root).first;
 
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint recvsize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &recvsize);
+
+    void *local_recvbuf;
+    if(ep_rank_loc==0)
+    {
+      local_recvbuf = new void*[recvsize*count];
+    }
+
+    MPI_Reduce_local2(sendbuf, local_recvbuf, count, datatype, op, comm);
+
+
+    if(ep_rank_loc==0)
+    {
+      ::MPI_Reduce(local_recvbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      innode_memcpy(0, recvbuf, root_ep_loc, recvbuf, count, datatype, comm);
+    }
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT) delete[] static_cast<int*>(local_recvbuf);
+      else if(datatype == MPI_FLOAT) delete[] static_cast<float*>(local_recvbuf);
+      else if(datatype == MPI_DOUBLE) delete[] static_cast<double*>(local_recvbuf);
+      else if(datatype == MPI_LONG) delete[] static_cast<long*>(local_recvbuf);
+      else if(datatype == MPI_UNSIGNED_LONG) delete[] static_cast<unsigned long*>(local_recvbuf);
+      else delete[] static_cast<char*>(local_recvbuf);
+    }
+
+    Message_Check(comm);
+
+    return 0;
+  }
+
+
+
+
+  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Allreduce(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op),
+                      static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint recvsize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &recvsize);
+
+    void *local_recvbuf;
+    if(ep_rank_loc==0)
+    {
+      local_recvbuf = new void*[recvsize*count];
+    }
+
+    MPI_Reduce_local2(sendbuf, local_recvbuf, count, datatype, op, comm);
+
+
+    if(ep_rank_loc==0)
+    {
+      ::MPI_Allreduce(local_recvbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    MPI_Bcast_local2(recvbuf, count, datatype, comm);
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT) delete[] static_cast<int*>(local_recvbuf);
+      else if(datatype == MPI_FLOAT) delete[] static_cast<float*>(local_recvbuf);
+      else if(datatype == MPI_DOUBLE) delete[] static_cast<double*>(local_recvbuf);
+      else if(datatype == MPI_LONG) delete[] static_cast<long*>(local_recvbuf);
+      else if(datatype == MPI_UNSIGNED_LONG) delete[] static_cast<unsigned long*>(local_recvbuf);
+      else delete[] static_cast<char*>(local_recvbuf);
+    }
+
+    Message_Check(comm);
+
+    return 0;
+  }
+
+
+  int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op),
+                           static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+    void* local_buf;
+    void* local_buf2;
+    int local_buf_size = accumulate(recvcounts, recvcounts+ep_size, 0);
+    int local_buf2_size = accumulate(recvcounts+ep_rank-ep_rank_loc, recvcounts+ep_rank-ep_rank_loc+num_ep, 0);
+
     ::MPI_Aint datasize, lb;
 
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
 
-    bool is_master = (ep_rank_loc==0 && mpi_rank != root_mpi_rank ) || ep_rank == root;
-    bool is_root = ep_rank == root;
-
-    void* local_recvbuf;
-
-    if(is_master)
-    {
-      local_recvbuf = new void*[datasize * count];
-    }
-
-    if(mpi_rank == root_mpi_rank) MPI_Reduce_local(sendbuf, local_recvbuf, count, datatype, op, root_ep_loc, comm);
-    else                          MPI_Reduce_local(sendbuf, local_recvbuf, count, datatype, op, 0, comm);
-
-
-
-    if(is_master)
-    {
-      ::MPI_Reduce(local_recvbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root_mpi_rank, to_mpi_comm(comm.mpi_comm));
-      
-    }
-
-    if(is_master)
-    {
-      delete[] local_recvbuf;
-    }
-
-    MPI_Barrier_local(comm);
-  }
-
-
+    if(ep_rank_loc == 0)
+    {
+      local_buf = new void*[local_buf_size*datasize];
+      local_buf2 = new void*[local_buf2_size*datasize];
+    }
+    MPI_Reduce_local2(sendbuf, local_buf, local_buf_size, MPI_INT, op, comm);
+
+
+    if(ep_rank_loc == 0)
+    {
+      int local_recvcnt[mpi_size];
+      for(int i=0; i<mpi_size; i++)
+      {
+        local_recvcnt[i] = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+      }
+
+      ::MPI_Reduce_scatter(local_buf, local_buf2, local_recvcnt, static_cast< ::MPI_Datatype>(datatype),
+                         static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    int displs[num_ep];
+    displs[0] = 0;
+    for(int i=1; i<num_ep; i++)
+    {
+      displs[i] = displs[i-1] + recvcounts[ep_rank-ep_rank_loc+i-1];
+    }
+
+    MPI_Scatterv_local2(local_buf2, recvcounts+ep_rank-ep_rank_loc, displs, datatype, recvbuf, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_buf);
+        delete[] static_cast<int*>(local_buf2);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_buf);
+        delete[] static_cast<float*>(local_buf2);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_buf);
+        delete[] static_cast<double*>(local_buf2);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_buf);
+        delete[] static_cast<long*>(local_buf2);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_buf);
+        delete[] static_cast<unsigned long*>(local_buf2);
+      }
+      else // if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<char*>(local_buf);
+        delete[] static_cast<char*>(local_buf2);
+      }
+    }
+
+    Message_Check(comm);
+    return 0;
+  }
 }
 

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_scan.cpp

@@ -9 +9 @@
 #include <mpi.h>
 #include "ep_declaration.hpp"
-#include "ep_mpi.hpp"
 
 using namespace std;
@@ -27 +26 @@
   }
 
-  template<typename T>
-  void reduce_max(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, max_op<T>);
-  }
-
-  template<typename T>
-  void reduce_min(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, min_op<T>);
-  }
-
-  template<typename T>
-  void reduce_sum(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, std::plus<T>());
-  }
-
-
-  int MPI_Scan_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-  {
-    valid_op(op);
-
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
-    
+
+  int MPI_Scan_local2(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Scan_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Scan_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Scan_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Scan_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Scan_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Scan_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Scan Datatype not supported!\n");
+      exit(0);
+    }
+
+  }
+
+
+
+
+  int MPI_Scan_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+
+  }
+
+  int MPI_Scan_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+
+      ::MPI_Scan(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
+                 static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
 
     ::MPI_Aint datasize, lb;
-    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
-
-    if(ep_rank_loc == 0 && mpi_rank != 0)
-    {
-      if(op == MPI_SUM)
-      {
-        if(datatype == MPI_INT && datasize == sizeof(int))
-          reduce_sum<int>(static_cast<int*>(const_cast<void*>(sendbuf)), static_cast<int*>(recvbuf), count);    
-          
-        else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-          reduce_sum<float>(static_cast<float*>(const_cast<void*>(sendbuf)), static_cast<float*>(recvbuf), count);    
-             
-        else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-          reduce_sum<double>(static_cast<double*>(const_cast<void*>(sendbuf)), static_cast<double*>(recvbuf), count);
-      
-        else if(datatype == MPI_CHAR && datasize == sizeof(char))
-          reduce_sum<char>(static_cast<char*>(const_cast<void*>(sendbuf)), static_cast<char*>(recvbuf), count);
-      
-        else if(datatype == MPI_LONG && datasize == sizeof(long))
-          reduce_sum<long>(static_cast<long*>(const_cast<void*>(sendbuf)), static_cast<long*>(recvbuf), count);
-            
-        else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-          reduce_sum<unsigned long>(static_cast<unsigned long*>(const_cast<void*>(sendbuf)), static_cast<unsigned long*>(recvbuf), count);    
-            
-        else printf("datatype Error\n");
-      }
-
-      else if(op == MPI_MAX)
-      {
-        if(datatype == MPI_INT && datasize == sizeof(int))
-          reduce_max<int>(static_cast<int*>(const_cast<void*>(sendbuf)), static_cast<int*>(recvbuf), count);    
-          
-        else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-          reduce_max<float>(static_cast<float*>(const_cast<void*>(sendbuf)), static_cast<float*>(recvbuf), count);    
-             
-        else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-          reduce_max<double>(static_cast<double*>(const_cast<void*>(sendbuf)), static_cast<double*>(recvbuf), count);
-      
-        else if(datatype == MPI_CHAR && datasize == sizeof(char))
-          reduce_max<char>(static_cast<char*>(const_cast<void*>(sendbuf)), static_cast<char*>(recvbuf), count);
-      
-        else if(datatype == MPI_LONG && datasize == sizeof(long))
-          reduce_max<long>(static_cast<long*>(const_cast<void*>(sendbuf)), static_cast<long*>(recvbuf), count);
-            
-        else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-          reduce_max<unsigned long>(static_cast<unsigned long*>(const_cast<void*>(sendbuf)), static_cast<unsigned long*>(recvbuf), count);    
-            
-        else printf("datatype Error\n");
-      }
-
-      else //(op == MPI_MIN)
-      {
-        if(datatype == MPI_INT && datasize == sizeof(int))
-          reduce_min<int>(static_cast<int*>(const_cast<void*>(sendbuf)), static_cast<int*>(recvbuf), count);    
-          
-        else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-          reduce_min<float>(static_cast<float*>(const_cast<void*>(sendbuf)), static_cast<float*>(recvbuf), count);    
-             
-        else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-          reduce_min<double>(static_cast<double*>(const_cast<void*>(sendbuf)), static_cast<double*>(recvbuf), count);
-      
-        else if(datatype == MPI_CHAR && datasize == sizeof(char))
-          reduce_min<char>(static_cast<char*>(const_cast<void*>(sendbuf)), static_cast<char*>(recvbuf), count);
-      
-        else if(datatype == MPI_LONG && datasize == sizeof(long))
-          reduce_min<long>(static_cast<long*>(const_cast<void*>(sendbuf)), static_cast<long*>(recvbuf), count);
-            
-        else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-          reduce_min<unsigned long>(static_cast<unsigned long*>(const_cast<void*>(sendbuf)), static_cast<unsigned long*>(recvbuf), count);    
-            
-        else printf("datatype Error\n");
-      }
-
-      comm.my_buffer->void_buffer[0] = recvbuf;
-    }
-    else
-    {
-      comm.my_buffer->void_buffer[ep_rank_loc] = const_cast<void*>(sendbuf);  
-      memcpy(recvbuf, sendbuf, datasize*count);
-    } 
-      
-
-
-    MPI_Barrier_local(comm);
-
-    memcpy(recvbuf, comm.my_buffer->void_buffer[0], datasize*count);
-
-
-    if(op == MPI_SUM)
-    {
-      if(datatype == MPI_INT && datasize == sizeof(int))
-      {
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_sum<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);    
-      }
-     
-      else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-      {
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_sum<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);    
-      }
-      
-
-      else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-      {
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_sum<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-      }
-
-      else if(datatype == MPI_CHAR && datasize == sizeof(char))
-      {
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_sum<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-      }
-
-      else if(datatype == MPI_LONG && datasize == sizeof(long))
-      {
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_sum<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-      }
-
-      else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-      {
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_sum<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);    
-      }
-
-      else printf("datatype Error\n");
-
-      
-    }
-
-    else if(op == MPI_MAX)
-    {
-      if(datatype == MPI_INT && datasize == sizeof(int))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_max<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);    
-
-      else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_max<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);    
-
-      else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_max<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-
-      else if(datatype == MPI_CHAR && datasize == sizeof(char))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_max<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-
-      else if(datatype == MPI_LONG && datasize == sizeof(long))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_max<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-
-      else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_max<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);    
-     
-      else printf("datatype Error\n");
-    }
-
-    else //if(op == MPI_MIN)
-    {
-      if(datatype == MPI_INT && datasize == sizeof(int))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_min<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);    
-
-      else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_min<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);    
-
-      else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_min<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-
-      else if(datatype == MPI_CHAR && datasize == sizeof(char))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_min<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-
-      else if(datatype == MPI_LONG && datasize == sizeof(long))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_min<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-
-      else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-        for(int i=1; i<ep_rank_loc+1; i++)
-          reduce_min<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);    
-
-      else printf("datatype Error\n");
-    }
-
-    MPI_Barrier_local(comm);
-
-  }
-
-
-  int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-  {
-    if(!comm.is_ep)
-    {
-      return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm));
-    }
-    
-    valid_type(datatype);
-
-    int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
-    int ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
-
-    ::MPI_Aint datasize, lb;
-    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
-    
-    void* tmp_sendbuf;
-    tmp_sendbuf = new void*[datasize * count];
-
-    int my_src = 0;
-    int my_dst = ep_rank;
-
-    std::vector<int> my_map(mpi_size, 0);
-
-    for(int i=0; i<comm.rank_map->size(); i++) my_map[comm.rank_map->at(i).second]++;
-
-    for(int i=0; i<mpi_rank; i++) my_src += my_map[i];
-    my_src += ep_rank_loc;
-
-     
-    for(int i=0; i<mpi_size; i++)
-    {
-      if(my_dst < my_map[i])
-      {
-        my_dst = get_ep_rank(comm, my_dst, i); 
-        break;
-      }
-      else
-        my_dst -= my_map[i];
-    }
-
-    //printf("ID = %d : send to %d, recv from %d\n", ep_rank, my_dst, my_src);
-    MPI_Barrier(comm);
-
-    if(my_dst == ep_rank && my_src == ep_rank) memcpy(tmp_sendbuf, sendbuf, datasize*count);
-
-    if(ep_rank != my_dst) 
-    {
-      MPI_Request request[2];
-      MPI_Status status[2];
-
-      MPI_Isend(sendbuf,     count, datatype, my_dst, my_dst,  comm, &request[0]);
-    
-      MPI_Irecv(tmp_sendbuf, count, datatype, my_src, ep_rank, comm, &request[1]);
-    
-      MPI_Waitall(2, request, status);
-    }
-    
-
-    void* tmp_recvbuf;
-    tmp_recvbuf = new void*[datasize * count];    
-
-    MPI_Reduce_local(tmp_sendbuf, tmp_recvbuf, count, datatype, op, 0, comm);
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void* local_scan_recvbuf;
+    local_scan_recvbuf = new void*[datasize * count];
+
+
+    // local scan
+    MPI_Scan_local2(sendbuf, recvbuf, count, datatype, op, comm);
+
+//     MPI_scan
+    void* local_sum;
+    void* mpi_scan_recvbuf;
+
+
+    mpi_scan_recvbuf = new void*[datasize*count];
 
     if(ep_rank_loc == 0)
-      ::MPI_Exscan(MPI_IN_PLACE, tmp_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm));
-
-    //printf(" ID=%d : %d  %d \n", ep_rank, static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]);
-    
-    MPI_Scan_local(tmp_sendbuf, tmp_recvbuf, count, datatype, op, comm);
-
-    // printf(" ID=%d : after local tmp_sendbuf = %d %d ; tmp_recvbuf = %d  %d \n", ep_rank, static_cast<int*>(tmp_sendbuf)[0], static_cast<int*>(tmp_sendbuf)[1], static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]);
-
-
-
-    if(ep_rank != my_src) 
-    {
-      MPI_Request request[2];
-      MPI_Status status[2];
-
-      MPI_Isend(tmp_recvbuf, count, datatype, my_src, my_src,  comm, &request[0]);
-    
-      MPI_Irecv(recvbuf,     count, datatype, my_dst, ep_rank, comm, &request[1]);
-    
-      MPI_Waitall(2, request, status);
-    }
-
-    else memcpy(recvbuf, tmp_recvbuf, datasize*count);
-    
-
-
-
-    delete[] tmp_sendbuf;
-    delete[] tmp_recvbuf;
+    {
+      local_sum = new void*[datasize*count];
+    }
+
+
+    MPI_Reduce_local2(sendbuf, local_sum, count, datatype, op, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      ::MPI_Exscan(local_sum, mpi_scan_recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    if(mpi_rank > 0)
+    {
+      MPI_Bcast_local2(mpi_scan_recvbuf, count, datatype, comm);
+    }
+
+
+    if(datatype == MPI_DOUBLE)
+    {
+      double* sum_buf = static_cast<double*>(mpi_scan_recvbuf);
+      double* recv_buf = static_cast<double*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<double*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<double*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_FLOAT)
+    {
+      float* sum_buf = static_cast<float*>(mpi_scan_recvbuf);
+      float* recv_buf = static_cast<float*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<float*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<float*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_INT)
+    {
+      int* sum_buf = static_cast<int*>(mpi_scan_recvbuf);
+      int* recv_buf = static_cast<int*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<int*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<int*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_LONG)
+    {
+      long* sum_buf = static_cast<long*>(mpi_scan_recvbuf);
+      long* recv_buf = static_cast<long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* sum_buf = static_cast<unsigned long*>(mpi_scan_recvbuf);
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<unsigned long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<unsigned long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_CHAR)
+    {
+      char* sum_buf = static_cast<char*>(mpi_scan_recvbuf);
+      char* recv_buf = static_cast<char*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<char*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<char*>(local_sum);
+      }
+    }
+
 
   }

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_scatter.cpp

@@ -9 +9 @@
 #include <mpi.h>
 #include "ep_declaration.hpp"
-#include "ep_mpi.hpp"
 
 using namespace std;
@@ -16 +15 @@
 {
 
-  int MPI_Scatter_local(void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int local_root, MPI_Comm comm)
-  {
-    assert(valid_type(sendtype) && valid_type(recvtype));
-    assert(recvcount == sendcount);
-
-    ::MPI_Aint datasize, lb;
-    ::MPI_Type_get_extent(to_mpi_type(sendtype), &lb, &datasize);
-
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-
-
-    if(ep_rank_loc == local_root)
-      comm.my_buffer->void_buffer[local_root] = const_cast<void*>(sendbuf);
-
-    MPI_Barrier_local(comm);
-
-    #pragma omp critical (_scatter)      
-    memcpy(recvbuf, comm.my_buffer->void_buffer[local_root]+datasize*ep_rank_loc*sendcount, datasize * recvcount);
-    
-
-    MPI_Barrier_local(comm);
-  }
+  int MPI_Scatter_local2(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Scatter_local_int(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Scatter_local_float(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Scatter_local_double(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Scatter_local_long(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Scatter_local_ulong(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Scatter_local_char(sendbuf, count, recvbuf, comm);
+    }
+    else
+    {
+      printf("MPI_Scatter Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Scatter_local_int(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatter_local_float(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatter_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatter_local_long(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatter_local_ulong(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatter_local_char(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+
+
 
   int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm)
@@ -44 +287 @@
     if(!comm.is_ep)
     {
-      return ::MPI_Scatter(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm.mpi_comm));
-    }
-   
-    assert(sendcount == recvcount);
-
-    int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
-    int ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+      ::MPI_Scatter(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+                    root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype) && sendcount == recvcount);
 
     int root_mpi_rank = comm.rank_map->at(root).second;
     int root_ep_loc = comm.rank_map->at(root).first;
 
-    bool is_master = (ep_rank_loc==0 && mpi_rank != root_mpi_rank ) || ep_rank == root;
-    bool is_root = ep_rank == root;
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
 
     MPI_Datatype datatype = sendtype;