Changeset 1289 for XIOS/dev/branch_openmp/extern/src_ep_dev/ep_exscan.cpp

Timestamp:

10/04/17 17:02:13 (7 years ago)

Author:

yushan

Message:

EP update part 2

File:

• XIOS/dev/branch_openmp/extern/src_ep_dev/ep_exscan.cpp (modified) (2 diffs)

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_exscan.cpp

@@ -9 +9 @@
 #include <mpi.h>
 #include "ep_declaration.hpp"
-#include "ep_mpi.hpp"
 
 using namespace std;
@@ -27 +26 @@
   }
 
-  template<typename T>
-  void reduce_max(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, max_op<T>);
-  }
-
-  template<typename T>
-  void reduce_min(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, min_op<T>);
-  }
-
-  template<typename T>
-  void reduce_sum(const T * buffer, T* recvbuf, int count)
-  {
-    transform(buffer, buffer+count, recvbuf, recvbuf, std::plus<T>());
-  }
-
-
-  int MPI_Exscan_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-  {
-    valid_op(op);
-
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+  int MPI_Exscan_local2(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Exscan_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Exscan_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Exscan_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Exscan_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Exscan_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Exscan_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Exscan Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+
+
+
+  int MPI_Exscan_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+
+  }
+
+  int MPI_Exscan_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Exscan(const_cast<void*>(sendbuf), recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
+                   static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    ::MPI_Aint datasize, lb;
     
-
-    ::MPI_Aint datasize, lb;
-    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
-
-    if(ep_rank_loc == 0 && mpi_rank != 0)
-    {
-      comm.my_buffer->void_buffer[0] = recvbuf;
-    }
-    if(ep_rank_loc == 0 && mpi_rank == 0)
-    {
-      comm.my_buffer->void_buffer[0] = const_cast<void*>(sendbuf);  
-    } 
-      
-
-    MPI_Barrier_local(comm);
-
-    memcpy(recvbuf, comm.my_buffer->void_buffer[0], datasize*count);
-
-    MPI_Barrier_local(comm);
-
-    comm.my_buffer->void_buffer[ep_rank_loc] = const_cast<void*>(sendbuf);  
-    
-    MPI_Barrier_local(comm);
-
-    if(op == MPI_SUM)
-    {
-      if(datatype == MPI_INT && datasize == sizeof(int))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);    
-      }
-     
-      else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);    
-      }
-      
-
-      else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-      }
-
-      else if(datatype == MPI_CHAR && datasize == sizeof(char))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-      }
-
-      else if(datatype == MPI_LONG && datasize == sizeof(long))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-      }
-
-      else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-      {
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_sum<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);    
-      }
-
-      else printf("datatype Error\n");
-
-      
-    }
-
-    else if(op == MPI_MAX)
-    {
-      if(datatype == MPI_INT && datasize == sizeof(int))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);    
-
-      else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);    
-
-      else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-
-      else if(datatype == MPI_CHAR && datasize == sizeof(char))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-
-      else if(datatype == MPI_LONG && datasize == sizeof(long))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-
-      else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_max<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);    
-     
-      else printf("datatype Error\n");
-    }
-
-    else //if(op == MPI_MIN)
-    {
-      if(datatype == MPI_INT && datasize == sizeof(int))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count);    
-
-      else if(datatype == MPI_FLOAT && datasize == sizeof(float))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<float>(static_cast<float*>(comm.my_buffer->void_buffer[i]), static_cast<float*>(recvbuf), count);    
-
-      else if(datatype == MPI_DOUBLE && datasize == sizeof(double))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<double>(static_cast<double*>(comm.my_buffer->void_buffer[i]), static_cast<double*>(recvbuf), count);
-
-      else if(datatype == MPI_CHAR && datasize == sizeof(char))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<char>(static_cast<char*>(comm.my_buffer->void_buffer[i]), static_cast<char*>(recvbuf), count);
-
-      else if(datatype == MPI_LONG && datasize == sizeof(long))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<long>(static_cast<long*>(comm.my_buffer->void_buffer[i]), static_cast<long*>(recvbuf), count);
-
-      else if(datatype == MPI_UNSIGNED_LONG && datasize == sizeof(unsigned long))
-        for(int i=0; i<ep_rank_loc; i++)
-          reduce_min<unsigned long>(static_cast<unsigned long*>(comm.my_buffer->void_buffer[i]), static_cast<unsigned long*>(recvbuf), count);    
-
-      else printf("datatype Error\n");
-    }
-
-    MPI_Barrier_local(comm);
-
-  }
-
-  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
-  {
-    if(!comm.is_ep)
-    {
-      return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm));
-    }
-    
-    valid_type(datatype);
-
-    int ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
-    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
-    int mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
-    int ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
-    int num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
-    int mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
-
-    ::MPI_Aint datasize, lb;
-    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
-    
-    void* tmp_sendbuf;
-    tmp_sendbuf = new void*[datasize * count];
-
-    int my_src = 0;
-    int my_dst = ep_rank;
-
-    std::vector<int> my_map(mpi_size, 0);
-
-    for(int i=0; i<comm.rank_map->size(); i++) my_map[comm.rank_map->at(i).second]++;
-
-    for(int i=0; i<mpi_rank; i++) my_src += my_map[i];
-    my_src += ep_rank_loc;
-
-     
-    for(int i=0; i<mpi_size; i++)
-    {
-      if(my_dst < my_map[i])
-      {
-        my_dst = get_ep_rank(comm, my_dst, i); 
-        break;
-      }
-      else
-        my_dst -= my_map[i];
-    }
-
-    if(ep_rank != my_dst) 
-    {
-      MPI_Request request[2];
-      MPI_Status status[2];
-
-      MPI_Isend(sendbuf,     count, datatype, my_dst, my_dst,  comm, &request[0]);
-    
-      MPI_Irecv(tmp_sendbuf, count, datatype, my_src, ep_rank, comm, &request[1]);
-    
-      MPI_Waitall(2, request, status);
-    }
-
-    else memcpy(tmp_sendbuf, sendbuf, datasize*count);
-    
-
-    void* tmp_recvbuf;
-    tmp_recvbuf = new void*[datasize * count];    
-
-    MPI_Reduce_local(tmp_sendbuf, tmp_recvbuf, count, datatype, op, 0, comm);
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void* local_scan_recvbuf;
+    local_scan_recvbuf = new void*[datasize * count];
+
+
+    // local scan
+    MPI_Exscan_local2(sendbuf, recvbuf, count, datatype, op, comm);
+
+//     MPI_scan
+    void* local_sum;
+    void* mpi_scan_recvbuf;
+
+
+    mpi_scan_recvbuf = new void*[datasize*count];
 
     if(ep_rank_loc == 0)
-      ::MPI_Exscan(MPI_IN_PLACE, tmp_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm.mpi_comm));
-
-    // printf(" ID=%d : %d  %d \n", ep_rank, static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]);
-    
-    MPI_Exscan_local(tmp_sendbuf, tmp_recvbuf, count, datatype, op, comm);
-
-     // printf(" ID=%d : after local tmp_sendbuf = %d %d ; tmp_recvbuf = %d  %d \n", ep_rank, static_cast<int*>(tmp_sendbuf)[0], static_cast<int*>(tmp_sendbuf)[1], static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]);
-
-
-
-    if(ep_rank != my_src) 
-    {
-      MPI_Request request[2];
-      MPI_Status status[2];
-
-      MPI_Isend(tmp_recvbuf, count, datatype, my_src, my_src,  comm, &request[0]);
-    
-      MPI_Irecv(recvbuf,     count, datatype, my_dst, ep_rank, comm, &request[1]);
-    
-      MPI_Waitall(2, request, status);
-    }
-
-    else memcpy(recvbuf, tmp_recvbuf, datasize*count);
-    
-
-
-
-    delete[] tmp_sendbuf;
-    delete[] tmp_recvbuf;
-
-  }
+    {
+      local_sum = new void*[datasize*count];
+    }
+
+
+    MPI_Reduce_local2(sendbuf, local_sum, count, datatype, op, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      ::MPI_Exscan(local_sum, mpi_scan_recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    if(mpi_rank > 0)
+    {
+      MPI_Bcast_local2(mpi_scan_recvbuf, count, datatype, comm);
+    }
+
+
+    if(datatype == MPI_DOUBLE)
+    {
+      double* sum_buf = static_cast<double*>(mpi_scan_recvbuf);
+      double* recv_buf = static_cast<double*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<double*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<double*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_FLOAT)
+    {
+      float* sum_buf = static_cast<float*>(mpi_scan_recvbuf);
+      float* recv_buf = static_cast<float*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<float*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<float*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_INT)
+    {
+      int* sum_buf = static_cast<int*>(mpi_scan_recvbuf);
+      int* recv_buf = static_cast<int*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<int*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<int*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_CHAR)
+    {
+      char* sum_buf = static_cast<char*>(mpi_scan_recvbuf);
+      char* recv_buf = static_cast<char*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<char*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<char*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_LONG)
+    {
+      long* sum_buf = static_cast<long*>(mpi_scan_recvbuf);
+      long* recv_buf = static_cast<long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* sum_buf = static_cast<unsigned long*>(mpi_scan_recvbuf);
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<unsigned long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<unsigned long*>(local_sum);
+      }
+    }
+
+
+  }
+
+
 
 }