Index: XIOS/trunk/extern/src_ep/ep_alltoall.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_alltoall.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_alltoall.cpp	(revision 1034)
@@ -0,0 +1,25 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+//#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)
+  {
+    ::MPI_Aint typesize, llb;
+    ::MPI_Type_get_extent(sendtype, &llb, &typesize);
+
+    for(int i=0; i<comm.ep_comm_ptr->size_rank_info[0].second; i++)
+    {
+      MPI_Gather((char*)sendbuf+i*sendcount*typesize, sendcount, sendtype, recvbuf, recvcount, recvtype, i, comm);
+    }
+
+    return 0;
+  }
+
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_barrier.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_barrier.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_barrier.cpp	(revision 1034)
@@ -0,0 +1,48 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+//#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Barrier(MPI_Comm comm)
+  {
+
+    if(comm.is_ep)
+    {
+      int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+
+      MPI_Barrier_local(comm);
+
+      if(ep_rank_loc == 0)
+      {
+        ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+        ::MPI_Barrier(mpi_comm);
+      }
+
+      MPI_Barrier_local(comm);
+
+      return 0;
+    }
+    else if(comm.mpi_comm !=0 )
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+      ::MPI_Barrier(mpi_comm);
+      return 0;
+    }
+
+    else return 0;
+  }
+
+  int MPI_Barrier_local(MPI_Comm comm)
+  {
+    //Message_Check(comm);
+    comm.ep_barrier->wait();
+  }
+
+
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_bcast.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_bcast.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_bcast.cpp	(revision 1034)
@@ -0,0 +1,307 @@
+/*!
+   \file ep_bcast.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Bcast
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib
+{
+  int MPI_Bcast_local(void *buffer, int count, MPI_Datatype datatype, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Bcast_local_int(buffer, count, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Bcast_local_float(buffer, count, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Bcast_local_double(buffer, count, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Bcast_local_char(buffer, count, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Bcast_local_long(buffer, count, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Bcast_local_char(buffer, count, comm);
+    }
+    else
+    {
+      printf("MPI_Bcast Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Bcast_local_int(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *tmp = static_cast<int*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_float(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *tmp = static_cast<float*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_double(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *tmp = static_cast<double*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Bcast_local_char(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *tmp = static_cast<char*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_long(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *tmp = static_cast<long*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_ulong(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *tmp = static_cast<unsigned long*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Bcast(buffer, count, static_cast< ::MPI_Datatype>(datatype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_rank_loc = comm.rank_map->at(root).first;
+
+
+    // if root is not master thread, send first to master
+    if(root_ep_rank_loc != 0 && mpi_rank == root_mpi_rank)
+    {
+      innode_memcpy(root_ep_rank_loc, buffer, 0, buffer, count, datatype, comm);
+    }
+
+
+    if(ep_rank_loc==0)
+    {
+      ::MPI_Bcast(buffer, count, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    MPI_Bcast_local(buffer, count, datatype, comm);
+
+    return 0;
+  }
+
+
+}
Index: XIOS/trunk/extern/src_ep/ep_create.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_create.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_create.cpp	(revision 1034)
@@ -0,0 +1,282 @@
+/*!
+   \file ep_create.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI endpoint function: MPI_Comm_create_endpoints
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+
+namespace ep_lib {
+
+  /*!
+    Dynamic creation of endpoints for each MPI process.
+    The output of this function is an array of communicator handles of length num_ep, where each handle
+    corresponds to a nez local randk in the output communicator.
+    Once created, endpoints behave as MPI processes.
+    \param [in] mpi_comm Parent MPI communicator.
+    \param [in] num_ep Number of endpoints per process.
+    \param [out] info Information of the EP creation.
+    \param [out] out_comm_hdls Handles of EP communicators.
+  */
+  #ifdef _intelmpi
+  int MPI_Comm_create_endpoints(int base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls)
+  {
+    int base_rank;
+    int base_size;
+
+    ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm_ptr);
+
+    ::MPI_Comm_size(mpi_base_comm, &base_size);  // ep_lib::mpi_comm_size
+    ::MPI_Comm_rank(mpi_base_comm, &base_rank);  // ep_lib::mpi_comm_rank
+									  // parent_comm can also be endpoints communicators
+
+    std::vector<int> recv_num_ep(base_size);
+
+    out_comm_hdls = new MPI_Comm[num_ep];
+
+    for (int idx = 0; idx < num_ep; ++idx)
+    {
+      out_comm_hdls[idx].is_ep = true;
+      out_comm_hdls[idx].is_intercomm = false;
+      out_comm_hdls[idx].ep_comm_ptr = new ep_communicator;
+      out_comm_hdls[idx].mpi_comm = base_comm_ptr;
+      out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls;
+      out_comm_hdls[idx].ep_comm_ptr->comm_label = 0;
+    }
+
+    ::MPI_Allgather(&num_ep, 1, MPI_INT_STD, &recv_num_ep[0], 1, MPI_INT_STD, mpi_base_comm);
+
+
+    int sum = 0;  // representing total ep number of process with smaller rank
+    for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; }
+
+    int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0);
+
+    out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep);
+
+    out_comm_hdls[0].my_buffer = new BUFFER;
+    out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
+
+    out_comm_hdls[0].rank_map = new RANK_MAP;
+    out_comm_hdls[0].rank_map->resize(ep_size);
+
+
+    for (int i = 1; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier;
+      out_comm_hdls[i].my_buffer  = out_comm_hdls[0].my_buffer;
+      out_comm_hdls[i].rank_map   = out_comm_hdls[0].rank_map;
+    }
+
+
+    for (int i = 0; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size);
+
+      out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list;
+    }
+
+
+    int ind = 0;
+
+    for(int i=0; i<base_size; i++)
+    {
+      for(int j=0; j<recv_num_ep[i]; j++)
+      {
+        out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i);
+        ind++;
+      }
+    }
+
+    printf("ep_lib::MPI_Comm_create_endpoints()       OK from int\n");
+
+    return 0;
+
+  } //MPI_Comm_create_endpoints
+
+  #elif _openmpi
+  int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls)
+  {
+
+    int base_rank;
+    int base_size;
+
+    ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm_ptr);
+
+    ::MPI_Comm_size(mpi_base_comm, &base_size);  // ep_lib::mpi_comm_size
+    ::MPI_Comm_rank(mpi_base_comm, &base_rank);  // ep_lib::mpi_comm_rank
+                                                 // parent_comm can also be endpoints communicators ?
+    std::vector<int> recv_num_ep(base_size);
+
+    out_comm_hdls = new MPI_Comm[num_ep];
+
+    for (int idx = 0; idx < num_ep; ++idx)
+    {
+      out_comm_hdls[idx].is_ep = true;
+      out_comm_hdls[idx].is_intercomm = false;
+      out_comm_hdls[idx].ep_comm_ptr = new ep_communicator;
+      out_comm_hdls[idx].mpi_comm = base_comm_ptr;
+      out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls;
+      out_comm_hdls[idx].ep_comm_ptr->comm_label = 0;
+    }
+
+    ::MPI_Allgather(&num_ep, 1, MPI_INT_STD, &recv_num_ep[0], 1, MPI_INT_STD, mpi_base_comm);
+
+    int sum = 0;  // representing total ep number of process with smaller rank
+    for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; }
+
+    int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0);
+
+    out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep);
+
+    out_comm_hdls[0].my_buffer = new BUFFER;
+    out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
+
+    out_comm_hdls[0].rank_map = new RANK_MAP;
+    out_comm_hdls[0].rank_map->resize(ep_size);
+
+
+    for (int i = 1; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier;
+      out_comm_hdls[i].my_buffer  = out_comm_hdls[0].my_buffer;
+      out_comm_hdls[i].rank_map   = out_comm_hdls[0].rank_map;
+    }
+
+
+    for (int i = 0; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size);
+
+      out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list;
+    }
+
+
+    int ind = 0;
+
+    for(int i=0; i<base_size; i++)
+    {
+      for(int j=0; j<recv_num_ep[i]; j++)
+      {
+        out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i);
+        ind++;
+      }
+    }
+
+    printf("ep_lib::MPI_Comm_create_endpoints()       OK from void*\n");
+
+    return 0;
+
+  } //MPI_Comm_create_endpoints
+
+  #endif
+  
+  
+  int MPI_Comm_create_endpoints(MPI_Comm base_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls)
+  {
+
+    int base_rank;
+    int base_size;
+    
+    assert(!base_comm.is_ep);
+
+    ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm.mpi_comm);
+
+    ::MPI_Comm_size(mpi_base_comm, &base_size);  // ep_lib::mpi_comm_size
+    ::MPI_Comm_rank(mpi_base_comm, &base_rank);  // ep_lib::mpi_comm_rank
+                    // parent_comm can also be endpoints communicators
+
+    std::vector<int> recv_num_ep(base_size);
+
+    out_comm_hdls = new MPI_Comm[num_ep];
+
+    for (int idx = 0; idx < num_ep; ++idx)
+    {
+      out_comm_hdls[idx].is_ep = true;
+      out_comm_hdls[idx].is_intercomm = false;
+      out_comm_hdls[idx].ep_comm_ptr = new ep_communicator;
+      out_comm_hdls[idx].mpi_comm = base_comm.mpi_comm;
+      out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls;
+      out_comm_hdls[idx].ep_comm_ptr->comm_label = 0;
+    }
+
+    ::MPI_Allgather(&num_ep, 1, MPI_INT_STD, &recv_num_ep[0], 1, MPI_INT_STD, mpi_base_comm);
+
+
+    int sum = 0;  // representing total ep number of process with smaller rank
+    for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; }
+
+    int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0);
+
+    out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep);
+
+    out_comm_hdls[0].my_buffer = new BUFFER;
+    out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
+
+    out_comm_hdls[0].rank_map = new RANK_MAP;
+    out_comm_hdls[0].rank_map->resize(ep_size);
+
+
+    for (int i = 1; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier;
+      out_comm_hdls[i].my_buffer  = out_comm_hdls[0].my_buffer;
+      out_comm_hdls[i].rank_map   = out_comm_hdls[0].rank_map;
+    }
+
+
+    for (int i = 0; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size);
+
+      out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list;
+    }
+
+
+    int ind = 0;
+
+    for(int i=0; i<base_size; i++)
+    {
+      for(int j=0; j<recv_num_ep[i]; j++)
+      {
+        out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i);
+        ind++;
+      }
+    }
+
+    printf("ep_lib::MPI_Comm_create_endpoints()       OK from MPI_Comm\n");
+
+    return 0;
+
+  } //MPI_Comm_create_endpoints
+
+
+} //namespace ep_lib
Index: XIOS/trunk/extern/src_ep/ep_declaration.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_declaration.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_declaration.cpp	(revision 1034)
@@ -0,0 +1,59 @@
+
+
+#include "ep_lib.hpp"
+
+#include <mpi.h>
+//#include "ep_declaration.hpp"
+
+#undef MPI_INT
+#undef MPI_FLOAT
+#undef MPI_DOUBLE
+#undef MPI_CHAR
+#undef MPI_LONG
+#undef MPI_UNSIGNED_LONG
+#undef MPI_UNSIGNED_CHAR
+
+#undef MPI_SUM
+#undef MPI_MAX
+#undef MPI_MIN
+
+#undef MPI_COMM_WORLD
+#undef MPI_COMM_NULL
+
+// _STD defined in ep_type.cpp
+
+extern ::MPI_Datatype MPI_INT_STD;
+extern ::MPI_Datatype MPI_FLOAT_STD;
+extern ::MPI_Datatype MPI_DOUBLE_STD;
+extern ::MPI_Datatype MPI_LONG_STD;
+extern ::MPI_Datatype MPI_CHAR_STD;
+extern ::MPI_Datatype MPI_UNSIGNED_LONG_STD;
+extern ::MPI_Datatype MPI_UNSIGNED_CHAR_STD;
+
+extern ::MPI_Op MPI_SUM_STD;
+extern ::MPI_Op MPI_MAX_STD;
+extern ::MPI_Op MPI_MIN_STD;
+
+extern ::MPI_Comm MPI_COMM_WORLD_STD;
+extern ::MPI_Comm MPI_COMM_NULL_STD;
+
+ep_lib::MPI_Datatype MPI_INT = MPI_INT_STD;
+ep_lib::MPI_Datatype MPI_FLOAT = MPI_FLOAT_STD;
+ep_lib::MPI_Datatype MPI_DOUBLE = MPI_DOUBLE_STD;
+ep_lib::MPI_Datatype MPI_CHAR = MPI_CHAR_STD;
+ep_lib::MPI_Datatype MPI_LONG = MPI_LONG_STD;
+ep_lib::MPI_Datatype MPI_UNSIGNED_LONG = MPI_UNSIGNED_LONG_STD;
+ep_lib::MPI_Datatype MPI_UNSIGNED_CHAR = MPI_UNSIGNED_CHAR_STD;
+
+ep_lib::MPI_Op MPI_SUM = MPI_SUM_STD;
+ep_lib::MPI_Op MPI_MAX = MPI_MAX_STD;
+ep_lib::MPI_Op MPI_MIN = MPI_MIN_STD;
+
+ep_lib::MPI_Comm MPI_COMM_WORLD(MPI_COMM_WORLD_STD);
+ep_lib::MPI_Comm MPI_COMM_NULL(MPI_COMM_NULL_STD);
+
+//ep_lib::MPI_Comm EP_COMM_WORLD;
+//ep_lib::MPI_Comm EP_COMM_NULL;
+
+
+
Index: XIOS/trunk/extern/src_ep/ep_declaration.hpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_declaration.hpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_declaration.hpp	(revision 1034)
@@ -0,0 +1,53 @@
+#ifndef EP_DECLARATION_HPP_INCLUDED
+#define EP_DECLARATION_HPP_INCLUDED
+
+extern ::MPI_Datatype MPI_INT_STD;
+extern ::MPI_Datatype MPI_FLOAT_STD;
+extern ::MPI_Datatype MPI_DOUBLE_STD;
+extern ::MPI_Datatype MPI_LONG_STD;
+extern ::MPI_Datatype MPI_CHAR_STD;
+extern ::MPI_Datatype MPI_UNSIGNED_LONG_STD;
+extern ::MPI_Datatype MPI_UNSIGNED_CHAR_STD;
+
+extern ::MPI_Op MPI_SUM_STD;
+extern ::MPI_Op MPI_MAX_STD;
+extern ::MPI_Op MPI_MIN_STD;
+
+extern ::MPI_Comm MPI_COMM_WORLD_STD;
+extern ::MPI_Comm MPI_COMM_NULL_STD;
+
+#undef MPI_INT
+#undef MPI_FLOAT
+#undef MPI_DOUBLE
+#undef MPI_CHAR
+#undef MPI_LONG
+#undef MPI_UNSIGNED_LONG
+#undef MPI_UNSIGNED_CHAR
+
+#undef MPI_SUM
+#undef MPI_MAX
+#undef MPI_MIN
+
+#undef MPI_COMM_WORLD
+#undef MPI_COMM_NULL
+
+extern ep_lib::MPI_Datatype MPI_INT;
+extern ep_lib::MPI_Datatype MPI_FLOAT;
+extern ep_lib::MPI_Datatype MPI_DOUBLE;
+extern ep_lib::MPI_Datatype MPI_CHAR;
+extern ep_lib::MPI_Datatype MPI_LONG;
+extern ep_lib::MPI_Datatype MPI_UNSIGNED_LONG;
+extern ep_lib::MPI_Datatype MPI_UNSIGNED_CHAR;
+
+extern ep_lib::MPI_Op MPI_SUM;
+extern ep_lib::MPI_Op MPI_MAX;
+extern ep_lib::MPI_Op MPI_MIN;
+
+extern ep_lib::MPI_Comm MPI_COMM_WORLD;
+extern ep_lib::MPI_Comm MPI_COMM_NULL;
+
+
+
+
+#endif // EP_DECLARATION_HPP_INCLUDED
+
Index: XIOS/trunk/extern/src_ep/ep_dup.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_dup.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_dup.cpp	(revision 1034)
@@ -0,0 +1,110 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Comm_dup_dev(MPI_Comm comm, MPI_Comm *newcomm)
+  {
+    if(!comm.is_ep)
+    {
+      Debug("Comm_dup MPI\n");
+      newcomm = new MPI_Comm;
+      newcomm->is_ep = comm.is_ep;
+
+      ::MPI_Comm input = static_cast< ::MPI_Comm>(comm.mpi_comm);
+      ::MPI_Comm output;
+
+      
+      ::MPI_Comm_dup(input, &output);
+
+      newcomm->mpi_comm = output;
+
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    if(0 == my_rank)
+    {
+      MPI_Info info;
+      ::MPI_Comm mpi_dup;
+
+      ::MPI_Comm in_comm = static_cast< ::MPI_Comm>(comm.mpi_comm);
+
+      ::MPI_Comm_dup(in_comm, &mpi_dup);
+
+      MPI_Comm_create_endpoints(mpi_dup, num_ep, info, newcomm);
+      comm.ep_comm_ptr->comm_list->mem_bridge = newcomm;
+    }
+
+    MPI_Barrier_local(comm);
+
+    newcomm = &(comm.ep_comm_ptr->comm_list->mem_bridge[my_rank]);
+
+    return MPI_SUCCESS;
+  }
+
+
+
+  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
+  {
+    if(!comm.is_ep)
+    {
+      Debug("Comm_dup MPI\n");
+      newcomm = new MPI_Comm;
+      newcomm->is_ep = comm.is_ep;
+
+      ::MPI_Comm input = static_cast< ::MPI_Comm>(comm.mpi_comm);
+      ::MPI_Comm output;
+
+      
+      ::MPI_Comm_dup(input, &output);
+
+      newcomm->mpi_comm = output;
+
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    if(0 == my_rank)
+    {
+      MPI_Info info;
+      MPI_Comm *out_comm;
+      ::MPI_Comm mpi_dup;
+
+      ::MPI_Comm in_comm = static_cast< ::MPI_Comm>(comm.mpi_comm);
+
+      ::MPI_Comm_dup(in_comm, &mpi_dup);
+
+      MPI_Comm_create_endpoints(mpi_dup, num_ep, info, out_comm);
+      comm.ep_comm_ptr->comm_list->mem_bridge = out_comm;
+    }
+
+    MPI_Barrier(comm);
+
+    *newcomm = (comm.ep_comm_ptr->comm_list->mem_bridge[my_rank]);
+
+    return MPI_SUCCESS;
+  }
+
+
+
+
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_exscan.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_exscan.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_exscan.cpp	(revision 1034)
@@ -0,0 +1,879 @@
+/*!
+   \file ep_scan.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Exscan
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+  template<typename T>
+  T max_op(T a, T b)
+  {
+    return max(a,b);
+  }
+
+  template<typename T>
+  T min_op(T a, T b)
+  {
+    return min(a,b);
+  }
+
+  int MPI_Exscan_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Exscan_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Exscan_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Exscan_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Exscan_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Exscan_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Exscan_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Exscan Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+
+
+
+  int MPI_Exscan_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+
+  }
+
+  int MPI_Exscan_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Exscan(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
+                   static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    ::MPI_Aint datasize, lb;
+    
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void* local_scan_recvbuf;
+    local_scan_recvbuf = new void*[datasize * count];
+
+
+    // local scan
+    MPI_Exscan_local(sendbuf, recvbuf, count, datatype, op, comm);
+
+//     MPI_scan
+    void* local_sum;
+    void* mpi_scan_recvbuf;
+
+
+    mpi_scan_recvbuf = new void*[datasize*count];
+
+    if(ep_rank_loc == 0)
+    {
+      local_sum = new void*[datasize*count];
+    }
+
+
+    MPI_Reduce_local(sendbuf, local_sum, count, datatype, op, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Exscan(local_sum, mpi_scan_recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    if(mpi_rank > 0)
+    {
+      MPI_Bcast_local(mpi_scan_recvbuf, count, datatype, comm);
+    }
+
+
+    if(datatype == MPI_DOUBLE)
+    {
+      double* sum_buf = static_cast<double*>(mpi_scan_recvbuf);
+      double* recv_buf = static_cast<double*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<double*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<double*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_FLOAT)
+    {
+      float* sum_buf = static_cast<float*>(mpi_scan_recvbuf);
+      float* recv_buf = static_cast<float*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<float*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<float*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_INT)
+    {
+      int* sum_buf = static_cast<int*>(mpi_scan_recvbuf);
+      int* recv_buf = static_cast<int*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<int*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<int*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_CHAR)
+    {
+      char* sum_buf = static_cast<char*>(mpi_scan_recvbuf);
+      char* recv_buf = static_cast<char*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<char*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<char*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_LONG)
+    {
+      long* sum_buf = static_cast<long*>(mpi_scan_recvbuf);
+      long* recv_buf = static_cast<long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* sum_buf = static_cast<unsigned long*>(mpi_scan_recvbuf);
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<unsigned long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<unsigned long*>(local_sum);
+      }
+    }
+
+
+  }
+
+
+
+}
Index: XIOS/trunk/extern/src_ep/ep_finalize.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_finalize.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_finalize.cpp	(revision 1034)
@@ -0,0 +1,43 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+namespace ep_lib
+{
+
+  int MPI_Finalize()
+  {
+    printf("calling EP Finalize\n");
+
+    int id = omp_get_thread_num();
+
+    if(id == 0)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Finalize();
+    }
+    return 0;
+  }
+
+  int MPI_Abort(MPI_Comm comm, int errorcode)
+  {
+    int id = omp_get_thread_num();
+
+    if(id == 0)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.mpi_comm);
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Abort(mpi_comm, errorcode);
+    }
+    return 0;
+  }
+
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_fortran.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_fortran.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_fortran.cpp	(revision 1034)
@@ -0,0 +1,61 @@
+//#include "ep_lib.hpp"
+#include "ep_lib_fortran.hpp"
+#include <mpi.h>
+#include <map>
+#include <utility>
+#undef MPI_Comm_f2c(comm)
+#undef MPI_Comm_c2f(comm)
+
+
+namespace ep_lib
+{
+
+
+  MPI_Fint MPI_Comm_c2f(MPI_Comm comm)
+  {
+    Debug("MPI_Comm_c2f");
+    int fint;
+    fint = (::MPI_Fint)(comm.mpi_comm);
+    
+    std::map<std::pair<int, int>, MPI_Comm  > ::iterator it;
+    
+    it = fc_comm_map.find(std::make_pair(fint, omp_get_thread_num()));
+    if(it == fc_comm_map.end())
+    {
+      fc_comm_map.insert(std::make_pair( std::make_pair( fint, omp_get_thread_num()) , comm));
+      printf("MAP insert: %d, %d, %p\n", fint, omp_get_thread_num(), &comm);
+    }
+    
+    MPI_Fint Fint;
+    Fint.mpi_fint = fint;
+    return Fint;
+    
+  }
+
+
+  MPI_Comm MPI_Comm_f2c(MPI_Fint comm)
+  {
+    Debug("MPI_Comm_f2c");
+    
+    
+    std::map<std::pair<int, int>, MPI_Comm  > ::iterator it;
+    
+    it = fc_comm_map.find(std::make_pair(comm.mpi_fint, omp_get_thread_num()));
+    if(it != fc_comm_map.end())
+    {
+      MPI_Comm comm_ptr;
+      comm_ptr =  it->second;
+      printf("MAP find: %d, %d, %p\n", it->first.first, it->first.second, &comm_ptr);
+      return  comm_ptr;
+    }
+    else
+    {      
+      MPI_Comm return_comm;
+      return_comm.mpi_comm = (::MPI_Comm)(comm.mpi_fint);
+      return return_comm;
+    }
+  }
+
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_free.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_free.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_free.cpp	(revision 1034)
@@ -0,0 +1,140 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Comm_free(MPI_Comm *comm)
+  {
+    //return 0;
+    
+    if(comm == NULL) return 0;
+
+    MPI_Barrier(*comm);
+
+
+    if(! comm->is_ep)
+    {
+      if(comm->mpi_comm)
+      {
+        ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->mpi_comm);
+
+        ::MPI_Comm_free(&mpi_comm);
+        Debug("comm is MPI, freed\n");
+      }
+      return 0;
+    }
+
+    else
+    {
+      if(! comm->mpi_comm) return 0;
+
+      int ep_rank, ep_rank_loc, mpi_rank;
+      int ep_size, num_ep, mpi_size;
+
+      ep_rank = comm->ep_comm_ptr->size_rank_info[0].first;
+      ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;
+      mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first;
+      ep_size = comm->ep_comm_ptr->size_rank_info[0].second;
+      num_ep = comm->ep_comm_ptr->size_rank_info[1].second;
+      mpi_size = comm->ep_comm_ptr->size_rank_info[2].second;
+
+      MPI_Barrier(*comm);
+
+
+      #pragma omp critical (memory_free)
+      if(comm->is_intercomm && comm->ep_comm_ptr->intercomm)
+      {
+        if(comm->ep_comm_ptr->intercomm->local_rank_map) comm->ep_comm_ptr->intercomm->local_rank_map->clear();
+        if(comm->ep_comm_ptr->intercomm->remote_rank_map) comm->ep_comm_ptr->intercomm->remote_rank_map->clear();
+        if(comm->ep_comm_ptr->intercomm->intercomm_rank_map) comm->ep_comm_ptr->intercomm->intercomm_rank_map->clear();
+        Debug("intercomm local/remote/intercomm_rank_map emptied\n");
+      }
+
+
+      if(ep_rank_loc == 0)
+      {
+        Debug("comm is EP, mpi_comm_ptr != NULL\n");
+
+
+/*
+        if(comm->my_buffer)
+        {
+          if(comm->my_buffer->buf_int) delete[] comm->my_buffer->buf_int; Debug("buf_int freed\n");
+          if(comm->my_buffer->buf_float) delete[] comm->my_buffer->buf_float; Debug("buf_float freed\n");
+          if(comm->my_buffer->buf_double) delete[] comm->my_buffer->buf_double; Debug("buf_double freed\n");
+          if(comm->my_buffer->buf_long) delete[] comm->my_buffer->buf_long; Debug("buf_long freed\n");
+          if(comm->my_buffer->buf_ulong) delete[] comm->my_buffer->buf_ulong; Debug("buf_ulong freed\n");
+          if(comm->my_buffer->buf_char) delete[] comm->my_buffer->buf_char; Debug("buf_char freed\n");
+        }
+*/
+        if(comm->ep_barrier)
+        {
+          comm->ep_barrier->~OMPbarrier();
+          Debug("ep_barrier freed\n");
+        }
+
+
+        if(comm->rank_map)
+        {
+          comm->rank_map->clear();
+          Debug("rank_map emptied\n");
+        }
+
+
+        if(comm->is_intercomm && comm->ep_comm_ptr->intercomm->mpi_inter_comm)
+        {
+          ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->ep_comm_ptr->intercomm->mpi_inter_comm);
+
+          ::MPI_Comm_free(&mpi_comm);
+          comm->ep_comm_ptr->intercomm->mpi_inter_comm = NULL;
+          Debug("mpi_intercomm freed\n");
+        }
+
+
+
+        for(int i=0; i<num_ep; i++)
+        {
+          if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue)
+          {
+            comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue->clear();
+            Debug("message queue freed\n");
+          }
+
+          if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr)
+          {
+            delete comm->ep_comm_ptr->comm_list[i].ep_comm_ptr;
+            Debug("ep_comm_ptr freed\n");
+          }
+        }
+
+        if(comm->mpi_comm)
+        {
+          ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->mpi_comm);
+          #ifdef _serialized
+          #pragma omp critical (_mpi_call)
+          #endif // _serialized
+          ::MPI_Comm_free(&mpi_comm);
+          comm->mpi_comm = NULL;
+//          printf("label = %d, mpi_comm freed\n", comm->ep_comm_ptr->comm_label);
+        }
+
+       if(comm) {delete[] comm->ep_comm_ptr->comm_list; Debug("comm freed\n");}
+
+      }
+      return 0;
+    }
+
+
+
+  }
+
+
+
+
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_gather.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_gather.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_gather.cpp	(revision 1034)
@@ -0,0 +1,551 @@
+/*!
+   \file ep_gather.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Gather, MPI_Allgather
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  int MPI_Gather_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Gather_local_int(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Gather_local_float(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Gather_local_double(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Gather_local_long(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Gather_local_ulong(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Gather_local_char(sendbuf, count, recvbuf, comm);
+    }
+    else
+    {
+      printf("MPI_Gather Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Gather_local_int(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_float(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_long(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_ulong(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Gather_local_char(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+
+  int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Gather(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+                   root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype) && sendcount == recvcount);
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+
+    ::MPI_Aint datasize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+
+    if(ep_rank_loc==0)
+    {
+      local_gather_recvbuf = new void*[datasize*num_ep*count];
+    }
+
+    // local gather to master
+    MPI_Gather_local(sendbuf, count, datatype, local_gather_recvbuf, comm);
+
+    //MPI_Gather
+
+    if(ep_rank_loc == 0)
+    {
+      int *gatherv_recvcnt;
+      int *gatherv_displs;
+      int gatherv_cnt = count*num_ep;
+
+      gatherv_recvcnt = new int[mpi_size];
+      gatherv_displs = new int[mpi_size];
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      gatherv_displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Gatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      delete[] gatherv_recvcnt;
+      delete[] gatherv_displs;
+    }
+
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      innode_memcpy(0, recvbuf, root_ep_loc, recvbuf, count*ep_size, datatype, comm);
+    }
+
+
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else// if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+    }
+
+
+  }
+
+
+  int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allgather(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+                      static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype) && sendcount == recvcount);
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint datasize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+
+    if(ep_rank_loc==0)
+    {
+      local_gather_recvbuf = new void*[datasize*num_ep*count];
+    }
+
+    // local gather to master
+    MPI_Gather_local(sendbuf, count, datatype, local_gather_recvbuf, comm);
+
+    //MPI_Gather
+
+    if(ep_rank_loc == 0)
+    {
+      int *gatherv_recvcnt;
+      int *gatherv_displs;
+      int gatherv_cnt = count*num_ep;
+
+      gatherv_recvcnt = new int[mpi_size];
+      gatherv_displs = new int[mpi_size];
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      gatherv_displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allgatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      delete[] gatherv_recvcnt;
+      delete[] gatherv_displs;
+    }
+
+    MPI_Bcast_local(recvbuf, count*ep_size, datatype, comm);
+
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else// if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+    }
+  }
+
+
+}
Index: XIOS/trunk/extern/src_ep/ep_gatherv.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_gatherv.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_gatherv.cpp	(revision 1034)
@@ -0,0 +1,578 @@
+/*!
+   \file ep_gather.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Gatherv, MPI_Allgatherv
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+using namespace std;
+
+namespace ep_lib
+{
+  int MPI_Gatherv_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Gatherv_local_int(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Gatherv_local_float(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Gatherv_local_double(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Gatherv_local_long(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Gatherv_local_ulong(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Gatherv_local_char(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else
+    {
+      printf("MPI_Gatherv Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Gatherv_local_int(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_float(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_double(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_long(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_ulong(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_char(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+  
+    
+    
+
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Gatherv(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcounts, displs,
+                    static_cast< ::MPI_Datatype>(recvtype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype));
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+    
+    MPI_Bcast(const_cast< int* >(recvcounts), ep_size, MPI_INT, root, comm);
+    MPI_Bcast(const_cast< int* >(displs), ep_size, MPI_INT, root, comm);
+
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+
+    ::MPI_Aint datasize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+
+    if(ep_rank_loc==0)
+    {
+      int buffer_size = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+      local_gather_recvbuf = new void*[datasize*buffer_size];
+    }
+
+    // local gather to master
+    int local_displs[num_ep];
+    local_displs[0] = 0;
+    for(int i=1; i<num_ep; i++)
+    {
+      local_displs[i] = displs[ep_rank-ep_rank_loc+i]-displs[ep_rank-ep_rank_loc];
+    }
+    MPI_Gatherv_local(sendbuf, count, datatype, local_gather_recvbuf, recvcounts+ep_rank-ep_rank_loc, local_displs, comm);
+
+    //MPI_Gather
+    if(ep_rank_loc == 0)
+    {
+
+      int gatherv_recvcnt[mpi_size];
+      int gatherv_displs[mpi_size];
+      int gatherv_cnt = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+
+      //gatherv_recvcnt = new int[mpi_size];
+      //gatherv_displs = new int[mpi_size];
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      gatherv_displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Gatherv(local_gather_recvbuf, gatherv_cnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      //delete[] gatherv_recvcnt;
+      //delete[] gatherv_displs;
+    }
+
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      innode_memcpy(0, recvbuf, root_ep_loc, recvbuf, accumulate(recvcounts, recvcounts+ep_size, 0), datatype, comm);
+    }
+
+
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+      else // if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+    }
+    return 0;
+  }
+
+
+
+  int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allgatherv(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcounts, displs,
+                       static_cast< ::MPI_Datatype>(recvtype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype));
+
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+    
+
+    assert(accumulate(recvcounts, recvcounts+ep_size-1, 0) == displs[ep_size-1]); // Only for contunuous gather.
+
+
+    ::MPI_Aint datasize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+
+    if(ep_rank_loc==0)
+    {
+      int buffer_size = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+      local_gather_recvbuf = new void*[datasize*buffer_size];
+    }
+
+    // local gather to master
+    int local_displs[num_ep];
+    local_displs[0] = 0;
+    for(int i=1; i<num_ep; i++)
+    {
+      local_displs[i] = displs[ep_rank-ep_rank_loc+i]-displs[ep_rank-ep_rank_loc];
+    }
+    MPI_Gatherv_local(sendbuf, count, datatype, local_gather_recvbuf, recvcounts+ep_rank-ep_rank_loc, local_displs, comm);
+
+    //MPI_Gather
+    if(ep_rank_loc == 0)
+    {
+      int *gatherv_recvcnt;
+      int *gatherv_displs;
+      int gatherv_cnt = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+
+      gatherv_recvcnt = new int[mpi_size];
+      gatherv_displs = new int[mpi_size];
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      gatherv_displs[0] = displs[0];
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allgatherv(local_gather_recvbuf, gatherv_cnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      delete[] gatherv_recvcnt;
+      delete[] gatherv_displs;
+    }
+
+    MPI_Bcast_local(recvbuf, accumulate(recvcounts, recvcounts+ep_size, 0), datatype, comm);
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+      else // if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+    }
+  }
+
+
+}
Index: XIOS/trunk/extern/src_ep/ep_init.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_init.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_init.cpp	(revision 1034)
@@ -0,0 +1,49 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Init_thread(int *argc, char*** argv, int required, int*provided)
+  {
+    printf("MPI_Init_thread\n");
+
+    int id = omp_get_thread_num();
+
+    if(id == 0)
+    {
+      ::MPI_Init_thread(argc, argv, required, provided);
+    }
+    return 0;
+  }
+
+  int MPI_Init(int *argc, char ***argv)
+  {
+    printf("MPI_init called\n");
+    int id = omp_get_thread_num();
+
+    if(id == 0)
+    {
+      ::MPI_Init(argc, argv);
+    }
+  	return 0;
+  }
+
+  int MPI_Initialized(int *flag)
+  {
+    printf("MPI_initialized called\n");
+
+    ::MPI_Initialized(flag);
+    
+    return 0;
+  }
+
+
+
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_intercomm.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_intercomm.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_intercomm.cpp	(revision 1034)
@@ -0,0 +1,209 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+    assert(local_comm.is_ep);
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    MPI_Barrier(local_comm);
+
+
+
+    int leader_ranks[6]; //! 0: rank in world, 1: mpi_size, 2: rank_in_peer.
+                         //! 3, 4, 5 : remote
+
+    bool is_decider = false;
+
+
+
+    if(ep_rank == local_leader)
+    {
+      ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &leader_ranks[0]);
+
+      leader_ranks[1] = mpi_size;
+      MPI_Comm_rank(peer_comm, &leader_ranks[2]);
+
+      MPI_Request req_s, req_r;
+      MPI_Isend(&leader_ranks[0], 3, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+      MPI_Status status;
+      MPI_Irecv(&leader_ranks[3], 3, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+    }
+
+    MPI_Bcast(leader_ranks, 6, MPI_INT_STD, local_leader, local_comm);
+
+    MPI_Barrier(local_comm);
+
+    if(leader_ranks[0] == leader_ranks[3])
+    {
+      if( leader_ranks[1] * leader_ranks[4] == 1)
+      {
+        if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_unique_leader\n");
+        local_comm.ep_comm_ptr->comm_label = -99;
+
+        return MPI_Intercomm_create_unique_leader(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
+      }
+      else // leader_ranks[1] * leader_ranks[4] != 1
+      {
+        // change leader
+        if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_from_world\n");
+
+        int new_local_leader;
+
+        if(leader_ranks[2] < leader_ranks[5])
+        {
+          if(leader_ranks[1] > 1) //! change leader
+          {
+            // change leader
+            is_decider = true;
+            int target = local_comm.rank_map->at(local_leader).second;
+            {
+              for(int i=0; i<ep_size; i++)
+              {
+                if(local_comm.rank_map->at(i).second != target && local_comm.rank_map->at(i).first == 0)
+                {
+                  new_local_leader = i;
+                  break;
+                }
+              }
+            }
+          }
+          else
+          {
+            new_local_leader = local_leader;
+          }
+        }
+        else
+        {
+          if(leader_ranks[4] == 1)
+          {
+            // change leader
+            is_decider = true;
+            int target = local_comm.rank_map->at(local_leader).second;
+            {
+              for(int i=0; i<ep_size; i++)
+              {
+                if(local_comm.rank_map->at(i).second != target && local_comm.rank_map->at(i).first == 0)
+                {
+                  new_local_leader = i;
+                  break;
+                }
+              }
+            }
+          }
+          else
+          {
+            new_local_leader = local_leader;
+          }
+        }
+
+
+        int new_tag_in_world;
+
+        int leader_in_world[2];
+
+
+        if(is_decider)
+        {
+          if(ep_rank == new_local_leader)
+          {
+            new_tag_in_world = TAG++;
+          }
+          MPI_Bcast(&new_tag_in_world, 1, MPI_INT_STD, new_local_leader, local_comm);
+          if(ep_rank == local_leader) MPI_Send(&new_tag_in_world, 1, MPI_INT_STD, remote_leader, tag, peer_comm);
+        }
+        else
+        {
+          if(ep_rank == local_leader)
+          {
+            MPI_Status status;
+            MPI_Recv(&new_tag_in_world, 1, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+          }
+          MPI_Bcast(&new_tag_in_world, 1, MPI_INT_STD, new_local_leader, local_comm);
+        }
+
+
+        if(ep_rank == new_local_leader)
+        {
+          #ifdef _serialized
+          #pragma omp critical (_mpi_call)
+          #endif // _serialized
+          ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &leader_in_world[0]);
+        }
+
+        MPI_Bcast(&leader_in_world[0], 1, MPI_INT_STD, new_local_leader, local_comm);
+
+
+        if(ep_rank == local_leader)
+        {
+          MPI_Request req_s, req_r;
+
+          MPI_Isend(&leader_in_world[0], 1, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+          MPI_Irecv(&leader_in_world[1], 1, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+
+          MPI_Status status;
+          MPI_Wait(&req_s, &status);
+          MPI_Wait(&req_r, &status);
+
+      /*
+          MPI_Send(&leader_in_world[0], 1, MPI_INT_STD, remote_leader, tag, peer_comm);
+          MPI_Status status;
+          MPI_Recv(&leader_in_world[1], 1, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+   */
+        }
+
+
+
+        MPI_Bcast(&leader_in_world[1], 1, MPI_INT, local_leader, local_comm);
+
+
+
+        local_comm.ep_comm_ptr->comm_label = tag;
+
+        return MPI_Intercomm_create_from_world(local_comm, new_local_leader, MPI_COMM_WORLD_STD, leader_in_world[1], new_tag_in_world, newintercomm);
+      }
+    }
+
+    if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_kernel\n");
+
+    return MPI_Intercomm_create_kernel(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
+
+  }
+
+  int MPI_Comm_test_inter(MPI_Comm comm, int *flag)
+  {
+    if(comm.is_ep)
+    {
+      *flag = comm.is_intercomm;
+      return 0;
+    } 
+    else
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif
+      ::MPI_Comm_test_inter(mpi_comm, flag);
+      return 0;  
+    }
+  }
+
+
+}
Index: XIOS/trunk/extern/src_ep/ep_intercomm_kernel.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_intercomm_kernel.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_intercomm_kernel.cpp	(revision 1034)
@@ -0,0 +1,788 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib
+{
+  int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
+                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
+
+    int rank_in_world;
+    int rank_in_local_parent;
+
+    int rank_in_peer_mpi[2];
+
+    int local_ep_size = ep_size;
+    int remote_ep_size;
+
+
+    ::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
+
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    {
+      ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &rank_in_world);
+      ::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
+    }
+
+    bool is_proc_master = false;
+    bool is_local_leader = false;
+    bool is_final_master = false;
+
+
+    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
+    if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
+
+
+    int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
+
+    int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
+
+
+    std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
+
+    std::vector<int> new_rank_info[4];
+    std::vector<int> new_ep_info[2];
+
+    std::vector<int> offset;
+
+    if(is_proc_master)
+    {
+
+      size_info[0] = mpi_size;
+
+      rank_info[0].resize(size_info[0]);
+      rank_info[1].resize(size_info[0]);
+
+
+
+      ep_info[0].resize(size_info[0]);
+
+      vector<int> send_buf(6);
+      vector<int> recv_buf(3*size_info[0]);
+
+      send_buf[0] = rank_in_world;
+      send_buf[1] = rank_in_local_parent;
+      send_buf[2] = num_ep;
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
+
+      for(int i=0; i<size_info[0]; i++)
+      {
+        rank_info[0][i] = recv_buf[3*i];
+        rank_info[1][i] = recv_buf[3*i+1];
+        ep_info[0][i]   = recv_buf[3*i+2];
+      }
+
+      if(is_local_leader)
+      {
+        leader_info[0] = rank_in_world;
+        leader_info[1] = remote_leader;
+
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Comm_rank(static_cast< ::MPI_Comm>(peer_comm.mpi_comm), &rank_in_peer_mpi[0]);
+
+        MPI_Status status;
+
+        send_buf[0] = size_info[0];
+        send_buf[1] = local_ep_size;
+        send_buf[2] = rank_in_peer_mpi[0];
+
+        MPI_Send(send_buf.data(), 3, MPI_INT_STD, remote_leader, tag, peer_comm);
+        MPI_Recv(recv_buf.data(), 3, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+
+        size_info[1] = recv_buf[0];
+        remote_ep_size = recv_buf[1];
+        rank_in_peer_mpi[1] = recv_buf[2];
+
+      }
+
+      send_buf[0] = size_info[1];
+      send_buf[1] = leader_info[0];
+      send_buf[2] = leader_info[1];
+      send_buf[3] = rank_in_peer_mpi[0];
+      send_buf[4] = rank_in_peer_mpi[1];
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Bcast(send_buf.data(), 5, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      size_info[1] = send_buf[0];
+      leader_info[0] = send_buf[1];
+      leader_info[1] = send_buf[2];
+      rank_in_peer_mpi[0] = send_buf[3];
+      rank_in_peer_mpi[1] = send_buf[4];
+
+
+      rank_info[2].resize(size_info[1]);
+      rank_info[3].resize(size_info[1]);
+
+      ep_info[1].resize(size_info[1]);
+
+      send_buf.resize(3*size_info[0]);
+      recv_buf.resize(3*size_info[1]);
+
+      if(is_local_leader)
+      {
+        MPI_Status status;
+
+        std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
+        std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
+        std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] );
+
+        MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, remote_leader, tag, peer_comm);
+        MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+
+      }
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() );
+
+
+      offset.resize(size_info[0]);
+
+      if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
+      {
+
+        bool found = false;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+
+            if(target == leader_info[0]) // the leader is doubled in remote
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else
+            {
+              offset[i] = ep_local;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+
+            offset[i] = 0;
+          }
+
+        }
+      }
+
+      else // erase rank doubled with remote leader
+      {
+
+        bool found = false;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+            if(target != leader_info[1])
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else // found remote leader
+            {
+              offset[i] = ep_remote;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+            offset[i] = 0;
+          }
+        }
+      }
+
+      if(offset[mpi_rank] == 0)
+      {
+        is_final_master = true;
+      }
+
+
+      //! size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
+
+      if(is_local_leader)
+      {
+        size_info[2] = new_ep_info[0].size();
+        MPI_Status status;
+        MPI_Send(&size_info[2], 1, MPI_INT_STD, remote_leader, tag, peer_comm);
+        MPI_Recv(&size_info[3], 1, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+      }
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      new_rank_info[2].resize(size_info[3]);
+      new_rank_info[3].resize(size_info[3]);
+      new_ep_info[1].resize(size_info[3]);
+
+      send_buf.resize(size_info[2]);
+      recv_buf.resize(size_info[3]);
+
+      if(is_local_leader)
+      {
+        MPI_Status status;
+
+        std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
+        std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
+        std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] );
+
+        MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, remote_leader, tag, peer_comm);
+        MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+      }
+
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() );
+
+    }
+
+
+
+    if(is_proc_master)
+    {
+      //! leader_info[4]: 2-> rank of local leader in new_group generated comm;
+                      // 3-> rank of remote leader in new_group generated comm;
+      ::MPI_Group local_group;
+      ::MPI_Group new_group;
+      ::MPI_Comm new_comm;
+      ::MPI_Comm intercomm;
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Comm_group(local_mpi_comm, &local_group);
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
+
+
+
+      if(is_local_leader)
+      {
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Comm_rank(new_comm, &leader_info[2]);
+      }
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      if(new_comm != MPI_COMM_NULL_STD)
+      {
+
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Barrier(new_comm);
+
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Intercomm_create(new_comm, leader_info[2], static_cast< ::MPI_Comm>(peer_comm.mpi_comm), rank_in_peer_mpi[1], tag, &intercomm);
+
+        int id;
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Comm_rank(new_comm, &id);
+        int my_num_ep = new_ep_info[0][id];
+
+        MPI_Comm *ep_intercomm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
+
+
+        for(int i= 0; i<my_num_ep; i++)
+        {
+          ep_intercomm[i].is_intercomm = true;
+
+          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
+          ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
+        }
+
+
+        #pragma omp critical (write_to_tag_list)
+        tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
+      }
+
+
+    }
+
+
+    MPI_Barrier_local(local_comm);
+
+    vector<int> bcast_buf(8);
+    if(is_local_leader)
+    {
+      std::copy(size_info, size_info+4, bcast_buf.begin());
+      std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
+    }
+
+    MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
+      std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
+    }
+
+    if(!is_local_leader)
+    {
+      new_rank_info[1].resize(size_info[2]);
+      ep_info[1].resize(size_info[1]);
+      offset.resize(size_info[0]);
+    }
+
+    bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
+
+    if(is_local_leader)
+    {
+      bcast_buf[0] = remote_ep_size;
+      std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
+      std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
+      std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
+    }
+
+    MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      remote_ep_size = bcast_buf[0];
+      std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
+    }
+
+    int my_position = offset[rank_in_local_parent]+ep_rank_loc;
+
+
+    MPI_Barrier_local(local_comm);
+    #pragma omp flush
+
+
+    #pragma omp critical (read_from_tag_list)
+    {
+      bool found = false;
+      while(!found)
+      {
+        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+        {
+          if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
+          {
+            *newintercomm =  iter->second[my_position];
+            found = true;
+            break;
+          }
+        }
+      }
+    }
+
+    MPI_Barrier_local(local_comm);
+
+    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
+    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
+
+    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
+    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
+    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
+    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
+    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
+    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
+
+    MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
+
+    int my_rank_map_elem[2];
+
+    my_rank_map_elem[0] = intercomm_ep_rank;
+    my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
+
+    vector<pair<int, int> > local_rank_map_array;
+    vector<pair<int, int> > remote_rank_map_array;
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
+
+    MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
+
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
+
+    int local_intercomm_size = intercomm_ep_size;
+    int remote_intercomm_size;
+
+    int new_bcast_root_0 = 0;
+    int new_bcast_root = 0;
+
+
+    if(is_local_leader)
+    {
+      MPI_Status status;
+      MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT_STD, remote_leader, tag, peer_comm);
+      MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+
+      MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, remote_leader, tag, peer_comm);
+      MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+
+      new_bcast_root_0 = intercomm_ep_rank;
+    }
+
+    MPI_Allreduce(&new_bcast_root_0, &new_bcast_root, 1, MPI_INT_STD, MPI_SUM_STD, *newintercomm);
+
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, local_leader, local_comm);
+    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, new_bcast_root, *newintercomm);
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
+
+
+
+
+    if(is_local_leader)
+    {
+      MPI_Status status;
+      MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, MPI_INT_STD, remote_leader, tag, peer_comm);
+      MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, new_bcast_root, *newintercomm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = tag;
+
+/*
+    for(int i=0; i<local_ep_size; i++)
+    if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, local_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+          (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).second);
+
+    for(int i=0; i<remote_ep_size; i++)
+    if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+          (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->at(i).second);
+
+    for(int i=0; i<remote_intercomm_size; i++)
+    if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+          (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second);
+*/
+
+//    for(int i=0; i<(*newintercomm).rank_map->size(); i++)
+//    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+//          (*newintercomm).rank_map->at(i).first, (*newintercomm).rank_map->at(i).second);
+
+//    MPI_Comm *test_comm = newintercomm->ep_comm_ptr->intercomm->local_comm;
+//    int test_rank;
+//    MPI_Comm_rank(*test_comm, &test_rank);
+//    printf("=================test_rank = %d\n", test_rank);
+
+    return MPI_SUCCESS;
+
+  }
+
+
+
+
+  int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+    //! mpi_size of local comm = 1
+    //! same world rank of leaders
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
+                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
+
+    int rank_in_world;
+
+    int rank_in_peer_mpi[2];
+
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &rank_in_world);
+
+
+    int local_num_ep = num_ep;
+    int remote_num_ep;
+    int total_num_ep;
+
+    int leader_rank_in_peer[2];
+
+    int my_position;
+    int tag_label[2];
+
+    vector<int> send_buf(4);
+    vector<int> recv_buf(4);
+
+
+    if(ep_rank == local_leader)
+    {
+      MPI_Status status;
+
+
+
+      MPI_Comm_rank(peer_comm, &leader_rank_in_peer[0]);
+
+      send_buf[0] = local_num_ep;
+      send_buf[1] = leader_rank_in_peer[0];
+
+      MPI_Request req_s, req_r;
+
+      MPI_Isend(send_buf.data(), 2, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+      MPI_Irecv(recv_buf.data(), 2, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+
+      recv_buf[2] = leader_rank_in_peer[0];
+
+    }
+
+    MPI_Bcast(recv_buf.data(), 3, MPI_INT_STD, local_leader, local_comm);
+
+    remote_num_ep = recv_buf[0];
+    leader_rank_in_peer[1] = recv_buf[1];
+    leader_rank_in_peer[0] = recv_buf[2];
+
+    total_num_ep = local_num_ep + remote_num_ep;
+
+
+    if(leader_rank_in_peer[0] < leader_rank_in_peer[1])
+    {
+      my_position = ep_rank_loc;
+      //! LEADER create EP
+      if(ep_rank == local_leader)
+      {
+        ::MPI_Comm mpi_dup;
+
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Comm_dup(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &mpi_dup);
+
+        MPI_Comm *ep_intercomm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(mpi_dup, total_num_ep, info, ep_intercomm);
+
+
+        for(int i=0; i<total_num_ep; i++)
+        {
+          ep_intercomm[i].is_intercomm = true;
+          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = 0;
+
+          ep_intercomm[i].ep_comm_ptr->comm_label = leader_rank_in_peer[0];
+        }
+
+        tag_label[0] = TAG++;
+        tag_label[1] = rank_in_world;
+
+        #pragma omp critical (write_to_tag_list)
+        tag_list.push_back(make_pair( make_pair(tag_label[0], tag_label[1]) , ep_intercomm));
+
+        MPI_Request req_s;
+        MPI_Status sta_s;
+        MPI_Isend(tag_label, 2, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+
+        MPI_Wait(&req_s, &sta_s);
+
+      }
+    }
+    else
+    {
+      //! Wait for EP creation
+      my_position = remote_num_ep + ep_rank_loc;
+      if(ep_rank == local_leader)
+      {
+        MPI_Status status;
+        MPI_Request req_r;
+        MPI_Irecv(tag_label, 2, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+        MPI_Wait(&req_r, &status);
+      }
+    }
+
+    MPI_Bcast(tag_label, 2, MPI_INT_STD, local_leader, local_comm);
+
+
+
+
+    #pragma omp critical (read_from_tag_list)
+    {
+      bool found = false;
+      while(!found)
+      {
+        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+        {
+          if((*iter).first == make_pair(tag_label[0], tag_label[1]))
+          {
+            *newintercomm =  iter->second[my_position];
+            found = true;
+            break;
+          }
+        }
+      }
+    }
+
+
+
+    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
+    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
+
+    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
+    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
+    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
+    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
+    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
+    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map  = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_num_ep);
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_num_ep);
+
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
+
+
+
+    int local_rank_map_ele[2];
+    local_rank_map_ele[0] = intercomm_ep_rank;
+    local_rank_map_ele[1] = (*newintercomm).ep_comm_ptr->comm_label;
+
+    MPI_Allgather(local_rank_map_ele, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
+
+    if(ep_rank == local_leader)
+    {
+      MPI_Status status;
+      MPI_Request req_s, req_r;
+
+      MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_num_ep, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT_STD, local_leader, local_comm);
+    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = tag;
+
+
+    return MPI_SUCCESS;
+  }
+
+
+}
Index: XIOS/trunk/extern/src_ep/ep_intercomm_world.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_intercomm_world.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_intercomm_world.cpp	(revision 1034)
@@ -0,0 +1,1029 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  #ifdef _openmpi
+
+  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
+                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
+
+    int rank_in_world;
+    int rank_in_local_parent;
+
+    int local_ep_size = ep_size;
+    int remote_ep_size;
+
+    ::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
+    ::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
+
+    ::MPI_Comm_rank(peer_comm, &rank_in_world);
+
+    ::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
+
+    bool is_proc_master = false;
+    bool is_local_leader = false;
+    bool is_final_master = false;
+
+
+    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
+    if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
+
+
+    int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
+
+    int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
+
+
+    std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
+
+    std::vector<int> new_rank_info[4];
+    std::vector<int> new_ep_info[2];
+
+    std::vector<int> offset;
+
+    if(is_proc_master)
+    {
+
+      size_info[0] = mpi_size;
+
+      rank_info[0].resize(size_info[0]);
+      rank_info[1].resize(size_info[0]);
+
+
+
+      ep_info[0].resize(size_info[0]);
+
+      vector<int> send_buf(6);
+      vector<int> recv_buf(3*size_info[0]);
+
+      send_buf[0] = rank_in_world;
+      send_buf[1] = rank_in_local_parent;
+      send_buf[2] = num_ep;
+
+      ::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
+
+      for(int i=0; i<size_info[0]; i++)
+      {
+        rank_info[0][i] = recv_buf[3*i];
+        rank_info[1][i] = recv_buf[3*i+1];
+        ep_info[0][i]   = recv_buf[3*i+2];
+      }
+
+      if(is_local_leader)
+      {
+        leader_info[0] = rank_in_world;
+        leader_info[1] = mpi_remote_leader;
+
+        ::MPI_Status mpi_status;
+
+        send_buf[0] = size_info[0];
+        send_buf[1] = local_ep_size;
+
+        ::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+        recv_buf[2] = rank_in_world;
+        recv_buf[3] = mpi_remote_leader;
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      size_info[1] = recv_buf[0];
+      remote_ep_size = recv_buf[1];
+      leader_info[0] = recv_buf[2];
+      leader_info[1] = recv_buf[3];
+
+      rank_info[2].resize(size_info[1]);
+      rank_info[3].resize(size_info[1]);
+
+      ep_info[1].resize(size_info[1]);
+
+      send_buf.resize(3*size_info[0]);
+      recv_buf.resize(3*size_info[1]);
+
+      if(is_local_leader)
+      {
+        ::MPI_Status mpi_status;
+
+
+        std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
+        std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
+        std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] );
+
+        ::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() );
+
+      offset.resize(size_info[0]);
+
+      if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
+      {
+
+        bool found = false;
+        int ep_tmp;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_tmp = ep_info[1][j];
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+
+            if(target == leader_info[0]) // the leader is doubled in remote
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else
+            {
+              offset[i] = ep_local;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+
+            offset[i] = 0;
+          }
+
+        }
+      }
+
+      else // erase rank doubled with remote leader
+      {
+
+        bool found = false;
+        int ep_tmp;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_tmp = ep_info[1][j];
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+            if(target != leader_info[1])
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else // found remote leader
+            {
+              offset[i] = ep_remote;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+            offset[i] = 0;
+          }
+        }
+      }
+
+      if(offset[mpi_rank] == 0)
+      {
+        is_final_master = true;
+      }
+
+
+      // size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
+
+      if(is_local_leader)
+      {
+        size_info[2] = new_ep_info[0].size();
+        ::MPI_Status mpi_status;
+
+        ::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+      }
+
+      ::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      new_rank_info[2].resize(size_info[3]);
+      new_rank_info[3].resize(size_info[3]);
+      new_ep_info[1].resize(size_info[3]);
+
+      send_buf.resize(size_info[2]);
+      recv_buf.resize(size_info[3]);
+
+      if(is_local_leader)
+      {
+        ::MPI_Status mpi_status;
+
+        std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
+        std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
+        std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] );
+
+        MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+        MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() );
+
+    }
+
+
+
+    if(is_proc_master)
+    {
+      // leader_info[4]: 2-> rank of local leader in new_group generated comm;
+                      // 3-> rank of remote leader in new_group generated comm;
+      ::MPI_Group local_group;
+      ::MPI_Group new_group;
+      ::MPI_Comm new_comm;
+      ::MPI_Comm intercomm;
+
+      ::MPI_Comm_group(local_mpi_comm, &local_group);
+
+      ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
+
+      ::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
+
+
+
+      if(is_local_leader)
+      {
+        ::MPI_Comm_rank(new_comm, &leader_info[2]);
+      }
+
+      ::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      if(new_comm != MPI_COMM_NULL_STD)
+      {
+        ::MPI_Barrier(new_comm);
+
+        ::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
+
+        int id;
+        MPI_Comm_rank(new_comm, &id);
+        int my_num_ep = new_ep_info[0][id];
+
+        MPI_Comm *ep_intercomm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
+
+
+
+        for(int i= 0; i<my_num_ep; i++)
+        {
+          ep_intercomm[i].is_intercomm = true;
+
+          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
+          ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
+        }
+
+        #pragma omp critical (write_to_tag_list)
+        tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
+      }
+
+
+    }
+
+
+    MPI_Barrier_local(local_comm);
+
+    vector<int> bcast_buf(8);
+    if(is_local_leader)
+    {
+      std::copy(size_info, size_info+4, bcast_buf.begin());
+      std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
+    }
+
+    MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
+      std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
+    }
+
+    if(!is_local_leader)
+    {
+      new_rank_info[1].resize(size_info[2]);
+      ep_info[1].resize(size_info[1]);
+      offset.resize(size_info[0]);
+    }
+
+    bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
+
+    if(is_local_leader)
+    {
+      bcast_buf[0] = remote_ep_size;
+      std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
+      std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
+      std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
+    }
+
+    MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      remote_ep_size = bcast_buf[0];
+      std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
+    }
+
+
+    int my_position = offset[rank_in_local_parent]+ep_rank_loc;
+
+
+    MPI_Barrier_local(local_comm);
+    #pragma omp flush
+
+
+    #pragma omp critical (read_from_tag_list)
+    {
+      bool found = false;
+      while(!found)
+      {
+        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+        {
+          if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
+          {
+            *newintercomm =  iter->second[my_position];
+
+            found = true;
+            break;
+          }
+        }
+      }
+    }
+
+    MPI_Barrier_local(local_comm);
+
+    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
+    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
+
+    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
+    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
+    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
+    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
+    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
+    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
+
+    MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
+
+    int my_rank_map_elem[2];
+
+
+    my_rank_map_elem[0] = intercomm_ep_rank;
+
+    my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
+
+    vector<pair<int, int> > local_rank_map_array;
+    vector<pair<int, int> > remote_rank_map_array;
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
+
+    MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
+
+    int local_intercomm_size = intercomm_ep_size;
+    int remote_intercomm_size;
+
+
+
+
+    if(is_local_leader)
+    {
+      ::MPI_Status status;
+
+      ::MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+
+      ::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, local_leader, local_comm);
+    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
+
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
+
+
+    if(is_local_leader)
+    {
+      ::MPI_Status status;
+
+      ::MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
+
+
+    return MPI_SUCCESS;
+
+  }
+
+
+
+  #elif _intelmpi
+  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
+                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
+
+    int rank_in_world;
+    int rank_in_local_parent;
+
+    int local_ep_size = ep_size;
+    int remote_ep_size;
+
+    ::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
+    ::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
+
+    ::MPI_Comm_rank(peer_comm, &rank_in_world);
+
+    ::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
+
+    bool is_proc_master = false;
+    bool is_local_leader = false;
+    bool is_final_master = false;
+
+
+    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
+    if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
+
+
+    int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
+
+    int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
+
+
+    std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
+
+    std::vector<int> new_rank_info[4];
+    std::vector<int> new_ep_info[2];
+
+    std::vector<int> offset;
+
+    if(is_proc_master)
+    {
+
+      size_info[0] = mpi_size;
+
+      rank_info[0].resize(size_info[0]);
+      rank_info[1].resize(size_info[0]);
+
+      ep_info[0].resize(size_info[0]);
+
+      vector<int> send_buf(6);
+      vector<int> recv_buf(3*size_info[0]);
+
+      send_buf[0] = rank_in_world;
+      send_buf[1] = rank_in_local_parent;
+      send_buf[2] = num_ep;
+
+      ::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
+
+      for(int i=0; i<size_info[0]; i++)
+      {
+        rank_info[0][i] = recv_buf[3*i];
+        rank_info[1][i] = recv_buf[3*i+1];
+        ep_info[0][i]   = recv_buf[3*i+2];
+      }
+
+
+      if(is_local_leader)
+      {
+        leader_info[0] = rank_in_world;
+        leader_info[1] = mpi_remote_leader;
+
+        ::MPI_Status mpi_status;
+
+        send_buf[0] = size_info[0];
+        send_buf[1] = local_ep_size;
+
+        ::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+        recv_buf[2] = rank_in_world;
+        recv_buf[3] = mpi_remote_leader;
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      size_info[1] = recv_buf[0];
+      remote_ep_size = recv_buf[1];
+      leader_info[0] = recv_buf[2];
+      leader_info[1] = recv_buf[3];
+
+      rank_info[2].resize(size_info[1]);
+      rank_info[3].resize(size_info[1]);
+
+      ep_info[1].resize(size_info[1]);
+
+      send_buf.resize(3*size_info[0]);
+      recv_buf.resize(3*size_info[1]);
+
+      if(is_local_leader)
+      {
+        ::MPI_Status mpi_status;
+
+
+        std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
+        std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
+        std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] );
+
+        ::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() );
+
+      offset.resize(size_info[0]);
+
+      if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
+      {
+
+        bool found = false;
+        int ep_tmp;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_tmp = ep_info[1][j];
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+
+            if(target == leader_info[0]) // the leader is doubled in remote
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else
+            {
+              offset[i] = ep_local;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+
+            offset[i] = 0;
+          }
+
+        }
+      }
+
+      else // erase rank doubled with remote leader
+      {
+
+        bool found = false;
+        int ep_tmp;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_tmp = ep_info[1][j];
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+            if(target != leader_info[1])
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else // found remote leader
+            {
+              offset[i] = ep_remote;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+            offset[i] = 0;
+          }
+        }
+      }
+
+      if(offset[mpi_rank] == 0)
+      {
+        is_final_master = true;
+      }
+
+
+      // size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
+
+      if(is_local_leader)
+      {
+        size_info[2] = new_ep_info[0].size();
+        ::MPI_Status mpi_status;
+
+        ::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+      }
+
+      ::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      new_rank_info[2].resize(size_info[3]);
+      new_rank_info[3].resize(size_info[3]);
+      new_ep_info[1].resize(size_info[3]);
+
+      send_buf.resize(size_info[2]);
+      recv_buf.resize(size_info[3]);
+
+      if(is_local_leader)
+      {
+        ::MPI_Status mpi_status;
+
+        std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
+        std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
+        std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] );
+
+        ::MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+      }
+
+      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() );
+
+    }
+
+
+
+    if(is_proc_master)
+    {
+      // leader_info[4]: 2-> rank of local leader in new_group generated comm;
+                      // 3-> rank of remote leader in new_group generated comm;
+      ::MPI_Group local_group;
+      ::MPI_Group new_group;
+      ::MPI_Comm new_comm;
+      ::MPI_Comm intercomm;
+
+      ::MPI_Comm_group(local_mpi_comm, &local_group);
+
+      ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
+
+      ::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
+
+
+
+      if(is_local_leader)
+      {
+        ::MPI_Comm_rank(new_comm, &leader_info[2]);
+      }
+
+      ::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      if(new_comm != MPI_COMM_NULL_STD)
+      {
+        ::MPI_Barrier(new_comm);
+
+        ::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
+
+        int id;
+        ::MPI_Comm_rank(new_comm, &id);
+        int my_num_ep = new_ep_info[0][id];
+
+        MPI_Comm *ep_intercomm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
+
+        for(int i= 0; i<my_num_ep; i++)
+        {
+          ep_intercomm[i].is_intercomm = true;
+
+          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
+          ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
+        }
+
+
+        #pragma omp critical (write_to_tag_list)
+        tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
+
+      }
+
+
+    }
+
+
+    MPI_Barrier_local(local_comm);
+
+
+    vector<int> bcast_buf(8);
+    if(is_local_leader)
+    {
+      std::copy(size_info, size_info+4, bcast_buf.begin());
+      std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
+    }
+
+
+
+    MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
+
+
+    if(!is_local_leader)
+    {
+      std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
+      std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
+    }
+
+
+
+    if(!is_local_leader)
+    {
+      new_rank_info[1].resize(size_info[2]);
+      ep_info[1].resize(size_info[1]);
+      offset.resize(size_info[0]);
+    }
+
+    bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
+
+    if(is_local_leader)
+    {
+      bcast_buf[0] = remote_ep_size;
+      std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
+      std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
+      std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
+    }
+
+    MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
+
+
+    if(!is_local_leader)
+    {
+      remote_ep_size = bcast_buf[0];
+      std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
+    }
+
+
+    int my_position = offset[rank_in_local_parent]+ep_rank_loc;
+
+
+    MPI_Barrier_local(local_comm);
+    #pragma omp flush
+
+
+    #pragma omp critical (read_from_tag_list)
+    {
+      bool found = false;
+      while(!found)
+      {
+        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+        {
+          if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
+          {
+            *newintercomm =  iter->second[my_position];
+
+            found = true;
+            break;
+          }
+        }
+      }
+    }
+
+    MPI_Barrier_local(local_comm);
+
+    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
+    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
+
+    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
+    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
+    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
+    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
+    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
+    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+    MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
+
+    int my_rank_map_elem[2];
+
+
+    my_rank_map_elem[0] = intercomm_ep_rank;
+
+    my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
+
+    vector<pair<int, int> > local_rank_map_array;
+    vector<pair<int, int> > remote_rank_map_array;
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
+
+    MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
+
+    int local_intercomm_size = intercomm_ep_size;
+    int remote_intercomm_size;
+
+
+    if(is_local_leader)
+    {
+      ::MPI_Status status;
+
+      ::MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+
+      ::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, local_leader, local_comm);
+    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
+
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
+
+
+    if(is_local_leader)
+    {
+      ::MPI_Status status;
+
+      ::MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
+
+
+//    for(int i=0; i<local_ep_size; i++)
+//    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+//          (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).second);
+
+//    for(int i=0; i<remote_intercomm_size; i++)
+//    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+//          (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second);
+
+//    for(int i=0; i<(*newintercomm).rank_map->size(); i++)
+//    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+//          (*newintercomm).rank_map->at(i).first, (*newintercomm).rank_map->at(i).second);
+
+
+
+
+
+    return MPI_SUCCESS;
+
+  }
+
+  #endif
+
+}
+
Index: XIOS/trunk/extern/src_ep/ep_lib.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_lib.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_lib.cpp	(revision 1034)
@@ -0,0 +1,376 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+#include <iostream>
+#include <fstream>
+
+using namespace std;
+
+
+namespace ep_lib
+{
+
+
+  int tag_combine(int real_tag, int src, int dest)
+  {
+    int a = real_tag << 16;
+    int b = src << 8;
+    int c = dest;
+
+    return a+b+c;
+  }
+
+  int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank)
+  {
+    for(int i=0; i<comm.rank_map->size(); i++)
+    {
+      if(   ( comm.rank_map->at(i).first  == ep_rank_loc )
+         && ( comm.rank_map->at(i).second == mpi_rank ) )
+      {
+        return i;
+      }
+    }
+    printf("rank not find\n");
+  }
+  
+  int get_ep_rank_intercomm(MPI_Comm comm, int ep_rank_loc, int mpi_rank)
+  {
+    // intercomm
+    int inter_rank;
+    for(int i=0; i<comm.ep_comm_ptr->intercomm->intercomm_rank_map->size(); i++)
+    {
+      if(   ( comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first  == ep_rank_loc )
+         && ( comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second == mpi_rank ) )
+      {
+        inter_rank =  i;
+        break;
+      }
+    }
+
+    for(int i=0; i<comm.ep_comm_ptr->intercomm->remote_rank_map->size(); i++)
+    {
+      if(  comm.ep_comm_ptr->intercomm->remote_rank_map->at(i).first  == inter_rank  )
+      {
+        printf("get_ep_rank for intercomm, ep_rank_loc = %d, mpi_rank = %d => ep_src = %d\n", ep_rank_loc, mpi_rank, i);
+        return i;
+      }
+    }
+
+    printf("rank not find\n");
+    
+  }
+
+
+  int innode_memcpy(int sender, const void* sendbuf, int receiver, void* recvbuf, int count, MPI_Datatype datatype, MPI_Comm comm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    if(datatype == MPI_INT)
+    {
+
+      int* send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+      int* recv_buf = static_cast<int*>(recvbuf);
+      int* buffer = comm.my_buffer->buf_int;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+
+      float* send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+      float* recv_buf = static_cast<float*>(recvbuf);
+      float* buffer = comm.my_buffer->buf_float;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+
+
+      double* send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+      double* recv_buf = static_cast<double*>(recvbuf);
+      double* buffer = comm.my_buffer->buf_double;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_LONG)
+    {
+      long* send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+      long* recv_buf = static_cast<long*>(recvbuf);
+      long* buffer = comm.my_buffer->buf_long;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+      unsigned long* buffer = comm.my_buffer->buf_ulong;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      char* send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+      char* recv_buf = static_cast<char*>(recvbuf);
+      char* buffer = comm.my_buffer->buf_char;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else
+    {
+      printf("datatype not supported!!\n");
+      exit(1);
+    }
+    return 0;
+  }
+
+
+  int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count)
+  {
+/*
+    ::MPI_Aint datasize, char_size, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+    ::MPI_Type_get_extent(MPI_CHAR, &lb, &char_size);
+
+    *count = status->char_count / ( datasize/ char_size);
+
+    //printf("MPI_Get_count, status_count  = %d\n", *count);
+    return 0;
+*/
+    ::MPI_Status *mpi_status = static_cast< ::MPI_Status* >(status->mpi_status);
+    ::MPI_Datatype mpi_datatype = static_cast< ::MPI_Datatype >(datatype);
+
+    ::MPI_Get_count(mpi_status, mpi_datatype, count);
+  }
+
+  double MPI_Wtime()
+  {
+    return ::MPI_Wtime();
+
+  }
+
+  void check_sum_send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, int type)
+  {
+    int src_rank;
+    int int_count;
+    ::MPI_Aint datasize, intsize, charsize, lb;
+    
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+    ::MPI_Type_get_extent(MPI_CHAR_STD, &lb, &intsize);
+
+    int_count = count * datasize / intsize ;
+
+    char *buffer = static_cast< char* >(const_cast< void*> (buf));
+    
+    unsigned long sum = 0;
+    for(int i = 0; i<int_count; i++)
+      sum += *(buffer+i); 
+
+
+    MPI_Comm_rank(comm, &src_rank);
+    
+    ofstream myfile;
+    myfile.open ("send_log.txt", ios::app);
+    if (myfile.is_open())
+    {
+      myfile << "type = " << type << " src = "<< src_rank<< " dest = "<< dest <<" tag = "<< tag << "  count = "<< count << " sum = "<< sum << "\n";
+      myfile.close();  
+    }
+    else printf("Unable to open file\n");
+
+  }
+
+
+  void check_sum_recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, int type)
+  {
+    int dest_rank;
+    int int_count;
+    ::MPI_Aint datasize, intsize, charsize, lb;
+    
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+    ::MPI_Type_get_extent(MPI_CHAR_STD, &lb, &intsize);
+
+    int_count = count * datasize / intsize ;
+
+    char *buffer = static_cast< char* >(buf);
+    
+    unsigned long sum = 0;
+    for(int i = 0; i<int_count; i++)
+      sum += *(buffer+i); 
+
+
+    MPI_Comm_rank(comm, &dest_rank);
+    
+    ofstream myfile;
+    myfile.open ("recv_log.txt", ios::app);
+    if (myfile.is_open())
+    {
+      myfile << "type = " << type << " src = "<< src << " dest = "<< dest_rank <<" tag = "<< tag << "  count = "<< count << " sum = "<< sum << "\n";
+      myfile.close();  
+    }
+    else printf("Unable to open file\n");
+
+  }
+}
+
+
+
+
+
+
Index: XIOS/trunk/extern/src_ep/ep_lib.hpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_lib.hpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_lib.hpp	(revision 1034)
@@ -0,0 +1,91 @@
+#ifndef EP_LIB_HPP_INCLUDED
+#define EP_LIB_HPP_INCLUDED
+
+#include "ep_type.hpp"
+#include "ep_lib_intercomm.hpp"
+#include "ep_lib_local.hpp"
+#include "ep_lib_collective.hpp"
+#include "ep_tag.hpp"
+#include "ep_lib_fortran.hpp"
+
+
+namespace ep_lib
+{
+#ifdef _intelmpi
+  typedef int MPI_Datatype;
+  typedef int MPI_Op;
+#elif _openmpi
+  typedef void* MPI_Datatype;
+  typedef void* MPI_Op;
+#endif
+
+  #define MPI_ANY_SOURCE -2 
+  #define MPI_ANY_TAG -1 
+
+  int MPI_Init_thread(int* argc, char*** argv, int required, int*provided);
+
+  int MPI_Init(int *argc, char ***argv);
+
+  int MPI_Initialized(int *flag);
+
+  int MPI_Comm_rank(MPI_Comm comm, int* rank);
+
+  int MPI_Comm_size(MPI_Comm comm, int* rank);
+
+  int MPI_Comm_free(MPI_Comm* comm);
+
+  int MPI_Finalize();
+  
+  double MPI_Wtime();
+
+  int MPI_Abort(MPI_Comm comm, int errorcode);
+
+  int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count);
+
+  #ifdef _openmpi
+  int MPI_Comm_create_endpoints(void* mpi_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from MPI to create endpoints
+  #elif _intelmpi
+  int MPI_Comm_create_endpoints(int mpi_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from MPI to create endpoints
+  #endif
+  
+  int MPI_Comm_create_endpoints(MPI_Comm base_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from EP to create endpoints
+
+  int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm);
+  int MPI_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm);
+  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request);
+  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request);
+
+  int tag_combine(int real_tag, int src, int dest);
+  int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank);
+  int get_ep_rank_intercomm(MPI_Comm comm, int ep_rank_loc, int mpi_rank);
+
+  int Message_Check(MPI_Comm comm);
+
+  int MPI_Recv  (void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status);
+  int MPI_Irecv (void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request);
+  int MPI_Mrecv (void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Status *status);
+  int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request);
+
+
+  int MPI_Test(MPI_Request *request, int *flag, MPI_Status *status);
+  int MPI_Testall(int count, MPI_Request *array_of_requests, int *flag, MPI_Status *array_of_statuses);
+
+  int MPI_Wait(MPI_Request *request, MPI_Status *status);
+  int MPI_Waitall(int count, MPI_Request *array_of_requests, MPI_Status *array_of_statuses);
+
+
+  int MPI_Iprobe(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status);
+  int MPI_Improbe(int source, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status);
+  
+  int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr);
+  
+  int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr);
+
+
+  void check_sum_send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, int type);
+  void check_sum_recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, int type);
+}
+
+
+
+#endif // EP_LIB_HPP_INCLUDED
Index: XIOS/trunk/extern/src_ep/ep_lib_collective.hpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_lib_collective.hpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_lib_collective.hpp	(revision 1034)
@@ -0,0 +1,44 @@
+#ifndef EP_LIB_COLLECTIVE_HPP_INCLUDED
+#define EP_LIB_COLLECTIVE_HPP_INCLUDED
+
+namespace ep_lib
+{
+  #ifdef _intelmpi
+  typedef int MPI_Datatype;
+  typedef int MPI_Op;
+  #elif _openmpi
+  typedef void* MPI_Datatype;
+  typedef void* MPI_Op;
+  #endif
+
+  int MPI_Barrier(MPI_Comm comm);
+
+  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
+
+  int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm);
+  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm);
+
+  int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm);
+  int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm);
+
+  int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, int root, MPI_Comm comm);
+  int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, MPI_Comm comm);
+
+  int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm);
+  int MPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount,
+                   MPI_Datatype recvtype, int root, MPI_Comm comm);
+
+  int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm);
+
+}
+
+#endif // EP_LIB_COLLECTIVE_HPP_INCLUDED
Index: XIOS/trunk/extern/src_ep/ep_lib_fortran.hpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_lib_fortran.hpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_lib_fortran.hpp	(revision 1034)
@@ -0,0 +1,20 @@
+#ifndef EP_LIB_FORTRAN_HPP_INCLUDED
+#define EP_LIB_FORTRAN_HPP_INCLUDED
+
+#include "ep_type.hpp"
+
+namespace ep_lib
+{
+  
+
+  MPI_Fint MPI_Comm_c2f(MPI_Comm comm);
+  //int MPI_Comm_c2f(MPI_Comm comm);
+
+
+  MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
+  //void MPI_Comm_f2c(MPI_Fint comm);
+
+}
+
+
+#endif // EP_LIB_FORTRAN_HPP_INCLUDED
Index: XIOS/trunk/extern/src_ep/ep_lib_intercomm.hpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_lib_intercomm.hpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_lib_intercomm.hpp	(revision 1034)
@@ -0,0 +1,49 @@
+#ifndef EP_LIB_INTERCOMM_HPP_INCLUDED
+#define EP_LIB_INTERCOMM_HPP_INCLUDED
+
+namespace ep_lib
+{
+  #ifdef _intelmpi
+  typedef int MPI_Datatype;
+  typedef int MPI_Op;
+  #elif _openmpi
+  typedef void* MPI_Datatype;
+  typedef void* MPI_Op;
+  #endif
+
+  int Message_Check_intercomm(MPI_Comm comm);
+
+  int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request);
+  int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request);
+
+
+  int MPI_Iprobe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status);
+  int MPI_Iprobe_any_source(int tag, MPI_Comm comm, int *flag, MPI_Status *status);
+  int MPI_Improbe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status);
+  int MPI_Improbe_any_source(int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status);
+
+  int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm);
+
+  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
+
+  int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
+  
+  #ifdef _intelmpi
+  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm);
+  #elif _openmpi
+  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm);
+  #endif
+  
+  int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
+
+  int MPI_Intercomm_merge(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm);
+
+  int MPI_Intercomm_merge_unique_leader(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm);
+
+  int MPI_Comm_remote_size(MPI_Comm comm, int *size);
+
+  int MPI_Comm_test_inter(MPI_Comm comm, int *flag);
+}
+
+
+#endif // EP_LIB_INTERCOMM_HPP_INCLUDED
Index: XIOS/trunk/extern/src_ep/ep_lib_local.hpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_lib_local.hpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_lib_local.hpp	(revision 1034)
@@ -0,0 +1,88 @@
+#ifndef EP_LIB_LOCAL_HPP_INCLUDED
+#define EP_LIB_LOCAL_HPP_INCLUDED
+
+namespace ep_lib
+{
+  #ifdef _intelmpi
+  typedef int MPI_Datatype;
+  typedef int MPI_Op;
+  #elif _openmpi
+  typedef void* MPI_Datatype;
+  typedef void* MPI_Op;
+  #endif
+
+
+  int MPI_Reduce_local       (const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_int   (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_float (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_long  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_ulong (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_char  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+
+
+  int MPI_Scan_local       (const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_int   (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_float (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_long  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_ulong (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_char  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Exscan_local       (const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_int   (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_float (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_long  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_ulong (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_char  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Bcast_local       (void *buffer, int count, MPI_Datatype datatype, MPI_Comm comm);
+  int MPI_Bcast_local_int   (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_float (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_double(void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_long  (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_ulong (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_char  (void *buffer, int count, MPI_Comm comm);
+
+  int MPI_Gather_local       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_int   (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_float (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_long  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_ulong (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_char  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+
+
+  int MPI_Gatherv_local       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf,
+                               const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_int   (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_float (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_double(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_long  (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_ulong (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_char  (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+
+  int MPI_Scatter_local       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_int   (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_float (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_long  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_ulong (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_char  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+
+  int MPI_Scatterv_local       (const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_int   (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_float (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_double(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_long  (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_ulong (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_char  (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+
+  int innode_memcpy(int sender, const void* sendbuf, int receiver, void* recvbuf, int count, MPI_Datatype datatype, MPI_Comm comm);
+
+  int MPI_Barrier_local(MPI_Comm comm);
+
+}
+
+#endif // EP_LIB_LOCAL_HPP_INCLUDED
Index: XIOS/trunk/extern/src_ep/ep_memory.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_memory.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_memory.cpp	(revision 1034)
@@ -0,0 +1,32 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr)
+  {
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Alloc_mem(size.mpi_aint, info.mpi_info, baseptr);
+    return 0;
+   }
+   
+  int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr)
+  {
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Alloc_mem(size, info.mpi_info, baseptr);
+    return 0;
+   }
+
+
+
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_merge.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_merge.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_merge.cpp	(revision 1034)
@@ -0,0 +1,307 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib {
+
+  int MPI_Intercomm_merge_unique_leader(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm)
+  {
+    Debug("intercomm_merge with unique leader\n");
+
+
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = inter_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = inter_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = inter_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = inter_comm.ep_comm_ptr->size_rank_info[2].second;
+
+    int local_high = high;
+    int remote_high;
+
+    int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size();
+
+    int local_ep_rank, local_ep_rank_loc, local_mpi_rank;
+    int local_ep_size, local_num_ep, local_mpi_size;
+
+    local_ep_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first;
+    local_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first;
+    local_mpi_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first;
+    local_ep_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second;
+    local_num_ep = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second;
+    local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second;
+
+
+//    if(local_ep_rank == 0 && high == false)
+//    {
+//      MPI_Status status;
+//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
+//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
+//    }
+//
+//    if(local_ep_rank == 0 && high == true)
+//    {
+//      MPI_Status status;
+//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
+//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
+//    }
+
+    if(local_ep_rank == 0)
+    {
+      MPI_Status status;
+      MPI_Request req_s, req_r;
+      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_s);
+      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_r);
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+    }
+
+
+    MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm));
+
+//    printf("%d, %d, %d, %d\n", local_ep_size, remote_ep_size, local_high, remote_high);
+
+
+    MPI_Comm_dup(inter_comm, newintracomm);
+
+    int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size;
+
+
+    int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank;
+    int intra_ep_size, intra_num_ep, intra_mpi_size;
+
+    intra_ep_rank = newintracomm->ep_comm_ptr->size_rank_info[0].first;
+    intra_ep_rank_loc = newintracomm->ep_comm_ptr->size_rank_info[1].first;
+    intra_mpi_rank = newintracomm->ep_comm_ptr->size_rank_info[2].first;
+    intra_ep_size = newintracomm->ep_comm_ptr->size_rank_info[0].second;
+    intra_num_ep = newintracomm->ep_comm_ptr->size_rank_info[1].second;
+    intra_mpi_size = newintracomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+    MPI_Barrier_local(*newintracomm);
+
+
+    int *reorder;
+    if(intra_ep_rank_loc == 0)
+    {
+      reorder = new int[intra_ep_size];
+    }
+
+
+    MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm);
+    if(intra_ep_rank_loc == 0)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Bcast(reorder, intra_ep_size, MPI_INT_STD, 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
+
+      vector< pair<int, int> > tmp_rank_map(intra_ep_size);
+
+
+      for(int i=0; i<intra_ep_size; i++)
+      {
+        tmp_rank_map[reorder[i]] = newintracomm->rank_map->at(i) ;
+      }
+
+      newintracomm->rank_map->swap(tmp_rank_map);
+
+      tmp_rank_map.clear();
+    }
+
+    MPI_Barrier_local(*newintracomm);
+
+    (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank;
+
+    if(intra_ep_rank_loc == 0)
+    {
+      delete[] reorder;
+    }
+
+    return MPI_SUCCESS;
+  }
+
+
+
+
+
+  int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm)
+  {
+
+    assert(inter_comm.is_intercomm);
+
+    if(inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99)
+    {
+        return MPI_Intercomm_merge_unique_leader(inter_comm, high, newintracomm);
+    }
+
+
+    Debug("intercomm_merge kernel\n");
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = inter_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = inter_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = inter_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = inter_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    int local_ep_rank, local_ep_rank_loc, local_mpi_rank;
+    int local_ep_size, local_num_ep, local_mpi_size;
+
+
+    local_ep_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first;
+    local_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first;
+    local_mpi_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first;
+    local_ep_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second;
+    local_num_ep = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second;
+    local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second;
+
+    int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size();
+
+    int local_high = high;
+    int remote_high;
+
+    MPI_Barrier(inter_comm);
+
+//    if(local_ep_rank == 0 && high == false)
+//    {
+//      MPI_Status status;
+//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
+//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
+//    }
+//
+//    if(local_ep_rank == 0 && high == true)
+//    {
+//      MPI_Status status;
+//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
+//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
+//    }
+
+    if(local_ep_rank == 0)
+    {
+      MPI_Status status;
+      MPI_Request req_s, req_r;
+      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_s);
+      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_r);
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+    }
+
+    MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm));
+
+    int intercomm_high;
+    if(ep_rank == 0) intercomm_high = local_high;
+    MPI_Bcast(&intercomm_high, 1, MPI_INT, 0, inter_comm);
+
+    //printf("remote_ep_size = %d, local_high = %d, remote_high = %d, intercomm_high = %d\n", remote_ep_size, local_high, remote_high, intercomm_high);
+
+
+    ::MPI_Comm mpi_intracomm;
+    MPI_Comm *ep_intracomm;
+
+    if(ep_rank_loc == 0)
+    {
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(inter_comm.ep_comm_ptr->intercomm->mpi_inter_comm);
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Intercomm_merge(mpi_comm, intercomm_high, &mpi_intracomm);
+      MPI_Info info;
+      MPI_Comm_create_endpoints(mpi_intracomm, num_ep, info, ep_intracomm);
+
+      inter_comm.ep_comm_ptr->comm_list->mem_bridge = ep_intracomm;
+
+    }
+
+
+
+    MPI_Barrier_local(inter_comm);
+
+    *newintracomm = inter_comm.ep_comm_ptr->comm_list->mem_bridge[ep_rank_loc];
+
+    int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size;
+
+    int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank;
+    int intra_ep_size, intra_num_ep, intra_mpi_size;
+
+    intra_ep_rank = newintracomm->ep_comm_ptr->size_rank_info[0].first;
+    intra_ep_rank_loc = newintracomm->ep_comm_ptr->size_rank_info[1].first;
+    intra_mpi_rank = newintracomm->ep_comm_ptr->size_rank_info[2].first;
+    intra_ep_size = newintracomm->ep_comm_ptr->size_rank_info[0].second;
+    intra_num_ep = newintracomm->ep_comm_ptr->size_rank_info[1].second;
+    intra_mpi_size = newintracomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    MPI_Barrier_local(*newintracomm);
+
+
+    int *reorder;
+    if(intra_ep_rank_loc == 0)
+    {
+      reorder = new int[intra_ep_size];
+    }
+
+
+
+    MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm);
+    if(intra_ep_rank_loc == 0)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Bcast(reorder, intra_ep_size, MPI_INT_STD, 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
+
+      vector< pair<int, int> > tmp_rank_map(intra_ep_size);
+
+
+      for(int i=0; i<intra_ep_size; i++)
+      {
+        tmp_rank_map[reorder[i]] = newintracomm->rank_map->at(i) ;
+      }
+
+      newintracomm->rank_map->swap(tmp_rank_map);
+
+      tmp_rank_map.clear();
+    }
+
+    MPI_Barrier_local(*newintracomm);
+
+    (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank;
+
+
+    if(intra_ep_rank_loc == 0)
+    {
+      delete[] reorder;
+
+    }
+
+    /*
+    if(intra_ep_rank == 0)
+    {
+      for(int i=0; i<intra_ep_size; i++)
+      {
+        printf("intra rank_map[%d] = (%d, %d)\n", i, newintracomm->rank_map->at(i).first, newintracomm->rank_map->at(i).second);
+      }
+    }
+*/
+    return MPI_SUCCESS;
+
+  }
+
+
+}
Index: XIOS/trunk/extern/src_ep/ep_message.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_message.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_message.cpp	(revision 1034)
@@ -0,0 +1,230 @@
+/*!
+   \file ep_message.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI endpoint function: Message_Check
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  int Message_Check(MPI_Comm comm)
+  {
+    int myRank;
+    MPI_Comm_rank(comm, &myRank);
+
+    //printf("myRank = %d, comm.is_ep = %d, comm.is_intercomm = %d\n", myRank, comm.is_ep, comm.is_intercomm);
+    if(!comm.is_ep) return 0;
+
+    if(comm.is_intercomm)
+	  {
+      return  Message_Check_intercomm(comm);
+    }
+
+    int flag = true;
+    ::MPI_Message message;
+    ::MPI_Status status;
+    int mpi_source;
+
+    while(flag) // loop until the end of global queue
+    {
+      Debug("Message probing for intracomm\n");
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+      #ifdef _openmpi
+      #pragma omp critical (_mpi_call)
+      {
+        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &status);
+        if(flag)
+        {
+          Debug("find message in mpi comm \n");
+          mpi_source = status.MPI_SOURCE;
+          int tag = status.MPI_TAG;
+          ::MPI_Mprobe(mpi_source, tag, mpi_comm, &message, &status);
+
+        }
+      }
+      #elif _intelmpi
+      //::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &message, &status); 
+      ::MPI_Improbe(-2, -1, mpi_comm, &flag, &message, &status); 
+      //printf("myRank = %d, ::MPI_Improbe flag = %d\n", myRank, flag);     
+      #endif
+
+      if(flag)
+      {
+
+        MPI_Message *msg_block = new MPI_Message; //printf("myRank = %d, new ok\n", myRank);
+        msg_block->mpi_message = message;  //printf("myRank = %d, msg_block->mpi_message = %d\n", myRank, msg_block->mpi_message);
+        msg_block->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong();  //printf("myRank = %d, msg_block->ep_tag = %d\n", myRank, msg_block->ep_tag);
+        int src_loc       = bitset<8> (status.MPI_TAG >> 8) .to_ulong();  //printf("myRank = %d, src_loc = %d\n",           myRank, src_loc);
+        int dest_loc      = bitset<8> (status.MPI_TAG)	    .to_ulong();  //printf("myRank = %d, dest_loc = %d\n",          myRank, dest_loc);
+        int src_mpi       = status.MPI_SOURCE;                            //printf("myRank = %d, src_mpi = %d\n",           myRank, src_mpi);
+             
+        msg_block->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);        //printf("myRank = %d, msg_block->ep_src = %d\n",  myRank, msg_block->ep_src);
+
+        //printf("myRank = %d, probed one message, ep_src = %d, ep_dest = %d, tag = %d, message = %d\n", myRank, msg_block->ep_src, msg_block->ep_dest, msg_block->ep_tag, msg_block->mpi_message);
+        msg_block->mpi_status = new ::MPI_Status(status);
+
+        MPI_Comm* ptr_comm_list = comm.ep_comm_ptr->comm_list;
+        MPI_Comm* ptr_comm_target = &ptr_comm_list[dest_loc];
+
+
+        #pragma omp critical (_query)
+        {
+          #pragma omp flush
+          ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);  
+          // printf("myRank = %d, push_back OK, ep_src = %d, ep_tag = %d, mpi_status = %p (%p)\n", myRank, 
+          //                                                                                   ptr_comm_target->ep_comm_ptr->message_queue->back().ep_src,
+          //                                                                                   ptr_comm_target->ep_comm_ptr->message_queue->back().ep_tag,
+          //                                                                                   msg_block->mpi_status, &status);
+    
+          #pragma omp flush
+        }
+        
+        delete msg_block;
+        //printf("myRank = %d, delete msg_block, queue size = %lu\n", myRank, ptr_comm_target->ep_comm_ptr->message_queue->size());
+        
+      }
+
+    }
+
+    return MPI_SUCCESS;
+  }
+
+
+
+  int Message_Check_intercomm(MPI_Comm comm)
+  {
+    if(!comm.ep_comm_ptr->intercomm->mpi_inter_comm) return 0;
+
+    Debug("Message probing for intercomm\n");
+
+    int flag = true;
+    ::MPI_Message message;
+    ::MPI_Status status;
+    int mpi_source;
+    int current_ep_rank;
+    MPI_Comm_rank(comm, &current_ep_rank);
+
+    while(flag) // loop until the end of global queue "comm.ep_comm_ptr->intercomm->mpi_inter_comm"
+    {
+      Debug("Message probing for intracomm\n");
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.ep_comm_ptr->intercomm->mpi_inter_comm);  // => mpi_intercomm
+      
+      #ifdef _openmpi
+      #pragma omp critical (_mpi_call)
+      {
+        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &status);
+        if(flag)
+        {
+          Debug("find message in mpi comm \n");
+          mpi_source = status.MPI_SOURCE;
+          int tag = status.MPI_TAG;
+          ::MPI_Mprobe(mpi_source, tag, mpi_comm, &message, &status);
+
+        }
+      }
+      #elif _intelmpi
+      //::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &message, &status);       
+      ::MPI_Improbe(-2, -1, mpi_comm, &flag, &message, &status);       
+      #endif
+
+      if(flag)
+      {
+
+        MPI_Message *msg_block = new MPI_Message;
+
+        msg_block->mpi_message = message;
+        msg_block->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong();
+        int src_loc       = bitset<8> (status.MPI_TAG >> 8) .to_ulong();
+        int dest_loc      = bitset<8> (status.MPI_TAG)	    .to_ulong();
+        int src_mpi       = status.MPI_SOURCE;
+        int current_inter = comm.ep_comm_ptr->intercomm->local_rank_map->at(current_ep_rank).first;
+             
+        msg_block->ep_src  = get_ep_rank_intercomm(comm, src_loc,  src_mpi);
+        msg_block->mpi_status = new ::MPI_Status(status);
+
+
+        MPI_Comm* ptr_comm_list = comm.ep_comm_ptr->comm_list;
+        MPI_Comm* ptr_comm_target = &ptr_comm_list[dest_loc];
+
+
+        #pragma omp critical (_query)
+        {
+          #pragma omp flush
+          ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);
+          //printf("probed one message, ep_src = %d, ep_dest = %d, tag = %d, queue = %p, message = %d\n", msg_block->ep_src, msg_block->ep_dest, msg_block->ep_tag, ptr_comm_target->ep_comm_ptr->message_queue, msg_block->mpi_message);
+          #pragma omp flush
+        }
+        
+        delete msg_block;
+        
+      }
+
+    }
+
+    flag = true;
+    while(flag) // loop until the end of global queue "comm.mpi_comm"
+    {
+      Debug("Message probing for intracomm\n");
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+      #ifdef _openmpi
+      #pragma omp critical (_mpi_call)
+      {
+        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &status);
+        if(flag)
+        {
+          Debug("find message in mpi comm \n");
+          mpi_source = status.MPI_SOURCE;
+          int tag = status.MPI_TAG;
+          ::MPI_Mprobe(mpi_source, tag, mpi_comm, &message, &status);
+
+        }
+      }
+      #elif _intelmpi
+      ::MPI_Improbe(-2, -1, mpi_comm, &flag, &message, &status);       
+      #endif
+
+      if(flag)
+      {
+
+        MPI_Message *msg_block = new MPI_Message;
+       
+        msg_block->mpi_message = message;
+        msg_block->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong();
+        int src_loc       = bitset<8> (status.MPI_TAG >> 8) .to_ulong();
+        int dest_loc      = bitset<8> (status.MPI_TAG)	    .to_ulong();
+        int src_mpi       = status.MPI_SOURCE;
+        int current_inter = comm.ep_comm_ptr->intercomm->local_rank_map->at(current_ep_rank).first;
+        
+        msg_block->ep_src  = get_ep_rank_intercomm(comm, src_loc, src_mpi);
+        msg_block->mpi_status = new ::MPI_Status(status);
+        
+
+        MPI_Comm* ptr_comm_list = comm.ep_comm_ptr->comm_list;
+        MPI_Comm* ptr_comm_target = &ptr_comm_list[dest_loc];
+
+
+        #pragma omp critical (_query)
+        {
+          #pragma omp flush
+          ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);
+          printf("probed one message, ep_src = %d, tag = %d, mpi_status = %p (%p), message = %d\n", msg_block->ep_src, msg_block->ep_tag, msg_block->mpi_status, &status, msg_block->mpi_message);
+          #pragma omp flush
+        }
+        
+        delete msg_block;
+        
+      }
+
+    }
+
+		return MPI_SUCCESS;
+  }
+
+}
Index: XIOS/trunk/extern/src_ep/ep_probe.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_probe.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_probe.cpp	(revision 1034)
@@ -0,0 +1,145 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Iprobe(int src, int tag, MPI_Comm comm, int *flag, MPI_Status *status)
+  {
+    *flag = false;
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.mpi_comm);
+      ::MPI_Status *mpi_status = static_cast< ::MPI_Status* >(status->mpi_status);
+      ::MPI_Iprobe(src, tag, mpi_comm, flag, mpi_status);
+
+      status->mpi_status = mpi_status;
+      status->ep_src = src;
+      status->ep_tag = tag;
+      return 0;
+    }
+
+    Debug("calling MPI_Iprobe EP\n");
+
+    Message_Check(comm);
+
+    #pragma omp flush
+
+    
+    //printf("iprobe, message queue size = %lu, queue = %p\n", comm.ep_comm_ptr->message_queue->size(), comm.ep_comm_ptr->message_queue);
+
+    #pragma omp flush
+    #pragma critical (_query)
+    if(comm.ep_comm_ptr->message_queue->size() > 0)
+    {
+      for(Message_list::iterator it = comm.ep_comm_ptr->message_queue->begin(); it!= comm.ep_comm_ptr->message_queue->end(); it++)
+      {
+        bool src_matched = src<0? true: it->ep_src == src;
+        bool tag_matched = tag<0? true: it->ep_tag == tag;
+        
+        if(src_matched && tag_matched)        
+        {
+          Debug("find message\n");
+          // printf("iprobe, find in local message queue %p, src = %d, tag = %d\n", comm.ep_comm_ptr->message_queue, it->ep_src, it->ep_tag);
+
+          *flag = true;
+
+          ::MPI_Status mpi_status = *(static_cast< ::MPI_Status *>(it->mpi_status));
+
+          status->mpi_status = new ::MPI_Status(mpi_status);
+          status->ep_src = it->ep_src;
+          status->ep_tag = it->ep_tag;
+
+          break;
+        }
+
+      }
+    }
+
+    return 0;
+  }
+
+
+
+  int MPI_Improbe(int src, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status)
+  {
+    *flag = false;
+    if(!comm.is_ep)
+    {
+      Debug("calling MPI_Improbe MPI\n");
+
+    	::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm.mpi_comm);
+    	::MPI_Status mpi_status;
+      ::MPI_Message mpi_message;
+
+    	#ifdef _openmpi
+      #pragma omp critical (_mpi_call)
+      {
+        ::MPI_Iprobe(src, tag, mpi_comm, flag, &mpi_status);
+        if(*flag)
+        {
+          ::MPI_Mprobe(src, tag, mpi_comm, &mpi_message, &mpi_status);
+        }
+      }
+      #elif _intelmpi
+    	::MPI_Improbe(src, tag, mpi_comm, flag, &mpi_message, &mpi_status);
+      #endif
+    	
+      status->mpi_status = new ::MPI_Status(mpi_status);
+      status->ep_src = src;
+      status->ep_tag = tag;
+
+      message->mpi_message = mpi_message;
+      message->ep_src = src;
+      message->ep_tag = tag;
+    	return 0;
+    }
+
+    
+    Message_Check(comm);
+
+    #pragma omp flush
+    #pragma critical (_query)
+    if(comm.ep_comm_ptr->message_queue->size() > 0)
+    {
+      for(Message_list::iterator it = comm.ep_comm_ptr->message_queue->begin(); it!= comm.ep_comm_ptr->message_queue->end(); it++)
+      {
+        bool src_matched = src<0? true: it->ep_src == src;
+        bool tag_matched = tag<0? true: it->ep_tag == tag;
+        
+        if(src_matched && tag_matched)
+        {
+          *flag = true;
+
+          ::MPI_Status mpi_status;
+          mpi_status = *(static_cast< ::MPI_Status *>(it->mpi_status));
+
+          status->mpi_status = new ::MPI_Status(mpi_status);
+          status->ep_src = it->ep_src;
+          status->ep_tag = it->ep_tag;
+
+          message->mpi_message = it->mpi_message;
+          message->ep_tag = it->ep_tag;
+          message->ep_src = it->ep_src;
+
+          #pragma omp critical (_query)
+          {              
+            //printf("local message erased. src = %d, dest = %d, tag = %d\n", it->ep_src, it->ep_dest, it->ep_tag);     
+            delete it->mpi_status;
+            comm.ep_comm_ptr->message_queue->erase(it);
+            #pragma omp flush
+          }
+
+          break;
+        }
+
+      }
+    }
+  }
+
+}
+
Index: XIOS/trunk/extern/src_ep/ep_rank.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_rank.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_rank.cpp	(revision 1034)
@@ -0,0 +1,50 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Comm_rank(MPI_Comm comm, int* rank)
+  {
+
+    if(comm.is_ep)
+    {
+      Debug("Calling EP_Comm_rank\n");
+
+      if(comm.is_intercomm)
+      {
+        *rank = comm.ep_comm_ptr->intercomm->size_rank_info[0].first;
+        return 0;
+      }
+      else
+      {
+        *rank = comm.ep_comm_ptr->size_rank_info[0].first;
+        return 0;
+      }
+    }
+
+
+    if(comm.mpi_comm)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Comm_rank(mpi_comm, rank);
+      return 0;
+    }
+    else
+    {
+      printf("rank is undefined\n");
+
+      *rank = MPI_UNDEFINED;
+      return 0;
+    }
+
+  }
+
+
+
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_recv.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_recv.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_recv.cpp	(revision 1034)
@@ -0,0 +1,141 @@
+/*!
+   \file ep_recv.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI receive functions: MPI_Recv, MPI_Mrecv, MPI_Irecv, MPI_Imrecv
+ */
+
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib {
+
+	int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status)
+	{
+    int dest_rank;
+    MPI_Comm_rank(comm, &dest_rank);
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.mpi_comm);
+      ::MPI_Status mpi_status;
+      ::MPI_Recv(buf, count, static_cast< ::MPI_Datatype >(datatype), src, tag, mpi_comm, &mpi_status);
+
+      status->ep_src = src;
+      status->ep_tag = tag;
+      status->ep_datatype = datatype;
+
+      return 0;  
+    }
+
+    Message_Check(comm);
+
+    MPI_Request request;
+    MPI_Irecv(buf, count, datatype, src, tag, comm, &request);
+    MPI_Wait(&request, status);
+
+    return 0;
+	}
+
+
+
+
+	int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request)
+	{
+
+		Debug("MPI_Irecv with EP");
+    int dest_rank;
+    MPI_Comm_rank(comm, &dest_rank);
+
+ 		if(!comm.is_ep)
+		{
+		  ::MPI_Request mpi_request;
+		  ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+		  ::MPI_Irecv(buf, count, static_cast< ::MPI_Datatype> (datatype), src, tag, mpi_comm, &mpi_request);
+
+		  request->mpi_request = mpi_request;
+      request->ep_src = src;
+      request->ep_datatype = datatype;
+      request->ep_tag = tag;
+		}
+
+		Message_Check(comm);
+		
+
+		request->mpi_request = NULL;
+		request->buf = buf;
+    request->comm = comm;
+		request->type = 2;
+
+		request->ep_src = src;
+    request->ep_tag = tag;
+    request->ep_datatype = datatype;
+
+
+    
+		/* With Improbe*/
+		Message_Check(comm);
+
+    int flag = false;
+    MPI_Message message;
+    MPI_Status status;
+    
+    MPI_Improbe(src, tag, comm, &flag, &message, &status);
+            
+    if(flag)
+    {
+      MPI_Imrecv(buf, count, datatype, &message, request);
+    }
+
+    return 0;
+	}
+
+	int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request)
+	{
+		Debug("MPI_Imrecv");
+
+		request->type = 3;
+
+    ::MPI_Request mpi_request;
+    ::MPI_Message mpi_message = static_cast< ::MPI_Message >(message->mpi_message);
+		
+		::MPI_Imrecv(buf, count, static_cast< ::MPI_Datatype>(datatype), &mpi_message, &mpi_request);
+
+    request->mpi_request = mpi_request;
+    request->ep_datatype = datatype;
+    request->ep_tag = message->ep_tag;
+    request->ep_src = message->ep_src;
+    request->buf = buf;
+
+    return 0;
+	}
+
+
+  int MPI_Mrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Status *status)
+	{
+    Debug("EP Mrecv called\n");
+
+    ::MPI_Status mpi_status;
+    ::MPI_Message mpi_message = static_cast< ::MPI_Message >(message->mpi_message);
+		
+		::MPI_Mrecv(buf, count, static_cast< ::MPI_Datatype>(datatype), &mpi_message, &mpi_status);
+
+    status->mpi_status = new ::MPI_Status(mpi_status);
+    status->ep_src = message->ep_src;
+    status->ep_datatype = datatype;
+    status->ep_tag = message->ep_tag;
+
+    //check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag);
+
+    return 0;
+	}
+
+
+  
+
+}
Index: XIOS/trunk/extern/src_ep/ep_reduce.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_reduce.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_reduce.cpp	(revision 1034)
@@ -0,0 +1,693 @@
+/*!
+   \file ep_reduce.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Reduce, MPI_Allreduce
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib {
+
+  template<typename T>
+  T max_op(T a, T b)
+  {
+    return max(a,b);
+  }
+
+  template<typename T>
+  T min_op(T a, T b)
+  {
+    return min(a,b);
+  }
+
+
+  int MPI_Reduce_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Reduce_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Reduce_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Reduce Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+
+  int MPI_Reduce_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+          }
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Reduce_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+          }
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Reduce(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), root,
+                   static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+
+    if(!comm.mpi_comm) return 0;
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint recvsize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &recvsize);
+
+    void *local_recvbuf;
+    if(ep_rank_loc==0)
+    {
+      local_recvbuf = new void*[recvsize*count];
+    }
+
+    MPI_Reduce_local(sendbuf, local_recvbuf, count, datatype, op, comm);
+
+
+    if(ep_rank_loc==0)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Reduce(local_recvbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      innode_memcpy(0, recvbuf, root_ep_loc, recvbuf, count, datatype, comm);
+    }
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT) delete[] static_cast<int*>(local_recvbuf);
+      else if(datatype == MPI_FLOAT) delete[] static_cast<float*>(local_recvbuf);
+      else if(datatype == MPI_DOUBLE) delete[] static_cast<double*>(local_recvbuf);
+      else if(datatype == MPI_LONG) delete[] static_cast<long*>(local_recvbuf);
+      else if(datatype == MPI_UNSIGNED_LONG) delete[] static_cast<unsigned long*>(local_recvbuf);
+      else delete[] static_cast<char*>(local_recvbuf);
+    }
+
+    Message_Check(comm);
+
+    return 0;
+  }
+
+
+
+
+  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allreduce(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op),
+                      static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint recvsize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &recvsize);
+
+    void *local_recvbuf;
+    if(ep_rank_loc==0)
+    {
+      local_recvbuf = new void*[recvsize*count];
+    }
+
+    MPI_Reduce_local(sendbuf, local_recvbuf, count, datatype, op, comm);
+
+
+    if(ep_rank_loc==0)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Allreduce(local_recvbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    MPI_Bcast_local(recvbuf, count, datatype, comm);
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT) delete[] static_cast<int*>(local_recvbuf);
+      else if(datatype == MPI_FLOAT) delete[] static_cast<float*>(local_recvbuf);
+      else if(datatype == MPI_DOUBLE) delete[] static_cast<double*>(local_recvbuf);
+      else if(datatype == MPI_LONG) delete[] static_cast<long*>(local_recvbuf);
+      else if(datatype == MPI_UNSIGNED_LONG) delete[] static_cast<unsigned long*>(local_recvbuf);
+      else delete[] static_cast<char*>(local_recvbuf);
+    }
+
+    Message_Check(comm);
+
+    return 0;
+  }
+
+
+  int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op),
+                           static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+    void* local_buf;
+    void* local_buf2;
+    int local_buf_size = accumulate(recvcounts, recvcounts+ep_size, 0);
+    int local_buf2_size = accumulate(recvcounts+ep_rank-ep_rank_loc, recvcounts+ep_rank-ep_rank_loc+num_ep, 0);
+
+    ::MPI_Aint datasize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    if(ep_rank_loc == 0)
+    {
+      local_buf = new void*[local_buf_size*datasize];
+      local_buf2 = new void*[local_buf2_size*datasize];
+    }
+    MPI_Reduce_local(sendbuf, local_buf, local_buf_size, MPI_INT, op, comm);
+
+
+    if(ep_rank_loc == 0)
+    {
+      int local_recvcnt[mpi_size];
+      for(int i=0; i<mpi_size; i++)
+      {
+        local_recvcnt[i] = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+      }
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Reduce_scatter(local_buf, local_buf2, local_recvcnt, static_cast< ::MPI_Datatype>(datatype),
+                         static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    int displs[num_ep];
+    displs[0] = 0;
+    for(int i=1; i<num_ep; i++)
+    {
+      displs[i] = displs[i-1] + recvcounts[ep_rank-ep_rank_loc+i-1];
+    }
+
+    MPI_Scatterv_local(local_buf2, recvcounts+ep_rank-ep_rank_loc, displs, datatype, recvbuf, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_buf);
+        delete[] static_cast<int*>(local_buf2);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_buf);
+        delete[] static_cast<float*>(local_buf2);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_buf);
+        delete[] static_cast<double*>(local_buf2);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_buf);
+        delete[] static_cast<long*>(local_buf2);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_buf);
+        delete[] static_cast<unsigned long*>(local_buf2);
+      }
+      else // if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<char*>(local_buf);
+        delete[] static_cast<char*>(local_buf2);
+      }
+    }
+
+    Message_Check(comm);
+    return 0;
+  }
+}
+
Index: XIOS/trunk/extern/src_ep/ep_scan.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_scan.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_scan.cpp	(revision 1034)
@@ -0,0 +1,754 @@
+/*!
+   \file ep_scan.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Scan
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+  template<typename T>
+  T max_op(T a, T b)
+  {
+    return max(a,b);
+  }
+
+  template<typename T>
+  T min_op(T a, T b)
+  {
+    return min(a,b);
+  }
+
+
+  int MPI_Scan_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Scan_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Scan_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Scan_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Scan_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Scan_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Scan_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Scan Datatype not supported!\n");
+      exit(0);
+    }
+
+  }
+
+
+
+
+  int MPI_Scan_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+
+  }
+
+  int MPI_Scan_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Scan(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
+                 static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint datasize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void* local_scan_recvbuf;
+    local_scan_recvbuf = new void*[datasize * count];
+
+
+    // local scan
+    MPI_Scan_local(sendbuf, recvbuf, count, datatype, op, comm);
+
+//     MPI_scan
+    void* local_sum;
+    void* mpi_scan_recvbuf;
+
+
+    mpi_scan_recvbuf = new void*[datasize*count];
+
+    if(ep_rank_loc == 0)
+    {
+      local_sum = new void*[datasize*count];
+    }
+
+
+    MPI_Reduce_local(sendbuf, local_sum, count, datatype, op, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Exscan(local_sum, mpi_scan_recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    if(mpi_rank > 0)
+    {
+      MPI_Bcast_local(mpi_scan_recvbuf, count, datatype, comm);
+    }
+
+
+    if(datatype == MPI_DOUBLE)
+    {
+      double* sum_buf = static_cast<double*>(mpi_scan_recvbuf);
+      double* recv_buf = static_cast<double*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<double*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<double*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_FLOAT)
+    {
+      float* sum_buf = static_cast<float*>(mpi_scan_recvbuf);
+      float* recv_buf = static_cast<float*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<float*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<float*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_INT)
+    {
+      int* sum_buf = static_cast<int*>(mpi_scan_recvbuf);
+      int* recv_buf = static_cast<int*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<int*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<int*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_LONG)
+    {
+      long* sum_buf = static_cast<long*>(mpi_scan_recvbuf);
+      long* recv_buf = static_cast<long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* sum_buf = static_cast<unsigned long*>(mpi_scan_recvbuf);
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<unsigned long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<unsigned long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_CHAR)
+    {
+      char* sum_buf = static_cast<char*>(mpi_scan_recvbuf);
+      char* recv_buf = static_cast<char*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<char*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<char*>(local_sum);
+      }
+    }
+
+
+  }
+
+}
Index: XIOS/trunk/extern/src_ep/ep_scatter.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_scatter.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_scatter.cpp	(revision 1034)
@@ -0,0 +1,410 @@
+/*!
+   \file ep_gather.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Scatter
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  int MPI_Scatter_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Scatter_local_int(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Scatter_local_float(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Scatter_local_double(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Scatter_local_long(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Scatter_local_ulong(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Scatter_local_char(sendbuf, count, recvbuf, comm);
+    }
+    else
+    {
+      printf("MPI_Scatter Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Scatter_local_int(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatter_local_float(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatter_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatter_local_long(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatter_local_ulong(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatter_local_char(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+
+
+
+  int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Scatter(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+                    root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype) && sendcount == recvcount);
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    ::MPI_Aint datasize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+
+    void *master_sendbuf;
+    void *local_recvbuf;
+
+    if(root_ep_loc!=0 && mpi_rank == root_mpi_rank)
+    {
+      if(ep_rank_loc == 0) master_sendbuf = new void*[datasize*count*ep_size];
+
+      innode_memcpy(root_ep_loc, sendbuf, 0, master_sendbuf, count*ep_size, datatype, comm);
+    }
+
+
+
+    if(ep_rank_loc == 0)
+    {
+      int mpi_sendcnt = count*num_ep;
+      int mpi_scatterv_sendcnt[mpi_size];
+      int displs[mpi_size];
+
+      local_recvbuf = new void*[datasize*mpi_sendcnt];
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Gather(&mpi_sendcnt, 1, MPI_INT_STD, mpi_scatterv_sendcnt, 1, MPI_INT_STD, root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+        displs[i] = displs[i-1] + mpi_scatterv_sendcnt[i-1];
+
+
+      if(root_ep_loc!=0)
+      {
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Scatterv(master_sendbuf, mpi_scatterv_sendcnt, displs, static_cast< ::MPI_Datatype>(datatype),
+                     local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+      else
+      {
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Scatterv(sendbuf, mpi_scatterv_sendcnt, displs, static_cast< ::MPI_Datatype>(datatype),
+                     local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+    }
+
+    MPI_Scatter_local(local_recvbuf, count, datatype, recvbuf, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      if(datatype == MPI_INT)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<int*>(master_sendbuf);
+        delete[] static_cast<int*>(local_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<float*>(master_sendbuf);
+        delete[] static_cast<float*>(local_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<double*>(master_sendbuf);
+        delete[] static_cast<double*>(local_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<long*>(master_sendbuf);
+        delete[] static_cast<long*>(local_recvbuf);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<unsigned long*>(master_sendbuf);
+        delete[] static_cast<unsigned long*>(local_recvbuf);
+      }
+      else //if(datatype == MPI_DOUBLE)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<char*>(master_sendbuf);
+        delete[] static_cast<char*>(local_recvbuf);
+      }
+    }
+
+  }
+}
Index: XIOS/trunk/extern/src_ep/ep_scatterv.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_scatterv.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_scatterv.cpp	(revision 1034)
@@ -0,0 +1,432 @@
+/*!
+   \file ep_gather.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Scatterv
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  int MPI_Scatterv_local(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Scatterv_local_int(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Scatterv_local_float(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Scatterv_local_double(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Scatterv_local_long(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Scatterv_local_ulong(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Scatterv_local_char(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else
+    {
+      printf("MPI_scatterv Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Scatterv_local_int(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatterv_local_float(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatterv_local_double(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatterv_local_long(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatterv_local_ulong(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatterv_local_char(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount,
+                   MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Scatterv(sendbuf, sendcounts, displs, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount,
+                     static_cast< ::MPI_Datatype>(recvtype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype));
+
+    MPI_Datatype datatype = sendtype;
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+    
+    MPI_Bcast(const_cast<int*>(sendcounts), ep_size, MPI_INT, root, comm);
+    MPI_Bcast(const_cast<int*>(displs), ep_size, MPI_INT, root, comm);
+
+
+    int count = recvcount;
+
+    ::MPI_Aint datasize, lb;
+    #ifdef _serialized
+    #pragma omp critical (_mpi_call)
+    #endif // _serialized
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    assert(accumulate(sendcounts, sendcounts+ep_size-1, 0) == displs[ep_size-1]); // Only for contunuous gather.
+
+
+    void *master_sendbuf;
+    void *local_recvbuf;
+
+    if(root_ep_loc!=0 && mpi_rank == root_mpi_rank)
+    {
+      int count_sum = accumulate(sendcounts, sendcounts+ep_size, 0);
+      if(ep_rank_loc == 0) master_sendbuf = new void*[datasize*count_sum];
+
+      innode_memcpy(root_ep_loc, sendbuf, 0, master_sendbuf, count_sum, datatype, comm);
+    }
+
+
+
+    if(ep_rank_loc == 0)
+    {
+      int mpi_sendcnt = accumulate(sendcounts+ep_rank, sendcounts+ep_rank+num_ep, 0);
+      int mpi_scatterv_sendcnt[mpi_size];
+      int mpi_displs[mpi_size];
+
+      local_recvbuf = new void*[datasize*mpi_sendcnt];
+
+      #ifdef _serialized
+      #pragma omp critical (_mpi_call)
+      #endif // _serialized
+      ::MPI_Gather(&mpi_sendcnt, 1, MPI_INT_STD, mpi_scatterv_sendcnt, 1, MPI_INT_STD, root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      mpi_displs[0] = displs[0];
+      for(int i=1; i<mpi_size; i++)
+        mpi_displs[i] = mpi_displs[i-1] + mpi_scatterv_sendcnt[i-1];
+
+
+      if(root_ep_loc!=0)
+      {
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Scatterv(master_sendbuf, mpi_scatterv_sendcnt, mpi_displs, static_cast< ::MPI_Datatype>(datatype),
+                     local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+      else
+      {
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Scatterv(sendbuf, mpi_scatterv_sendcnt, mpi_displs, static_cast< ::MPI_Datatype>(datatype),
+                     local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+    }
+
+    int local_displs[num_ep];
+    local_displs[0] = 0;
+    for(int i=1; i<num_ep; i++)
+    {
+      local_displs[i] = displs[ep_rank-ep_rank_loc+i]-displs[ep_rank-ep_rank_loc];
+    }
+
+    MPI_Scatterv_local(local_recvbuf, sendcounts+ep_rank-ep_rank_loc, local_displs, datatype, recvbuf, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      if(datatype == MPI_INT)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<int*>(master_sendbuf);
+        delete[] static_cast<int*>(local_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<float*>(master_sendbuf);
+        delete[] static_cast<float*>(local_recvbuf);
+      }
+      else  if(datatype == MPI_DOUBLE)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<double*>(master_sendbuf);
+        delete[] static_cast<double*>(local_recvbuf);
+      }
+      else  if(datatype == MPI_LONG)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<long*>(master_sendbuf);
+        delete[] static_cast<long*>(local_recvbuf);
+      }
+      else  if(datatype == MPI_UNSIGNED_LONG)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<unsigned long*>(master_sendbuf);
+        delete[] static_cast<unsigned long*>(local_recvbuf);
+      }
+      else // if(datatype == MPI_DOUBLE)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<char*>(master_sendbuf);
+        delete[] static_cast<char*>(local_recvbuf);
+      }
+    }
+
+  }
+}
Index: XIOS/trunk/extern/src_ep/ep_send.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_send.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_send.cpp	(revision 1034)
@@ -0,0 +1,317 @@
+/*!
+   \file ep_send.hpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI send functions: MPI_Send, MPI_Ssend, MPI_Isend, MPI_Issend
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+namespace ep_lib {
+
+
+  int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Send(buf, count, static_cast< ::MPI_Datatype>(datatype), dest, tag, mpi_comm);
+      return 0;
+    }
+
+    MPI_Request request;
+    MPI_Status status;
+    MPI_Isend(buf, count, datatype, dest, tag, comm, &request);
+    MPI_Wait(&request, &status);
+
+    //check_sum_send(buf, count, datatype, dest, tag, comm);
+
+    return 0;
+  }
+
+
+  int MPI_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Ssend(buf, count, static_cast< ::MPI_Datatype>(datatype), dest, tag, mpi_comm);
+      return 0;
+    }
+
+    MPI_Request request;
+    MPI_Status status;
+    MPI_Issend(buf, count, datatype, dest, tag, comm, &request);
+    MPI_Wait(&request, &status);
+    //check_sum_send(buf, count, datatype, dest, tag, comm);
+    return 0;
+  }
+
+
+
+  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("\nMPI_Isend with EP\n");
+    int src_rank;
+    MPI_Comm_rank(comm, &src_rank);
+
+    
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Request mpi_request;
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype> (datatype), dest, tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+
+      request->ep_src = src_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+      request->type = 1;
+      request->comm = comm;
+
+      return 0;
+    }
+
+    if(comm.is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request);
+
+    // EP intracomm
+
+    check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int ep_src_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first;
+    int mpi_tag 		= tag_combine(tag, ep_src_loc, ep_dest_loc);
+    int mpi_dest 	= comm.ep_comm_ptr->comm_list->rank_map->at(dest).second;
+
+    request->ep_src  = src_rank;
+    request->ep_tag  = tag;
+    request->ep_datatype = datatype;
+
+    ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+    ::MPI_Request mpi_request;
+
+    ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype>(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+    request->mpi_request = mpi_request;
+    request->type = 1; 		// used in wait
+    request->comm = comm;
+
+    Message_Check(comm);
+
+    return 0;
+  }
+
+
+
+
+  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("\nMPI_Issend with EP\n");
+
+    int src_rank;
+    MPI_Comm_rank(comm, &src_rank);
+
+    
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Request mpi_request;
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Issend(buf, count, static_cast< ::MPI_Datatype> (datatype), dest, tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->ep_src = src_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+      request->type = 1;
+      request->comm = comm;
+
+      return 0;
+    }
+
+    if(comm.is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request);
+
+    // EP intracomm
+
+    check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int ep_src_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first;
+    int mpi_tag 		= tag_combine(tag, ep_src_loc, ep_dest_loc);
+    int mpi_dest 	= comm.ep_comm_ptr->comm_list->rank_map->at(dest).second;
+    
+    request->ep_src = src_rank;
+    request->ep_tag = tag;
+    request->ep_datatype = datatype;
+
+    ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+    ::MPI_Request mpi_request;
+
+    ::MPI_Issend(buf, count, static_cast< ::MPI_Datatype>(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+    request->mpi_request = mpi_request;
+    request->type = 1; 		// used in wait
+    request->comm = comm;
+    
+
+    //Message_Check(comm);
+
+    return 0;
+  }
+
+
+
+  int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("MPI_Isend with intercomm\n");
+
+    check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first;
+    int dest_remote_comm_label = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).second;
+
+    int src_ep_rank    = comm.ep_comm_ptr->intercomm->size_rank_info[0].first;
+    int src_comm_label;
+
+    for(int i=0; i<comm.ep_comm_ptr->intercomm->local_rank_map->size(); i++)
+    {
+      if(comm.ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank)
+      {
+        src_comm_label = comm.ep_comm_ptr->intercomm->local_rank_map->at(i).second;
+        break;
+      }
+    }
+
+    Message_Check(comm);
+
+
+    if(dest_remote_comm_label == src_comm_label)       // mpi_dest differs
+    {
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Request mpi_request;
+  
+      ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    else   // dest_remote_comm_label != src_comm_label
+    { 
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.ep_comm_ptr->intercomm->mpi_inter_comm);
+      ::MPI_Request mpi_request;
+
+      ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+   
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    return 0;
+
+  }
+
+
+  int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("MPI_Issend with intercomm\n");
+
+    check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first;
+    int dest_remote_comm_label = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).second;
+
+    int src_ep_rank    = comm.ep_comm_ptr->intercomm->size_rank_info[0].first;
+    int src_comm_label;
+
+    for(int i=0; i<comm.ep_comm_ptr->intercomm->local_rank_map->size(); i++)
+    {
+      if(comm.ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank)
+      {
+        src_comm_label = comm.ep_comm_ptr->intercomm->local_rank_map->at(i).second;
+        break;
+      }
+    }
+
+    Message_Check(comm);
+
+
+    if(dest_remote_comm_label == src_comm_label)       // dest rank (loc, mpi) differs
+    {
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Request mpi_request;
+  
+      ::MPI_Issend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    else   // dest_remote_comm_label != src_comm_label
+    { 
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.ep_comm_ptr->intercomm->mpi_inter_comm);
+      ::MPI_Request mpi_request;
+
+      ::MPI_Issend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+   
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    return 0;
+
+  }
+}
+
+
+
+
Index: XIOS/trunk/extern/src_ep/ep_size.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_size.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_size.cpp	(revision 1034)
@@ -0,0 +1,83 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+
+  int MPI_Comm_size(MPI_Comm comm, int* size)
+  {
+    
+    if(comm.is_ep)
+    {
+      if(!comm.is_intercomm)
+      {
+        *size = comm.ep_comm_ptr->size_rank_info[0].second;
+
+        return 0;
+      }
+      else
+      {
+        *size = comm.ep_comm_ptr->intercomm->size_rank_info[0].second;
+
+        return 0;
+      }
+
+    }
+
+    Debug("Calling EP_Comm_size\n");
+
+    if(comm.mpi_comm)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm); 
+      int mpi_size;
+
+      ::MPI_Comm_size(mpi_comm, &mpi_size);
+      printf("============ ep comm size called for non ep comm, %d\n", mpi_size);
+      *size = mpi_size;
+      return 0;
+    }
+
+    else
+    {
+      *size = MPI_UNDEFINED;
+      printf("============ ep comm size called for non ep comm not defined\n");
+      return 0;
+    }
+  }
+
+  int MPI_Comm_remote_size(MPI_Comm comm, int *size)
+  {
+    if(!comm.is_ep)
+    {
+      if(comm.mpi_comm)
+      {
+        ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm); 
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Comm_remote_size(mpi_comm, size);
+        return 0;
+      }
+
+      else
+      {
+        *size = MPI_UNDEFINED;
+        return 0;
+      }
+    }
+    
+    if(comm.is_intercomm)
+    {
+      *size = comm.ep_comm_ptr->intercomm->remote_rank_map->size();
+      return 0;
+    }
+    *size = MPI_UNDEFINED;
+    return 0; 
+  }
+}
+
+
Index: XIOS/trunk/extern/src_ep/ep_split.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_split.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_split.cpp	(revision 1034)
@@ -0,0 +1,181 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  void vec_simplify(std::vector<int> *inout_vector)
+  {
+    std::vector<int> out_vec;
+    int found=false;
+    for(std::vector<int>::iterator it_in = inout_vector->begin() ; it_in != inout_vector->end(); ++it_in)
+    {
+      for(std::vector<int>::iterator it = out_vec.begin() ; it != out_vec.end(); ++it)
+      {
+        if(*it_in == *it)
+        {
+          found=true;
+          break;
+        }
+        else found=false;
+      }
+      if(found == false)
+      {
+        out_vec.push_back(*it_in);
+      }
+    }
+    inout_vector->swap(out_vec);
+  }
+
+
+
+  int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+
+    int num_color = 0;
+
+    int color_index;
+
+    vector<int> matched_number;
+    vector<int> matched_number_loc;
+
+
+    vector<int> all_color(ep_size);
+    vector<int> all_color_loc(num_ep);
+
+    MPI_Gather_local(&color, 1, MPI_INT, all_color_loc.data(), comm);
+    MPI_Bcast_local(all_color_loc.data(), num_ep, MPI_INT, comm);
+
+
+    MPI_Allgather(&color, 1, MPI_INT, all_color.data(), 1, MPI_INT, comm);
+
+    list<int> color_list(all_color.begin(), all_color.end());
+    list<int> color_list_loc(all_color_loc.begin(), all_color_loc.end());
+
+    vec_simplify(&all_color);
+
+    matched_number.resize(all_color.size(), 0);
+    matched_number_loc.resize(all_color.size(), 0);
+
+
+    while(color_list.size())
+    {
+      int target_color = color_list.front();
+      for(list<int>::iterator it = color_list.begin(); it != color_list.end(); ++it)
+      {
+        if(*it == target_color)
+        {
+          matched_number[num_color]++;
+        }
+      }
+      for(list<int>::iterator it = color_list_loc.begin(); it != color_list_loc.end(); ++it)
+      {
+        if(*it == target_color)
+        {
+          matched_number_loc[num_color]++;
+        }
+      }
+      color_list.remove(target_color);
+      color_list_loc.remove(target_color);
+      num_color++;
+    }
+
+
+    vector<int> colored_key_loc[num_color];
+    vector<int> key_loc(num_ep);
+
+    MPI_Gather_local(&key, 1, MPI_INT, key_loc.data(), comm);
+    MPI_Bcast_local(key_loc.data(), num_ep, MPI_INT, comm);
+
+    for(int i=0; i<num_ep; i++)
+    {
+      for(int j = 0; j<num_color; j++)
+      {
+        if(all_color_loc[i] == all_color[j])
+        {
+          colored_key_loc[j].push_back(key_loc[i]);
+        }
+      }
+    }
+
+    int new_ep_rank_loc;
+
+    for(int i=0; i<num_color; i++)
+    {
+      if(colored_key_loc[i].size()>1)
+      {
+        std::sort(colored_key_loc[i].begin(), colored_key_loc[i].end());
+      }
+      if(color == all_color[i])
+      {
+        color_index = i;
+        for(int j=0; j<colored_key_loc[i].size(); j++)
+        {
+          if(key == colored_key_loc[i][j])
+          {
+            new_ep_rank_loc = j;
+            break;
+          }
+        }
+      }
+    }
+
+    ::MPI_Comm split_mpi_comm[num_color];
+
+    for(int j=0; j<num_color; j++)
+    {
+      if(ep_rank_loc == 0)
+      {
+        int master_color = 1;
+        if(matched_number_loc[j] == 0) master_color = MPI_UNDEFINED;
+        #ifdef _serialized
+        #pragma omp critical (_mpi_call)
+        #endif // _serialized
+        ::MPI_Comm_split(static_cast< ::MPI_Comm>(comm.mpi_comm), master_color, mpi_rank, &split_mpi_comm[j]);
+        
+        comm.ep_comm_ptr->comm_list->mpi_bridge = split_mpi_comm[j];
+      }
+      MPI_Barrier_local(comm);
+      int num_new_ep = 0;
+
+      if(new_ep_rank_loc == 0 && color == all_color[j])
+      {
+        num_new_ep = matched_number_loc[j];
+        MPI_Info info;
+        MPI_Comm *ep_comm;
+
+        MPI_Comm_create_endpoints(comm.ep_comm_ptr->comm_list->mpi_bridge, num_new_ep, info, ep_comm);
+
+
+        comm.ep_comm_ptr->comm_list->mem_bridge = ep_comm;
+      }
+      MPI_Barrier_local(comm);
+      if(color == all_color[j])
+      {
+        *newcomm = comm.ep_comm_ptr->comm_list->mem_bridge[new_ep_rank_loc];
+//        newcomm = &(comm.ep_comm_ptr->comm_list->mem_bridge[new_ep_rank_loc]);
+        (*newcomm).ep_comm_ptr->comm_label = color;
+      }
+    }
+
+
+
+    return 0;
+  }
+
+}
Index: XIOS/trunk/extern/src_ep/ep_tag.hpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_tag.hpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_tag.hpp	(revision 1034)
@@ -0,0 +1,25 @@
+#ifndef EP_TAG_HPP_INCLUDED
+#define EP_TAG_HPP_INCLUDED
+
+
+
+namespace ep_lib
+{
+  int print_comm_info(MPI_Comm comm);
+
+
+
+//  int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request);
+//  int MPI_Mrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message);
+
+
+//  int MPI_Testall(int count, MPI_Request *array_of_requests, int *flag, MPI_Status *array_of_statuses);
+//  int MPI_Testany(int count, MPI_Request *array_of_requests, int *index, int *flag, MPI_Status *status);
+//  int MPI_Testsome(int incount, MPI_Request *array_of_requests, int *outcount, int *array_of_indices, MPI_Status *array_of_statuses);
+
+
+//  int MPI_Waitany(int count, MPI_Request *array_of_requests,  int *index, MPI_Status *array_of_statuses);
+//  int MPI_Waitsome(int incount, MPI_Request *array_of_requests, int *outcount, int *array_of_indices, MPI_Status *array_of_statuses);
+}
+
+#endif // EP_TAG_HPP_INCLUDED
Index: XIOS/trunk/extern/src_ep/ep_test.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_test.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_test.cpp	(revision 1034)
@@ -0,0 +1,97 @@
+/*!
+   \file ep_test.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI test function: MPI_Test, MPI_Testsome, MPI_Testany, MPI_Testall
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib {
+
+	int MPI_Test(MPI_Request *request, int *flag, MPI_Status *status)
+	{
+		Debug("MPI_Test with EP");
+
+    *flag = false;
+
+    if(request->type == 1)      // isend
+    {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Test(&mpi_request, flag, &mpi_status);
+      
+      if(*flag) 
+      {
+        status->mpi_status = new ::MPI_Status(mpi_status);
+        status->ep_src = request->ep_src;
+        status->ep_tag = request->ep_tag;
+        status->ep_datatype = request->ep_datatype;
+      }
+
+      return 0;
+    }
+
+    if(request->type == 2)   // irecv message not probed
+    {
+      Message_Check(request->comm);
+      #pragma omp flush
+      MPI_Message message;
+      MPI_Improbe(request->ep_src, request->ep_tag, request->comm, flag, &message, status);
+      if(*flag)
+      {
+        int count;
+        MPI_Get_count(status, request->ep_datatype, &count);
+        MPI_Imrecv(request->buf, count, request->ep_datatype, &message, request);
+        MPI_Test(request, flag, status);
+      }
+      return 0;
+    }
+
+    if(request->type == 3)  // imrecv
+    {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Test(&mpi_request, flag, &mpi_status);
+      if(*flag)
+      {
+        status->mpi_status = new ::MPI_Status(mpi_status);
+        status->ep_src = request->ep_src;
+      status->ep_tag = request->ep_tag;
+      status->ep_datatype = request->ep_datatype;
+        int count;
+        MPI_Get_count(status, request->ep_datatype, &count);
+        check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+      }
+
+      status->ep_src = request->ep_src;
+      status->ep_tag = request->ep_tag;
+      status->ep_datatype = request->ep_datatype;
+
+      
+
+      return 0;
+    }
+		
+	}
+
+
+	int MPI_Testall(int count, MPI_Request *array_of_requests, int *flag, MPI_Status *array_of_statuses)
+	{
+	  Debug("MPI_Testall with EP");
+    *flag = true;
+    int i=0;
+    while(*flag && i<count )
+    {
+      MPI_Test(&array_of_requests[i], flag, &array_of_statuses[i]);
+      i++;
+    }
+	}
+
+
+}
Index: XIOS/trunk/extern/src_ep/ep_type.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_type.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_type.cpp	(revision 1034)
@@ -0,0 +1,98 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+
+
+using namespace std;
+
+::MPI_Comm MPI_COMM_WORLD_STD = MPI_COMM_WORLD;
+//#undef MPI_COMM_WORLD
+
+
+::MPI_Comm MPI_COMM_NULL_STD = MPI_COMM_NULL;
+//#undef MPI_COMM_NULL
+
+
+::MPI_Datatype MPI_INT_STD = MPI_INT;
+::MPI_Datatype MPI_FLOAT_STD = MPI_FLOAT;
+::MPI_Datatype MPI_DOUBLE_STD = MPI_DOUBLE;
+::MPI_Datatype MPI_LONG_STD = MPI_LONG;
+::MPI_Datatype MPI_CHAR_STD = MPI_CHAR;
+::MPI_Datatype MPI_UNSIGNED_LONG_STD = MPI_UNSIGNED_LONG;
+::MPI_Datatype MPI_UNSIGNED_CHAR_STD = MPI_UNSIGNED_CHAR;
+
+#undef MPI_INT
+#undef MPI_FLOAT
+#undef MPI_DOUBLE
+#undef MPI_LONG
+#undef MPI_CHAR
+#undef MPI_UNSIGNED_LONG
+#undef MPI_UNSIGNED_CHAR
+
+
+//ep_lib::MPI_Datatype MPI_INT = MPI_INT_STD;
+//ep_lib::MPI_Datatype MPI_FLOAT = MPI_FLOAT_STD;
+//ep_lib::MPI_Datatype MPI_DOUBLE = MPI_DOUBLE_STD;
+//ep_lib::MPI_Datatype MPI_LONG = MPI_LONG_STD;
+//ep_lib::MPI_Datatype MPI_CHAR = MPI_CHAR_STD;
+//ep_lib::MPI_Datatype MPI_UNSIGNED_LONG = MPI_UNSIGNED_LONG_STD;
+
+
+
+
+::MPI_Op MPI_SUM_STD = MPI_SUM;
+::MPI_Op MPI_MAX_STD = MPI_MAX;
+::MPI_Op MPI_MIN_STD = MPI_MIN;
+
+#undef MPI_SUM
+#undef MPI_MAX
+#undef MPI_MIN
+
+//ep_lib::MPI_Op MPI_SUM = MPI_SUM_STD;
+//ep_lib::MPI_Op MPI_MAX = MPI_MAX_STD;
+//ep_lib::MPI_Op MPI_MIN = MPI_MIN_STD;
+
+//  ep_lib::MPI_Comm::MPI_Comm(const MPI_Comm & comm)
+//  {
+//    printf("calling MPI_Comm copy constructor\n");
+//    is_ep = comm.is_ep;
+//    is_intercomm = comm.is_intercomm;
+
+//    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+//    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+//    if(0 == my_rank)
+//    {
+//      MPI_Info info;
+//      MPI_Comm *out_comm;
+//      ::MPI_Comm mpi_dup;
+
+//      ::MPI_Comm in_comm = static_cast< ::MPI_Comm>(comm.mpi_comm);
+
+//      ::MPI_Comm_dup(in_comm, &mpi_dup);
+
+//      MPI_Comm_create_endpoints(mpi_dup, num_ep, info, out_comm);
+//      comm.ep_comm_ptr->comm_list->mem_bridge = out_comm;
+//    }
+
+//    MPI_Barrier(comm);
+
+//    *this = (comm.ep_comm_ptr->comm_list->mem_bridge[my_rank]);
+// //       // my_buffer = NULL;
+// //       // ep_barrier = NULL;
+// //       // rank_map = NULL;
+// //       // ep_comm_ptr = NULL;
+// //       // mem_bridge = NULL;
+// //       // mpi_bridge = NULL;
+// //       // mpi_comm = comm;
+// }
+
+
+
+
+
+
+
+
+
+
Index: XIOS/trunk/extern/src_ep/ep_type.hpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_type.hpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_type.hpp	(revision 1034)
@@ -0,0 +1,404 @@
+#ifndef EP_TYPE_HPP_INCLUDED
+#define EP_TYPE_HPP_INCLUDED
+
+
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <list>
+#include <map>
+#include <omp.h>
+#include <vector>
+#include <numeric>
+#include <bitset>
+#include <memory.h>
+#include <algorithm>
+#include <assert.h>
+#include <math.h>
+
+#ifdef _Debug
+#define Debug(x) std::cout << x << std::endl
+#else
+#define Debug(x)
+#endif
+
+#define BUFFER_SIZE 10000
+
+typedef std::pair< int, int > SIZE_RANK_INFO; // < rank, size>
+
+typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank>
+
+typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
+
+
+typedef struct
+{
+  int first;
+  int second;
+  int third;
+} Triple_int;
+
+namespace ep_lib
+{
+  #define MPI_UNDEFINED -32766
+  #define MPI_STATUS_IGNORE NULL
+  #define MPI_INFO_NULL MPI_Info()
+
+  class ep_communicator;
+  class ep_intercomm;
+  class OMPbarrier;
+  typedef ep_communicator* EP_Comm;
+  class MPI_Comm;
+
+
+  class MPI_Status
+  {
+    public:
+
+      #ifdef _intelmpi
+      int ep_datatype;
+      #elif _openmpi
+      void * ep_datatype;
+      #endif
+      
+      int ep_src;
+      int ep_tag;
+
+      
+      void* mpi_status;
+  };
+
+  class MPI_Message
+  {
+    public:
+
+      #ifdef _intelmpi
+      int mpi_message;
+      #elif _openmpi
+      void * mpi_message;
+      #endif
+
+      int ep_src;
+      int ep_tag;
+
+      void* mpi_status;
+
+      MPI_Message()
+      {
+        mpi_message = 0;
+        mpi_status = 0;
+      }
+  };
+
+  typedef std::list<MPI_Message > Message_list;
+
+
+  class OMPbarrier
+  {
+    private:
+      int nbThreads;          //<The number of threads for this barrier
+      int currentNbThread;    //<The current number of threads waiting
+      bool sense;             //<Direct barrier feedback protection
+      omp_lock_t mutex;       //<To have an atomic int
+
+      OMPbarrier(OMPbarrier&){}
+      OMPbarrier& operator=(OMPbarrier&){return *this;}
+
+    public:
+      /** Constructor with the number of threads */
+      explicit OMPbarrier(const int inNbThreads)
+          : nbThreads(inNbThreads), currentNbThread(0), sense(false) {
+          omp_init_lock( &mutex );
+      }
+
+      /** Destructor, release the omp lock */
+      ~OMPbarrier(){
+          omp_destroy_lock( &mutex );
+      }
+
+      /** Perform a barrier */
+      void wait(){
+          const bool mySense = sense;
+          omp_set_lock( &mutex );
+          const int nbThreadsArrived = (++currentNbThread);
+          omp_unset_lock( &mutex );
+
+          if(nbThreadsArrived == nbThreads) {
+              currentNbThread = 0;
+              sense = !sense;
+              #pragma omp flush
+          }
+          else {
+              volatile const bool* const ptSense = &sense;
+              while( (*ptSense) == mySense){
+              }
+          }
+      }
+
+
+      /** Change the number of threads */
+      void setNbThreads(const int inNbThread){
+          omp_set_lock( &mutex );
+          nbThreads = inNbThread;
+          omp_unset_lock( &mutex );
+      }
+  };
+
+  class ep_intercomm
+  {
+    public:
+
+
+    #ifdef _intelmpi
+    int mpi_inter_comm;
+    #elif _openmpi
+    void * mpi_inter_comm;
+    #endif
+
+    RANK_MAP *intercomm_rank_map;
+    RANK_MAP *local_rank_map;
+    RANK_MAP *remote_rank_map;
+
+
+    SIZE_RANK_INFO size_rank_info[3];
+
+
+    MPI_Comm *local_comm;
+    int intercomm_tag;
+
+    ep_intercomm()
+    {
+      intercomm_rank_map = NULL;
+      local_rank_map = NULL;
+      remote_rank_map = NULL;
+    }
+
+    bool operator == (ep_intercomm right)
+    {
+      bool a = intercomm_rank_map == right.intercomm_rank_map;
+      bool b = local_rank_map == right.local_rank_map;
+      bool c = remote_rank_map == right.remote_rank_map;
+      bool d = mpi_inter_comm == right.mpi_inter_comm;
+      bool e = size_rank_info == right.size_rank_info;
+      bool f = intercomm_tag == right.intercomm_tag;
+      return a&&b&&c&&d&&e&&f;
+    }
+
+    bool operator != (ep_intercomm right)
+    {
+      bool a = intercomm_rank_map != right.intercomm_rank_map;
+      bool b = local_rank_map != right.local_rank_map;
+      bool c = remote_rank_map != right.remote_rank_map;
+      bool d = mpi_inter_comm != right.mpi_inter_comm;
+      bool e = size_rank_info != right.size_rank_info;
+      bool f = intercomm_tag != right.intercomm_tag;
+      return a||b||c||d||e||f;
+    }
+  };
+
+
+  class ep_communicator
+  {
+    public:
+
+    SIZE_RANK_INFO size_rank_info[3]; // 0: ep_rank,     ep_size
+                                      // 1: ep_rank_loc, num_ep
+                                      // 2: mpi_rank,    mpi_size
+
+
+    MPI_Comm *comm_list;
+
+    Message_list *message_queue;
+
+
+    int comm_label;
+
+    ep_intercomm *intercomm;
+
+    ep_communicator()
+    {
+      comm_list = NULL;
+      message_queue = NULL;
+      intercomm = NULL;
+    }
+
+    bool operator == (ep_communicator right)
+    {
+      bool a = size_rank_info == right.size_rank_info;
+      bool b = comm_label == right.comm_label;
+      bool c = intercomm == right.intercomm;
+      return a&&b&&c;
+    }
+
+    bool operator != (ep_communicator right)
+    {
+      bool a = size_rank_info != right.size_rank_info;
+      bool b = comm_label != right.comm_label;
+      bool c = intercomm != right.intercomm;
+      return a||b||c;
+    }
+  };
+
+
+  struct BUFFER
+  {
+    double *buf_double;
+    float  *buf_float;
+    int    *buf_int;
+    long    *buf_long;
+    unsigned long    *buf_ulong;
+    char    *buf_char;
+  };
+
+
+  class MPI_Comm
+  {
+    public:
+
+    #ifdef _intelmpi
+    int mpi_comm;
+    #elif _openmpi
+    void * mpi_comm;
+    #endif
+
+    bool is_ep;
+    bool is_intercomm;
+
+    BUFFER     *my_buffer;
+    OMPbarrier *ep_barrier;
+    RANK_MAP   *rank_map;
+
+    EP_Comm ep_comm_ptr;
+
+    MPI_Comm *mem_bridge;
+
+    #ifdef _intelmpi
+    int mpi_bridge;
+    #elif _openmpi
+    void * mpi_bridge;
+    #endif
+
+    MPI_Comm()
+    {
+      is_ep = false;
+      is_intercomm = false;
+      my_buffer = NULL;
+      ep_barrier = NULL;
+      rank_map = NULL;
+      ep_comm_ptr = NULL;
+      mem_bridge = NULL;
+      mpi_bridge = NULL;
+      mpi_comm = 0;
+    }
+
+    MPI_Comm(int comm)
+    {
+      is_ep = false;
+      is_intercomm = false;
+      my_buffer = NULL;
+      ep_barrier = NULL;
+      rank_map = NULL;
+      ep_comm_ptr = NULL;
+      mem_bridge = NULL;
+      mpi_bridge = NULL;
+      mpi_comm = comm;
+    }
+
+    //MPI_Comm(const MPI_Comm &comm);
+
+
+    bool operator == (MPI_Comm right)
+    {
+      bool a = is_ep == right.is_ep;
+      bool b = is_intercomm == right.is_intercomm;
+      bool c = mpi_comm == right.mpi_comm;
+      bool d = is_ep ? ep_comm_ptr == right.ep_comm_ptr : true;
+      return a&&b&&c&&d;
+    }
+
+    bool operator != (MPI_Comm right)
+    {
+      bool a = is_ep != right.is_ep;
+      bool b = is_intercomm != right.is_intercomm;
+      bool c = mpi_comm != right.mpi_comm;
+      bool d = is_ep ? ep_comm_ptr != right.ep_comm_ptr : true;
+      return a||b||c||d;
+    }
+  };
+
+
+  class MPI_Info
+  {
+    public:
+
+      #ifdef _intelmpi
+      int mpi_info;
+      #elif _openmpi
+      void * mpi_info;
+      #endif
+
+      MPI_Info()
+      {
+        mpi_info = 0;
+      }
+  };
+
+
+  class MPI_Request
+  {
+    public:
+
+      #ifdef _intelmpi
+      int mpi_request;
+      #elif _openmpi
+      void * mpi_request;
+      #endif
+
+      int type;	//! type of the non-blocking communication. 1: Isend; 2:Irecv; 3:Imrecv; 4:Issend
+      void* buf;
+
+      int ep_src;
+      int ep_tag;
+      #ifdef _intelmpi
+      int ep_datatype;
+      #elif _openmpi
+      void * ep_datatype;
+      #endif
+
+      MPI_Comm comm;	//! EP communicator related to the communication
+
+      MPI_Request()
+      {
+        mpi_request = 0;
+      }
+  };
+
+  class MPI_Fint
+  {
+    public:
+
+    int mpi_fint;
+  };
+
+  class MPI_Aint
+  {
+    public:
+
+    unsigned long mpi_aint;
+  };
+
+
+  static MPI_Comm *passage;
+
+  static int TAG = 40000;
+
+  static std::list<std::pair<std::pair<int, int>, MPI_Comm * > > tag_list;
+
+  static std::map<std::pair<int, int>, MPI_Comm  >  fc_comm_map;
+            //    <MPI_Fint,thread_num>   EP_Comm
+
+}
+
+
+
+#endif // EP_TYPE_HPP_INCLUDED
Index: XIOS/trunk/extern/src_ep/ep_wait.cpp
===================================================================
--- XIOS/trunk/extern/src_ep/ep_wait.cpp	(revision 1034)
+++ XIOS/trunk/extern/src_ep/ep_wait.cpp	(revision 1034)
@@ -0,0 +1,142 @@
+/*!
+   \file ep_wait.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI wait function: MPI_Wait, MPI_Waitall, MPI_Waitsome, MPI_Waitany
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+
+namespace ep_lib {	
+
+	int MPI_Wait(MPI_Request *request, MPI_Status *status)
+	{
+
+    if(request->type == 1)
+    {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Wait(&mpi_request, &mpi_status);
+
+
+      
+      status->mpi_status = &mpi_status;
+      status->ep_src = request->ep_src;
+      status->ep_tag = request->ep_tag;
+      status->ep_datatype = request->ep_datatype;
+
+      return 0;
+    }
+
+    if(request->type == 2)
+    {
+      int flag = false;
+      MPI_Message message;
+
+      while(!flag)
+      {
+        Message_Check(request->comm);
+        #pragma omp flush
+        MPI_Improbe(request->ep_src, request->ep_tag, request->comm, &flag, &message, status);
+      }
+
+      int count;
+      MPI_Get_count(status, request->ep_datatype, &count);
+      MPI_Mrecv(request->buf, count, request->ep_datatype, &message, status);
+      status->ep_datatype = request->ep_datatype;
+
+      check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+
+      return 0;
+    }
+
+    if(request->type == 3)
+    {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Wait(&mpi_request, &mpi_status);
+      
+      status->mpi_status = new ::MPI_Status(mpi_status);
+      status->ep_src = request->ep_src;
+      status->ep_tag = request->ep_tag;
+      status->ep_datatype = request->ep_datatype;
+
+      int count;
+      MPI_Get_count(status, request->ep_datatype, &count);
+      check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+    }
+	  return MPI_SUCCESS;
+	}
+
+
+
+	
+
+
+	int MPI_Waitall(int count, MPI_Request *array_of_requests, MPI_Status *array_of_statuses)
+	{
+    int dest_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &dest_rank);
+    
+    int finished = 0;
+    bool finished_index[count];
+
+	  for(int i=0; i<count; i++)
+    {
+      finished_index[i] = false;
+    }
+
+    while(finished < count)
+    {
+      for(int i=0; i<count; i++)
+      {
+        if(finished_index[i] == false) // this request has not been tested.
+        {
+          if(array_of_requests[i].type != 2) // isend or imrecv
+          {      
+            MPI_Wait(&array_of_requests[i], &array_of_statuses[i]);
+            if(array_of_requests[i].type == 3)
+            {
+              int check_count;
+              MPI_Get_count(&array_of_statuses[i], array_of_requests[i].ep_datatype, &check_count);
+              check_sum_recv(array_of_requests[i].buf, count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
+            }
+            finished++;
+            finished_index[i] = true;
+          }
+          else // irecv
+          {
+            int flag = false;
+            MPI_Message message;
+            
+            MPI_Improbe(array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, &flag, &message, &array_of_statuses[i]);
+            
+            if(flag)
+            {
+              //printf("dest_rank = %d, Waiting one message with src = %d, tag = %d, buf = %p\n", dest_rank, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].buf);
+              int recv_count;
+              MPI_Get_count(&array_of_statuses[i], array_of_requests[i].ep_datatype, &recv_count);
+              MPI_Mrecv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, &message, &array_of_statuses[i]);
+              check_sum_recv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
+
+              finished++;
+              finished_index[i] = true;
+            }
+          }
+        }
+      }    
+    }
+	  return MPI_SUCCESS;
+	}
+
+
+
+
+
+}
