Index: XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.cpp
===================================================================
--- XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.cpp	(revision 1146)
+++ XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.cpp	(revision 1147)
@@ -21,7 +21,7 @@
 void cptOffsetsFromLengths(const int *lengths, int *offsets, int sz)
 {
-	offsets[0] = 0;
-	for (int i = 1; i < sz; i++)
-		offsets[i] = offsets[i-1] + lengths[i-1];
+  offsets[0] = 0;
+  for (int i = 1; i < sz; i++)
+    offsets[i] = offsets[i-1] + lengths[i-1];
 }
 
@@ -31,10 +31,10 @@
   srcGrid.pole = Coord(pole[0], pole[1], pole[2]);
 
-	int mpiRank, mpiSize;
-	MPI_Comm_rank(communicator, &mpiRank);
-	MPI_Comm_size(communicator, &mpiSize);
-
-	sourceElements.reserve(nbCells);
-	sourceMesh.reserve(nbCells);
+  int mpiRank, mpiSize;
+  MPI_Comm_rank(communicator, &mpiRank);
+  MPI_Comm_size(communicator, &mpiSize);
+
+  sourceElements.reserve(nbCells);
+  sourceMesh.reserve(nbCells);
   sourceGlobalId.resize(nbCells) ;
 
@@ -49,15 +49,15 @@
   else sourceGlobalId.assign(globalId,globalId+nbCells);
 
-	for (int i = 0; i < nbCells; i++)
-	{
-		int offs = i*nVertex;
-		Elt elt(boundsLon + offs, boundsLat + offs, nVertex);
-		elt.src_id.rank = mpiRank;
-		elt.src_id.ind = i;
+  for (int i = 0; i < nbCells; i++)
+  {
+    int offs = i*nVertex;
+    Elt elt(boundsLon + offs, boundsLat + offs, nVertex);
+    elt.src_id.rank = mpiRank;
+    elt.src_id.ind = i;
     elt.src_id.globalId = sourceGlobalId[i];
-		sourceElements.push_back(elt);
-		sourceMesh.push_back(Node(elt.x, cptRadius(elt), &sourceElements.back()));
-		cptEltGeom(sourceElements[i], Coord(pole[0], pole[1], pole[2]));
-	}
+    sourceElements.push_back(elt);
+    sourceMesh.push_back(Node(elt.x, cptRadius(elt), &sourceElements.back()));
+    cptEltGeom(sourceElements[i], Coord(pole[0], pole[1], pole[2]));
+  }
 
 }
@@ -485,5 +485,4 @@
     mpiRoute.init(routes);
     int nRecv = mpiRoute.getTotalSourceElement();
-    // cout << mpiRank << " NRECV " << nRecv << "(" << routes.size() << ")"<< endl;
 
     int *nbSendNode = new int[mpiSize];
@@ -879,4 +878,5 @@
         {
             MPI_Issend(sendBuffer2[rank], sentMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
+            printf("proc %d send %d elements to proc %d\n", mpiRank, sentMessageSize[rank], rank);
             nbSendRequest++;
         }
@@ -884,4 +884,5 @@
         {
             MPI_Irecv(recvBuffer2[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+            printf("proc %d recv %d elements from proc %d\n", mpiRank, recvMessageSize[rank], rank);
             nbRecvRequest++;
         }
Index: XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_alltoall.cpp
===================================================================
--- XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_alltoall.cpp	(revision 1146)
+++ XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_alltoall.cpp	(revision 1147)
@@ -16,9 +16,11 @@
     int ep_size;
     MPI_Comm_size(comm, &ep_size);
+    
 
     for(int i=0; i<ep_size; i++)
     {
-      MPI_Gather(sendbuf+i*sendcount*typesize, sendcount, sendtype, recvbuf, recvcount, recvtype, i, comm);
+      ep_lib::MPI_Gather(sendbuf+i*sendcount*typesize, sendcount, sendtype, recvbuf, recvcount, recvtype, i, comm);
     }
+    
 
     return 0;
Index: XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gather.cpp
===================================================================
--- XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gather.cpp	(revision 1146)
+++ XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gather.cpp	(revision 1147)
@@ -352,4 +352,6 @@
 
     void *local_gather_recvbuf;
+    void *master_recvbuf;
+    if(ep_rank_loc == 0 && mpi_rank == root_mpi_rank && root_ep_loc != 0) master_recvbuf = new void*[sizeof(recvbuf)];
 
     if(ep_rank_loc==0)
@@ -381,7 +383,14 @@
       }
 
-
-      ::MPI_Gatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+      if(root_ep_loc != 0) // gather to root_master
+      {
+        ::MPI_Gatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), master_recvbuf, gatherv_recvcnt,
                     gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+      else
+      {
+        ::MPI_Gatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
 
       delete[] gatherv_recvcnt;
@@ -392,5 +401,6 @@
     if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
     {
-      innode_memcpy(0, recvbuf, root_ep_loc, recvbuf, count*ep_size, datatype, comm);
+      innode_memcpy(0, master_recvbuf, root_ep_loc, recvbuf, count*ep_size, datatype, comm);
+      if(ep_rank_loc == 0 ) delete[] master_recvbuf;
     }
 
@@ -399,4 +409,5 @@
     if(ep_rank_loc==0)
     {
+
       if(datatype == MPI_INT)
       {
Index: XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gatherv.cpp
===================================================================
--- XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gatherv.cpp	(revision 1146)
+++ XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gatherv.cpp	(revision 1147)
@@ -386,4 +386,7 @@
     void *local_gather_recvbuf;
     int buffer_size;
+    void *master_recvbuf;
+
+    if(ep_rank_loc == 0 && mpi_rank == root_mpi_rank && root_ep_loc != 0) master_recvbuf = new void*[sizeof(recvbuf)];
 
     if(ep_rank_loc==0)
@@ -411,7 +414,12 @@
       ::MPI_Gather(&buff_start,  1, MPI_INT_STD, mpi_displs,  1, MPI_INT_STD, root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
 
-
-      ::MPI_Gatherv(local_gather_recvbuf + datasize*buff_start, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, mpi_recvcnt,
+      if(root_ep_loc == 0)
+      {  ::MPI_Gatherv(local_gather_recvbuf + datasize*buff_start, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, mpi_recvcnt,
                        mpi_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+      else  // gatherv to master_recvbuf
+      {  ::MPI_Gatherv(local_gather_recvbuf + datasize*buff_start, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), master_recvbuf, mpi_recvcnt,
+                       mpi_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
 
       delete[] mpi_recvcnt;
@@ -425,5 +433,6 @@
     if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
     {
-      innode_memcpy(0, recvbuf+datasize*global_min_displs, root_ep_loc, recvbuf+datasize*global_min_displs, global_recvcnt, datatype, comm);
+      innode_memcpy(0, master_recvbuf+datasize*global_min_displs, root_ep_loc, recvbuf+datasize*global_min_displs, global_recvcnt, datatype, comm);
+      if(ep_rank_loc == 0) delete[] master_recvbuf;
     }
 
Index: XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_recv.cpp
===================================================================
--- XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_recv.cpp	(revision 1146)
+++ XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_recv.cpp	(revision 1147)
@@ -14,8 +14,9 @@
 
 
-namespace ep_lib {
+namespace ep_lib 
+{
 
-	int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status)
-	{
+  int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status)
+  {
     int dest_rank;
     MPI_Comm_rank(comm, &dest_rank);
@@ -41,44 +42,42 @@
 
     return 0;
-	}
+  }
 
 
 
 
-	int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request)
-	{
+  int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request)
+  {
 
-		Debug("MPI_Irecv with EP");
+    Debug("MPI_Irecv with EP");
     int dest_rank;
     MPI_Comm_rank(comm, &dest_rank);
 
- 		if(!comm.is_ep)
-		{
-		  ::MPI_Request mpi_request;
-		  ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
-		  ::MPI_Irecv(buf, count, static_cast< ::MPI_Datatype> (datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, mpi_comm, &mpi_request);
+    if(!comm.is_ep)
+    {
+      ::MPI_Request mpi_request;
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Irecv(buf, count, static_cast< ::MPI_Datatype> (datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, mpi_comm, &mpi_request);
 
-		  request->mpi_request = mpi_request;
+      request->mpi_request = mpi_request;
       request->ep_src = src;
       request->ep_datatype = datatype;
       request->ep_tag = tag;
-		}
+    }
 
-		Message_Check(comm);
+    Message_Check(comm);
 		
 
-		request->mpi_request = MPI_REQUEST_NULL_STD;
-		request->buf = buf;
+    request->mpi_request = MPI_REQUEST_NULL_STD;
+    request->buf = buf;
     request->comm = comm;
-		request->type = 2;
+    request->type = 2;
 
-		request->ep_src = src;
+    request->ep_src = src;
     request->ep_tag = tag;
     request->ep_datatype = datatype;
 
-
-    
-		/* With Improbe*/
-		Message_Check(comm);
+    /* With Improbe*/
+    Message_Check(comm);
 
     int flag = false;
@@ -94,16 +93,16 @@
 
     return 0;
-	}
+  }
 
-	int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request)
-	{
-		Debug("MPI_Imrecv");
+  int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request)
+  {
+    Debug("MPI_Imrecv");
 
-		request->type = 3;
+    request->type = 3;
 
     ::MPI_Request mpi_request;
     ::MPI_Message mpi_message = static_cast< ::MPI_Message >(message->mpi_message);
 		
-		::MPI_Imrecv(buf, count, static_cast< ::MPI_Datatype>(datatype), &mpi_message, &mpi_request);
+    ::MPI_Imrecv(buf, count, static_cast< ::MPI_Datatype>(datatype), &mpi_message, &mpi_request);
 
     request->mpi_request = mpi_request;
@@ -114,15 +113,15 @@
 
     return 0;
-	}
+  }
 
 
   int MPI_Mrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Status *status)
-	{
+  {
     Debug("EP Mrecv called\n");
 
     ::MPI_Status mpi_status;
     ::MPI_Message mpi_message = static_cast< ::MPI_Message >(message->mpi_message);
-		
-		::MPI_Mrecv(buf, count, static_cast< ::MPI_Datatype>(datatype), &mpi_message, &mpi_status);
+    
+    ::MPI_Mrecv(buf, count, static_cast< ::MPI_Datatype>(datatype), &mpi_message, &mpi_status);
 
     status->mpi_status = new ::MPI_Status(mpi_status);
@@ -134,8 +133,7 @@
 
     return 0;
-	}
+  }
+
+}
 
 
-  
-
-}
Index: XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_send.cpp
===================================================================
--- XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_send.cpp	(revision 1146)
+++ XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_send.cpp	(revision 1147)
@@ -158,4 +158,5 @@
     request->type = 1; 		// used in wait
     request->comm = comm;
+    request->buf = NULL;
     
 
Index: XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp
===================================================================
--- XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp	(revision 1146)
+++ XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp	(revision 1147)
@@ -14,64 +14,63 @@
 
 
-namespace ep_lib {	
+namespace ep_lib 
+{	
 
-    int MPI_Wait(MPI_Request *request, MPI_Status *status)
+  int MPI_Wait(MPI_Request *request, MPI_Status *status)
+  {
+
+    if(request->type == 1)
     {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Wait(&mpi_request, &mpi_status);
 
-        if(request->type == 1)
-        {
-            ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
-            ::MPI_Status mpi_status;
-            ::MPI_Wait(&mpi_request, &mpi_status);
+      status->mpi_status = &mpi_status;
+      status->ep_src = request->ep_src;
+      status->ep_tag = request->ep_tag;
+      status->ep_datatype = request->ep_datatype;
 
+      return 0;
+    }
 
+    if(request->type == 2)
+    {
+      int flag = false;
+      MPI_Message message;
 
-            status->mpi_status = &mpi_status;
-            status->ep_src = request->ep_src;
-            status->ep_tag = request->ep_tag;
-            status->ep_datatype = request->ep_datatype;
+      while(!flag)
+      {
+        Message_Check(request->comm);
+        #pragma omp flush
+        MPI_Improbe(request->ep_src, request->ep_tag, request->comm, &flag, &message, status);
+      }
 
-            return 0;
-        }
+      int count;
+      MPI_Get_count(status, request->ep_datatype, &count);
+      MPI_Mrecv(request->buf, count, request->ep_datatype, &message, status);
+      status->ep_datatype = request->ep_datatype;
 
-        if(request->type == 2)
-        {
-            int flag = false;
-            MPI_Message message;
+      //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
 
-            while(!flag)
-            {
-                Message_Check(request->comm);
-#pragma omp flush
-                MPI_Improbe(request->ep_src, request->ep_tag, request->comm, &flag, &message, status);
-            }
+      return 0;
+    }
 
-            int count;
-            MPI_Get_count(status, request->ep_datatype, &count);
-            MPI_Mrecv(request->buf, count, request->ep_datatype, &message, status);
-            status->ep_datatype = request->ep_datatype;
+    if(request->type == 3)
+    {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Wait(&mpi_request, &mpi_status);
 
-            //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+      status->mpi_status = new ::MPI_Status(mpi_status);
+      status->ep_src = request->ep_src;
+      status->ep_tag = request->ep_tag;
+      status->ep_datatype = request->ep_datatype;
 
-            return 0;
-        }
+      //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+    }
 
-        if(request->type == 3)
-        {
-            ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
-            ::MPI_Status mpi_status;
-            ::MPI_Wait(&mpi_request, &mpi_status);
+    return MPI_SUCCESS;
 
-            status->mpi_status = new ::MPI_Status(mpi_status);
-            status->ep_src = request->ep_src;
-            status->ep_tag = request->ep_tag;
-            status->ep_datatype = request->ep_datatype;
-
-            //int count;
-            //MPI_Get_count(status, request->ep_datatype, &count);
-            //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
-        }
-        return MPI_SUCCESS;
-    }
+  }   /*end of mpi_wait*/
 
 
@@ -80,57 +79,52 @@
 
 
-    int MPI_Waitall(int count, MPI_Request *array_of_requests, MPI_Status *array_of_statuses)
+  int MPI_Waitall(int count, MPI_Request *array_of_requests, MPI_Status *array_of_statuses)
+  {
+    int dest_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &dest_rank);
+
+    int finished = 0;
+    int finished_index[count];
+
+    for(int i=0; i<count; i++)
     {
-        int dest_rank;
-        MPI_Comm_rank(MPI_COMM_WORLD, &dest_rank);
-
-        int finished = 0;
-        int finished_index[count];
-
-        for(int i=0; i<count; i++)
-        {
-            finished_index[i] = false;
-        }
-
-        while(finished < count)
-        {
-            for(int i=0; i<count; i++)
-            {
-                //MPI_Test(&array_of_requests[i], &finished_index[i], &array_of_statuses[i]);
-                if(finished_index[i] == false) // this request has not been tested.
-                {
-                    if(array_of_requests[i].type != 2) // isend or imrecv
-                    {      
-                        MPI_Wait(&array_of_requests[i], &array_of_statuses[i]);
-                        finished++;
-                        finished_index[i] = true;
-                    }
-                    else // irecv
-                    {
-                        int flag = false;
-                        MPI_Message message;
-
-                        MPI_Improbe(array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, &flag, &message, &array_of_statuses[i]);
-
-                        if(flag)
-                        {
-                            //printf("dest_rank = %d, Waiting one message with src = %d, tag = %d, buf = %p\n", dest_rank, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].buf);
-                            int recv_count;
-                            MPI_Get_count(&array_of_statuses[i], array_of_requests[i].ep_datatype, &recv_count);
-                            MPI_Mrecv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, &message, &array_of_statuses[i]);
-                            //check_sum_recv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
-
-                            finished++;
-                            finished_index[i] = true;
-                        }
-                    }
-                }
-            }    
-        }
-        return MPI_SUCCESS;
+      finished_index[i] = false;
     }
 
+    while(finished < count)
+    {
+      for(int i=0; i<count; i++)
+      {
+        if(finished_index[i] == false) // this request has not been tested.
+        {
+          if(array_of_requests[i].type != 2) // isend or imrecv
+          {      
+            MPI_Wait(&array_of_requests[i], &array_of_statuses[i]);
+            finished++;
+            finished_index[i] = true;
+          }
+          else // irecv
+          {
+            int flag = false;
+            MPI_Message message;
 
+            MPI_Improbe(array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, &flag, &message, &array_of_statuses[i]);
 
+            if(flag)
+            {
+              int recv_count;
+              MPI_Get_count(&array_of_statuses[i], array_of_requests[i].ep_datatype, &recv_count);
+              MPI_Mrecv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, &message, &array_of_statuses[i]);
+              //check_sum_recv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
+
+              finished++;
+              finished_index[i] = true;
+            }
+          }
+        }
+      }    
+    }
+    return MPI_SUCCESS;
+  }  /* end of mpi_waitall*/
 
 
