Changeset 1503 for XIOS/dev

Timestamp:

05/28/18 17:28:07 (6 years ago)

Author:

yushan

Message:

rank_map is passed from vector to map, in order to have more flexibility in comm_split

Location:

XIOS/dev/branch_openmp/extern/ep_dev

Files:

• ep_accumulate.cpp (modified) (4 diffs)
• ep_allgather.cpp (modified) (1 diff)
• ep_allgatherv.cpp (modified) (2 diffs)
• ep_bcast.cpp (modified) (1 diff)
• ep_comm.hpp (modified) (2 diffs)
• ep_compare.cpp (modified) (1 diff)
• ep_create.cpp (modified) (3 diffs)
• ep_exscan.cpp (modified) (1 diff)
• ep_fetch.cpp (modified) (1 diff)
• ep_free.cpp (modified) (2 diffs)
• ep_gather.cpp (modified) (3 diffs)
• ep_gatherv.cpp (modified) (4 diffs)
• ep_get.cpp (modified) (2 diffs)
• ep_intercomm.cpp (modified) (2 diffs)
• ep_intercomm.hpp (modified) (1 diff)
• ep_intercomm_kernel.cpp (modified) (7 diffs)
• ep_intercomm_world.cpp (modified) (7 diffs)
• ep_lib.cpp (modified) (4 diffs)
• ep_lib.hpp (modified) (1 diff)
• ep_merge.cpp (modified) (2 diffs)
• ep_message.cpp (modified) (11 diffs)
• ep_probe.cpp (modified) (8 diffs)
• ep_put.cpp (modified) (2 diffs)
• ep_recv.cpp (modified) (3 diffs)
• ep_reduce.cpp (modified) (1 diff)
• ep_request.hpp (modified) (2 diffs)
• ep_scan.cpp (modified) (1 diff)
• ep_scatter.cpp (modified) (2 diffs)
• ep_scatterv.cpp (modified) (2 diffs)
• ep_send.cpp (modified) (7 diffs)
• ep_size.cpp (modified) (2 diffs)
• ep_type.cpp (modified) (2 diffs)
• ep_type.hpp (modified) (1 diff)
• ep_wait.cpp (modified) (2 diffs)
• main.cpp (modified) (5 diffs)
• makefile (modified) (1 diff)

XIOS/dev/branch_openmp/extern/ep_dev/ep_accumulate.cpp

@@ -12 +12 @@
                      int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
     if(num_ep==1)
@@ -27 +27 @@
                      int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
     ::MPI_Request mpi_request;
@@ -60 +60 @@
                          int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
     if(num_ep==1)
@@ -75 +75 @@
                          int target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
     ::MPI_Request mpi_request;

XIOS/dev/branch_openmp/extern/ep_dev/ep_allgather.cpp

@@ -86 +86 @@
       for(int i=0; i<ep_size; i++)
       {
-        offset = mpi_displs[comm->rank_map->at(i).second] + comm->rank_map->at(i).first * sendcount; 
+        offset = mpi_displs[comm->ep_rank_map->at(i).second] + comm->ep_rank_map->at(i).first * sendcount; 
         memcpy(recvbuf + i*sendcount*datasize, tmp_recvbuf+offset*datasize, sendcount*datasize);
       }

XIOS/dev/branch_openmp/extern/ep_dev/ep_allgatherv.cpp

@@ -92 +92 @@
       {
         int extra = 0;
-        for(int j=0, k=0; j<ep_size, k<comm->rank_map->at(i).first; j++)
-          if(comm->rank_map->at(i).second == comm->rank_map->at(j).second)
+        for(int j=0, k=0; j<ep_size, k<comm->ep_rank_map->at(i).first; j++)
+          if(comm->ep_rank_map->at(i).second == comm->ep_rank_map->at(j).second)
           {
             extra += recvcounts[j];
@@ -99 +99 @@
           }  
 
-        offset = mpi_displs[comm->rank_map->at(i).second] +  extra;
+        offset = mpi_displs[comm->ep_rank_map->at(i).second] +  extra;
 
         memcpy(recvbuf+displs[i]*datasize, tmp_recvbuf+offset*datasize, recvcounts[i]*datasize);

XIOS/dev/branch_openmp/extern/ep_dev/ep_bcast.cpp

@@ -57 +57 @@
     int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first;
 
-    int root_mpi_rank = comm->rank_map->at(root).second;
-    int root_ep_rank_loc = comm->rank_map->at(root).first;
+    int root_mpi_rank = comm->ep_rank_map->at(root).second;
+    int root_ep_rank_loc = comm->ep_rank_map->at(root).first;
 
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_comm.hpp

@@ -10 +10 @@
 typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank>
 typedef std::map<int, std::pair<int, int> > EP_RANK_MAP;  // key(ep_rank) = <ep_rank_local, mpi_rank>
-typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
+//typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
 typedef std::list<ep_lib::MPI_Message > Message_list;
 
@@ -52 +52 @@
     BUFFER     *my_buffer;
     ep_barrier *ep_barrier;
-    RANK_MAP   *rank_map;
+    //RANK_MAP   *rank_map;
+    EP_RANK_MAP   *ep_rank_map;
     void* mpi_comm;
     ep_communicator *ep_comm_ptr;

XIOS/dev/branch_openmp/extern/ep_dev/ep_compare.cpp

@@ -11 +11 @@
         int MPI_Compare_and_swap(const void *origin_addr, const void *compare_addr, void *result_addr, MPI_Datatype datatype, int target_rank, MPI_Aint target_disp, MPI_Win win)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
     if(num_ep==1)

XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp

@@ -64 +64 @@
     out_comm_hdls[0]->my_buffer = new BUFFER;
 
-    out_comm_hdls[0]->rank_map = new RANK_MAP;
-    out_comm_hdls[0]->rank_map->resize(ep_size);
+    //out_comm_hdls[0]->rank_map = new RANK_MAP;
+    //out_comm_hdls[0]->rank_map->resize(ep_size);
+
+    out_comm_hdls[0]->ep_rank_map = new EP_RANK_MAP;
+    //out_comm_hdls[0]->ep_rank_map->resize(ep_size);
 
 
@@ -72 +75 @@
       out_comm_hdls[i]->ep_barrier = out_comm_hdls[0]->ep_barrier;
       out_comm_hdls[i]->my_buffer  = out_comm_hdls[0]->my_buffer;
-      out_comm_hdls[i]->rank_map   = out_comm_hdls[0]->rank_map;
+      //out_comm_hdls[i]->rank_map   = out_comm_hdls[0]->rank_map;
+      out_comm_hdls[i]->ep_rank_map= out_comm_hdls[0]->ep_rank_map;
     }
 
@@ -92 +96 @@
       for(int j=0; j<recv_num_ep[i]; j++)
       {
-        out_comm_hdls[0]->rank_map->at(ind) = make_pair(j, i);
+        //out_comm_hdls[0]->rank_map->at(ind) = make_pair(j, i);
+        out_comm_hdls[0]->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(ind, j, i));
         ind++;
       }

XIOS/dev/branch_openmp/extern/ep_dev/ep_exscan.cpp

@@ -253 +253 @@
     std::vector<int> my_map(mpi_size, 0);
 
-    for(int i=0; i<comm->rank_map->size(); i++) my_map[comm->rank_map->at(i).second]++;
+    for(int i=0; i<comm->ep_rank_map->size(); i++) my_map[comm->ep_rank_map->at(i).second]++;
 
     for(int i=0; i<mpi_rank; i++) my_src += my_map[i];

XIOS/dev/branch_openmp/extern/ep_dev/ep_fetch.cpp

@@ -11 +11 @@
         int MPI_Fetch_and_op(const void *origin_addr, void *result_addr, MPI_Datatype datatype, int target_rank, MPI_Aint target_disp, MPI_Op op, MPI_Win win)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
     if(num_ep==1)

XIOS/dev/branch_openmp/extern/ep_dev/ep_free.cpp

@@ -49 +49 @@
 
 
-        if( ! (*comm)->rank_map->empty() )
+        if( ! (*comm)->ep_rank_map->empty() )
         {
-          (*comm)->rank_map->clear();
-          delete (*comm)->rank_map;
-          Debug("rank_map emptied\n");
+          (*comm)->ep_rank_map->clear();
+          delete (*comm)->ep_rank_map;
+          Debug("ep_rank_map emptied and freed\n");
         }
 
@@ -110 +110 @@
 
 
-      if( ! (*comm)->rank_map->empty() )
+      if( ! (*comm)->ep_rank_map->empty() )
       {
-        (*comm)->rank_map->clear();
-        Debug("rank_map emptied\n");
+        (*comm)->ep_rank_map->clear();
+        delete (*comm)->ep_rank_map;
+        Debug("ep_rank_map emptied and freed\n");
       }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_gather.cpp

@@ -59 +59 @@
     int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second;
 
-    int root_mpi_rank = comm->rank_map->at(root).second;
-    int root_ep_loc = comm->rank_map->at(root).first;
+    int root_mpi_rank = comm->ep_rank_map->at(root).second;
+    int root_ep_loc = comm->ep_rank_map->at(root).first;
 
     ::MPI_Aint datasize, lb;
@@ -90 +90 @@
       for(int i=0; i<ep_size; i++)
       {
-        recvcounts[comm->rank_map->at(i).second]+=sendcount;
+        recvcounts[comm->ep_rank_map->at(i).second]+=sendcount;
       }
 
@@ -111 +111 @@
       for(int i=0; i<ep_size; i++)
       {
-        offset = displs[comm->rank_map->at(i).second] + comm->rank_map->at(i).first * sendcount; 
+        offset = displs[comm->ep_rank_map->at(i).second] + comm->ep_rank_map->at(i).first * sendcount; 
         memcpy(recvbuf + i*sendcount*datasize, tmp_recvbuf+offset*datasize, sendcount*datasize);
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_gatherv.cpp

@@ -63 +63 @@
     int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second;
 
-    int root_mpi_rank = comm->rank_map->at(root).second;
-    int root_ep_loc = comm->rank_map->at(root).first;
+    int root_mpi_rank = comm->ep_rank_map->at(root).second;
+    int root_ep_loc = comm->ep_rank_map->at(root).first;
 
     ::MPI_Aint datasize, lb;
@@ -126 +126 @@
       for(int i=0; i<ep_size; i++)
       {
-        mpi_recvcounts[comm->rank_map->at(i).second]+=recvcounts[i];
+        mpi_recvcounts[comm->ep_rank_map->at(i).second]+=recvcounts[i];
       }
 
@@ -144 +144 @@
       {
         int extra = 0;
-        for(int j=0, k=0; j<ep_size, k<comm->rank_map->at(i).first; j++)
-          if(comm->rank_map->at(i).second == comm->rank_map->at(j).second)
+        for(int j=0, k=0; j<ep_size, k<comm->ep_rank_map->at(i).first; j++)
+          if(comm->ep_rank_map->at(i).second == comm->ep_rank_map->at(j).second)
           {
             extra += recvcounts[j];
@@ -151 +151 @@
           }  
 
-        offset = mpi_displs[comm->rank_map->at(i).second] +  extra;
+        offset = mpi_displs[comm->ep_rank_map->at(i).second] +  extra;
 
         memcpy(recvbuf+displs[i]*datasize, tmp_recvbuf+offset*datasize, recvcounts[i]*datasize);

XIOS/dev/branch_openmp/extern/ep_dev/ep_get.cpp

@@ -12 +12 @@
                   int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
     if(num_ep==1)
@@ -27 +27 @@
               int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp

@@ -77 +77 @@
             // change leader
             is_decider = true;
-            int target = local_comm->rank_map->at(local_leader).second;
+            int target = local_comm->ep_rank_map->at(local_leader).second;
             {
               for(int i=0; i<ep_size; i++)
               {
-                if(local_comm->rank_map->at(i).second != target && local_comm->rank_map->at(i).first == 0)
+                if(local_comm->ep_rank_map->at(i).second != target && local_comm->ep_rank_map->at(i).first == 0)
                 {
                   new_local_leader = i;
@@ -100 +100 @@
             // change leader
             is_decider = true;
-            int target = local_comm->rank_map->at(local_leader).second;
+            int target = local_comm->ep_rank_map->at(local_leader).second;
             {
               for(int i=0; i<ep_size; i++)
               {
-                if(local_comm->rank_map->at(i).second != target && local_comm->rank_map->at(i).first == 0)
+                if(local_comm->ep_rank_map->at(i).second != target && local_comm->ep_rank_map->at(i).first == 0)
                 {
                   new_local_leader = i;

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.hpp

@@ -5 +5 @@
 typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank>
 typedef std::map<int, std::pair<int, int> > EP_RANK_MAP;  // key(ep_rank) = <ep_rank_local, mpi_rank>
-typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
+//typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
 
 namespace ep_lib

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp

@@ -46 +46 @@
 
     if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
-    if(ep_rank_loc == 0 && mpi_rank != local_comm->rank_map->at(local_leader).second) is_proc_master = true;
+    if(ep_rank_loc == 0 && mpi_rank != local_comm->ep_rank_map->at(local_leader).second) is_proc_master = true;
 
 
@@ -125 +125 @@
       send_buf[4] = rank_in_peer_mpi[1];
 
-      ::MPI_Bcast(send_buf.data(), 5, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(send_buf.data(), 5, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       size_info[1] = send_buf[0];
@@ -157 +157 @@
       }
 
-      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
@@ -281 +281 @@
       }
 
-      ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       new_rank_info[2].resize(size_info[3]);
@@ -305 +305 @@
       }
 
-      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
@@ -337 +337 @@
       }
 
-      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       if(new_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm))
@@ -542 +542 @@
       MPI_Status statuses[2];
       
-      MPI_Isend((*newintercomm)->rank_map->data(), 2*local_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[0]);
-      MPI_Irecv((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[1]);
+      std::vector<std::pair<int, std::pair<int, int> > > map2vec((*newintercomm)->ep_rank_map->size());
+      std::vector<std::pair<int, std::pair<int, int> > > vec2map((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->size());
+      
+      int ii=0;
+      for(std::map<int, std::pair<int, int> >::iterator it = (*newintercomm)->ep_rank_map->begin(); it != (*newintercomm)->ep_rank_map->end(); it++)
+      {
+        map2vec[ii++] = make_pair(it->first, make_pair(it->second.first, it->second.second));
+      }
+      
+      
+      MPI_Isend(map2vec.data(), 3*local_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[0]);
+      MPI_Irecv(vec2map.data(), 3*remote_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[1]);
+      
+     
+      for(ii=0; ii<vec2map.size(); ii++)
+      {
+        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(vec2map[ii].first) = make_pair(vec2map[ii].second.first, vec2map[ii].second.second);
+      }
       
       MPI_Waitall(2, requests, statuses);

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_world.cpp

@@ -44 +44 @@
 
     if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
-    if(ep_rank_loc == 0 && mpi_rank != local_comm->rank_map->at(local_leader).second) is_proc_master = true;
+    if(ep_rank_loc == 0 && mpi_rank != local_comm->ep_rank_map->at(local_leader).second) is_proc_master = true;
 
 
@@ -105 +105 @@
       }
 
-      ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       size_info[1] = recv_buf[0];
@@ -135 +135 @@
       }
 
-      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
@@ -261 +261 @@
       }
 
-      ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       new_rank_info[2].resize(size_info[3]);
@@ -283 +283 @@
       }
 
-      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
@@ -315 +315 @@
       }
 
-      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->rank_map->at(local_leader).second, local_mpi_comm);
+      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, local_mpi_comm);
 
       if(new_comm != static_cast< ::MPI_Comm* >(MPI_COMM_NULL->mpi_comm))
@@ -512 +512 @@
     {
       ::MPI_Status status;
-
-      ::MPI_Send((*newintercomm)->rank_map->data(), 2*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm);
-
-      ::MPI_Recv((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status);
+      
+      std::vector<std::pair<int, std::pair<int, int> > > map2vec((*newintercomm)->ep_rank_map->size());
+      std::vector<std::pair<int, std::pair<int, int> > > vec2map((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->size());
+      
+      int ii=0;
+      for(std::map<int, std::pair<int, int> >::iterator it = (*newintercomm)->ep_rank_map->begin(); it != (*newintercomm)->ep_rank_map->end(); it++)
+      {
+        map2vec[ii++] = make_pair(it->first, make_pair(it->second.first, it->second.second));
+      }
+      
+      
+      ::MPI_Send(map2vec.data(), 3*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm);
+      ::MPI_Recv(vec2map.data(), 3*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status);
+      
+     
+      for(ii=0; ii<vec2map.size(); ii++)
+      {
+        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(vec2map[ii].first) = make_pair(vec2map[ii].second.first, vec2map[ii].second.second);
+      }
     }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.cpp

@@ -33 +33 @@
   int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank)
   {
-    for(int i=0; i<comm->rank_map->size(); i++)
-    {
-      if(   ( comm->rank_map->at(i).first  == ep_rank_loc )
-         && ( comm->rank_map->at(i).second == mpi_rank ) )
+    for(int i=0; i<comm->ep_rank_map->size(); i++)
+    {
+      if(   ( comm->ep_rank_map->at(i).first  == ep_rank_loc )
+         && ( comm->ep_rank_map->at(i).second == mpi_rank ) )
       {
         return i;
@@ -89 +89 @@
   void check_sum_send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, int type)
   {
+
     int src_rank;
     int int_count;
@@ -121 +122 @@
   void check_sum_recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, int type)
   {
+
     int dest_rank;
     int int_count;
@@ -147 +149 @@
     }
     else printf("Unable to open file\n");
+
 
   }

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.hpp

@@ -9 +9 @@
 #include "ep_lib_fortran.hpp"
 #include "ep_lib_win.hpp"
+#include "ep_lib_mpi.hpp"
+
 
 namespace ep_lib

XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp

@@ -92 +92 @@
       for(int i=0; i<intra_ep_size; i++)
       {
-        tmp_rank_map[reorder[i]] = (*newintracomm)->rank_map->at(i) ;
+        tmp_rank_map[reorder[i]] = (*newintracomm)->ep_rank_map->at(i) ;
       }
 
-      (*newintracomm)->rank_map->swap(tmp_rank_map);
+      //(*newintracomm)->rank_map->swap(tmp_rank_map);
+      (*newintracomm)->ep_rank_map->clear();
+      for(int i=0; i<tmp_rank_map.size(); i++)
+      {
+        (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(i, tmp_rank_map[i].first, tmp_rank_map[i].second));
+      }
+      
 
       tmp_rank_map.clear();
@@ -206 +212 @@
     for(int i=0; i<newintracomm_ep_size; i++)
     {
-      (*newintracomm)->rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1];
-      (*newintracomm)->rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2];
+      (*newintracomm)->ep_rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1];
+      (*newintracomm)->ep_rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2];
     }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_message.cpp

@@ -20 +20 @@
   int Request_Check()
   {
+    if(EP_PendingRequests == 0 ) EP_PendingRequests = new std::list< MPI_Request* >;
+    
+    if(EP_PendingRequests->size() == 0) return 0;
+    
     MPI_Status status;
     MPI_Message *message;
@@ -26 +30 @@
     std::list<MPI_Request* >::iterator it;
     
-    if(EP_PendingRequests == 0 ) return 0;
     
     for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); it++)
@@ -38 +41 @@
       if((*(*it))->state == 0)
       {
-          
-        message = new MPI_Message;
-        *message = new ep_message;
-        
-        printf("new %p : in ep_lib::Request_Check, message = new MPI_Message\n", message);
-        printf("new %p : in ep_lib::Request_Check, *message = new ep_message\n", *message);
+        #pragma omp critical (_query0)
+        {
+          MPI_Iprobe((*(*it))->ep_src, (*(*it))->ep_tag, ((*(*it))->comm), &probed, &status);
+          if(probed)
+          {
+            message = new MPI_Message;
+            *message = new ep_message;
+        
+            memcheck("new "<< message <<" : in ep_lib::Request_Check, message = new MPI_Message");
+            memcheck("new "<< *message <<" : in ep_lib::Request_Check, *message = new ep_message");
           
           
-        MPI_Improbe((*(*it))->ep_src, (*(*it))->ep_tag, ((*(*it))->comm), &probed, message, &status);
-        printf("in Request_Check, after improbe, mpi_status = %p\n", to_mpi_status_ptr(status));
+            MPI_Improbe((*(*it))->ep_src, (*(*it))->ep_tag, (*(*it))->comm, &probed, message, &status);
+        
+          }
+        }
       
         
@@ -54 +63 @@
           ::MPI_Get_count(to_mpi_status_ptr(status), to_mpi_type((*(*it))->ep_datatype), &recv_count);
           
-          printf("in Request_Check, imrecv, buf = %p, recv_count = %d, status = %p\n", (*(*it))->buf, recv_count, to_mpi_status_ptr(status));
           MPI_Imrecv((*(*it))->buf, recv_count, (*(*it))->ep_datatype, message, *it);
           (*(*it))->type = 3;
           (*(*it))->state = 1;
-          
-
-          printf("delete %p : in ep_lib::Request_Check, delete *message\n", *message);
-          printf("delete %p : in ep_lib::Request_Check, delete message\n", message);
+
+          memcheck("delete "<< status.mpi_status <<" : in ep_lib::Request_Check, delete status.mpi_status");
+          delete status.mpi_status;          
+
+          memcheck("delete "<< *message <<" : in ep_lib::Request_Check, delete *message");
+          memcheck("delete "<< message <<" : in ep_lib::Request_Check, delete message");
 
           delete *message;
@@ -68 +78 @@
           it++;
           continue;      
-        }      
+        }              
       }
       
       if((*(*it))->state == 2)
       {
-        printf("delete %p : in ep_lib::Request_Check, delete (*(*it))\n", (*(*it)));
+        int ep_rank = ((*(*it))->comm)->ep_comm_ptr->size_rank_info[0].first;
+        memcheck("delete "<< (*(*it)) <<" : in ep_lib::Request_Check, delete (*(*it))");
         delete (*(*it));
         
         EP_PendingRequests->erase(it);
-        printf("EP_PendingRequests->size() = %lu\n", EP_PendingRequests->size());
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
         it = EP_PendingRequests->begin();
         continue;
@@ -128 +139 @@
         msg->mpi_message = new ::MPI_Message(message);
 
-        printf("new %p : in ep_lib::Message_Check, msg = new ep_message\n", msg);
-        printf("new %p : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message\n", msg->mpi_message);
+        memcheck("new "<< msg <<" : in ep_lib::Message_Check, msg = new ep_message");
+        memcheck("new "<< msg->mpi_message <<" : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message");
               
 
@@ -139 +150 @@
         msg->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);       
         msg->mpi_status = new ::MPI_Status(status);  
-        printf("new %p : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status\n", msg->mpi_status);
+        memcheck("new "<< msg->mpi_status <<" : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status");
 
         #pragma omp critical (_query)
@@ -145 +156 @@
           #pragma omp flush
           comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->push_back(msg);  
-          int test_count;
-          ::MPI_Get_count(static_cast< ::MPI_Status* >(msg->mpi_status), 1275070475, &test_count);
-          printf("status1 = %p, test_count2 = %d\n", static_cast< ::MPI_Status* >(msg->mpi_status), test_count);
-          
-          ::MPI_Get_count(static_cast< ::MPI_Status* >(comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->back()->mpi_status), 1275070475, &test_count);
-          printf("status2 = %p, test_count2 = %d\n", static_cast< ::MPI_Status* >(comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->back()->mpi_status), test_count);
-                   
+          int dest_mpi = comm->ep_comm_ptr->size_rank_info[2].first;
+          memcheck("message_queue["<<dest_mpi<<","<<dest_loc<<"]->size = "<<comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->size());
           #pragma omp flush
         }
@@ -199 +205 @@
         msg->mpi_message = new ::MPI_Message(message);
 
-        printf("new %p : in ep_lib::Message_Check, msg = new ep_message\n", msg);
-        printf("new %p : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message\n", msg->mpi_message);
+        memcheck("new "<< msg <<" : in ep_lib::Message_Check, msg = new ep_message");
+        memcheck("new "<< msg->mpi_message <<" : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message");
               
 
@@ -210 +216 @@
              
         msg->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);       
-        msg->mpi_status = new ::MPI_Status(status);  
-        printf("new %p : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status\n", msg->mpi_status);
+        msg->mpi_status = new ::MPI_Status(status); 
+        memcheck("new "<< msg->mpi_status <<" : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status");
 
         #pragma omp critical (_query)
@@ -217 +223 @@
           #pragma omp flush
           comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->push_back(msg);
+          memcheck("comm->ep_comm_ptr->comm_list["<<dest_loc<<"]->ep_comm_ptr->message_queue->size = "<<comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->size());
           #pragma omp flush
         }

XIOS/dev/branch_openmp/extern/ep_dev/ep_probe.cpp

@@ -13 +13 @@
     if(!comm->is_ep)
     {
-      ::MPI_Status *mpi_status = static_cast< ::MPI_Status* >(status->mpi_status);
-      ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, mpi_status);
+      Debug("calling MPI_Iprobe MPI\n");
+      ::MPI_Status mpi_status;
+      ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status);
 
-      status->mpi_status = mpi_status;
+      status->mpi_status = &mpi_status;
       status->ep_src = src;
       status->ep_tag = tag;
@@ -23 +24 @@
 
     Debug("calling MPI_Iprobe EP\n");
-
     Message_Check(comm);
 
@@ -43 +43 @@
           ::MPI_Status mpi_status = *(static_cast< ::MPI_Status *>((*it)->mpi_status));
 
-          status->mpi_status = new ::MPI_Status(mpi_status);
+          status->mpi_status = (*it)->mpi_status;
           status->ep_src = (*it)->ep_src;
           status->ep_tag = (*it)->ep_tag;
@@ -49 +49 @@
           break;
         }
-
       }
     }
@@ -60 +59 @@
   int MPI_Improbe(int src, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status)
   {
+    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;
+    int mpi_rank    = comm->ep_comm_ptr->size_rank_info[2].first;
     *flag = false;
     if(!comm->is_ep)
@@ -102 +103 @@
       for(Message_list::iterator it = comm->ep_comm_ptr->message_queue->begin(); it!= comm->ep_comm_ptr->message_queue->end(); ++it)
       {
-
-        //printf("in ep_lib::Improbe, it->mpi_message = %p, it->mpi_status = %p, it->ep_src = %d, it->ep_tag = %d\n", 
-        //                            (*(*it))->mpi_message,(*(*it))->mpi_status,(*(*it))->ep_src,(*(*it))->ep_tag);
                                           
         bool src_matched = src<0? true: (*it)->ep_src == src;
@@ -113 +111 @@
           *flag = true;
 
-          status->mpi_status = (*it)->mpi_status;
+          status->mpi_status = new ::MPI_Status(*static_cast< ::MPI_Status*>((*it)->mpi_status));
+          memcheck("new "<< status->mpi_status << " : in ep_lib::MPI_Improbe, status->mpi_status = new ::MPI_Status");
           status->ep_src = (*it)->ep_src;
           status->ep_tag = (*it)->ep_tag;
 
-          (*message)->mpi_message = (*it)->mpi_message;
+          (*message)->mpi_message = new ::MPI_Message(*static_cast< ::MPI_Message*>((*it)->mpi_message));
+          memcheck("new "<< (*message)->mpi_message <<" : in ep_lib::MPI_Improbe, (*message)->mpi_message = new ::MPI_Message");
           (*message)->ep_src = (*it)->ep_src;
           (*message)->ep_tag = (*it)->ep_tag;
-          
-          int test_count;
-          ::MPI_Get_count(to_mpi_status_ptr(status), 1275070475, &test_count);
-          printf("status = %p, test_count = %d\n", to_mpi_status_ptr(status), test_count);
-          
-          
-          ::MPI_Get_count(static_cast< ::MPI_Status* >(comm->ep_comm_ptr->message_queue->back()->mpi_status), 1275070475, &test_count);
-          printf("in ep_lib::Improbe, status = %p, test_count = %d\n", static_cast< ::MPI_Status* >(comm->ep_comm_ptr->message_queue->back()->mpi_status), test_count);
                                       
 
           #pragma omp critical (_query2)
           {              
-            printf("delete %p : in ep_lib::Message_Check, delete (*it)->mpi_message\n", (*it)->mpi_message);
-            printf("delete %p : in ep_lib::Message_Check, delete (*it)->mpi_status\n", (*it)->mpi_status);
-            printf("delete %p : in ep_lib::Message_Check, delete (*it)\n", (*it));
+            memcheck("delete "<< (*it)->mpi_message <<" : in ep_lib::Message_Check, delete (*it)->mpi_message");
+            memcheck("delete "<< (*it)->mpi_status <<" : in ep_lib::Message_Check, delete (*it)->mpi_status");
+            memcheck("delete "<< (*it) <<" : in ep_lib::Message_Check, delete (*it)");
+            
             
             delete (*it)->mpi_message;     
@@ -140 +133 @@
             delete *it;
             
+                       
             comm->ep_comm_ptr->message_queue->erase(it);
-            printf("message queue siez = %lu\n", comm->ep_comm_ptr->message_queue->size());
+            memcheck("message_queue["<<mpi_rank<<","<<ep_rank_loc<<"]->size = "<<comm->ep_comm_ptr->message_queue->size());
             #pragma omp flush
           }

XIOS/dev/branch_openmp/extern/ep_dev/ep_put.cpp

@@ -12 +12 @@
                   int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
     if(num_ep==1)
@@ -27 +27 @@
               int target_rank, MPI_Aint target_disp, int target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request)
   {
-    int target_mpi_rank   = win->comm->rank_map->at(target_rank).second;
-    int target_local_rank = win->comm->rank_map->at(target_rank).first;
+    int target_mpi_rank   = win->comm->ep_rank_map->at(target_rank).second;
+    int target_local_rank = win->comm->ep_rank_map->at(target_rank).first;
     int num_ep = win->comm->ep_comm_ptr->size_rank_info[1].second;
     

XIOS/dev/branch_openmp/extern/ep_dev/ep_recv.cpp

@@ -19 +19 @@
 namespace ep_lib 
 {
-  
 
   int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status)
   {
 
-    if(!comm->is_ep)
-    {
-      ::MPI_Status mpi_status;
-      ::MPI_Recv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_status);
-
-      status->ep_src = src;
-      status->ep_tag = tag;
-      status->ep_datatype = datatype;
-
-      return 0;  
-    }
-
-    //Message_Check(comm);
+    if(!comm->is_ep) return MPI_Recv_mpi(buf, count, datatype, src, tag, comm, status);
+    
+    Debug("MPI_Recv with EP");
 
     MPI_Request request;
@@ -45 +34 @@
   }
 
-
+  
 
 
   int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request)
   {
-
+    if(!comm->is_ep) return MPI_Irecv_mpi(buf, count, datatype, src, tag, comm, request);
+    
     Debug("MPI_Irecv with EP");
     int dest_rank;
     MPI_Comm_rank(comm, &dest_rank);
     
-
-    if(!comm->is_ep)
-    {
-      ::MPI_Request mpi_request;
-      ::MPI_Irecv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_request);
-
-      (*request)->mpi_request = new ::MPI_Request(mpi_request);
-      (*request)->ep_src = src;
-      (*request)->ep_datatype = datatype;
-      (*request)->ep_tag = tag;
-    }
-    
-    *request = new ep_request;
-    printf("new %p : in ep_lib::MPI_Irecv, *request = new ep_request\n", (*request));
+    *request = new ep_request;   
+    memcheck("new " << *request <<" : in ep_lib::MPI_Irecv, *request = new ep_request");
 
     (*request)->mpi_request = new ::MPI_Request;
-    printf("new %p : in ep_lib::MPI_Irecv, (*request)->mpi_request = new ::MPI_Request\n", (*request)->mpi_request);
+    memcheck("new " << (*request)->mpi_request << " : in ep_lib::MPI_Irecv, (*request)->mpi_request = new ::MPI_Request");
     
     (*request)->buf = buf;
@@ -82 +60 @@
     (*request)->ep_tag = tag;
     (*request)->ep_datatype = datatype;
-    
-    
-    /* With Improbe*/
+  
+    if(EP_PendingRequests == 0 ) EP_PendingRequests = new std::list< MPI_Request* >;
 
-    if(EP_PendingRequests == 0 ) 
-    {
-      EP_PendingRequests = new std::list< MPI_Request* >;
-    }
-
-    EP_PendingRequests->push_back(request);                                                                
+    EP_PendingRequests->push_back(request);  
+    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first;
+    memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());                                                              
                                           
-    Request_Check();
-
-    return 0;
+    return Request_Check();
   }
-
-  int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request)
-  {
-    Debug("MPI_Imrecv");
-
-    (*request)->type = 3;
-        
-    ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request));                
-
-    (*request)->ep_datatype = datatype;
-    (*request)->ep_tag = (*message)->ep_tag;
-    (*request)->ep_src = (*message)->ep_src;
-    
-    
-    return 0;
-  }
-
-
+  
   int MPI_Mrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Status *status)
   {
-    Debug("EP Mrecv called\n");
+    Debug("MPI_Mrecv with MPI/EP");
 
-    ::MPI_Status mpi_status;
-    ::MPI_Mrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >((*message)->mpi_message), &mpi_status);
+    status->mpi_status = new ::MPI_Status;
+    memcheck("new " << status->mpi_status << " : in ep_lib::MPI_Mrecv, status->mpi_status = new ::MPI_Status");
+    
+    ::MPI_Mrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >((*message)->mpi_message), to_mpi_status_ptr(*status));
 
-    status->mpi_status = new ::MPI_Status(mpi_status);
+    
     status->ep_src = (*message)->ep_src;
     status->ep_datatype = datatype;
     status->ep_tag = (*message)->ep_tag;
 
+    memcheck("delete " << (*message)->mpi_message << " : in ep_lib::MPI_Mrecv, delete (*message)->mpi_message");
     delete (*message)->mpi_message;
 
-    //check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag);
+#ifdef _check_sum
+    check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag);
+#endif
 
-    return 0;
+    return Request_Check();
   }
 
+
+  int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request)
+  {
+    Debug("MPI_Imrecv with MPI/EP");
+
+    (*request)->type = 3;
+    (*request)->ep_datatype = datatype;
+    (*request)->ep_tag = (*message)->ep_tag;
+    (*request)->ep_src = (*message)->ep_src;
+        
+    ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request));                
+    
+    memcheck("delete " << (*message)->mpi_message << " : in ep_lib::MPI_Imrecv, delete (*message)->mpi_message");
+    delete (*message)->mpi_message;
+
+#ifdef _check_sum
+    check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag);
+#endif
+
+    
+    return Request_Check();
+  }
+
+
+   int MPI_Recv_mpi(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status)
+  {
+    Debug("MPI_Recv with MPI");
+    status->ep_src = src;
+    status->ep_tag = tag;
+    status->ep_datatype = datatype;
+    
+    return ::MPI_Recv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), to_mpi_status_ptr(*status)); 
+  }
+  
+  int MPI_Irecv_mpi(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("MPI_Irecv with MPI");
+    int dest_rank;
+    MPI_Comm_rank(comm, &dest_rank);
+    
+    *request = new ep_request;
+    memcheck("new "<< *request <<" : in ep_lib::MPI_Irecv, *request = new ep_request");
+
+    (*request)->mpi_request = new ::MPI_Request;
+    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Irecv, (*request)->mpi_request = new ::MPI_Request");
+      
+    (*request)->ep_src = src;
+    (*request)->ep_datatype = datatype;
+    (*request)->type = 2;
+    (*request)->ep_tag = tag;
+    
+    return ::MPI_Irecv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));
+  }
 }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_reduce.cpp

@@ -302 +302 @@
     int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second;
 
-    int root_mpi_rank = comm->rank_map->at(root).second;
-    int root_ep_loc = comm->rank_map->at(root).first;
+    int root_mpi_rank = comm->ep_rank_map->at(root).second;
+    int root_ep_loc = comm->ep_rank_map->at(root).first;
 
     ::MPI_Aint datasize, lb;

XIOS/dev/branch_openmp/extern/ep_dev/ep_request.hpp

@@ -14 +14 @@
 
       int type; //! type of the non-blocking communication. 
-                // 1: Isend 
-                // 2:Irecv 
-                // 3:Imrecv 
-                // 4:Issend
+                // 1: Isend or Issend
+                // 2: Irecv 
+                // 3: Imrecv 
+                
       int state; // 0: new 
                  // 1: imrecvd
@@ -27 +27 @@
       int ep_tag;
       void* ep_datatype;
+      
+      
 
       ep_comm *comm;    //! EP communicator related to the communication

XIOS/dev/branch_openmp/extern/ep_dev/ep_scan.cpp

@@ -375 +375 @@
     std::vector<int> my_map(mpi_size, 0);
 
-    for(int i=0; i<comm->rank_map->size(); i++) my_map[comm->rank_map->at(i).second]++;
+    for(int i=0; i<comm->ep_rank_map->size(); i++) my_map[comm->ep_rank_map->at(i).second]++;
 
     for(int i=0; i<mpi_rank; i++) my_src += my_map[i];

XIOS/dev/branch_openmp/extern/ep_dev/ep_scatter.cpp

@@ -56 +56 @@
     int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second;
 
-    int root_mpi_rank = comm->rank_map->at(root).second;
-    int root_ep_loc = comm->rank_map->at(root).first;
+    int root_mpi_rank = comm->ep_rank_map->at(root).second;
+    int root_ep_loc = comm->ep_rank_map->at(root).first;
 
     bool is_master = (ep_rank_loc==0 && mpi_rank != root_mpi_rank ) || ep_rank == root;
@@ -87 +87 @@
       for(int i=0; i<ep_size; i++)
       {
-        recvcounts[comm->rank_map->at(i).second]++;
+        recvcounts[comm->ep_rank_map->at(i).second]++;
       }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_scatterv.cpp

@@ -59 +59 @@
     int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second;
 
-    int root_mpi_rank = comm->rank_map->at(root).second;
-    int root_ep_loc = comm->rank_map->at(root).first;
+    int root_mpi_rank = comm->ep_rank_map->at(root).second;
+    int root_ep_loc = comm->ep_rank_map->at(root).first;
 
     bool is_master = (ep_rank_loc==0 && mpi_rank != root_mpi_rank ) || ep_rank == root;
@@ -90 +90 @@
       for(int i=0; i<ep_size; i++)
       {
-        recvcounts[comm->rank_map->at(i).second]++;
+        recvcounts[comm->ep_rank_map->at(i).second]++;
       }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_send.cpp

@@ -12 +12 @@
 
 
-namespace ep_lib {
-
-
+namespace ep_lib 
+{
+  
   int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
   {
-    if(!comm->is_ep)
-      return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm));
-    if(comm->is_intercomm)
-    {
-      MPI_Request request;
-      MPI_Status status;
-      MPI_Isend(buf, count, datatype, dest, tag, comm, &request);
-      MPI_Wait(&request, &status);
-    }
-    else
-    {
-      int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first;
-      int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first;
-      int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc);
-      int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second;
-
-      ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm));
-    }
-    //check_sum_send(buf, count, datatype, dest, tag, comm);
-
-    return 0;
+    if(!comm->is_ep)       return MPI_Send_mpi(buf, count, datatype, dest, tag, comm);
+    if(comm->is_intercomm) return MPI_Send_intercomm(buf, count, datatype, dest, tag, comm);
+    
+    Debug("\nMPI_Send with EP\n");
+    
+    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first;
+    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first;
+    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc);
+    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second;
+
+#ifdef _check_sum    
+    check_sum_send(buf, count, datatype, dest, tag, comm);
+#endif
+
+    return ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm));
   }
 
@@ -43 +37 @@
   int MPI_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
   {
-    if(!comm->is_ep)
-    {
-      return ::MPI_Ssend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm));
-    }
-
+    if(!comm->is_ep)       return MPI_Ssend_mpi(buf, count, datatype, dest, tag, comm);
+    if(comm->is_intercomm) return MPI_Ssend_intercomm(buf, count, datatype, dest, tag, comm);
+    
+    Debug("\nMPI_Ssend with EP\n");
+
+    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first;
+    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first;
+    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc);
+    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second;
+    
+#ifdef _check_sum    
+    check_sum_send(buf, count, datatype, dest, tag, comm);
+#endif
+    
+    return ::MPI_Ssend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm));
+  }
+  
+
+
+
+
+  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    if(!comm->is_ep)       return MPI_Isend_mpi(buf, count, datatype, dest, tag, comm, request);
+    if(comm->is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request);
+
+    Debug("\nMPI_Isend with EP\n");
+   
+    int src_rank;
+    MPI_Comm_rank(comm, &src_rank);
+
+
+#ifdef _check_sum    
+    check_sum_send(buf, count, datatype, dest, tag, comm);
+#endif
+    
+    *request = new ep_request;
+    memcheck("new "<< *request <<" : in ep_lib::MPI_Isend, *request = new ep_request");
+
+    (*request)->mpi_request = new ::MPI_Request;
+    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request");
+    
+
+    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first;
+    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first;
+    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc);
+    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second;
+
+    (*request)->ep_src  = src_rank;
+    (*request)->ep_tag  = tag;
+    (*request)->ep_datatype = datatype;
+
+    (*request)->type = 1;    // used in wait
+    (*request)->comm = comm;
+    (*request)->buf = const_cast<void*>(buf);
+
+
+    return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));
+  }
+
+
+
+
+  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+ 
+    if(!comm->is_ep) return MPI_Issend_mpi(buf, count, datatype, dest, tag, comm, request);
+    if(comm->is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request);
+
+    Debug("\nMPI_Issend with EP\n");
+   
+    int src_rank;
+    MPI_Comm_rank(comm, &src_rank);
+
+
+#ifdef _check_sum    
+    check_sum_send(buf, count, datatype, dest, tag, comm);
+#endif
+    
+    *request = new ep_request;
+    memcheck("new "<< *request <<" : in ep_lib::MPI_Issend, *request = new ep_request");
+
+    (*request)->mpi_request = new ::MPI_Request;
+    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend, (*request)->mpi_request = new ::MPI_Request");
+    
+
+    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first;
+    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first;
+    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc);
+    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second;
+
+    (*request)->ep_src  = src_rank;
+    (*request)->ep_tag  = tag;
+    (*request)->ep_datatype = datatype;
+
+    (*request)->type = 1;    // used in wait
+    (*request)->comm = comm;
+    (*request)->buf = const_cast<void*>(buf);
+    
+    return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));    
+  }
+  
+  int MPI_Send_mpi(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+  {
+    Debug("\nMPI_Send with MPI\n");
+    return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm));
+  }
+  
+  int MPI_Send_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+  {
+    Debug("\nMPI_Send with EP_Intercomm\n");
+    MPI_Request request;
+    MPI_Status status;
+    MPI_Isend(buf, count, datatype, dest, tag, comm, &request);
+    MPI_Wait(&request, &status);
+    return MPI_SUCCESS;
+  }
+  
+  
+  int MPI_Ssend_mpi(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+  {
+    Debug("\nMPI_Ssend with MPI\n");
+    return ::MPI_Ssend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm));
+  }
+  
+
+  int MPI_Ssend_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+  {
+    Debug("\nMPI_Ssend with EP_Intercomm\n");
     MPI_Request request;
     MPI_Status status;
     MPI_Issend(buf, count, datatype, dest, tag, comm, &request);
     MPI_Wait(&request, &status);
-    //check_sum_send(buf, count, datatype, dest, tag, comm);
-    return 0;
-  }
-  
-
-
-
-
-  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
-  {
-    Debug("\nMPI_Isend with EP\n");
+    return MPI_SUCCESS;
+  }
+  
+  int MPI_Isend_mpi(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("\nMPI_Isend with MPI\n");
+    
     int src_rank;
     MPI_Comm_rank(comm, &src_rank);
-
-    
-
-    if(!comm->is_ep)
-    {
-      ::MPI_Request mpi_request;
-      ::MPI_Isend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), &mpi_request);
-
-      (*request)->mpi_request = new ::MPI_Request(mpi_request);
-
-      (*request)->ep_src = src_rank;
-      (*request)->ep_tag = tag;
-      (*request)->ep_datatype = datatype;
-      (*request)->type = 1;
-      (*request)->comm = comm;
-
-      return 0;
-    }
-
-    if(comm->is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request);
-
-    // EP intracomm
-
-    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
     
     *request = new ep_request;
-    printf("new %p : in ep_lib::MPI_Isend, *request = new ep_request\n", (*request));
+    memcheck("new "<< *request <<" : in ep_lib::MPI_Isend, *request = new ep_request");
 
     (*request)->mpi_request = new ::MPI_Request;
-    printf("new %p : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request\n", (*request)->mpi_request);
-    
-
-    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first;
-    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first;
-    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc);
-    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second;
-
-    (*request)->ep_src  = src_rank;
-    (*request)->ep_tag  = tag;
-    (*request)->ep_datatype = datatype;
-
-    ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));
-
-    (*request)->type = 1;    // used in wait
-    (*request)->comm = comm;
-    (*request)->buf = const_cast<void*>(buf);
-
-
-    return 0;
-  }
-
-
-
-
-  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
-  {
-    Debug("\nMPI_Issend with EP\n");
-
-    int src_rank;
-    MPI_Comm_rank(comm, &src_rank);
-
-    
-
-    if(!comm->is_ep)
-    {
-      ::MPI_Request mpi_request;
-      ::MPI_Issend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), &mpi_request);
-
-      (*request)->mpi_request = new ::MPI_Request(mpi_request);
-      (*request)->ep_src = src_rank;
-      (*request)->ep_tag = tag;
-      (*request)->ep_datatype = datatype;
-      (*request)->type = 1;
-      (*request)->comm = comm;
-
-      return 0;
-    }
-
-    if(comm->is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request);
-
-    // EP intracomm
-
-    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
-
-    int ep_src_loc  = comm->ep_comm_ptr->size_rank_info[1].first;
-    int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).first;
-    int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc);
-    int mpi_dest    = comm->ep_comm_ptr->comm_list[0]->rank_map->at(dest).second;
-    
+    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request");
+      
     (*request)->ep_src = src_rank;
     (*request)->ep_tag = tag;
     (*request)->ep_datatype = datatype;
-
-    ::MPI_Request mpi_request;
-
-    ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request);
-
-    (*request)->mpi_request = new ::MPI_Request(mpi_request);
-    (*request)->type = 1;    // used in wait
+    (*request)->type = 1;
     (*request)->comm = comm;
-    (*request)->buf = NULL;
-    
-
-    //Message_Check(comm);
-
-    return 0;
-  }
-  
-
-
-
+    
+    return ::MPI_Isend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));
+  }
+  
   int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
   {
-    Debug("MPI_Isend with intercomm\n");
-
-    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+    Debug("MPI_Isend with EP_Intercomm\n");
+
+#ifdef _check_sum    
+    check_sum_send(buf, count, datatype, dest, tag, comm);
+#endif
 
     int dest_remote_ep_rank    = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first;
@@ -189 +210 @@
 
     src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second;
-
-
-    
-
     //Message_Check(comm);
-
 
     if(dest_remote_comm_label == src_comm_label)       // mpi_dest differs
     {
       int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
-      int ep_src_loc = comm->rank_map->at(inter_src).first;
-      int ep_dest_loc = comm->rank_map->at(dest_remote_ep_rank).first;
-      int mpi_dest    = comm->rank_map->at(dest_remote_ep_rank).second;
+      int ep_src_loc = comm->ep_rank_map->at(inter_src).first;
+      int ep_dest_loc = comm->ep_rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm->ep_rank_map->at(dest_remote_ep_rank).second;
       int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
 
-      ::MPI_Request mpi_request;
-  
-      ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request);
-
-      (*request)->mpi_request = new ::MPI_Request(mpi_request);
+      *request = new ep_request;
+      memcheck("new "<< *request <<" : in ep_lib::MPI_Isend_intercomm, *request = new ep_request");
+
+      (*request)->mpi_request = new ::MPI_Request;
+      memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend_intercomm, (*request)->mpi_request = new ::MPI_Request");
+      
       (*request)->type = 1;    // used in wait
       (*request)->comm = comm;
-
       (*request)->ep_src = src_ep_rank;
       (*request)->ep_tag = tag;
       (*request)->ep_datatype = datatype;
+
+      return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));
+
     }
 
@@ -220 +239 @@
     { 
       int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
-      int ep_src_loc = comm->rank_map->at(inter_src).first;
+      int ep_src_loc = comm->ep_rank_map->at(inter_src).first;
       int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first;
       int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second;
       int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
 
-      ::MPI_Request mpi_request;
-
-      ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request);
-
-      (*request)->mpi_request = new ::MPI_Request(mpi_request);
+      *request = new ep_request;
+      memcheck("new "<< *request <<" : in ep_lib::MPI_Isend_intercomm, *request = new ep_request");
+
+      (*request)->mpi_request = new ::MPI_Request;
+      memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend_intercomm, (*request)->mpi_request = new ::MPI_Request");
+
       (*request)->type = 1;    // used in wait
       (*request)->comm = comm;
-   
       (*request)->ep_src = src_ep_rank;
       (*request)->ep_tag = tag;
       (*request)->ep_datatype = datatype;
+      
+      return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request)); 
     }
-
-    return 0;
-
+  }
+
+  int MPI_Issend_mpi(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("\nMPI_Issend with MPI\n");
+    
+    int src_rank;
+    MPI_Comm_rank(comm, &src_rank);
+    
+    *request = new ep_request;
+    memcheck("new "<< *request <<" : in ep_lib::MPI_Issend, *request = new ep_request");
+
+    (*request)->mpi_request = new ::MPI_Request;
+    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend, (*request)->mpi_request = new ::MPI_Request");
+      
+    (*request)->ep_src = src_rank;
+    (*request)->ep_tag = tag;
+    (*request)->ep_datatype = datatype;
+    (*request)->type = 1;
+    (*request)->comm = comm;
+    
+    return ::MPI_Issend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));
   }
 
@@ -245 +285 @@
   int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
   {
-    Debug("MPI_Issend with intercomm\n");
-
-    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+    Debug("MPI_Issend with EP_Intercomm\n");
+
+#ifdef _check_sum    
+    check_sum_send(buf, count, datatype, dest, tag, comm);
+#endif
 
     int dest_remote_ep_rank    = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first;
@@ -255 +297 @@
     int src_comm_label;
 
-    for(int i=0; i<comm->ep_comm_ptr->intercomm->local_rank_map->size(); i++)
-    {
-      if(comm->ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank)
-      {
-        src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(i).second;
-        break;
-      }
-    }
-
+    src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second;
     //Message_Check(comm);
 
-
-    if(dest_remote_comm_label == src_comm_label)       // dest rank (loc, mpi) differs
+    if(dest_remote_comm_label == src_comm_label)       // mpi_dest differs
     {
       int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
-      int ep_src_loc = comm->rank_map->at(inter_src).first;
-      int ep_dest_loc = comm->rank_map->at(dest_remote_ep_rank).first;
-      int mpi_dest    = comm->rank_map->at(dest_remote_ep_rank).second;
+      int ep_src_loc = comm->ep_rank_map->at(inter_src).first;
+      int ep_dest_loc = comm->ep_rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm->ep_rank_map->at(dest_remote_ep_rank).second;
       int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
 
-      ::MPI_Request mpi_request;
-  
-      ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), &mpi_request);
-
-      (*request)->mpi_request = new ::MPI_Request(mpi_request);
+      *request = new ep_request;
+      memcheck("new "<< *request <<" : in ep_lib::MPI_Issend_intercomm, *request = new ep_request");
+
+      (*request)->mpi_request = new ::MPI_Request;
+      memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend_intercomm, (*request)->mpi_request = new ::MPI_Request");
+      
       (*request)->type = 1;    // used in wait
       (*request)->comm = comm;
-
       (*request)->ep_src = src_ep_rank;
       (*request)->ep_tag = tag;
       (*request)->ep_datatype = datatype;
+
+      return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));
+
     }
 
@@ -291 +327 @@
     { 
       int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
-      int ep_src_loc = comm->rank_map->at(inter_src).first;
+      int ep_src_loc = comm->ep_rank_map->at(inter_src).first;
       int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first;
       int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second;
       int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
 
-      ::MPI_Request mpi_request;
-
-      ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_request);
-
-      (*request)->mpi_request = new ::MPI_Request(mpi_request);
+      *request = new ep_request;
+      memcheck("new "<< *request <<" : in ep_lib::MPI_Issend_intercomm, *request = new ep_request");
+
+      (*request)->mpi_request = new ::MPI_Request;
+      memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend_intercomm, (*request)->mpi_request = new ::MPI_Request");
+
       (*request)->type = 1;    // used in wait
       (*request)->comm = comm;
-   
       (*request)->ep_src = src_ep_rank;
       (*request)->ep_tag = tag;
       (*request)->ep_datatype = datatype;
+      
+      return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request)); 
     }
-
-    return 0;
-
   }
   

XIOS/dev/branch_openmp/extern/ep_dev/ep_size.cpp

@@ -38 +38 @@
 
       ::MPI_Comm_size(mpi_comm, &mpi_size);
-      printf("============ ep comm size called for non ep comm, %d\n", mpi_size);
       *size = mpi_size;
       return 0;
@@ -46 +45 @@
     {
       *size = MPI_UNDEFINED;
-      printf("============ ep comm size called for non ep comm not defined\n");
       return 0;
     }

XIOS/dev/branch_openmp/extern/ep_dev/ep_type.cpp

@@ -125 +125 @@
     my_buffer = NULL;
     ep_barrier = NULL;
-    rank_map = NULL;
+    ep_rank_map = NULL;
     ep_comm_ptr = NULL;
     mem_bridge = NULL;
@@ -140 +140 @@
     my_buffer = NULL;
     ep_barrier = NULL;
-    rank_map = NULL;
+    ep_rank_map = NULL;
     ep_comm_ptr = NULL;
     mem_bridge = NULL;

XIOS/dev/branch_openmp/extern/ep_dev/ep_type.hpp

@@ -23 +23 @@
 #endif
 
+#ifdef _Memory_check
+#define memcheck(x) std::cout << x << std::endl
+#else
+#define memcheck(x)
+#endif
+
+
 #define BUFFER_SIZE 10000
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_wait.cpp

@@ -41 +41 @@
     (*request)->state = 2;
 
-    printf("delete %p : in ep_lib::MPI_Wait, delete (*request)->mpi_request\n", (*request)->mpi_request);
+    memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Wait, delete (*request)->mpi_request");
     delete (*request)->mpi_request;
     Request_Check();
@@ -91 +91 @@
       array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype;
       array_of_requests[i]->state = 2;
-      printf("delete %p : in ep_lib::MPI_Wait, array_of_requests[i]->mpi_request\n", array_of_requests[i]->mpi_request);
+      memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]->mpi_request");
       delete array_of_requests[i]->mpi_request;
       if(array_of_requests[i]->type == 1) 
       {
-        printf("delete %p : in ep_lib::MPI_Wait, array_of_requests[i]\n", array_of_requests[i]);
+        memcheck("delete "<< array_of_requests[i] <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]");
         delete array_of_requests[i];
       }

XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

@@ -52 +52 @@
 
 
+    MPI_Comm comm_for_dup; // this should act as EP_COMM_WORLD
     MPI_Comm comm; // this should act as EP_COMM_WORLD
-    comm = passage[omp_get_thread_num()];
+        
+    comm_for_dup = passage[omp_get_thread_num()];
+    MPI_Comm_dup(comm_for_dup, &comm);
+    
+    MPI_Comm_free(&comm_for_dup);
+    
+    MPI_Barrier(comm);
 
     int rank, size;
     MPI_Comm_rank(comm, &rank);
     MPI_Comm_size(comm, &size);
+    
+    if(rank == 0) printf("           \t test MPI_Comm_dup \t OK \n");
 /*
     // TIMING SYCHRONIZATION
@@ -109 +118 @@
       MPI_Barrier(comm);
       
+      MPI_Comm equal_comm = comm;
+      
+      
+      
+      
       double sendbuf[10];
       double recvbuf[20];
       
-      int sender = 1;
-      int receiver = 2;
+      int sender;
+      if(rank == 0) sender = rand() % size;
+      MPI_Bcast(&sender, 1, MPI_INT, 0, comm);
+      
+      int receiver = sender;
+      if(rank == 0) while(sender == receiver) {receiver = rand() % size;}
+      MPI_Bcast(&receiver, 1, MPI_INT, 0, comm);
+      
+      
+      
       
       if(rank == sender)
       {
         for(int i=0; i<10; i++) sendbuf[i] = 99.99;
-        MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 99, comm);
+        MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 99, equal_comm);
         for(int i=0; i<10; i++) sendbuf[i] = -99.99;
-        MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 11, comm);
+        MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 11, equal_comm);
       }
       
@@ -131 +153 @@
         
         for(int i=0; i<20; i++) std::cout << "recvbuf["<< i <<"] = "<< recvbuf[i] << std::endl;
+        printf("sender = %d\nreceiver = %d \tTEST of p2p blocking communication\tOK\n", sender, receiver);
       }
       
@@ -145 +168 @@
       double recvbuf[20];
       
-      int sender = 3;
-      int receiver = 7;
-      
+      int sender;
+      if(rank == 0) sender = rand() % size;
+      MPI_Bcast(&sender, 1, MPI_INT, 0, comm);
+      
+      int receiver = sender;
+      if(rank == 0) receiver = rand() % size;
+      MPI_Bcast(&receiver, 1, MPI_INT, 0, comm);
+
+
+            
       MPI_Request request[2];
       
@@ -179 +209 @@
       {
         for(int i=0; i<20; i++) std::cout << "recvbuf["<< i <<"] = "<< recvbuf[i] << std::endl;
+        printf("sender = %d\nreceiver = %d \tTEST of p2p non-blocking communication\tOK\n", sender, receiver);
       }
     

XIOS/dev/branch_openmp/extern/ep_dev/makefile

@@ -1 +1 @@
 appname := EP_test
 
-CXX := mpiicc -D_intelmpi -openmp
+CXX := mpiicc -D_intelmpi -D_noMemory_check -D_nocheck_sum -openmp
 CXXFLAGS := -std=c++11  -g
 #CXXFLAGS := -std=c++11  -g