Changeset 1053 for XIOS/dev/branch_yushan/extern

Timestamp:

02/17/17 17:55:37 (7 years ago)

Author:

yushan

Message:

ep_lib namespace specified when netcdf involved

Location:

XIOS/dev/branch_yushan/extern

Files:

• remap/src/mapper.cpp (modified) (1 diff)
• remap/src/mapper.hpp (modified) (3 diffs)
• remap/src/mpi_routing.cpp (modified) (2 diffs)
• remap/src/parallel_tree.hpp (modified) (3 diffs)
• src_ep_dev/ep_alltoall.cpp (modified) (1 diff)
• src_ep_dev/ep_declaration.cpp (modified) (3 diffs)
• src_ep_dev/ep_declaration.hpp (modified) (3 diffs)
• src_ep_dev/ep_exscan.cpp (modified) (2 diffs)
• src_ep_dev/ep_finalize.cpp (modified) (2 diffs)
• src_ep_dev/ep_fortran.cpp (modified) (3 diffs)
• src_ep_dev/ep_free.cpp (modified) (10 diffs)
• src_ep_dev/ep_gather.cpp (modified) (8 diffs)
• src_ep_dev/ep_gatherv.cpp (modified) (8 diffs)
• src_ep_dev/ep_init.cpp (modified) (3 diffs)
• src_ep_dev/ep_intercomm.cpp (modified) (3 diffs)
• src_ep_dev/ep_intercomm_kernel.cpp (modified) (12 diffs)
• src_ep_dev/ep_intercomm_world.cpp (modified) (1 diff)
• src_ep_dev/ep_lib.hpp (modified) (1 diff)
• src_ep_dev/ep_lib_fortran.hpp (modified) (1 diff)
• src_ep_dev/ep_memory.cpp (modified) (2 diffs)
• src_ep_dev/ep_merge.cpp (modified) (5 diffs)
• src_ep_dev/ep_probe.cpp (modified) (3 diffs)
• src_ep_dev/ep_rank.cpp (modified) (1 diff)
• src_ep_dev/ep_recv.cpp (modified) (1 diff)
• src_ep_dev/ep_reduce.cpp (modified) (9 diffs)
• src_ep_dev/ep_scan.cpp (modified) (3 diffs)
• src_ep_dev/ep_scatter.cpp (modified) (5 diffs)
• src_ep_dev/ep_scatterv.cpp (modified) (5 diffs)
• src_ep_dev/ep_send.cpp (modified) (4 diffs)
• src_ep_dev/ep_size.cpp (modified) (2 diffs)
• src_ep_dev/ep_split.cpp (modified) (1 diff)
• src_ep_dev/ep_test.cpp (modified) (1 diff)
• src_ep_dev/ep_type.cpp (modified) (4 diffs)
• src_ep_dev/ep_type.hpp (modified) (8 diffs)
• src_ep_dev/ep_wait.cpp (modified) (4 diffs)

XIOS/dev/branch_yushan/extern/remap/src/mapper.cpp

@@ -12 +12 @@
 
 #include "mapper.hpp"
+
 
 namespace sphereRemap {

XIOS/dev/branch_yushan/extern/remap/src/mapper.hpp

@@ -3 +3 @@
 #include "parallel_tree.hpp"
 #include "mpi.hpp"
+
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace sphereRemap {
@@ -18 +22 @@
 {
 public:
-       Mapper(MPI_Comm comm=MPI_COMM_WORLD) : communicator(comm), verbose(SILENT), neighbourElements(NULL), sstree(comm) {}
+       Mapper(ep_lib::MPI_Comm comm=MPI_COMM_WORLD) : communicator(comm), verbose(SILENT), neighbourElements(NULL), sstree(comm) {}
        ~Mapper();
        void setVerbosity(verbosity v) {verbose=v ;}
@@ -67 +71 @@
 
        CParallelTree sstree;
-       MPI_Comm communicator ;
+       ep_lib::MPI_Comm communicator ;
        std::vector<Elt>  sourceElements ;
        std::vector<Node> sourceMesh ;

XIOS/dev/branch_yushan/extern/remap/src/mpi_routing.cpp

@@ -5 +5 @@
 #include "timerRemap.hpp"
 #include <iostream>
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace sphereRemap {
@@ -122 +125 @@
         CTimer::get("CMPIRouting::init(reduce_scatter)").print();
 
-        MPI_Alloc_mem(nbTarget *sizeof(int), MPI_INFO_NULL, &targetRank);
-        MPI_Alloc_mem(nbSource *sizeof(int), MPI_INFO_NULL, &sourceRank);
+        MPI_Info info_null;
+
+        MPI_Alloc_mem(nbTarget *sizeof(int), info_null, &targetRank);
+        MPI_Alloc_mem(nbSource *sizeof(int), info_null, &sourceRank);
 
         targetRankToIndex = new int[mpiSize];

XIOS/dev/branch_yushan/extern/remap/src/parallel_tree.hpp

@@ -6 +6 @@
 #include "mpi_cascade.hpp"
 #include "mpi.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace sphereRemap {
@@ -12 +15 @@
 {
 public:
-        CParallelTree(MPI_Comm comm);
+        CParallelTree(ep_lib::MPI_Comm comm);
         ~CParallelTree();
 
@@ -34 +37 @@
         vector<CSampleTree> treeCascade; // first for sample tree, then for routing tree
         CMPICascade cascade;
-  MPI_Comm communicator ;
+  ep_lib::MPI_Comm communicator ;
 
 };

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_alltoall.cpp

@@ -11 +11 @@
   {
     ::MPI_Aint typesize, llb;
-    ::MPI_Type_get_extent(sendtype, &llb, &typesize);
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(sendtype), &llb, &typesize);
 
     for(int i=0; i<comm.ep_comm_ptr->size_rank_info[0].second; i++)

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_declaration.cpp

@@ -21 +21 @@
 #undef MPI_COMM_NULL
 
+#undef MPI_STATUS_IGNORE
+//#undef MPI_INFO_NULL
+#undef MPI_REQUEST_NULL
+
+#ifdef _openmpi
+//#undef MPI_Fint
+#endif
+
 // _STD defined in ep_type.cpp
 
@@ -38 +46 @@
 extern ::MPI_Comm MPI_COMM_NULL_STD;
 
+extern ::MPI_Status MPI_STATUS_IGNORE_STD;
+//extern ::MPI_Info MPI_INFO_NULL_STD;
+extern ::MPI_Request MPI_REQUEST_NULL_STD;
+
 ep_lib::MPI_Datatype MPI_INT = MPI_INT_STD;
 ep_lib::MPI_Datatype MPI_FLOAT = MPI_FLOAT_STD;
@@ -53 +65 @@
 ep_lib::MPI_Comm MPI_COMM_NULL(MPI_COMM_NULL_STD);
 
+//ep_lib::MPI_Info MPI_INFO_NULL(MPI_INFO_NULL_STD);
+ep_lib::MPI_Request MPI_REQUEST_NULL(MPI_REQUEST_NULL_STD);
+
+//ep_lib::MPI_Status MPI_STATUS_IGNORE_STD = MPI_STATUS_IGNORE_STD;
+
 //ep_lib::MPI_Comm EP_COMM_WORLD;
 //ep_lib::MPI_Comm EP_COMM_NULL;

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_declaration.hpp

@@ -17 +17 @@
 extern ::MPI_Comm MPI_COMM_NULL_STD;
 
+extern ::MPI_Status MPI_STATUS_IGNORE_STD;
+//extern ::MPI_Info MPI_INFO_NULL_STD;
+extern ::MPI_Request MPI_REQUEST_NULL_STD;
+
 #undef MPI_INT
 #undef MPI_FLOAT
@@ -31 +35 @@
 #undef MPI_COMM_WORLD
 #undef MPI_COMM_NULL
+
+//#undef MPI_INFO_NULL
+#undef MPI_REQUEST_NULL
+
+#ifdef _openmpi
+//#undef MPI_Fint
+#endif
+
+#undef MPI_STATUS_IGNORE
 
 extern ep_lib::MPI_Datatype MPI_INT;
@@ -47 +60 @@
 extern ep_lib::MPI_Comm MPI_COMM_NULL;
 
-
-
+extern ep_lib::MPI_Status MPI_STATUS_IGNORE;
+//extern ep_lib::MPI_Info MPI_INFO_NULL;
+extern ep_lib::MPI_Request MPI_REQUEST_NULL;
 
 #endif // EP_DECLARATION_HPP_INCLUDED

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_exscan.cpp

@@ -435 +435 @@
     if(!comm.is_ep)
     {
-      ::MPI_Exscan(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
+      ::MPI_Exscan(const_cast<void*>(sendbuf), recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
                    static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
       return 0;
@@ -481 +481 @@
     if(ep_rank_loc == 0)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Exscan(local_sum, mpi_scan_recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
     }

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_finalize.cpp

@@ -15 +15 @@
     if(id == 0)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Finalize();
     }
@@ -30 +27 @@
     {
       ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.mpi_comm);
-
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Abort(mpi_comm, errorcode);
     }

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_fortran.cpp

@@ -4 +4 @@
 #include <map>
 #include <utility>
+
+#ifdef _intelmpi
 #undef MPI_Comm_f2c(comm)
 #undef MPI_Comm_c2f(comm)
+#endif
 
+#ifdef _openmpi
+//#undef MPI_Fint
+#endif
 
 namespace ep_lib
 {
 
+  int EP_Comm_c2f(MPI_Comm comm)
+  {
+    Debug("MPI_Comm_c2f");
+    int fint;
+    #ifdef _intelmpi
+    fint = (::MPI_Fint)(comm.mpi_comm);
+    #elif _openmpi
+    fint = ::MPI_Comm_c2f(static_cast< ::MPI_Comm>(comm.mpi_comm));
+    #endif
+    std::map<std::pair<int, int>, MPI_Comm  > ::iterator it;
+    
+    it = fc_comm_map.find(std::make_pair(fint, omp_get_thread_num()));
+    if(it == fc_comm_map.end())
+    {
+      fc_comm_map.insert(std::make_pair( std::make_pair( fint, omp_get_thread_num()) , comm));
+      printf("MAP insert: %d, %d, %p\n", fint, omp_get_thread_num(), &comm);
+    }
+    
+    
+    return fint;
+    
+  }
+
+  MPI_Comm EP_Comm_f2c(int comm)
+  {
+    Debug("MPI_Comm_f2c");
+    
+    
+    std::map<std::pair<int, int>, MPI_Comm  > ::iterator it;
+    
+    it = fc_comm_map.find(std::make_pair(comm, omp_get_thread_num()));
+    if(it != fc_comm_map.end())
+    {
+      MPI_Comm comm_ptr;
+      comm_ptr =  it->second;
+      printf("MAP find: %d, %d, %p\n", it->first.first, it->first.second, &comm_ptr);
+      return  comm_ptr;
+    }
+    else
+    {      
+      MPI_Comm return_comm;
+      return_comm.mpi_comm = ::MPI_Comm_f2c(comm);
+      return return_comm;
+    }
+  }
+
+  #ifdef _intelmpi
 
   MPI_Fint MPI_Comm_c2f(MPI_Comm comm)
@@ -32 +85 @@
     
   }
+
+  
 
 
@@ -57 +112 @@
   }
 
+  
+
+  #elif _openmpi
+  
+  int MPI_Comm_c2f(MPI_Comm comm)
+  {
+    Debug("MPI_Comm_c2f");
+    int fint;
+    fint = ::MPI_Comm_c2f(static_cast< ::MPI_Comm>(comm.mpi_comm));
+    
+    std::map<std::pair<int, int>, MPI_Comm  > ::iterator it;
+    
+    it = fc_comm_map.find(std::make_pair(fint, omp_get_thread_num()));
+    if(it == fc_comm_map.end())
+    {
+      fc_comm_map.insert(std::make_pair( std::make_pair( fint, omp_get_thread_num()) , comm));
+      printf("MAP insert: %d, %d, %p\n", fint, omp_get_thread_num(), &comm);
+    }
+    
+    return fint;
+    
+  }
+
+  ep_lib::MPI_Comm MPI_Comm_f2c(MPI_Fint comm)
+  {
+    Debug("MPI_Comm_f2c");
+    
+    
+    std::map<std::pair<int, int>, MPI_Comm  > ::iterator it;
+    
+    it = fc_comm_map.find(std::make_pair(comm, omp_get_thread_num()));
+    if(it != fc_comm_map.end())
+    {
+      MPI_Comm comm_ptr;
+      comm_ptr =  it->second;
+      printf("MAP find: %d, %d, %p\n", it->first.first, it->first.second, &comm_ptr);
+      return  comm_ptr;
+    }
+    else
+    {      
+      MPI_Comm return_comm;
+      return_comm.mpi_comm = (::MPI_Comm)(comm);
+      return return_comm;
+    }
+  }
+  #endif
+
 }
 

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_free.cpp

@@ -10 +10 @@
   int MPI_Comm_free(MPI_Comm *comm)
   {
-    //return 0;
-    
-    if(comm == NULL) return 0;
+    //if(comm == NULL) return 0;
 
     MPI_Barrier(*comm);
@@ -19 +17 @@
     if(! comm->is_ep)
     {
-      if(comm->mpi_comm)
+      if(comm->mpi_comm != MPI_COMM_NULL_STD)
       {
         ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->mpi_comm);
@@ -31 +29 @@
     else
     {
-      if(! comm->mpi_comm) return 0;
+      if(comm->mpi_comm == MPI_COMM_NULL_STD ) return 0;
 
       int ep_rank, ep_rank_loc, mpi_rank;
@@ -47 +45 @@
 
       #pragma omp critical (memory_free)
-      if(comm->is_intercomm && comm->ep_comm_ptr->intercomm)
+      if(comm->is_intercomm && comm->ep_comm_ptr->intercomm != NULL)
       {
         if(comm->ep_comm_ptr->intercomm->local_rank_map) comm->ep_comm_ptr->intercomm->local_rank_map->clear();
@@ -62 +60 @@
 
 /*
-        if(comm->my_buffer)
+        if(comm->my_buffer != NULL)
         {
-          if(comm->my_buffer->buf_int) delete[] comm->my_buffer->buf_int; Debug("buf_int freed\n");
-          if(comm->my_buffer->buf_float) delete[] comm->my_buffer->buf_float; Debug("buf_float freed\n");
-          if(comm->my_buffer->buf_double) delete[] comm->my_buffer->buf_double; Debug("buf_double freed\n");
-          if(comm->my_buffer->buf_long) delete[] comm->my_buffer->buf_long; Debug("buf_long freed\n");
-          if(comm->my_buffer->buf_ulong) delete[] comm->my_buffer->buf_ulong; Debug("buf_ulong freed\n");
-          if(comm->my_buffer->buf_char) delete[] comm->my_buffer->buf_char; Debug("buf_char freed\n");
+          if(comm->my_buffer->buf_int != NULL) delete[] comm->my_buffer->buf_int; Debug("buf_int freed\n");
+          if(comm->my_buffer->buf_float != NULL) delete[] comm->my_buffer->buf_float; Debug("buf_float freed\n");
+          if(comm->my_buffer->buf_double != NULL) delete[] comm->my_buffer->buf_double; Debug("buf_double freed\n");
+          if(comm->my_buffer->buf_long != NULL) delete[] comm->my_buffer->buf_long; Debug("buf_long freed\n");
+          if(comm->my_buffer->buf_ulong != NULL) delete[] comm->my_buffer->buf_ulong; Debug("buf_ulong freed\n");
+          if(comm->my_buffer->buf_char != NULL) delete[] comm->my_buffer->buf_char; Debug("buf_char freed\n");
         }
 */
-        if(comm->ep_barrier)
+        if(comm->ep_barrier != NULL)
         {
           comm->ep_barrier->~OMPbarrier();
@@ -79 +77 @@
 
 
-        if(comm->rank_map)
+        if(comm->rank_map != NULL)
         {
           comm->rank_map->clear();
@@ -86 +84 @@
 
 
-        if(comm->is_intercomm && comm->ep_comm_ptr->intercomm->mpi_inter_comm)
+        if(comm->is_intercomm && comm->ep_comm_ptr->intercomm->mpi_inter_comm != MPI_COMM_NULL_STD)
         {
           ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->ep_comm_ptr->intercomm->mpi_inter_comm);
 
           ::MPI_Comm_free(&mpi_comm);
-          comm->ep_comm_ptr->intercomm->mpi_inter_comm = NULL;
+          //comm->ep_comm_ptr->intercomm->mpi_inter_comm = NULL;
           Debug("mpi_intercomm freed\n");
         }
@@ -99 +97 @@
         for(int i=0; i<num_ep; i++)
         {
-          if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue)
+          if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue != NULL)
           {
             comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue->clear();
@@ -105 +103 @@
           }
 
-          if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr)
+          if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr != NULL)
           {
             delete comm->ep_comm_ptr->comm_list[i].ep_comm_ptr;
@@ -112 +110 @@
         }
 
-        if(comm->mpi_comm)
+        if(comm->mpi_comm != MPI_COMM_NULL_STD)
         {
           ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->mpi_comm);
-          #ifdef _serialized
-          #pragma omp critical (_mpi_call)
-          #endif // _serialized
           ::MPI_Comm_free(&mpi_comm);
-          comm->mpi_comm = NULL;
+          //comm->mpi_comm = NULL;
 //          printf("label = %d, mpi_comm freed\n", comm->ep_comm_ptr->comm_label);
         }
 
-       if(comm) {delete[] comm->ep_comm_ptr->comm_list; Debug("comm freed\n");}
+       if(comm != NULL) {delete[] comm->ep_comm_ptr->comm_list; Debug("comm freed\n");}
 
       }

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_gather.cpp

@@ -320 +320 @@
     if(!comm.is_ep && comm.mpi_comm)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
-      ::MPI_Gather(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+      ::MPI_Gather(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
                    root, static_cast< ::MPI_Comm>(comm.mpi_comm));
       return 0;
@@ -351 +348 @@
 
     ::MPI_Aint datasize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
 
@@ -377 +372 @@
       gatherv_displs = new int[mpi_size];
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
+
       ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
 
@@ -388 +381 @@
       }
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
+
       ::MPI_Gatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
                     gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -442 +433 @@
     if(!comm.is_ep && comm.mpi_comm)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
-      ::MPI_Allgather(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+      ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
                       static_cast< ::MPI_Comm>(comm.mpi_comm));
       return 0;
@@ -469 +457 @@
 
     ::MPI_Aint datasize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
 
@@ -495 +481 @@
       gatherv_displs = new int[mpi_size];
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
 
@@ -506 +489 @@
       }
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Allgatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
                     gatherv_displs, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Comm>(comm.mpi_comm));

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_gatherv.cpp

@@ -328 +328 @@
     if(!comm.is_ep && comm.mpi_comm)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
-      ::MPI_Gatherv(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcounts, displs,
+      ::MPI_Gatherv(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, const_cast<int*>(recvcounts), const_cast<int*>(displs),
                     static_cast< ::MPI_Datatype>(recvtype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
       return 0;
@@ -362 +359 @@
 
     ::MPI_Aint datasize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
 
@@ -395 +390 @@
       //gatherv_displs = new int[mpi_size];
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
+
       ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
 
@@ -406 +399 @@
       }
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
+
       ::MPI_Gatherv(local_gather_recvbuf, gatherv_cnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
                     gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -462 +453 @@
     if(!comm.is_ep && comm.mpi_comm)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Allgatherv(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcounts, displs,
                        static_cast< ::MPI_Datatype>(recvtype), static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -493 +481 @@
 
     ::MPI_Aint datasize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
 
@@ -525 +511 @@
       gatherv_displs = new int[mpi_size];
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
       gatherv_displs[0] = displs[0];
@@ -534 +517 @@
         gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
       }
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
+
       ::MPI_Allgatherv(local_gather_recvbuf, gatherv_cnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
                     gatherv_displs, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Comm>(comm.mpi_comm));

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_init.cpp

@@ -11 +11 @@
   int MPI_Init_thread(int *argc, char*** argv, int required, int*provided)
   {
-    printf("MPI_Init_thread\n");
+    //printf("MPI_Init_thread\n");
 
     int id = omp_get_thread_num();
@@ -24 +24 @@
   int MPI_Init(int *argc, char ***argv)
   {
-    printf("MPI_init called\n");
+    //printf("MPI_init called\n");
     int id = omp_get_thread_num();
 
@@ -36 +36 @@
   int MPI_Initialized(int *flag)
   {
-    printf("MPI_initialized called\n");
+    //printf("MPI_initialized called\n");
 
     ::MPI_Initialized(flag);

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_intercomm.cpp

@@ -143 +143 @@
         if(ep_rank == new_local_leader)
         {
-          #ifdef _serialized
-          #pragma omp critical (_mpi_call)
-          #endif // _serialized
           ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &leader_in_world[0]);
         }
@@ -190 +187 @@
   int MPI_Comm_test_inter(MPI_Comm comm, int *flag)
   {
+    *flag = false;
     if(comm.is_ep)
     {
@@ -195 +193 @@
       return 0;
     } 
-    else
+    else if(comm.mpi_comm != MPI_COMM_NULL_STD)
     {
       ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif
+      
       ::MPI_Comm_test_inter(mpi_comm, flag);
       return 0;  
     }
+    return 0;
   }
 

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_intercomm_kernel.cpp

@@ -35 +35 @@
     ::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
 
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
-    {
-      ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &rank_in_world);
-      ::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
-    }
+    
+    ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &rank_in_world);
+    ::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
+    
 
     bool is_proc_master = false;
@@ -83 +80 @@
       send_buf[2] = num_ep;
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
 
@@ -100 +94 @@
         leader_info[1] = remote_leader;
 
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
         ::MPI_Comm_rank(static_cast< ::MPI_Comm>(peer_comm.mpi_comm), &rank_in_peer_mpi[0]);
 
@@ -126 +117 @@
       send_buf[4] = rank_in_peer_mpi[1];
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Bcast(send_buf.data(), 5, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
 
@@ -156 +144 @@
         MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, remote_leader, tag, peer_comm);
         MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, remote_leader, tag, peer_comm, &status);
-
-      }
-
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
+      }
+
       ::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
 
@@ -283 +267 @@
       }
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
 
@@ -307 +288 @@
       }
 
-
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Bcast(recv_buf.data(),   3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
 
@@ -330 +307 @@
       ::MPI_Comm intercomm;
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Comm_group(local_mpi_comm, &local_group);
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
 
@@ -349 +317 @@
       if(is_local_leader)
       {
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
         ::MPI_Comm_rank(new_comm, &leader_info[2]);
       }
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
 
@@ -363 +325 @@
       {
 
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
         ::MPI_Barrier(new_comm);
 
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
         ::MPI_Intercomm_create(new_comm, leader_info[2], static_cast< ::MPI_Comm>(peer_comm.mpi_comm), rank_in_peer_mpi[1], tag, &intercomm);
 
         int id;
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
+
         ::MPI_Comm_rank(new_comm, &id);
         int my_num_ep = new_ep_info[0][id];
@@ -604 +558 @@
     int rank_in_peer_mpi[2];
 
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
     ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &rank_in_world);
 
@@ -663 +614 @@
       {
         ::MPI_Comm mpi_dup;
-
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
+        
         ::MPI_Comm_dup(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &mpi_dup);
 

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_intercomm_world.cpp

@@ -281 +281 @@
         std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] );
 
-        MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
-        MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+        ::MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+        ::MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
 
       }

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_lib.hpp

@@ -20 +20 @@
 #endif
 
+#ifdef _intelmpi
   #define MPI_ANY_SOURCE -2 
   #define MPI_ANY_TAG -1 
+#elif _openmpi
+  #define MPI_ANY_SOURCE -1 
+  #define MPI_ANY_TAG -1 
+#endif
 
   int MPI_Init_thread(int* argc, char*** argv, int required, int*provided);

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_lib_fortran.hpp

@@ -7 +7 @@
 {
   
+  #ifdef _intelmpi
+  
+  MPI_Fint MPI_Comm_c2f(MPI_Comm comm);
+  MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
+  
+  #elif _openmpi
+  
+  int MPI_Comm_c2f(MPI_Comm comm);
+  ep_lib::MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
+  
+  #endif
 
-  MPI_Fint MPI_Comm_c2f(MPI_Comm comm);
-  //int MPI_Comm_c2f(MPI_Comm comm);
-
-
-  MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
-  //void MPI_Comm_f2c(MPI_Fint comm);
-
+  int EP_Comm_c2f(MPI_Comm comm);
+  MPI_Comm EP_Comm_f2c(int comm);
 }
 

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_memory.cpp

@@ -10 +10 @@
   int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr)
   {
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
-    ::MPI_Alloc_mem(size.mpi_aint, info.mpi_info, baseptr);
+    ::MPI_Alloc_mem(size.mpi_aint, static_cast< ::MPI_Info>(info.mpi_info), baseptr);
     return 0;
    }
@@ -19 +16 @@
   int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr)
   {
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
-    ::MPI_Alloc_mem(size, info.mpi_info, baseptr);
+    ::MPI_Alloc_mem(size, static_cast< ::MPI_Info>(info.mpi_info), baseptr);
     return 0;
-   }
+  }
 
 

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_merge.cpp

@@ -39 +39 @@
     local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second;
 
+
+    if(local_ep_rank == 0)
+    {
+      MPI_Status status;
+      MPI_Request req_s, req_r;
+      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_s);
+      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_r);
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+    }
+
+
+    MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm));
+
+//    printf("%d, %d, %d, %d\n", local_ep_size, remote_ep_size, local_high, remote_high);
+
+
+    MPI_Comm_dup(inter_comm, newintracomm);
+
+    int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size;
+
+
+    int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank;
+    int intra_ep_size, intra_num_ep, intra_mpi_size;
+
+    intra_ep_rank = newintracomm->ep_comm_ptr->size_rank_info[0].first;
+    intra_ep_rank_loc = newintracomm->ep_comm_ptr->size_rank_info[1].first;
+    intra_mpi_rank = newintracomm->ep_comm_ptr->size_rank_info[2].first;
+    intra_ep_size = newintracomm->ep_comm_ptr->size_rank_info[0].second;
+    intra_num_ep = newintracomm->ep_comm_ptr->size_rank_info[1].second;
+    intra_mpi_size = newintracomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+    MPI_Barrier_local(*newintracomm);
+
+
+    int *reorder;
+    if(intra_ep_rank_loc == 0)
+    {
+      reorder = new int[intra_ep_size];
+    }
+
+
+    MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm);
+    if(intra_ep_rank_loc == 0)
+    {
+      ::MPI_Bcast(reorder, intra_ep_size, MPI_INT_STD, 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
+
+      vector< pair<int, int> > tmp_rank_map(intra_ep_size);
+
+
+      for(int i=0; i<intra_ep_size; i++)
+      {
+        tmp_rank_map[reorder[i]] = newintracomm->rank_map->at(i) ;
+      }
+
+      newintracomm->rank_map->swap(tmp_rank_map);
+
+      tmp_rank_map.clear();
+    }
+
+    MPI_Barrier_local(*newintracomm);
+
+    (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank;
+
+    if(intra_ep_rank_loc == 0)
+    {
+      delete[] reorder;
+    }
+
+    return MPI_SUCCESS;
+  }
+
+
+
+
+
+  int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm)
+  {
+
+    assert(inter_comm.is_intercomm);
+
+    if(inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99)
+    {
+        return MPI_Intercomm_merge_unique_leader(inter_comm, high, newintracomm);
+    }
+
+
+    Debug("intercomm_merge kernel\n");
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = inter_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = inter_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = inter_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = inter_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    int local_ep_rank, local_ep_rank_loc, local_mpi_rank;
+    int local_ep_size, local_num_ep, local_mpi_size;
+
+
+    local_ep_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first;
+    local_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first;
+    local_mpi_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first;
+    local_ep_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second;
+    local_num_ep = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second;
+    local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second;
+
+    int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size();
+
+    int local_high = high;
+    int remote_high;
+
+    MPI_Barrier(inter_comm);
 
 //    if(local_ep_rank == 0 && high == false)
@@ -65 +184 @@
     }
 
-
     MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm));
 
-//    printf("%d, %d, %d, %d\n", local_ep_size, remote_ep_size, local_high, remote_high);
-
-
-    MPI_Comm_dup(inter_comm, newintracomm);
+    int intercomm_high;
+    if(ep_rank == 0) intercomm_high = local_high;
+    MPI_Bcast(&intercomm_high, 1, MPI_INT, 0, inter_comm);
+
+    //printf("remote_ep_size = %d, local_high = %d, remote_high = %d, intercomm_high = %d\n", remote_ep_size, local_high, remote_high, intercomm_high);
+
+
+    ::MPI_Comm mpi_intracomm;
+    MPI_Comm *ep_intracomm;
+
+    if(ep_rank_loc == 0)
+    {
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(inter_comm.ep_comm_ptr->intercomm->mpi_inter_comm);
+
+      ::MPI_Intercomm_merge(mpi_comm, intercomm_high, &mpi_intracomm);
+      MPI_Info info;
+      MPI_Comm_create_endpoints(mpi_intracomm, num_ep, info, ep_intracomm);
+
+      inter_comm.ep_comm_ptr->comm_list->mem_bridge = ep_intracomm;
+
+    }
+
+
+
+    MPI_Barrier_local(inter_comm);
+
+    *newintracomm = inter_comm.ep_comm_ptr->comm_list->mem_bridge[ep_rank_loc];
 
     int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size;
-
 
     int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank;
@@ -87 +228 @@
 
 
+
     MPI_Barrier_local(*newintracomm);
 
@@ -97 +239 @@
 
 
+
     MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm);
     if(intra_ep_rank_loc == 0)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
+
       ::MPI_Bcast(reorder, intra_ep_size, MPI_INT_STD, 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
 
@@ -122 +263 @@
     (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank;
 
-    if(intra_ep_rank_loc == 0)
-    {
-      delete[] reorder;
-    }
-
-    return MPI_SUCCESS;
-  }
-
-
-
-
-
-  int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm)
-  {
-
-    assert(inter_comm.is_intercomm);
-
-    if(inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99)
-    {
-        return MPI_Intercomm_merge_unique_leader(inter_comm, high, newintracomm);
-    }
-
-
-    Debug("intercomm_merge kernel\n");
-
-    int ep_rank, ep_rank_loc, mpi_rank;
-    int ep_size, num_ep, mpi_size;
-
-    ep_rank = inter_comm.ep_comm_ptr->size_rank_info[0].first;
-    ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first;
-    mpi_rank = inter_comm.ep_comm_ptr->size_rank_info[2].first;
-    ep_size = inter_comm.ep_comm_ptr->size_rank_info[0].second;
-    num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second;
-    mpi_size = inter_comm.ep_comm_ptr->size_rank_info[2].second;
-
-
-    int local_ep_rank, local_ep_rank_loc, local_mpi_rank;
-    int local_ep_size, local_num_ep, local_mpi_size;
-
-
-    local_ep_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first;
-    local_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first;
-    local_mpi_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first;
-    local_ep_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second;
-    local_num_ep = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second;
-    local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second;
-
-    int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size();
-
-    int local_high = high;
-    int remote_high;
-
-    MPI_Barrier(inter_comm);
-
-//    if(local_ep_rank == 0 && high == false)
-//    {
-//      MPI_Status status;
-//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
-//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
-//    }
-//
-//    if(local_ep_rank == 0 && high == true)
-//    {
-//      MPI_Status status;
-//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
-//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
-//    }
-
-    if(local_ep_rank == 0)
-    {
-      MPI_Status status;
-      MPI_Request req_s, req_r;
-      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_s);
-      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_r);
-
-      MPI_Wait(&req_s, &status);
-      MPI_Wait(&req_r, &status);
-    }
-
-    MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm));
-
-    int intercomm_high;
-    if(ep_rank == 0) intercomm_high = local_high;
-    MPI_Bcast(&intercomm_high, 1, MPI_INT, 0, inter_comm);
-
-    //printf("remote_ep_size = %d, local_high = %d, remote_high = %d, intercomm_high = %d\n", remote_ep_size, local_high, remote_high, intercomm_high);
-
-
-    ::MPI_Comm mpi_intracomm;
-    MPI_Comm *ep_intracomm;
-
-    if(ep_rank_loc == 0)
-    {
-
-      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(inter_comm.ep_comm_ptr->intercomm->mpi_inter_comm);
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
-      ::MPI_Intercomm_merge(mpi_comm, intercomm_high, &mpi_intracomm);
-      MPI_Info info;
-      MPI_Comm_create_endpoints(mpi_intracomm, num_ep, info, ep_intracomm);
-
-      inter_comm.ep_comm_ptr->comm_list->mem_bridge = ep_intracomm;
-
-    }
-
-
-
-    MPI_Barrier_local(inter_comm);
-
-    *newintracomm = inter_comm.ep_comm_ptr->comm_list->mem_bridge[ep_rank_loc];
-
-    int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size;
-
-    int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank;
-    int intra_ep_size, intra_num_ep, intra_mpi_size;
-
-    intra_ep_rank = newintracomm->ep_comm_ptr->size_rank_info[0].first;
-    intra_ep_rank_loc = newintracomm->ep_comm_ptr->size_rank_info[1].first;
-    intra_mpi_rank = newintracomm->ep_comm_ptr->size_rank_info[2].first;
-    intra_ep_size = newintracomm->ep_comm_ptr->size_rank_info[0].second;
-    intra_num_ep = newintracomm->ep_comm_ptr->size_rank_info[1].second;
-    intra_mpi_size = newintracomm->ep_comm_ptr->size_rank_info[2].second;
-
-
-
-    MPI_Barrier_local(*newintracomm);
-
-
-    int *reorder;
-    if(intra_ep_rank_loc == 0)
-    {
-      reorder = new int[intra_ep_size];
-    }
-
-
-
-    MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm);
-    if(intra_ep_rank_loc == 0)
-    {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
-      ::MPI_Bcast(reorder, intra_ep_size, MPI_INT_STD, 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
-
-      vector< pair<int, int> > tmp_rank_map(intra_ep_size);
-
-
-      for(int i=0; i<intra_ep_size; i++)
-      {
-        tmp_rank_map[reorder[i]] = newintracomm->rank_map->at(i) ;
-      }
-
-      newintracomm->rank_map->swap(tmp_rank_map);
-
-      tmp_rank_map.clear();
-    }
-
-    MPI_Barrier_local(*newintracomm);
-
-    (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank;
-
 
     if(intra_ep_rank_loc == 0)

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_probe.cpp

@@ -34 +34 @@
 
     #pragma omp flush
-    #pragma critical (_query)
+
+    #pragma omp critical (_query)
     if(comm.ep_comm_ptr->message_queue->size() > 0)
     {
@@ -104 +105 @@
 
     #pragma omp flush
-    #pragma critical (_query)
+
+    #pragma omp critical (_query)
     if(comm.ep_comm_ptr->message_queue->size() > 0)
     {
@@ -127 +129 @@
           message->ep_src = it->ep_src;
 
-          #pragma omp critical (_query)
+          #pragma omp critical (_query2)
           {              
             //printf("local message erased. src = %d, dest = %d, tag = %d\n", it->ep_src, it->ep_dest, it->ep_tag);     

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_rank.cpp

@@ -28 +28 @@
 
 
-    if(comm.mpi_comm)
+    if(comm != MPI_COMM_NULL)
     {
       ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_recv.cpp

@@ -68 +68 @@
                 
 
-                request->mpi_request = NULL;
+                request->mpi_request = MPI_REQUEST_NULL_STD;
                 request->buf = buf;
     request->comm = comm;

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_reduce.cpp

@@ -447 +447 @@
     if(!comm.is_ep && comm.mpi_comm)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Reduce(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), root,
                    static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -473 +470 @@
 
     ::MPI_Aint recvsize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &recvsize);
 
@@ -489 +484 @@
     if(ep_rank_loc==0)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Reduce(local_recvbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
     }
@@ -522 +514 @@
     if(!comm.is_ep && comm.mpi_comm)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Allreduce(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op),
                       static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -545 +534 @@
 
     ::MPI_Aint recvsize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &recvsize);
 
@@ -561 +548 @@
     if(ep_rank_loc==0)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Allreduce(local_recvbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
     }
@@ -590 +574 @@
     if(!comm.is_ep && comm.mpi_comm)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op),
                            static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -616 +597 @@
 
     ::MPI_Aint datasize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
 
@@ -636 +615 @@
         local_recvcnt[i] = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
       }
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
+
       ::MPI_Reduce_scatter(local_buf, local_buf2, local_recvcnt, static_cast< ::MPI_Datatype>(datatype),
                          static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_scan.cpp

@@ -432 +432 @@
     if(!comm.is_ep)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
+
       ::MPI_Scan(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
                  static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -455 +453 @@
 
     ::MPI_Aint datasize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
 
@@ -484 +480 @@
     if(ep_rank_loc == 0)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Exscan(local_sum, mpi_scan_recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
     }

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_scatter.cpp

@@ -287 +287 @@
     if(!comm.is_ep)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Scatter(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
                     root, static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -317 +314 @@
 
     ::MPI_Aint datasize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
 
@@ -343 +338 @@
       local_recvbuf = new void*[datasize*mpi_sendcnt];
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Gather(&mpi_sendcnt, 1, MPI_INT_STD, mpi_scatterv_sendcnt, 1, MPI_INT_STD, root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
 
@@ -355 +347 @@
       if(root_ep_loc!=0)
       {
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
         ::MPI_Scatterv(master_sendbuf, mpi_scatterv_sendcnt, displs, static_cast< ::MPI_Datatype>(datatype),
                      local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -363 +352 @@
       else
       {
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
         ::MPI_Scatterv(sendbuf, mpi_scatterv_sendcnt, displs, static_cast< ::MPI_Datatype>(datatype),
                      local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_scatterv.cpp

@@ -296 +296 @@
     if(!comm.is_ep)
     {
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Scatterv(sendbuf, sendcounts, displs, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount,
                      static_cast< ::MPI_Datatype>(recvtype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -329 +326 @@
 
     ::MPI_Aint datasize, lb;
-    #ifdef _serialized
-    #pragma omp critical (_mpi_call)
-    #endif // _serialized
+
     ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
 
@@ -358 +353 @@
       local_recvbuf = new void*[datasize*mpi_sendcnt];
 
-      #ifdef _serialized
-      #pragma omp critical (_mpi_call)
-      #endif // _serialized
       ::MPI_Gather(&mpi_sendcnt, 1, MPI_INT_STD, mpi_scatterv_sendcnt, 1, MPI_INT_STD, root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
 
@@ -370 +362 @@
       if(root_ep_loc!=0)
       {
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
         ::MPI_Scatterv(master_sendbuf, mpi_scatterv_sendcnt, mpi_displs, static_cast< ::MPI_Datatype>(datatype),
                      local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
@@ -378 +367 @@
       else
       {
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
         ::MPI_Scatterv(sendbuf, mpi_scatterv_sendcnt, mpi_displs, static_cast< ::MPI_Datatype>(datatype),
                      local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_send.cpp

@@ -82 +82 @@
     // EP intracomm
 
-    check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
 
     int ep_src_loc = comm.ep_comm_ptr->size_rank_info[1].first;
@@ -139 +139 @@
     // EP intracomm
 
-    check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
 
     int ep_src_loc = comm.ep_comm_ptr->size_rank_info[1].first;
@@ -171 +171 @@
     Debug("MPI_Isend with intercomm\n");
 
-    check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
 
     int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first;
@@ -244 +244 @@
     Debug("MPI_Issend with intercomm\n");
 
-    check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
 
     int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first;

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_size.cpp

@@ -31 +31 @@
     Debug("Calling EP_Comm_size\n");
 
-    if(comm.mpi_comm)
+    if(comm.mpi_comm != MPI_COMM_NULL_STD)
     {
       ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm); 
@@ -54 +54 @@
     if(!comm.is_ep)
     {
-      if(comm.mpi_comm)
+      if(comm.mpi_comm != MPI_COMM_NULL_STD)
       {
         ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm); 
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
         ::MPI_Comm_remote_size(mpi_comm, size);
         return 0;

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_split.cpp

@@ -144 +144 @@
         int master_color = 1;
         if(matched_number_loc[j] == 0) master_color = MPI_UNDEFINED;
-        #ifdef _serialized
-        #pragma omp critical (_mpi_call)
-        #endif // _serialized
+
         ::MPI_Comm_split(static_cast< ::MPI_Comm>(comm.mpi_comm), master_color, mpi_rank, &split_mpi_comm[j]);
         

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_test.cpp

@@ -63 +63 @@
         status->mpi_status = new ::MPI_Status(mpi_status);
         status->ep_src = request->ep_src;
-      status->ep_tag = request->ep_tag;
-      status->ep_datatype = request->ep_datatype;
+        status->ep_tag = request->ep_tag;
+        status->ep_datatype = request->ep_datatype;
         int count;
         MPI_Get_count(status, request->ep_datatype, &count);
-        check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+        //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
       }
 

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_type.cpp

@@ -6 +6 @@
 
 ::MPI_Comm MPI_COMM_WORLD_STD = MPI_COMM_WORLD;
-//#undef MPI_COMM_WORLD
+#undef MPI_COMM_WORLD
 
 
 ::MPI_Comm MPI_COMM_NULL_STD = MPI_COMM_NULL;
-//#undef MPI_COMM_NULL
+#undef MPI_COMM_NULL
 
+//::MPI_Info MPI_INFO_NULL_STD = MPI_INFO_NULL;
+//#undef MPI_INFO_NULL
+
+::MPI_Request MPI_REQUEST_NULL_STD = MPI_REQUEST_NULL;
+#undef MPI_REQUEST_NULL
 
 ::MPI_Datatype MPI_INT_STD = MPI_INT;
@@ -30 +35 @@
 
 
-//ep_lib::MPI_Datatype MPI_INT = MPI_INT_STD;
-//ep_lib::MPI_Datatype MPI_FLOAT = MPI_FLOAT_STD;
-//ep_lib::MPI_Datatype MPI_DOUBLE = MPI_DOUBLE_STD;
-//ep_lib::MPI_Datatype MPI_LONG = MPI_LONG_STD;
-//ep_lib::MPI_Datatype MPI_CHAR = MPI_CHAR_STD;
-//ep_lib::MPI_Datatype MPI_UNSIGNED_LONG = MPI_UNSIGNED_LONG_STD;
-
-
-
-
 ::MPI_Op MPI_SUM_STD = MPI_SUM;
 ::MPI_Op MPI_MAX_STD = MPI_MAX;
@@ -48 +43 @@
 #undef MPI_MIN
 
-//ep_lib::MPI_Op MPI_SUM = MPI_SUM_STD;
-//ep_lib::MPI_Op MPI_MAX = MPI_MAX_STD;
-//ep_lib::MPI_Op MPI_MIN = MPI_MIN_STD;
-
-//  ep_lib::MPI_Comm::MPI_Comm(const MPI_Comm & comm)
-//  {
-//    printf("calling MPI_Comm copy constructor\n");
-//    is_ep = comm.is_ep;
-//    is_intercomm = comm.is_intercomm;
-
-//    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
-//    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
-
-
-//    if(0 == my_rank)
-//    {
-//      MPI_Info info;
-//      MPI_Comm *out_comm;
-//      ::MPI_Comm mpi_dup;
-
-//      ::MPI_Comm in_comm = static_cast< ::MPI_Comm>(comm.mpi_comm);
-
-//      ::MPI_Comm_dup(in_comm, &mpi_dup);
-
-//      MPI_Comm_create_endpoints(mpi_dup, num_ep, info, out_comm);
-//      comm.ep_comm_ptr->comm_list->mem_bridge = out_comm;
-//    }
-
-//    MPI_Barrier(comm);
-
-//    *this = (comm.ep_comm_ptr->comm_list->mem_bridge[my_rank]);
-// //       // my_buffer = NULL;
-// //       // ep_barrier = NULL;
-// //       // rank_map = NULL;
-// //       // ep_comm_ptr = NULL;
-// //       // mem_bridge = NULL;
-// //       // mpi_bridge = NULL;
-// //       // mpi_comm = comm;
-// }
+#ifdef _openmpi
+//#undef MPI_Fint
+#endif
 
 
@@ -97 +56 @@
 
 
+

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_type.hpp

@@ -12 +12 @@
 #include <numeric>
 #include <bitset>
-#include <memory.h>
+//#include <memory.h>
 #include <algorithm>
 #include <assert.h>
@@ -42 +42 @@
 {
   #define MPI_UNDEFINED -32766
-  #define MPI_STATUS_IGNORE NULL
-  #define MPI_INFO_NULL MPI_Info()
+  //#define MPI_STATUS_IGNORE NULL
+  //#define MPI_INFO_NULL MPI_Info(MPI_INFO_NULL_STD)
 
   class ep_communicator;
@@ -84 +84 @@
       void* mpi_status;
 
-      MPI_Message()
-      {
-        mpi_message = 0;
-        mpi_status = 0;
-      }
+      MPI_Message() {}
+      #ifdef _intelmpi
+      MPI_Message(int message): mpi_message(message) {}
+      #elif _openmpi
+      MPI_Message(void* message): mpi_message(message) {}
+      #endif
   };
 
@@ -289 +290 @@
       mem_bridge = NULL;
       mpi_bridge = NULL;
-      mpi_comm = 0;
-    }
-
+    }
+
+    #ifdef _intelmpi
     MPI_Comm(int comm)
     {
@@ -305 +306 @@
     }
 
-    //MPI_Comm(const MPI_Comm &comm);
+    #elif _openmpi
+
+    MPI_Comm(void* comm)
+    {
+      is_ep = false;
+      is_intercomm = false;
+      my_buffer = NULL;
+      ep_barrier = NULL;
+      rank_map = NULL;
+      ep_comm_ptr = NULL;
+      mem_bridge = NULL;
+      mpi_bridge = NULL;
+      mpi_comm = comm;
+    }
+    #endif
 
 
@@ -338 +353 @@
       #endif
 
-      MPI_Info()
-      {
-        mpi_info = 0;
-      }
+      MPI_Info() {}
+      
+      #ifdef _intelmpi
+      MPI_Info(int info): mpi_info(info) {}
+      #elif _openmpi
+      MPI_Info(void* info): mpi_info(info) {}
+      #endif
   };
 
@@ -368 +386 @@
       MPI_Comm comm;    //! EP communicator related to the communication
 
-      MPI_Request()
-      {
-        mpi_request = 0;
-      }
+      MPI_Request() {}
+
+      #ifdef _intelmpi
+      MPI_Request(int request): mpi_request(request) {}
+      #elif _openmpi
+      MPI_Request(void* request): mpi_request(request) {}
+      #endif
+  };
+
+  
+  class MPI_Aint
+  {
+    public:
+
+    unsigned long mpi_aint;
+
+    MPI_Aint() {}
+    MPI_Aint(int a): mpi_aint(a) {}
   };
 
@@ -379 +411 @@
 
     int mpi_fint;
-  };
-
-  class MPI_Aint
-  {
-    public:
-
-    unsigned long mpi_aint;
+
+    MPI_Fint() {}
+    MPI_Fint(int f): mpi_fint(f) {}
+    
   };
 

XIOS/dev/branch_yushan/extern/src_ep_dev/ep_wait.cpp

@@ -52 +52 @@
       status->ep_datatype = request->ep_datatype;
 
-      check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+      //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
 
       return 0;
@@ -70 +70 @@
       int count;
       MPI_Get_count(status, request->ep_datatype, &count);
-      check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+      //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
     }
           return MPI_SUCCESS;
@@ -106 +106 @@
               int check_count;
               MPI_Get_count(&array_of_statuses[i], array_of_requests[i].ep_datatype, &check_count);
-              check_sum_recv(array_of_requests[i].buf, count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
+              //check_sum_recv(array_of_requests[i].buf, count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
             }
             finished++;
@@ -124 +124 @@
               MPI_Get_count(&array_of_statuses[i], array_of_requests[i].ep_datatype, &recv_count);
               MPI_Mrecv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, &message, &array_of_statuses[i]);
-              check_sum_recv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
+              //check_sum_recv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
 
               finished++;