Changeset 1520 for XIOS/dev/branch_openmp/extern/src_ep_dev/ep_recv.cpp

Timestamp:

06/04/18 19:25:08 (6 years ago)

Author:

yushan

Message:

save dev. TO DO : test with xios

File:

• XIOS/dev/branch_openmp/extern/src_ep_dev/ep_recv.cpp (modified) (2 diffs)

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_recv.cpp

@@ -19 +19 @@
 namespace ep_lib 
 {
-  
 
   int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status)
   {
 
-    if(!comm.is_ep)
-    {
-      ::MPI_Status mpi_status;
-      ::MPI_Recv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm.mpi_comm), &mpi_status);
-
-      status->ep_src = src;
-      status->ep_tag = tag;
-      status->ep_datatype = datatype;
-
-      return 0;  
-    }
-
-    Message_Check(comm);
+    if(!comm->is_ep) return MPI_Recv_mpi(buf, count, datatype, src, tag, comm, status);
+    
+    Debug("MPI_Recv with EP");
 
     MPI_Request request;
@@ -45 +34 @@
   }
 
-
+  
 
 
   int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request)
   {
-
+    if(!comm->is_ep) return MPI_Irecv_mpi(buf, count, datatype, src, tag, comm, request);
+    
     Debug("MPI_Irecv with EP");
     int dest_rank;
     MPI_Comm_rank(comm, &dest_rank);
     
+    *request = new ep_request;   
+    memcheck("new " << *request <<" : in ep_lib::MPI_Irecv, *request = new ep_request");
 
-    if(!comm.is_ep)
+    (*request)->mpi_request = new ::MPI_Request;
+    memcheck("new " << (*request)->mpi_request << " : in ep_lib::MPI_Irecv, (*request)->mpi_request = new ::MPI_Request");
+    
+    (*request)->buf = buf;
+    (*request)->comm = comm;
+    (*request)->type = 2;
+    (*request)->state = 0;
+    
+
+    (*request)->ep_src = src;
+    (*request)->ep_tag = tag;
+    (*request)->ep_datatype = datatype;
+  
+    if(EP_PendingRequests == 0 ) EP_PendingRequests = new std::list< MPI_Request* >;
+
+    EP_PendingRequests->push_back(request);  
+    
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+    if(world_rank==2) 
     {
-      ::MPI_Request mpi_request;
-      ::MPI_Irecv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm.mpi_comm), &mpi_request);
+      printf("ep %d inserted one pending request %p\n", world_rank, *request);
+    }
+    
+    memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());    
 
-      request->mpi_request = new ::MPI_Request(mpi_request);
-      request->ep_src = src;
-      request->ep_datatype = datatype;
-      request->ep_tag = tag;
-    }
+#ifdef _showinfo
+    if(comm->is_intercomm)
+    {
+      int ep_dest_loc  = comm->ep_rank_map->at(dest_rank).first;
+      int ep_src_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(src).first;
+      int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc);
+      int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(src).second.first;
 
-    request->buf = buf;
-    request->comm = comm;
-    request->type = 2;
+      printf("Irecv : ep_src_loc = %d, ep_dest_loc = %d, mpi_src = %d, mpi_dest = %d, mpi_tag = %d\n", ep_src_loc, ep_dest_loc, comm->ep_comm_ptr->size_rank_info[2].first, mpi_dest, mpi_tag);
+    }                                                          
+#endif                                      
 
-    request->ep_src = src;
-    request->ep_tag = tag;
-    request->ep_datatype = datatype;
+    return Request_Check();
+  }
+  
+  int MPI_Mrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Status *status)
+  {
+    Debug("MPI_Mrecv with MPI/EP");
 
+    status->mpi_status = new ::MPI_Status;
+    memcheck("new " << status->mpi_status << " : in ep_lib::MPI_Mrecv, status->mpi_status = new ::MPI_Status");
+    
+    ::MPI_Mrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >((*message)->mpi_message), to_mpi_status_ptr(*status));
 
+    
+    status->ep_src = (*message)->ep_src;
+    status->ep_datatype = datatype;
+    status->ep_tag = (*message)->ep_tag;
 
-    /* With Improbe*/
-    Message_Check(comm);
+    memcheck("delete " << (*message)->mpi_message << " : in ep_lib::MPI_Mrecv, delete (*message)->mpi_message");
+    delete (*message)->mpi_message;
 
-    if(EP_PendingRequests == 0 ) 
-    {
-      EP_PendingRequests = new std::list< MPI_Request* >;
-    }
+#ifdef _check_sum
+    check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag);
+#endif
 
+    return Request_Check();
+  }
 
-    EP_PendingRequests->push_back(request);
-
-
-    Request_Check();
-
-    return 0;
-  }
 
   int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request)
   {
-    Debug("MPI_Imrecv");
+    Debug("MPI_Imrecv with MPI/EP");
 
-    request->type = 3;
+    (*request)->type = 3;
+    (*request)->ep_datatype = datatype;
+    (*request)->ep_tag = (*message)->ep_tag;
+    (*request)->ep_src = (*message)->ep_src;
+    
+    (*request)->state = 1;
+        
+    ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request));                
+    
+    memcheck("delete " << (*message)->mpi_message << " : in ep_lib::MPI_Imrecv, delete (*message)->mpi_message");
+    delete (*message)->mpi_message;
 
-    ::MPI_Request mpi_request;
-    ::MPI_Imrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >(message->mpi_message), &mpi_request);         
+#ifdef _check_sum
+    check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag);
+#endif
 
-    request->mpi_request = new ::MPI_Request(mpi_request);
-    request->ep_datatype = datatype;
-    request->ep_tag = message->ep_tag;
-    request->ep_src = message->ep_src;
     
-    delete message->mpi_message;
-
-    return 0;
+    return Request_Check();
   }
 
 
-  int MPI_Mrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Status *status)
+   int MPI_Recv_mpi(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status)
   {
-    Debug("EP Mrecv called\n");
+    Debug("MPI_Recv with MPI");
+    status->ep_src = src;
+    status->ep_tag = tag;
+    status->ep_datatype = datatype;
+    
+    return ::MPI_Recv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), to_mpi_status_ptr(*status)); 
+  }
+  
+  int MPI_Irecv_mpi(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("MPI_Irecv with MPI");
+    int dest_rank;
+    MPI_Comm_rank(comm, &dest_rank);
+    
+    *request = new ep_request;
+    memcheck("new "<< *request <<" : in ep_lib::MPI_Irecv, *request = new ep_request");
 
-    ::MPI_Status mpi_status;
-    ::MPI_Mrecv(buf, count, to_mpi_type(datatype), static_cast< ::MPI_Message* >(message->mpi_message), &mpi_status);
-
-    status->mpi_status = new ::MPI_Status(mpi_status);
-    status->ep_src = message->ep_src;
-    status->ep_datatype = datatype;
-    status->ep_tag = message->ep_tag;
-
-    delete message->mpi_message;
-
-    //check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag);
-
-    return 0;
+    (*request)->mpi_request = new ::MPI_Request;
+    memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Irecv, (*request)->mpi_request = new ::MPI_Request");
+      
+    (*request)->ep_src = src;
+    (*request)->ep_datatype = datatype;
+    (*request)->type = 2;
+    (*request)->ep_tag = tag;
+    
+    return ::MPI_Irecv(buf, count, to_mpi_type(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));
   }
-
 }