Changeset 1533 for XIOS/dev/branch_openmp

Timestamp:

06/08/18 19:34:51 (6 years ago)

Author:

yushan

Message:

save dev

Location:

XIOS/dev/branch_openmp

Files:

• bld.cfg (modified) (3 diffs)
• extern/ep_dev/ep_intercomm.cpp (modified) (1 diff)
• extern/src_ep_dev/ep_free.cpp (modified) (1 diff)
• extern/src_ep_dev/ep_init.cpp (modified) (1 diff)
• extern/src_ep_dev/ep_lib.hpp (modified) (1 diff)
• extern/src_ep_dev/ep_message.cpp (modified) (5 diffs)
• extern/src_ep_dev/ep_recv.cpp (modified) (1 diff)
• extern/src_ep_dev/ep_test.cpp (modified) (6 diffs)
• extern/src_ep_dev/ep_wait.cpp (modified) (5 diffs)
• src/cxios.cpp (modified) (1 diff)
• src/event_scheduler.cpp (modified) (1 diff)
• src/server.cpp (modified) (2 diffs)

XIOS/dev/branch_openmp/bld.cfg

@@ -32 +32 @@
 #src::src_ep_dev $PWD/extern/ep_dev
 bld::lib xios
-bld::target libxios.a 
+#bld::target libxios.a 
 #bld::target generate_fortran_interface.exe 
-bld::target xios_server.exe 
+#bld::target xios_server.exe 
 #bld::target test_regular.exe
 #bld::target test_expand_domain.exe
@@ -40 +40 @@
 #bld::target test_unstruct_complete.exe 
 bld::target test_omp.exe 
-bld::target test_complete_omp.exe 
+#bld::target test_complete_omp.exe 
 #bld::target test_remap.exe 
 #bld::target test_remap_ref.exe 
-bld::target test_remap_omp.exe 
+#bld::target test_remap_omp.exe 
 #bld::target test_unstruct_omp.exe
 #bld::target test_netcdf_omp.exe
@@ -52 +52 @@
 #bld::target test_connectivity_expand.exe
 #bld::target toy_cmip6.exe
-bld::target toy_cmip6_omp.exe
+#bld::target toy_cmip6_omp.exe
 bld::exe_dep
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp

@@ -296 +296 @@
     
 
-    int remote_num_ep;
+    int remote_num_ep = 0;
     for(int i=0; i<remote_mpi_size; i++)
     {

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_free.cpp

@@ -118 +118 @@
     int newcomm_num_ep = (*comm)->ep_comm_ptr->intercomm->size_rank_info[1].second;
 
+    return 0;
     MPI_Barrier(*comm);
 

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_init.cpp

@@ -28 +28 @@
     }
   }
+  
+  int EP_group_init()
+  {
+  
+  }
 
   int MPI_Initialized(int *flag)

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_lib.hpp

@@ -82 +82 @@
 
   int Request_Check();
+  
+  void show_EP_PendingRequests(std::list< ep_lib::MPI_Request* > * EP_PendingRequest);
 
   int test_sendrecv(MPI_Comm comm);
+  
+  int EP_group_init();
 
 

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_message.cpp

@@ -23 +23 @@
     
     if(EP_PendingRequests->size() == 0) return 0;
+    
+    show_EP_PendingRequests(EP_PendingRequests);
     
     MPI_Status status;
@@ -30 +32 @@
     std::list<MPI_Request* >::iterator it;
     
+//    show_EP_PendingRequests(EP_PendingRequests);
+    
     
     for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); it++)
     { 
+      if(*(*it) == 0)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
+      if((*(*it))->state == 2)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
       Message_Check(((*(*it))->comm));
     }
+    
+    
 
 
     for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); )
     {
+      if(*(*it) == 0)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
+      if((*(*it))->state == 2)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
       if((*(*it))->state == 0)
       {
@@ -81 +123 @@
       }
       
-      if((*(*it))->state == 2)
-      {
-        int ep_rank = ((*(*it))->comm)->ep_comm_ptr->size_rank_info[0].first;
-        memcheck("delete "<< (*(*it)) <<" : in ep_lib::Request_Check, delete (*(*it))");
-        
-        
-        int world_rank;
-        MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
-        if(world_rank==2) 
-        {
-          printf("ep %d erased one pending request %p\n", world_rank,*(*it));
-        }
-        
-        EP_PendingRequests->erase(it);
-        
-        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
-        it = EP_PendingRequests->begin();
-        continue;
-      }
-      else it++;
-    }
+      it++;
+    }
+    
+    show_EP_PendingRequests(EP_PendingRequests);
   }
   
@@ -178 +203 @@
     return MPI_SUCCESS;
   }
-
-
-  
-  
-  
 
 
@@ -249 +269 @@
     return MPI_SUCCESS;
   }
+  
+  
+  
+  void show_EP_PendingRequests(std::list< ep_lib::MPI_Request* > * EP_PendingRequest)
+  {
+#ifdef _showinfo
+    std::list<MPI_Request* >::iterator it;
+    
+    int i=0;
+    
+    for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); it++)
+    { 
+      if(*(*it) == 0)
+      {
+        EP_PendingRequests->erase(it);
+            
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
+      if((*(*it))->state == 2)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+          
+      if((*(*it))->ep_src>6)
+        printf("EP_PendingRequests[%d] : ep_src = %d, ep_tag = %d, type = %d, state = %d, comm = %d\n", i, (*(*it))->ep_src, (*(*it))->ep_tag, (*(*it))->type, (*(*it))->state, to_mpi_comm(((*(*it))->comm)->mpi_comm));
+      i++;
+    }
+#endif
+
+  }
 
   

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_recv.cpp

@@ -65 +65 @@
     EP_PendingRequests->push_back(request);  
     
-    int world_rank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
-    if(world_rank==2) 
-    {
-      printf("ep %d inserted one pending request %p\n", world_rank, *request);
-    }
+    memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());    
     
-    memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());    
+    show_EP_PendingRequests(EP_PendingRequests);
 
 #ifdef _showinfo

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_test.cpp

@@ -9 +9 @@
 #include <mpi.h>
 #include "ep_declaration.hpp"
+#include "ep_mpi.hpp"
 
 using namespace std;
@@ -18 +19 @@
   {
     Debug("MPI_Test with EP");
-
-    *flag = false;
     
-
-    if((*request)->type == 1)      // isend
+    if(*request==0) return *flag=true;
+    
+    if((*request)->type !=1 && (*request)->type !=2 && (*request)->type !=3) 
     {
-      ::MPI_Status mpi_status;
-      ::MPI_Test(static_cast< ::MPI_Request*>((*request)->mpi_request), flag, &mpi_status);
-      
-      if(*flag) 
-      {
-        status->mpi_status = new ::MPI_Status(mpi_status);
-        status->ep_src = (*request)->ep_src;
-        status->ep_tag = (*request)->ep_tag;
-        status->ep_datatype = (*request)->ep_datatype;
-        (*request)->state = 2;
-        delete (*request)->mpi_request;
-        delete *request;
-      }
-
-      return 0;
+      printf("MPI_Test : Error in request type\n");
+ 
+      exit(1);
     }
 
     if((*request)->type == 2)   // irecv message not probed
     {
-      Request_Check();
+      *flag = false;
+      return Request_Check();
+    }
+    
+    else //(*request)->type == 1 || (*request)->type == 3      // isend or imrecv
+    {
       
-      #pragma omp flush
-      
-    }
-
-    if((*request)->type == 3)  // imrecv
-    {
-      if((*request)->state == 2)
-      {
-        return *flag = true;
-      }
       ::MPI_Status mpi_status;
       
-      ::MPI_Test(static_cast< ::MPI_Request*>((*request)->mpi_request), flag, &mpi_status);
       
+      ::MPI_Test(to_mpi_request_ptr(*request), flag, &mpi_status);
       
-      if(*flag)
+      if(*flag) 
       {
-        status->mpi_status = new ::MPI_Status(mpi_status);
+
+        status->mpi_status=new ::MPI_Status(mpi_status);
         status->ep_src = (*request)->ep_src;
         status->ep_tag = (*request)->ep_tag;
@@ -68 +52 @@
         
         (*request)->state = 2;
-        
+                
+        memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Test, delete (*request)->mpi_request");
         delete (*request)->mpi_request;
         delete *request;
-        //int count;
-        //MPI_Get_count(status, request->ep_datatype, &count);
-        //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
-      }  
-
-      return Request_Check();
+        *request=0;
+      }
     }
+    
+    return Request_Check();
 
   }
@@ -85 +68 @@
   {
     Debug("MPI_Testall with EP");
-
+    *flag = false;
+   
+    int rank = array_of_requests[0]->comm->ep_comm_ptr->size_rank_info[0].first;
+    
+    for(int i=0; i<count; i++)
+    {
+      if(array_of_requests[i]->type == 2)
+      {
+        *flag=false;
+        return Request_Check();
+      }
+    }
+    
     ::MPI_Request* mpi_request = new ::MPI_Request[count];
     ::MPI_Status* mpi_status = new ::MPI_Status[count];
@@ -101 +96 @@
       for(int i=0; i<count; i++)
       {
-        array_of_statuses[i].mpi_status = &mpi_status[i];
+        array_of_statuses[i].mpi_status = new ::MPI_Status(mpi_status[i]);
         array_of_statuses[i].ep_src = array_of_requests[i]->ep_src;
         array_of_statuses[i].ep_tag = array_of_requests[i]->ep_tag;
@@ -108 +103 @@
         array_of_requests[i]->state = 2;
       
-        memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]->mpi_request");
+        memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Testall, array_of_requests["<<i<<"]->mpi_request");
         delete array_of_requests[i]->mpi_request;
-        delete array_of_requests[i]; 
+        delete array_of_requests[i];
+        array_of_requests[i]=0;
+        
       }
       
     }
+    
+    return Request_Check();
 
-    return Request_Check();
   }
 

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_wait.cpp

@@ -25 +25 @@
     if((*request)->type !=1 && (*request)->type !=2 && (*request)->type !=3) 
     {
-      printf("Error in request type\n");
+      printf("MPI_Wait : Error in request type\n");
  
       exit(1);
@@ -32 +32 @@
     while((*request)->type == 2) Request_Check();
     
-    ::MPI_Wait(to_mpi_request_ptr(*request), to_mpi_status_ptr(status));
+    ::MPI_Status mpi_status;
+    
+    ::MPI_Wait(to_mpi_request_ptr(*request), &mpi_status);
 
-    
+    status->mpi_status = new ::MPI_Status(mpi_status);
     status->ep_src = (*request)->ep_src;
     status->ep_tag = (*request)->ep_tag;
@@ -42 +44 @@
 
     memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Wait, delete (*request)->mpi_request");
+
     delete (*request)->mpi_request;
     delete *request;
+    *request=0;
     
-
     return Request_Check();
 
@@ -87 +90 @@
     for(int i=0; i<count; i++)
     {
-      array_of_statuses[i].mpi_status = &mpi_status[i];
+      array_of_statuses[i].mpi_status = new ::MPI_Status(mpi_status[i]);
       array_of_statuses[i].ep_src = array_of_requests[i]->ep_src;
       array_of_statuses[i].ep_tag = array_of_requests[i]->ep_tag;
@@ -97 +100 @@
       delete array_of_requests[i]->mpi_request;
       delete array_of_requests[i];
+      array_of_requests[i] = 0;
     }
 

XIOS/dev/branch_openmp/src/cxios.cpp

@@ -107 +107 @@
       MPI_Comm *ep_comm;
       MPI_Comm_create_endpoints(MPI_COMM_WORLD->mpi_comm, num_ep, info, ep_comm);  // servers should reach here too.
-      passage = ep_comm;  
+      passage = ep_comm; 
+      EP_group_init();
+      //::MPI_Comm_group(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &MPI_GROUP_WORLD); 
     }
         

XIOS/dev/branch_openmp/src/event_scheduler.cpp

@@ -140 +140 @@
       if (received)
       {
-        printf("probed a message coming from parent\n"),
         recvRequest=new SPendingRequest ;
         MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, -2, 1, communicator, &(recvRequest->request)) ;

XIOS/dev/branch_openmp/src/server.cpp

@@ -658 +658 @@
          MPI_Get_count(&status,MPI_CHAR,&(counts.back())) ;
          buffers.push_back(new char[counts.back()]) ;
+         //requests.push_back(request);
+         //MPI_Irecv((void*)(buffers.back()),counts.back(),MPI_CHAR,root,2,intraComm,&(requests.back())) ;
+         MPI_Irecv((void*)(buffers.back()),counts.back(),MPI_CHAR,root,2,intraComm,&request) ;
          requests.push_back(request);
-         MPI_Irecv((void*)(buffers.back()),counts.back(),MPI_CHAR,root,2,intraComm,&(requests.back())) ;
          isEventRegistered.push_back(false);
          isEventQueued.push_back(false);
@@ -668 +670 @@
        {
          // (2) If context id is received, register an event
-         MPI_Test(&requests[ctxNb],&flag,&status) ;
+         if(!isEventRegistered[ctxNb]) MPI_Test(&requests[ctxNb],&flag,&status) ;
          if (flag==true && !isEventRegistered[ctxNb])
          {