Changeset 1196 for XIOS/dev/branch_yushan_merged/src

Timestamp:

07/05/17 14:14:09 (7 years ago)

Author:

yushan

Message:

add request_check. test client and complete OK

Location:

XIOS/dev/branch_yushan_merged/src

Files:

• client.cpp (modified) (1 diff)
• client_client_dht_template_impl.hpp (modified) (12 diffs)
• server.cpp (modified) (1 diff)
• test/test_omp.f90 (modified) (1 diff)
• transformation/grid_transformation.cpp (modified) (2 diffs)

XIOS/dev/branch_yushan_merged/src/client.cpp

@@ -115 +115 @@
 
             
-            test_sendrecv(CXios::globalComm);
+            //test_sendrecv(CXios::globalComm);
             MPI_Intercomm_create(intraComm,0,CXios::globalComm,serverLeader,0,&interComm) ;
 

XIOS/dev/branch_yushan_merged/src/client_client_dht_template_impl.hpp

@@ -175 +175 @@
     recvIndexBuff = new unsigned long[recvNbIndexCount];
 
-  std::vector<ep_lib::MPI_Request> request;
+  int request_size = 0;
+
+  int currentIndex = 0;
+  int nbRecvClient = recvRankClient.size();
+
+  int position = 0;
+
+  for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
+    if (0 != recvNbIndexClientCount[idx])
+    {
+      request_size++;
+    }
+  }
+
+  request_size += client2ClientIndex.size();
+
+
+  std::vector<ep_lib::MPI_Request> request(request_size);
+
   std::vector<int>::iterator itbRecvIndex = recvRankClient.begin(), itRecvIndex,
                              iteRecvIndex = recvRankClient.end(),
                            itbRecvNbIndex = recvNbIndexClientCount.begin(),
                            itRecvNbIndex;
-  int currentIndex = 0;
-  int nbRecvClient = recvRankClient.size();
+  
   
   boost::unordered_map<int, size_t* >::iterator itbIndex = client2ClientIndex.begin(), itIndex,
                                                 iteIndex = client2ClientIndex.end();
   for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
-    sendIndexToClients(itIndex->first, (itIndex->second), sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
-
-
-
+  {
+    MPI_Isend(itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], MPI_UNSIGNED_LONG, itIndex->first, MPI_DHT_INDEX, commLevel, &request[position]);
+    position++;
+    //sendIndexToClients(itIndex->first, (itIndex->second), sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
+  }
+    
   for (int idx = 0; idx < nbRecvClient; ++idx)
   {
     if (0 != recvNbIndexClientCount[idx])
     {
-      recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+      MPI_Irecv(recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], MPI_UNSIGNED_LONG,
+            recvRankClient[idx], MPI_DHT_INDEX, commLevel, &request[position]);
+      position++;
+      //recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
     }
     currentIndex += recvNbIndexClientCount[idx];
@@ -200 +223 @@
 
   
-  std::vector<ep_lib::MPI_Status> status(request.size());
+  std::vector<ep_lib::MPI_Status> status(request_size);
   MPI_Waitall(request.size(), &request[0], &status[0]);
-  
+
 
   CArray<size_t,1>* tmpGlobalIndex;
@@ -256 +279 @@
   }
 
-  std::vector<ep_lib::MPI_Request> requestOnReturn;
+  request_size = 0;
+  for (int idx = 0; idx < recvRankOnReturn.size(); ++idx)
+  {
+    if (0 != recvNbIndexOnReturn[idx])
+    {
+      request_size += 2;
+    }
+  }
+
+  for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
+    if (0 != sendNbIndexOnReturn[idx])
+    {
+      request_size += 2;
+    }
+  }
+
+  std::vector<ep_lib::MPI_Request> requestOnReturn(request_size);
   currentIndex = 0;
+  position = 0;
   for (int idx = 0; idx < recvRankOnReturn.size(); ++idx)
   {
     if (0 != recvNbIndexOnReturn[idx])
     {
-      recvIndexFromClients(recvRankOnReturn[idx], recvIndexBuffOnReturn+currentIndex, recvNbIndexOnReturn[idx], commLevel, requestOnReturn);
-      recvInfoFromClients(recvRankOnReturn[idx],
-                          recvInfoBuffOnReturn+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
-                          recvNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(),
-                          commLevel, requestOnReturn);
+      //recvIndexFromClients(recvRankOnReturn[idx], recvIndexBuffOnReturn+currentIndex, recvNbIndexOnReturn[idx], commLevel, requestOnReturn);
+      MPI_Irecv(recvIndexBuffOnReturn+currentIndex, recvNbIndexOnReturn[idx], MPI_UNSIGNED_LONG,
+            recvRankOnReturn[idx], MPI_DHT_INDEX, commLevel, &requestOnReturn[position]);
+      position++;
+      //recvInfoFromClients(recvRankOnReturn[idx],
+      //                    recvInfoBuffOnReturn+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
+      //                    recvNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(),
+      //                    commLevel, requestOnReturn);
+      MPI_Irecv(recvInfoBuffOnReturn+currentIndex*ProcessDHTElement<InfoType>::typeSize(), 
+                recvNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), MPI_CHAR,
+                recvRankOnReturn[idx], MPI_DHT_INFO, commLevel, &requestOnReturn[position]);
+      position++;
     }
     currentIndex += recvNbIndexOnReturn[idx];
@@ -299 +347 @@
       }
 
-      sendIndexToClients(rank, client2ClientIndexOnReturn[rank],
-                         sendNbIndexOnReturn[idx], commLevel, requestOnReturn);
-      sendInfoToClients(rank, client2ClientInfoOnReturn[rank],
-                        sendNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), commLevel, requestOnReturn);
+      //sendIndexToClients(rank, client2ClientIndexOnReturn[rank],
+      //                   sendNbIndexOnReturn[idx], commLevel, requestOnReturn);
+      MPI_Isend(client2ClientIndexOnReturn[rank], sendNbIndexOnReturn[idx], MPI_UNSIGNED_LONG,
+            rank, MPI_DHT_INDEX, commLevel, &requestOnReturn[position]);
+      position++;
+      //sendInfoToClients(rank, client2ClientInfoOnReturn[rank],
+      //                  sendNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), commLevel, requestOnReturn);
+      MPI_Isend(client2ClientInfoOnReturn[rank], sendNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), MPI_CHAR,
+            rank, MPI_DHT_INFO, commLevel, &requestOnReturn[position]);
+      position++;
     }
     currentIndex += recvNbIndexClientCount[idx];
@@ -440 +494 @@
   int recvNbIndexCount = 0;
   for (int idx = 0; idx < recvNbIndexClientCount.size(); ++idx)
+  { 
     recvNbIndexCount += recvNbIndexClientCount[idx];
+  }
 
   unsigned long* recvIndexBuff;
@@ -453 +509 @@
   // it will send a message to the correct clients.
   // Contents of the message are index and its corresponding informatioin
-  std::vector<ep_lib::MPI_Request> request;
+  int request_size = 0;  
   int currentIndex = 0;
   int nbRecvClient = recvRankClient.size();
+  int current_pos = 0; 
+
   for (int idx = 0; idx < nbRecvClient; ++idx)
   {
     if (0 != recvNbIndexClientCount[idx])
     {
-      recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
-      recvInfoFromClients(recvRankClient[idx],
-                          recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
-                          recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(),
-                          commLevel, request);
-    }
-    currentIndex += recvNbIndexClientCount[idx];
-  }
+      request_size += 2;
+    }
+    //currentIndex += recvNbIndexClientCount[idx];
+  }
+
+  request_size += client2ClientIndex.size();
+  request_size += client2ClientInfo.size();
+
+
+
+  std::vector<ep_lib::MPI_Request> request(request_size);
+  
+  //unsigned long* tmp_send_buf_long[client2ClientIndex.size()];
+  //unsigned char* tmp_send_buf_char[client2ClientInfo.size()];
+  
+  int info_position = 0;
+  int index_position = 0;
+
 
   boost::unordered_map<int, size_t* >::iterator itbIndex = client2ClientIndex.begin(), itIndex,
@@ -473 +541 @@
   for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
   {
-    sendIndexToClients(itIndex->first, itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
+    //sendIndexToClients(itIndex->first, itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
+
+    //tmp_send_buf_long[index_position] = new unsigned long[sendNbIndexBuff[itIndex->first-groupRankBegin]];
+    //for(int i=0; i<sendNbIndexBuff[itIndex->first-groupRankBegin]; i++)
+    //{
+    //  tmp_send_buf_long[index_position][i] = (static_cast<unsigned long * >(itIndex->second))[i];
+    //}
+    //MPI_Isend(tmp_send_buf_long[current_pos], sendNbIndexBuff[itIndex->first-groupRankBegin], MPI_UNSIGNED_LONG,
+    MPI_Isend(itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], MPI_UNSIGNED_LONG,
+              itIndex->first, MPI_DHT_INDEX, commLevel, &request[current_pos]);
+    current_pos++; 
+    index_position++;
+
   }
 
@@ -480 +560 @@
   for (itInfo = itbInfo; itInfo != iteInfo; ++itInfo)
   {
-    sendInfoToClients(itInfo->first, itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], commLevel, request);
-
+    //sendInfoToClients(itInfo->first, itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], commLevel, request);
+
+    //tmp_send_buf_char[info_position] = new unsigned char[sendNbInfo[itInfo->first-groupRankBegin]];
+    //for(int i=0; i<sendNbInfo[itInfo->first-groupRankBegin]; i++)
+    //{
+    //  tmp_send_buf_char[info_position][i] = (static_cast<unsigned char * >(itInfo->second))[i];
+    //}
+
+    MPI_Isend(itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], MPI_CHAR,
+              itInfo->first, MPI_DHT_INFO, commLevel, &request[current_pos]);
+
+    current_pos++;
+    info_position++;
+  }
+  
+  for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
+    if (0 != recvNbIndexClientCount[idx])
+    {
+      //recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+      MPI_Irecv(recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], MPI_UNSIGNED_LONG,
+                recvRankClient[idx], MPI_DHT_INDEX, commLevel, &request[current_pos]);
+      current_pos++;
+      
+      
+      MPI_Irecv(recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(), 
+                recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(), 
+                MPI_CHAR, recvRankClient[idx], MPI_DHT_INFO, commLevel, &request[current_pos]);
+      
+      current_pos++;
+      
+
+
+      // recvInfoFromClients(recvRankClient[idx],
+      //                     recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
+      //                     recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(),
+      //                     commLevel, request);
+    }
+    currentIndex += recvNbIndexClientCount[idx];
   }
 
   std::vector<ep_lib::MPI_Status> status(request.size());
-
+  
   MPI_Waitall(request.size(), &request[0], &status[0]);
+  
+ 
+  //for(int i=0; i<client2ClientInfo.size(); i++)
+  //  delete[] tmp_send_buf_char[i];
+
+  
+
+  //for(int i=0; i<client2ClientIndex.size(); i++)
+  //  delete[] tmp_send_buf_long[i];
+
 
   Index2VectorInfoTypeMap indexToInfoMapping;
@@ -527 +654 @@
   else
     index2InfoMapping_.swap(indexToInfoMapping);
+  
 }
 
@@ -720 +848 @@
   std::vector<ep_lib::MPI_Request> request(sendBuffSize+recvBuffSize);
   std::vector<ep_lib::MPI_Status> requestStatus(sendBuffSize+recvBuffSize);
-
+  //ep_lib::MPI_Request request[sendBuffSize+recvBuffSize];
+  //ep_lib::MPI_Status requestStatus[sendBuffSize+recvBuffSize];
+  
   int my_rank;
   MPI_Comm_rank(this->internalComm_, &my_rank);
   
   int nRequest = 0;
+  for (int idx = 0; idx < recvBuffSize; ++idx)
+  {
+    MPI_Irecv(&recvBuff[2*idx], 2, MPI_INT,
+              recvRank[idx], MPI_DHT_INDEX_0, this->internalComm_, &request[nRequest]);
+    ++nRequest;
+  }
   
 
@@ -743 +879 @@
   }
   
-  for (int idx = 0; idx < recvBuffSize; ++idx)
-  {
-    MPI_Irecv(&recvBuff[0]+2*idx, 2, MPI_INT,
-              recvRank[idx], MPI_DHT_INDEX_0, this->internalComm_, &request[nRequest]);
-    ++nRequest;
-  }
+  
 
   //MPI_Barrier(this->internalComm_);
 
   MPI_Waitall(sendBuffSize+recvBuffSize, &request[0], &requestStatus[0]);
-
+  //MPI_Waitall(sendBuffSize+recvBuffSize, request, requestStatus);
+
+  
   int nbRecvRank = 0, nbRecvElements = 0;
   recvNbRank.clear();
@@ -765 +898 @@
     }
   }
-}
-
-}
+
+
+  
+  
+}
+
+}
+

XIOS/dev/branch_yushan_merged/src/server.cpp

@@ -93 +93 @@
                      <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
 
-             test_sendrecv(CXios::globalComm);
+            // test_sendrecv(CXios::globalComm);
              MPI_Intercomm_create(intraComm,0,CXios::globalComm,clientLeader,0,&newComm) ;
              interComm.push_back(newComm) ;

XIOS/dev/branch_yushan_merged/src/test/test_omp.f90

@@ -47 +47 @@
     if(rank < size-2) then
 
-    !$omp parallel default(private)
+    !$omp parallel default(firstprivate)
   
     CALL xios_initialize(id,return_comm=comm)

XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation.cpp

@@ -474 +474 @@
   // Sending global index of grid source to corresponding process as well as the corresponding mask
   std::vector<ep_lib::MPI_Request> requests;
+  requests.reserve(2*recvRankSizeMap.size()+2*globaIndexWeightFromSrcToDst.size());
   std::vector<ep_lib::MPI_Status> status;
   boost::unordered_map<int, unsigned char* > recvMaskDst;
@@ -534 +535 @@
   std::vector<ep_lib::MPI_Request>().swap(requests);
   std::vector<ep_lib::MPI_Status>().swap(status);
+  requests.reserve(sendRankSizeMap.size()+recvRankSizeMap.size());
   // Okie, on destination side, we will wait for information of masked index of source
   for (std::map<int,int>::const_iterator itSend = sendRankSizeMap.begin(); itSend != sendRankSizeMap.end(); ++itSend)