Changeset 1037 for XIOS/dev/branch_yushan/src/client_client_dht_template_impl.hpp

Timestamp:

01/25/17 16:25:17 (7 years ago)

Author:

yushan

Message:

initialize the branch

File:

• XIOS/dev/branch_yushan/src/client_client_dht_template_impl.hpp (modified) (20 diffs)

XIOS/dev/branch_yushan/src/client_client_dht_template_impl.hpp

@@ -10 +10 @@
 #include "utils.hpp"
 #include "mpi_tag.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
+
 
 namespace xios
@@ -37 +41 @@
   : H(clientIntraComm), index2InfoMapping_(), indexToInfoMappingLevel_(), nbClient_(0)
 {
+  
   MPI_Comm_size(clientIntraComm, &nbClient_);
   this->computeMPICommLevel();
-  int nbLvl = this->getNbLevel();
+  int nbLvl = this->getNbLevel();  
   sendRank_.resize(nbLvl);
   recvRank_.resize(nbLvl);
   Index2VectorInfoTypeMap indexToVecInfoMap;
-  indexToVecInfoMap.rehash(std::ceil(indexInfoMap.size()/indexToVecInfoMap.max_load_factor()));
-  typename Index2InfoTypeMap::const_iterator it = indexInfoMap.begin(), ite = indexInfoMap.end();
-  for (; it != ite; ++it) indexToVecInfoMap[it->first].push_back(it->second);
-  computeDistributedIndex(indexToVecInfoMap, clientIntraComm, nbLvl-1);
+  indexToVecInfoMap.rehash(std::ceil(indexInfoMap.size()/indexToVecInfoMap.max_load_factor())); 
+  typename Index2InfoTypeMap::const_iterator it = indexInfoMap.begin(), ite = indexInfoMap.end(); 
+  for (; it != ite; ++it)
+  {
+    indexToVecInfoMap[it->first].push_back(it->second);
+  } 
+  
+  computeDistributedIndex(indexToVecInfoMap, clientIntraComm, nbLvl-1); 
 }
 
@@ -179 +188 @@
   {
     if (0 != recvNbIndexClientCount[idx])
+    {
       recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+    }
     currentIndex += recvNbIndexClientCount[idx];
   }
@@ -189 +200 @@
 
   std::vector<MPI_Status> status(request.size());
+
+  //printf("1(%d): calling wait all for %lu requests\n", clientRank, request.size());
+
   MPI_Waitall(request.size(), &request[0], &status[0]);
+
+
+  //printf("               1(%d): calling wait all for %lu requests OK\n", clientRank, request.size());
 
   CArray<size_t,1>* tmpGlobalIndex;
@@ -294 +311 @@
 
   std::vector<MPI_Status> statusOnReturn(requestOnReturn.size());
+  //printf("2(%d): calling wait all for %lu requests\n", clientRank, requestOnReturn.size());
   MPI_Waitall(requestOnReturn.size(), &requestOnReturn[0], &statusOnReturn[0]);
+
+  //printf("            2(%d): calling wait all for %lu requests OK\n", clientRank, requestOnReturn.size());
 
   Index2VectorInfoTypeMap indexToInfoMapping;
@@ -363 +383 @@
                                                             int level)
 {
+  //printf("in computeDistributedIndex(const Index2VectorInfoTypeMap& indexInfoMap, const MPI_Comm& commLevel, int level)\n");
   int clientRank;
   MPI_Comm_rank(commLevel,&clientRank);
@@ -418 +439 @@
   }
 
+  //printf("check 4 OK. clientRank = %d\n", clientRank);
+
   // Calculate from how many clients each client receive message.
   // Calculate size of buffer for receiving message
@@ -423 +446 @@
   sendRecvRank(level, sendBuff, sendNbIndexBuff,
                recvRankClient, recvNbIndexClientCount);
+  //printf("sendRecvRank OK\n");
 
   int recvNbIndexCount = 0;
@@ -435 +459 @@
     recvInfoBuff = new unsigned char[recvNbIndexCount*ProcessDHTElement<InfoType>::typeSize()];
   }
+
+  //printf("check 5 OK. clientRank = %d\n", clientRank);
 
   // If a client holds information about index and the corresponding which don't belong to it,
@@ -447 +473 @@
     {
       recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+
       recvInfoFromClients(recvRankClient[idx],
                           recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
                           recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(),
                           commLevel, request);
+
     }
     currentIndex += recvNbIndexClientCount[idx];
   }
+
+  //printf("check 6 OK. clientRank = %d\n", clientRank);
 
   boost::unordered_map<int, size_t* >::iterator itbIndex = client2ClientIndex.begin(), itIndex,
                                                 iteIndex = client2ClientIndex.end();
   for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
+  {
     sendIndexToClients(itIndex->first, itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
+
+  }
+
+  //printf("check 7 OK. clientRank = %d\n", clientRank);
+
   boost::unordered_map<int, unsigned char*>::iterator itbInfo = client2ClientInfo.begin(), itInfo,
                                                       iteInfo = client2ClientInfo.end();
   for (itInfo = itbInfo; itInfo != iteInfo; ++itInfo)
+  {
     sendInfoToClients(itInfo->first, itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], commLevel, request);
 
+  }
+
+  //printf("check 8 OK. clientRank = %d\n", clientRank);
   std::vector<MPI_Status> status(request.size());
+
   MPI_Waitall(request.size(), &request[0], &status[0]);
 
@@ -480 +521 @@
     {
       ProcessDHTElement<InfoType>::unpackElement(infoValue, infoBuff, infoIndex);
+      unsigned long pp = *(recvIndexBuff+currentIndex+i);
+      
       indexToInfoMapping[*(recvIndexBuff+currentIndex+i)].push_back(infoValue);
     }
     currentIndex += count;
   }
+
+  //printf("check 9 OK. clientRank = %d\n", clientRank);
 
   if (0 != recvNbIndexCount)
@@ -498 +543 @@
       delete [] it->second;
 
+  //printf("check 10 OK. clientRank = %d\n", clientRank);
   // Ok, now do something recursive
   if (0 < level)
@@ -523 +569 @@
   MPI_Request request;
   requestSendIndex.push_back(request);
+
   MPI_Isend(indices, indiceSize, MPI_UNSIGNED_LONG,
             clientDestRank, MPI_DHT_INDEX, clientIntraComm, &(requestSendIndex.back()));
@@ -541 +588 @@
   MPI_Request request;
   requestRecvIndex.push_back(request);
+
   MPI_Irecv(indices, indiceSize, MPI_UNSIGNED_LONG,
             clientSrcRank, MPI_DHT_INDEX, clientIntraComm, &(requestRecvIndex.back()));
@@ -560 +608 @@
   MPI_Request request;
   requestSendInfo.push_back(request);
-
+  //printf("MPI_IsendInfo(info, infoSize, MPI_CHAR,... char count = %d, dest = %d, buf_size = %d\n", infoSize, clientDestRank, sizeof(*info) );
   MPI_Isend(info, infoSize, MPI_CHAR,
             clientDestRank, MPI_DHT_INFO, clientIntraComm, &(requestSendInfo.back()));
@@ -668 +716 @@
     ++nRequest;
   }
-
+  
+  int clientRank;
+  MPI_Comm_rank(this->internalComm_,&clientRank);
+  //printf("4(%d): calling wait all for %lu requests\n", clientRank, sendNbRank.size()+recvNbRank.size());
   MPI_Waitall(sendNbRank.size()+recvNbRank.size(), &request[0], &requestStatus[0]);
+  //printf("        4(%d): calling wait all for %lu requests OK\n", clientRank, sendNbRank.size()+recvNbRank.size());
 }
 
@@ -686 +738 @@
                                                  std::vector<int>& recvNbRank, std::vector<int>& recvNbElements)
 {
+  int myRank;
+  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+  //printf("myRank = %d, in sendRecvRank(int level, const std::vector<int>& sendNbRank, const std::vector<int>& sendNbElements, std::vector<int>& recvNbRank, std::vector<int>& recvNbElements)\n", myRank);
   int groupBegin = this->getGroupBegin()[level];
 
@@ -702 +757 @@
   for (int idx = 0; idx < recvBuffSize; ++idx)
   {
+    //printf("myRank = %d starts irecv with src = %d, tag = %d, idx = %d\n", myRank, recvRank[idx], MPI_DHT_INDEX_0, idx);
     MPI_Irecv(&recvBuff[0]+2*idx, 2, MPI_INT,
               recvRank[idx], MPI_DHT_INDEX_0, this->internalComm_, &request[nRequest]);
+    //printf("myRank = %d MPI_Irecv OK, idx = %d, nRequest = %d\n", myRank, idx, nRequest);
     ++nRequest;
   }
+
+  //printf("myRank = %d, check 1 OK\n", myRank);
 
   for (int idx = 0; idx < sendBuffSize; ++idx)
@@ -716 +775 @@
   for (int idx = 0; idx < sendBuffSize; ++idx)
   {
+    //printf("myRank = %d starts isend with dest = %d, tag = %d, idx = %d\n", myRank, sendRank[idx], MPI_DHT_INDEX_0, idx);
     MPI_Isend(&sendBuff[idx*2], 2, MPI_INT,
               sendRank[idx], MPI_DHT_INDEX_0, this->internalComm_, &request[nRequest]);
+    //printf("myRank = %d MPI_Isend OK, idx = %d, nRequest = %d\n", myRank, idx, nRequest);
     ++nRequest;
   }
 
+  MPI_Barrier(this->internalComm_);
+
+  //printf("myRank = %d, check 2 OK\n", myRank);
+
+  int clientRank;
+  MPI_Comm_rank(this->internalComm_,&clientRank);
+
+  //printf("5(%d): calling wait all for %lu requests\n", myRank, sendBuffSize+recvBuffSize);
   MPI_Waitall(sendBuffSize+recvBuffSize, &request[0], &requestStatus[0]);
+  //printf("            5(%d): calling wait all for %lu requests OK\n", myRank, sendBuffSize+recvBuffSize);
+  //printf("check 3 OK\n");
+
   int nbRecvRank = 0, nbRecvElements = 0;
   recvNbRank.clear();
@@ -733 +805 @@
     }
   }
-}
-
-}
+  //printf("check 4 OK\n");
+}
+
+}