Changeset 833

Timestamp:

04/08/16 15:00:15 (8 years ago)

Author:

mhnguyen

Message:

Improvements for dht

+) Implement adaptive hierarchy for dht, level of hierarchy depends on number of processes
+) Remove some redundant codes

Test
+) On Curie
+) All tests are correct

Location:

XIOS/trunk/src

Files:

• client_client_dht_template.hpp (modified) (4 diffs)
• client_client_dht_template_impl.hpp (modified) (15 diffs)
• dht_data_types.hpp (modified) (1 diff)
• distribution_client.cpp (modified) (4 diffs)
• io/netCdfInterface.cpp (modified) (3 diffs)
• io/netCdfInterface_impl.hpp (modified) (4 diffs)
• mpi_tag.hpp (modified) (1 diff)
• policy.cpp (modified) (1 diff)
• policy.hpp (modified) (1 diff)
• transformation/axis_algorithm_interpolate.cpp (modified) (1 diff)
• transformation/axis_algorithm_inverse.cpp (modified) (3 diffs)
• transformation/axis_algorithm_zoom.cpp (modified) (2 diffs)
• transformation/domain_algorithm_interpolate.cpp (modified) (2 diffs)
• transformation/domain_algorithm_zoom.cpp (modified) (2 diffs)
• transformation/generic_algorithm_transformation.cpp (modified) (1 diff)
• transformation/generic_algorithm_transformation.hpp (modified) (1 diff)

XIOS/trunk/src/client_client_dht_template.hpp

@@ -20 +20 @@
 namespace xios
 {
-template<typename T, class HierarchyPolicy = DivideCommByTwo> class CClientClientDHTTemplate;
+template<typename T, class HierarchyPolicy = DivideAdaptiveComm> class CClientClientDHTTemplate;
 
 /*!
@@ -61 +61 @@
                                       int level);
 
+    void computeSendRecvRank(int level, int rank);
+
+    void sendRecvRank(int level,
+                      const std::vector<int>& sendNbRank, const std::vector<int>& sendNbElements,
+                      int& recvNbRank, int& recvNbElements);
+
   protected:
     void probeIndexMessageFromClients(unsigned long* recvIndexGlobalBuff,
@@ -81 +87 @@
 
     // Send global index to clients
-    void sendIndexToClients(int clientDestRank, std::vector<size_t>& indexGlobal,
+    void sendIndexToClients(int clientDestRank, size_t* indices, size_t indiceSize,
                             const MPI_Comm& clientIntraComm, std::list<MPI_Request>& requestSendIndexGlobal);
 
@@ -100 +106 @@
     Index2InfoTypeMap indexToInfoMappingLevel_;
 
-    //! intracommuntion of clients
-    MPI_Comm intraCommRoot_;
+    std::vector<std::vector<int> > sendRank_;
+
+    std::vector<std::vector<int> > recvRank_;
 
     //! Flag to specify whether data is distributed or not

XIOS/trunk/src/client_client_dht_template_impl.hpp

@@ -25 +25 @@
                                                         const MPI_Comm& clientIntraComm,
                                                         int hierarLvl)
-  : index2InfoMapping_(), indexToInfoMappingLevel_()
-{
-  this->computeMPICommLevel(clientIntraComm, hierarLvl);
-  int lvl = this->commLevel_.size() - 1;
-  computeDistributedIndex(indexInfoMap, this->commLevel_[lvl], lvl);
+  : H(clientIntraComm), index2InfoMapping_(), indexToInfoMappingLevel_()
+{
+  this->computeMPICommLevel();
+  int nbLvl = this->getNbLevel();
+  sendRank_.resize(nbLvl);
+  recvRank_.resize(nbLvl);
+  computeDistributedIndex(indexInfoMap, clientIntraComm, nbLvl-1);
 }
 
@@ -44 +46 @@
 void CClientClientDHTTemplate<T,H>::computeIndexInfoMapping(const CArray<size_t,1>& indices)
 {
-  int lvl = this->commLevel_.size() - 1;
-  computeIndexInfoMappingLevel(indices, this->commLevel_[lvl], lvl);
+  int nbLvl = this->getNbLevel();
+  computeIndexInfoMappingLevel(indices, this->internalComm_, nbLvl-1);
 }
 
@@ -60 +62 @@
                                                                  int level)
 {
-  int nbClient, clientRank;
-  MPI_Comm_size(commLevel,&nbClient);
+  int clientRank;
   MPI_Comm_rank(commLevel,&clientRank);
+  int groupRankBegin = this->getGroupBegin()[level];
+  int nbClient = this->getNbInGroup()[level];
   std::vector<size_t> hashedIndex;
   computeHashIndex(hashedIndex, nbClient);
@@ -70 +73 @@
   std::vector<size_t>::const_iterator itbClientHash = hashedIndex.begin(), itClientHash,
                                       iteClientHash = hashedIndex.end();
-  std::map<int, std::vector<size_t> > client2ClientIndex;
+  std::vector<int> sendBuff(nbClient,0);
+  std::vector<int> sendNbIndexBuff(nbClient,0);
 
   // Number of global index whose mapping server are on other clients
   int nbIndexToSend = 0;
+  size_t index;
   HashXIOS<size_t> hashGlobalIndex;
   for (int i = 0; i < ssize; ++i)
   {
-    size_t index = indices(i);
+    index = indices(i);
     hashedVal  = hashGlobalIndex(index);
     itClientHash = std::upper_bound(itbClientHash, iteClientHash, hashedVal);
-    if (iteClientHash != itClientHash)
+    int indexClient = std::distance(itbClientHash, itClientHash)-1;
+    ++sendNbIndexBuff[indexClient];
+  }
+
+  std::map<int, size_t* > client2ClientIndex;
+  for (int idx = 0; idx < nbClient; ++idx)
+  {
+    if (0 != sendNbIndexBuff[idx])
+    {
+      client2ClientIndex[idx+groupRankBegin] = new unsigned long [sendNbIndexBuff[idx]];
+      nbIndexToSend += sendNbIndexBuff[idx];
+      sendBuff[idx] = 1;
+      sendNbIndexBuff[idx] = 0;
+    }
+  }
+
+  for (int i = 0; i < ssize; ++i)
+  {
+    index = indices(i);
+    hashedVal  = hashGlobalIndex(index);
+    itClientHash = std::upper_bound(itbClientHash, iteClientHash, hashedVal);
     {
       int indexClient = std::distance(itbClientHash, itClientHash)-1;
       {
-        client2ClientIndex[indexClient].push_back(index);
-        ++nbIndexToSend;
+        client2ClientIndex[indexClient+groupRankBegin][sendNbIndexBuff[indexClient]] = index;;
+        ++sendNbIndexBuff[indexClient];
       }
     }
   }
 
-  int* sendBuff = new int[nbClient];
-  for (int i = 0; i < nbClient; ++i) sendBuff[i] = 0;
-  std::map<int, std::vector<size_t> >::iterator itb  = client2ClientIndex.begin(), it,
-                                                ite  = client2ClientIndex.end();
-  for (it = itb; it != ite; ++it) sendBuff[it->first] = 1;
-  int* recvBuff = new int[nbClient];
-  MPI_Allreduce(sendBuff, recvBuff, nbClient, MPI_INT, MPI_SUM, commLevel);
+  int recvNbClient, recvNbIndexCount;
+  sendRecvRank(level, sendBuff, sendNbIndexBuff,
+               recvNbClient, recvNbIndexCount);
+
+  std::map<int, size_t* >::iterator itbIndex = client2ClientIndex.begin(), itIndex,
+                                                iteIndex = client2ClientIndex.end();
 
   std::list<MPI_Request> sendIndexRequest;
-  if (0 != nbIndexToSend)
-      for (it = itb; it != ite; ++it)
-         sendIndexToClients(it->first, it->second, commLevel, sendIndexRequest);
-
-  int nbDemandingClient = recvBuff[clientRank], nbSendBuffInfoReceived = 0;
+  for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
+     sendIndexToClients(itIndex->first, (itIndex->second), sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, sendIndexRequest);
+
+  int nbDemandingClient = recvNbClient; //recvBuff[clientRank],
+  int nbSendBuffInfoReceived = 0;
 
   // Receiving demand as well as the responds from other clients
@@ -109 +133 @@
   // Buffer to receive demand from other clients, it can be allocated or not depending whether it has demand(s)
   // There are some cases we demand duplicate index so need to determine maxium size of demanding buffer
-  for (it = itb; it != ite; ++it) sendBuff[it->first] = (it->second).size();
-  MPI_Allreduce(sendBuff, recvBuff, nbClient, MPI_INT, MPI_SUM, commLevel);
-
   unsigned long* recvBuffIndex = 0;
-  int maxNbIndexDemandedFromOthers = recvBuff[clientRank];
-
+  int maxNbIndexDemandedFromOthers = recvNbIndexCount;
   if (0 != maxNbIndexDemandedFromOthers)
     recvBuffIndex = new unsigned long[maxNbIndexDemandedFromOthers];
 
   // Buffer to receive respond from other clients, it can be allocated or not depending whether it demands other clients
-//  InfoType* recvBuffInfo = 0;
   unsigned char* recvBuffInfo = 0;
   int nbIndexReceivedFromOthers = nbIndexToSend;
@@ -181 +200 @@
   {
     --level;
-    computeIndexInfoMappingLevel(tmpGlobalIndexOnClient, this->commLevel_[level], level);
+    computeIndexInfoMappingLevel(tmpGlobalIndexOnClient, this->internalComm_, level);
   }
   else
@@ -234 +253 @@
         int clientSourceRank = statusInfo.MPI_SOURCE;
         unsigned char* beginBuff = infoBuffBegin[clientSourceRank];
-        std::vector<size_t>& indexTmp = client2ClientIndex[clientSourceRank];
+        size_t* indexTmp = client2ClientIndex[clientSourceRank];
         int infoIndex = 0;
         for (int i = 0; i < actualCountInfo; ++i)
@@ -253 +272 @@
   if (0 != maxNbIndexDemandedFromOthers) delete [] recvBuffIndex;
   if (0 != nbIndexReceivedFromOthers) delete [] recvBuffInfo;
+  for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex) delete [] itIndex->second;
   for (int idx = 0; idx < infoToSend.size(); ++idx) delete [] infoToSend[idx];
-  delete [] sendBuff;
-  delete [] recvBuff;
 }
 
@@ -292 +310 @@
                                                             int level)
 {
-  int nbClient, clientRank;
-  MPI_Comm_size(commLevel,&nbClient);
+  int clientRank;
   MPI_Comm_rank(commLevel,&clientRank);
+  computeSendRecvRank(level, clientRank);
+
+  int groupRankBegin = this->getGroupBegin()[level];
+  int nbClient = this->getNbInGroup()[level];
   std::vector<size_t> hashedIndex;
   computeHashIndex(hashedIndex, nbClient);
 
-  int* sendBuff = new int[nbClient];
-  int* sendNbIndexBuff = new int[nbClient];
-  for (int i = 0; i < nbClient; ++i)
-  {
-    sendBuff[i] = 0; sendNbIndexBuff[i] = 0;
-  }
-
-  // Compute size of sending and receving buffer
-  std::map<int, std::vector<size_t> > client2ClientIndex;
-  std::map<int, std::vector<InfoType> > client2ClientInfo;
-
+  std::vector<int> sendBuff(nbClient,0);
+  std::vector<int> sendNbIndexBuff(nbClient,0);
   std::vector<size_t>::const_iterator itbClientHash = hashedIndex.begin(), itClientHash,
                                       iteClientHash = hashedIndex.end();
-  typename boost::unordered_map<size_t,InfoType>::const_iterator it  = indexInfoMap.begin(),
+  typename boost::unordered_map<size_t,InfoType>::const_iterator itb = indexInfoMap.begin(),it,
                                                                  ite = indexInfoMap.end();
   HashXIOS<size_t> hashGlobalIndex;
-  for (; it != ite; ++it)
+
+  // Compute size of sending and receving buffer
+  for (it = itb; it != ite; ++it)
   {
     size_t hashIndex = hashGlobalIndex(it->first);
     itClientHash = std::upper_bound(itbClientHash, iteClientHash, hashIndex);
-    if (itClientHash != iteClientHash)
     {
       int indexClient = std::distance(itbClientHash, itClientHash)-1;
       {
-        sendBuff[indexClient] = 1;
         ++sendNbIndexBuff[indexClient];
-        client2ClientIndex[indexClient].push_back(it->first);
-        client2ClientInfo[indexClient].push_back(it->second);
       }
     }
   }
 
+  std::map<int, size_t*> client2ClientIndex;
+  std::map<int, unsigned char*> client2ClientInfo;
+  for (int idx = 0; idx < nbClient; ++idx)
+  {
+    if (0 != sendNbIndexBuff[idx])
+    {
+      client2ClientIndex[idx+groupRankBegin] = new unsigned long [sendNbIndexBuff[idx]];
+      client2ClientInfo[idx+groupRankBegin] = new unsigned char [sendNbIndexBuff[idx]*ProcessDHTElement<InfoType>::typeSize()];
+      sendNbIndexBuff[idx] = 0;
+      sendBuff[idx] = 1;
+    }
+  }
+
+  std::vector<int> sendNbInfo(nbClient,0);
+  for (it = itb; it != ite; ++it)
+  {
+    size_t hashIndex = hashGlobalIndex(it->first);
+    itClientHash = std::upper_bound(itbClientHash, iteClientHash, hashIndex);
+    {
+      int indexClient = std::distance(itbClientHash, itClientHash)-1;
+      {
+        client2ClientIndex[indexClient + groupRankBegin][sendNbIndexBuff[indexClient]] = it->first;;
+        ProcessDHTElement<InfoType>::packElement(it->second, client2ClientInfo[indexClient + groupRankBegin], sendNbInfo[indexClient]);
+        ++sendNbIndexBuff[indexClient];
+      }
+    }
+  }
+
   // Calculate from how many clients each client receive message.
-  int* recvBuff = new int[nbClient];
-  MPI_Allreduce(sendBuff, recvBuff, nbClient, MPI_INT, MPI_SUM, commLevel);
-  int recvNbClient = recvBuff[clientRank];
-
   // Calculate size of buffer for receiving message
-  int* recvNbIndexBuff = new int[nbClient];
-  MPI_Allreduce(sendNbIndexBuff, recvNbIndexBuff, nbClient, MPI_INT, MPI_SUM, commLevel);
-  int recvNbIndexCount = recvNbIndexBuff[clientRank];
-  unsigned long* recvIndexBuff = new unsigned long[recvNbIndexCount];
-  unsigned char* recvInfoBuff = new unsigned char[recvNbIndexCount*ProcessDHTElement<InfoType>::typeSize()];
+  int recvNbClient, recvNbIndexCount;
+  sendRecvRank(level, sendBuff, sendNbIndexBuff,
+               recvNbClient, recvNbIndexCount);
 
   // If a client holds information about index and the corresponding which don't belong to it,
@@ -346 +378 @@
   // Contents of the message are index and its corresponding informatioin
   std::list<MPI_Request> sendRequest;
-  std::map<int, std::vector<size_t> >::iterator itIndex  = client2ClientIndex.begin(),
-                                                iteIndex = client2ClientIndex.end();
-  for (; itIndex != iteIndex; ++itIndex)
-    sendIndexToClients(itIndex->first, itIndex->second, commLevel, sendRequest);
-  typename std::map<int, std::vector<InfoType> >::iterator itbInfo = client2ClientInfo.begin(), itInfo,
-                                                           iteInfo = client2ClientInfo.end();
-
-  std::vector<int> infoSizeToSend(client2ClientInfo.size(),0);
-  std::vector<unsigned char*> infoToSend(client2ClientInfo.size());
-  itInfo = itbInfo;
-  for (int idx = 0; itInfo != iteInfo; ++itInfo, ++idx)
-  {
-    const std::vector<InfoType>& infoVec = itInfo->second;
-    int infoVecSize = infoVec.size();
-    std::vector<int> infoIndex(infoVecSize);
-    for (int i = 0; i < infoVecSize; ++i)
-    {
-      infoIndex[i] = infoSizeToSend[idx];
-      ProcessDHTElement<InfoType>::packElement(infoVec[i], NULL, infoSizeToSend[idx]);
-    }
-
-    infoToSend[idx] = new unsigned char[infoSizeToSend[idx]];
-    infoSizeToSend[idx] = 0;
-    for (int i = 0; i < infoVecSize; ++i)
-    {
-      ProcessDHTElement<InfoType>::packElement(infoVec[i], infoToSend[idx], infoSizeToSend[idx]);
-    }
-
-    sendInfoToClients(itInfo->first, infoToSend[idx], infoSizeToSend[idx], commLevel, sendRequest);
-  }
-
+  std::map<int, size_t* >::iterator itbIndex = client2ClientIndex.begin(), itIndex,
+                                    iteIndex = client2ClientIndex.end();
+  for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
+    sendIndexToClients(itIndex->first, itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, sendRequest);
+  std::map<int, unsigned char*>::iterator itbInfo = client2ClientInfo.begin(), itInfo,
+                                          iteInfo = client2ClientInfo.end();
+  for (itInfo = itbInfo; itInfo != iteInfo; ++itInfo)
+    sendInfoToClients(itInfo->first, itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], commLevel, sendRequest);
+
+
+  unsigned long* recvIndexBuff = new unsigned long[recvNbIndexCount];
+  unsigned char* recvInfoBuff = new unsigned char[recvNbIndexCount*ProcessDHTElement<InfoType>::typeSize()];
 
   std::map<int, MPI_Request>::iterator itRequestIndex, itRequestInfo;
@@ -465 +478 @@
   }
 
-  for (int idx = 0; idx < infoToSend.size(); ++idx) delete [] infoToSend[idx];
-  delete [] sendBuff;
-  delete [] sendNbIndexBuff;
-  delete [] recvBuff;
-  delete [] recvNbIndexBuff;
+  for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex) delete [] itIndex->second;
+  for (itInfo = itbInfo; itInfo != iteInfo; ++itInfo) delete [] itInfo->second;
   delete [] recvIndexBuff;
   delete [] recvInfoBuff;
@@ -477 +487 @@
   {
     --level;
-    computeDistributedIndex(indexToInfoMapping, this->commLevel_[level], level);
+    computeDistributedIndex(indexToInfoMapping, this->internalComm_, level);
   }
   else
@@ -563 +573 @@
 */
 template<typename T, typename H>
-void CClientClientDHTTemplate<T,H>::sendIndexToClients(int clientDestRank, std::vector<size_t>& indices,
+void CClientClientDHTTemplate<T,H>::sendIndexToClients(int clientDestRank, size_t* indices, size_t indiceSize,
                                                        const MPI_Comm& clientIntraComm,
                                                        std::list<MPI_Request>& requestSendIndex)
@@ -569 +579 @@
   MPI_Request request;
   requestSendIndex.push_back(request);
-  MPI_Isend(&(indices)[0], (indices).size(), MPI_UNSIGNED_LONG,
+  MPI_Isend(indices, indiceSize, MPI_UNSIGNED_LONG,
             clientDestRank, MPI_DHT_INDEX, clientIntraComm, &(requestSendIndex.back()));
 }
@@ -658 +668 @@
 }
 
-}
+/*!
+  Compute how many processes one process needs to send to and from how many processes it will receive
+*/
+template<typename T, typename H>
+void CClientClientDHTTemplate<T,H>::computeSendRecvRank(int level, int rank)
+{
+  int groupBegin = this->getGroupBegin()[level];
+  int nbInGroup  = this->getNbInGroup()[level];
+  const std::vector<int>& groupParentBegin = this->getGroupParentsBegin()[level];
+  const std::vector<int>& nbInGroupParents = this->getNbInGroupParents()[level];
+
+  std::vector<size_t> hashedIndexGroup;
+  computeHashIndex(hashedIndexGroup, nbInGroup);
+  size_t a = hashedIndexGroup[rank-groupBegin];
+  size_t b = hashedIndexGroup[rank-groupBegin+1]-1;
+
+  int currentGroup, offset;
+  size_t e,f;
+
+  // Do a simple math [a,b) intersect [c,d)
+  for (int idx = 0; idx < groupParentBegin.size(); ++idx)
+  {
+    std::vector<size_t> hashedIndexGroupParent;
+    int nbInGroupParent = nbInGroupParents[idx];
+    if (0 != nbInGroupParent)
+      computeHashIndex(hashedIndexGroupParent, nbInGroupParent);
+    for (int i = 0; i < nbInGroupParent; ++i)
+    {
+      size_t c = hashedIndexGroupParent[i];
+      size_t d = hashedIndexGroupParent[i+1]-1;
+
+    if (!((d < a) || (b <c)))
+        recvRank_[level].push_back(groupParentBegin[idx]+i);
+    }
+
+    offset = rank - groupParentBegin[idx];
+    if ((offset<nbInGroupParents[idx]) && (0 <= offset))
+    {
+      e = hashedIndexGroupParent[offset];
+      f = hashedIndexGroupParent[offset+1]-1;
+    }
+  }
+
+  std::vector<size_t>::const_iterator itbHashGroup = hashedIndexGroup.begin(), itHashGroup,
+                                      iteHashGroup = hashedIndexGroup.end();
+  itHashGroup = std::lower_bound(itbHashGroup, iteHashGroup, e+1);
+  int begin = std::distance(itbHashGroup, itHashGroup)-1;
+  itHashGroup = std::upper_bound(itbHashGroup, iteHashGroup, f);
+  int end = std::distance(itbHashGroup, itHashGroup) -1;
+  sendRank_[level].resize(end-begin+1);
+  for (int idx = 0; idx < sendRank_[level].size(); ++idx) sendRank_[level][idx] = idx + groupBegin + begin;
+}
+
+/*!
+  Send and receive number of process each process need to listen to as well as number
+  of index it will receive
+*/
+template<typename T, typename H>
+void CClientClientDHTTemplate<T,H>::sendRecvRank(int level,
+                                                 const std::vector<int>& sendNbRank, const std::vector<int>& sendNbElements,
+                                                 int& recvNbRank, int& recvNbElements)
+{
+  int groupBegin = this->getGroupBegin()[level];
+
+  int offSet = 0;
+  std::vector<int>& sendRank = sendRank_[level];
+  std::vector<int>& recvRank = recvRank_[level];
+  int sendBuffSize = sendRank.size();
+  int* sendBuff = new int [sendBuffSize*2];
+  std::vector<MPI_Request> request(sendBuffSize);
+  std::vector<MPI_Status> requestStatus(sendBuffSize);
+  int recvBuffSize = recvRank.size();
+  int* recvBuff = new int [2];
+
+  for (int idx = 0; idx < sendBuffSize; ++idx)
+  {
+    offSet = sendRank[idx]-groupBegin;
+    sendBuff[idx*2] = sendNbRank[offSet];
+    sendBuff[idx*2+1] = sendNbElements[offSet];
+  }
+
+  for (int idx = 0; idx < sendBuffSize; ++idx)
+  {
+    MPI_Isend(&sendBuff[idx*2], 2, MPI_INT,
+              sendRank[idx], MPI_DHT_INDEX_1, this->internalComm_, &request[idx]);
+  }
+
+  MPI_Status status;
+  int nbRecvRank = 0, nbRecvElements = 0;
+  for (int idx = 0; idx < recvBuffSize; ++idx)
+  {
+    MPI_Recv(recvBuff, 2, MPI_INT,
+             recvRank[idx], MPI_DHT_INDEX_1, this->internalComm_, &status);
+    nbRecvRank += *(recvBuff);
+    nbRecvElements += *(recvBuff+1);
+  }
+
+  MPI_Waitall(sendBuffSize, &request[0], &requestStatus[0]);
+
+  recvNbRank = nbRecvRank;
+  recvNbElements = nbRecvElements;
+
+  delete [] sendBuff;
+  delete [] recvBuff;
+}
+
+}

XIOS/trunk/src/dht_data_types.hpp

@@ -15 +15 @@
 namespace xios
 {
-  typedef std::pair<int,int> PairIntInt;
-//struct PairIntInt
-//{
-////  PairIntInt(int f, int s) : first(f), second(s) {}
-////  PairIntInt(const PairIntInt& p)
-////  {
-////    this->first = p.first;
-////    this->second=p.second;
-////  }
-//
-//  int first;
-//  int second;
-//};
+typedef std::pair<int,int> PairIntInt;
 
 template<typename T>

XIOS/trunk/src/distribution_client.cpp

@@ -18 +18 @@
    , gridMask_(), localDomainIndex_(), localAxisIndex_(), indexMap_(), indexDomainData_(), indexAxisData_()
    , isDataDistributed_(true), axisNum_(0), domainNum_(0), nIndexDomain_(), nIndexAxis_()
-   , localDataIndex_(), localMaskIndex_()//, globalDataSendToServer_(), localDataIndexSendToServer_()
+   , localDataIndex_(), localMaskIndex_()
    , globalLocalDataSendToServerMap_()
    , infoIndex_()
@@ -31 +31 @@
    , gridMask_(), localDomainIndex_(), localAxisIndex_(), indexMap_(), indexDomainData_(), indexAxisData_()
    , isDataDistributed_(true), axisNum_(0), domainNum_(0), nIndexDomain_(), nIndexAxis_()
-   , localDataIndex_(), localMaskIndex_()//, globalDataSendToServer_(), localDataIndexSendToServer_()
+   , localDataIndex_(), localMaskIndex_()
    , globalLocalDataSendToServerMap_()
    , infoIndex_()
@@ -507 +507 @@
 
   // Now allocate these arrays
-//  globalDataSendToServer_.resize(indexSend2ServerCount);
-//  localDataIndexSendToServer_.resize(indexSend2ServerCount);
   localDataIndex_.resize(indexLocalDataOnClientCount);
-
   localMaskIndex_.resize(indexSend2ServerCount);
 
@@ -658 +655 @@
             }
           }
-//          globalDataSendToServer_[indexSend2ServerCount] = globalIndex;
-//          localDataIndexSendToServer_[indexSend2ServerCount] = indexLocalDataOnClientCount;
           globalLocalDataSendToServerMap_[globalIndex] = indexLocalDataOnClientCount;
           localMaskIndex_[indexSend2ServerCount] = gridMaskIndex;

XIOS/trunk/src/io/netCdfInterface.cpp

@@ -279 +279 @@
     sstr << "Error when calling function nc_inq_varname(ncid, varId, varNameBuff)" << std::endl
          << errormsg << std::endl
-         << "Unable to get variable name given its id: " << varId << std::endl;
+         << "Unable to get variable name: "<< varName << " given its id: " << varId << std::endl;
     StdString e = sstr.str();
     throw CNetCdfException(e);
@@ -329 +329 @@
     sstr << "Error when calling function nc_inq_dimname(ncid, dimId, fullNameIn)" << std::endl
          << errormsg << std::endl
-         << "Unable to get dimension name given its id: " << dimId << std::endl;
+         << "Unable to get dimension name: " << dimName << " given its id: " << dimId << std::endl;
     StdString e = sstr.str();
     throw CNetCdfException(e);
@@ -612 +612 @@
     sstr << "Error when calling function nc_inq_attname(ncid, varid, attnum, attName)" << std::endl;
     sstr << errormsg << std::endl;
-    sstr << "Unable to query the name of attribute " << attnum << " given the location id:" << ncid << " and the variable id:" << varid << std::endl;
+    sstr << "Unable to query the name: " << name << " of attribute " << attnum << " given the location id:" << ncid << " and the variable id:" << varid << std::endl;
     StdString e = sstr.str();
     throw CNetCdfException(e);

XIOS/trunk/src/io/netCdfInterface_impl.hpp

@@ -32 +32 @@
     {
       StdStringStream sstr;
+      StdString varName;
       sstr << "Error when calling function ncGetAttType(ncid, varId, attrName.c_str(), data)" << std::endl;
       sstr << nc_strerror(status) << std::endl;
-      sstr << "Unable to read attribute " << attrName << " given the location id: " << ncid << " and the variable id: " << varId << std::endl;
+      inqVarName(ncid, varId, varName);
+      sstr << "Unable to read attribute " << attrName << " given the location id: " << ncid << " and the variable whose id: " << varId << " and name: " << varName << std::endl;
       throw CNetCdfException(sstr.str());
     }
@@ -59 +61 @@
     {
       StdStringStream sstr;
+      StdString varName;
       sstr << "Error when calling function ncPutAttType(ncid, varId, attrName.c_str(), numVal, data)" << std::endl;
       sstr << nc_strerror(status) << std::endl;
-      sstr << "Unable to set attribute " << attrName << " given the location id: " << ncid << " and the variable id: " << varId
+      inqVarName(ncid, varId, varName);
+      sstr << "Unable to set attribute " << attrName << " given the location id: " << ncid << " and the variable whose id: " << varId << " and name: " << varName << std::endl
            << " with " << numVal << " elements." << std::endl;
       throw CNetCdfException(sstr.str());
@@ -85 +89 @@
     {
       StdStringStream sstr;
+      StdString varName;
       sstr << "Error when calling function ncGetVaraType(ncid, varId, start, count, data)" << std::endl;
       sstr << nc_strerror(status) << std::endl;
-      sstr << "Unable to read data given the location id: " << ncid << " and the variable id: " << varId << std::endl;
+      inqVarName(ncid, varId, varName);
+      sstr << "Unable to read data given the location id: " << ncid << " and the variable whose id: " << varId << " and name: " << varName << std::endl;
       throw CNetCdfException(sstr.str());
     }
@@ -110 +116 @@
     {
       StdStringStream sstr;
+      StdString varName;
       sstr << "Error when calling function ncPutVaraType(ncid, varId, start, count, data)" << std::endl;
       sstr << nc_strerror(status) << std::endl;
-      sstr << "Unable to write data given the location id: " << ncid << " and the variable id: " << varId << std::endl;
+      inqVarName(ncid, varId, varName);
+      sstr << "Unable to write data given the location id: " << ncid << " and the variable whose id: " << varId << " and name: " << varName << std::endl;
       throw CNetCdfException(sstr.str());
     }

XIOS/trunk/src/mpi_tag.hpp

@@ -13 +13 @@
 /* Tag for mpi communication to send and receive info in distributed hashed table version 1*/
 #define MPI_DHT_INFO_0 22
+
+/* Tag for mpi communication to send and receive index in distributed hashed table version 1*/
+#define MPI_DHT_INDEX_1 27
+
+/* Tag for mpi communication to send and receive info in distributed hashed table version 1*/
+#define MPI_DHT_INFO_1 24
 
 /* Tag for mpi communication to send and receive info of current grid source in grid transformation*/

XIOS/trunk/src/policy.cpp

@@ -49 +49 @@
 }
 
+DivideAdaptiveComm::DivideAdaptiveComm(const MPI_Comm& mpiComm)
+  : internalComm_(mpiComm), level_(0), groupBegin_(), nbInGroup_(), computed_(false)
+{
+
 }
 
+void DivideAdaptiveComm::computeMPICommLevel()
+{
+  if (computed_) return;
+  computed_ = true;
+
+  int mpiSize, mpiRank;
+  MPI_Comm_size(internalComm_,&mpiSize);
+  MPI_Comm_rank(internalComm_,&mpiRank);
+
+  int maxChild=1;
+  int m;
+  do
+  {
+    m=1;
+    ++maxChild;
+    for(int i=0;i<maxChild;++i) m *= maxChild;
+   } while(m<mpiSize);
+
+  int maxLevel=0;
+  for(int size=1; size<=mpiSize; size*=maxChild) ++maxLevel;
+
+  int pos, n, idx;
+  level_=0;
+  int begin=0;
+  int end=mpiSize-1;
+  int nb=end-begin+1;
+
+  nbInGroup_ = groupBegin_= std::vector<int>(maxLevel);
+  nbInGroupParents_ = groupParentsBegin_= std::vector<std::vector<int> >(maxLevel,std::vector<int>(maxChild));
+
+  groupBegin_[level_] = begin;
+  nbInGroup_[level_] = nb;
+  ++level_;
+  while (nb>2 && (level_<maxLevel))
+  {
+    n = 0; idx = 0;
+    pos = begin;
+    for(int i=0;i<maxChild && i<nb;i++)
+    {
+      if (i<nb%maxChild) n = nb/maxChild + 1;
+      else n = nb/maxChild;
+
+      if (mpiRank>=pos && mpiRank<pos+n)
+      {
+        begin=pos;
+        end=pos+n-1;
+      }
+      groupParentsBegin_[level_-1][idx] = pos;
+      nbInGroupParents_[level_-1][idx] = n;
+      ++idx;
+      pos=pos+n;
+    }
+    groupBegin_[level_] = begin;
+    nbInGroup_[level_] = nb = end-begin+1;
+
+    ++level_;
+  }
+
+  for (int i = 0; i < nbInGroup_[level_-1];++i)
+  {
+    groupParentsBegin_[level_-1][i] = groupBegin_[level_-1]+i;
+    nbInGroupParents_[level_-1][i] = 1;
+  }
+
+
+
+//  parent=vector<int>(maxLevel+1);
+//  child=vector<vector<int> >(maxLevel+1,vector<int>(maxChild));
+//  nbChild=vector<int> (maxLevel+1);
+
+//  do
+//  {
+//    n=0;
+//    pos=begin;
+//    nbChild[level_]=0;
+//    parent[level_+1]=begin;
+//    for(int i=0;i<maxChild && i<nb;i++)
+//    {
+//      if (i<nb%maxChild) n = nb/maxChild + 1;
+//      else n = nb/maxChild;
+//
+//      if (mpiRank>=pos && mpiRank<pos+n)
+//      {
+//        begin=pos;
+//        end=pos+n-1;
+//      }
+//      child[level_][i]=pos;
+//      pos=pos+n;
+//      nbChild[level_]++;
+//    }
+//    nb=end-begin+1;
+//    level_=level_+1;
+//  } while (nb>1);
+}
+
+//void DivideAdaptiveComm::divideMPICommLevel(const MPI_Comm& mpiCommLevel, int color, int level)
+//{
+////  int clientRank;
+////  MPI_Comm_rank(mpiCommLevel,&clientRank);
+//
+//   --level;
+//  if (0 < level)
+//  {
+//   int color = clientRank % 2;
+//   commLevel_.push_back(MPI_Comm());
+//   MPI_Comm_split(mpiCommLevel, color, 0, &(commLevel_.back()));
+//   divideMPICommLevel(commLevel_.back(), level);
+//  }
+//}
+
+
+}
+

XIOS/trunk/src/policy.hpp

@@ -28 +28 @@
 };
 
+class DivideAdaptiveComm
+{
+protected:
+  DivideAdaptiveComm(const MPI_Comm& mpiComm);
+
+  void computeMPICommLevel();
+  const std::vector<int>& getGroupBegin() { return groupBegin_; }
+  const std::vector<int>& getNbInGroup() { return nbInGroup_; }
+  const std::vector<std::vector<int> >& getGroupParentsBegin() { return groupParentsBegin_; }
+  const std::vector<std::vector<int> >& getNbInGroupParents() { return nbInGroupParents_; }
+  int getNbLevel() { return level_; }
+
+protected:
+  const MPI_Comm& internalComm_;
+  std::vector<std::vector<int> > groupParentsBegin_;
+  std::vector<std::vector<int> > nbInGroupParents_;
+
+  int level_;
+  std::vector<int> groupBegin_;  //! Rank beginning of a group
+  std::vector<int> nbInGroup_; //! Number of process in each group
+  bool computed_;
+//  std::vector<std::vector<int> > child_; /*!< List of child rank for each level */
+//  std::vector<int> nbChild_;         /*!< Number of child for each level */
+};
+
 }
 

XIOS/trunk/src/transformation/axis_algorithm_interpolate.cpp

@@ -128 +128 @@
 void CAxisAlgorithmInterpolate::computeWeightedValueAndMapping(const std::map<int, std::vector<std::pair<int,double> > >& interpolatingIndexValues, int transPos)
 {
-  std::map<int, std::vector<int> >& transMap = this->transformationMapping_[transPos];
-  std::map<int, std::vector<double> >& transWeight = this->transformationWeight_[transPos];
+  TransformationIndexMap& transMap = this->transformationMapping_[transPos];
+  TransformationWeightMap& transWeight = this->transformationWeight_[transPos];
   std::map<int, std::vector<std::pair<int,double> > >::const_iterator itb = interpolatingIndexValues.begin(), it,
                                                                       ite = interpolatingIndexValues.end();

XIOS/trunk/src/transformation/axis_algorithm_inverse.cpp

@@ -24 +24 @@
            << "Size of axis destionation " <<axisDestination->getId() << " is " << axisDestination->n_glo.getValue());
   }
-
-//  this->computeIndexSourceMapping();
 }
 
@@ -33 +31 @@
   this->transformationWeight_.resize(1);
 
-  std::map<int, std::vector<int> >& transMap = this->transformationMapping_[0];
-  std::map<int, std::vector<double> >& transWeight = this->transformationWeight_[0];
+  TransformationIndexMap& transMap = this->transformationMapping_[0];
+  TransformationWeightMap& transWeight = this->transformationWeight_[0];
 
   int globalIndexSize = axisDestGlobalIndex_.size();
@@ -69 +67 @@
   CTransformationMapping transformationMap(axisDest_, axisSrc_);
 
-  std::map<int, std::vector<int> >& transMap = this->transformationMapping_[0];
-  std::map<int, std::vector<double> >& transWeight = this->transformationWeight_[0];
+  TransformationIndexMap& transMap = this->transformationMapping_[0];
+  TransformationWeightMap& transWeight = this->transformationWeight_[0];
 
   CTransformationMapping::DestinationIndexMap globaIndexMapFromDestToSource;
-  std::map<int, std::vector<int> >::const_iterator it = transMap.begin(), ite = transMap.end();
+  TransformationIndexMap::const_iterator it = transMap.begin(), ite = transMap.end();
   int localIndex = 0;
   for (; it != ite; ++it)

XIOS/trunk/src/transformation/axis_algorithm_zoom.cpp

@@ -47 +47 @@
   this->transformationWeight_.resize(1);
 
-  std::map<int, std::vector<int> >& transMap = this->transformationMapping_[0];
-  std::map<int, std::vector<double> >& transWeight = this->transformationWeight_[0];
+  TransformationIndexMap& transMap = this->transformationMapping_[0];
+  TransformationWeightMap& transWeight = this->transformationWeight_[0];
 
   for (StdSize idx = 0; idx < ni; ++idx)
@@ -79 +79 @@
   StdSize iBeginMask = axisDest_->begin.getValue();
   StdSize globalIndexMask = 0;
-  std::map<int, std::vector<int> >& transMap = this->transformationMapping_[0];
-  std::map<int, std::vector<int> >::const_iterator ite = (transMap).end();
+  TransformationIndexMap& transMap = this->transformationMapping_[0];
+  TransformationIndexMap::const_iterator ite = (transMap).end();
   for (StdSize idx = 0; idx < niMask; ++idx)
   {

XIOS/trunk/src/transformation/domain_algorithm_interpolate.cpp

@@ -23 +23 @@
 {
   interpDomain_->checkValid(domainSource);
-//  computeIndexSourceMapping();
 }
 
@@ -385 +384 @@
   this->transformationWeight_.resize(1);
 
-  std::map<int, std::vector<int> >& transMap = this->transformationMapping_[0];
-  std::map<int, std::vector<double> >& transWeight = this->transformationWeight_[0];
+  TransformationIndexMap& transMap = this->transformationMapping_[0];
+  TransformationWeightMap& transWeight = this->transformationWeight_[0];
 
   boost::unordered_map<size_t,int> globalIndexOfDomainDest;

XIOS/trunk/src/transformation/domain_algorithm_zoom.cpp

@@ -72 +72 @@
   this->transformationWeight_.resize(1);
 
-  std::map<int, std::vector<int> >& transMap = this->transformationMapping_[0];
-  std::map<int, std::vector<double> >& transWeight = this->transformationWeight_[0];
+  TransformationIndexMap& transMap = this->transformationMapping_[0];
+  TransformationWeightMap& transWeight = this->transformationWeight_[0];
 
 //  std::map<int, std::vector<int> >& transMap = this->transformationMapping_;
@@ -117 +117 @@
   int globalIndexMask = 0;
 
-  std::map<int, std::vector<int> >& transMap = this->transformationMapping_[0];
-  std::map<int, std::vector<int> >::const_iterator ite = (transMap).end();
+  TransformationIndexMap& transMap = this->transformationMapping_[0];
+  TransformationIndexMap::const_iterator ite = (transMap).end();
   for (int j = 0; j < njMask; ++j)
   {

XIOS/trunk/src/transformation/generic_algorithm_transformation.cpp

@@ -35 +35 @@
   for (size_t idxTrans = 0; idxTrans < transformationMapping_.size(); ++idxTrans)
   {
-    std::map<int, std::vector<int> >::const_iterator itbTransMap = transformationMapping_[idxTrans].begin(), itTransMap,
+    TransformationIndexMap::const_iterator itbTransMap = transformationMapping_[idxTrans].begin(), itTransMap,
                                                      iteTransMap = transformationMapping_[idxTrans].end();
-    std::map<int, std::vector<double> >::const_iterator itTransWeight = transformationWeight_[idxTrans].begin();
+    TransformationWeightMap::const_iterator itTransWeight = transformationWeight_[idxTrans].begin();
 
     // If transformation position exists
-    std::map<int, std::vector<int> >::const_iterator itTransPos, iteTransPos;
+    TransformationIndexMap::const_iterator itTransPos, iteTransPos;
     if (!isTransPosEmpty)
     {

XIOS/trunk/src/transformation/generic_algorithm_transformation.hpp

@@ -70 +70 @@
 
 protected:
+  typedef boost::unordered_map<int, std::vector<int> > TransformationIndexMap;
+  typedef boost::unordered_map<int, std::vector<double> > TransformationWeightMap;
+  typedef boost::unordered_map<int, std::vector<int> > TransformationPositionMap;
+
   //! Map between global index of destination element and source element
-  std::vector<std::map<int, std::vector<int> > > transformationMapping_;
+  std::vector<TransformationIndexMap> transformationMapping_;
   //! Weight corresponding of source to destination
-  std::vector<std::map<int, std::vector<double> > > transformationWeight_;
+  std::vector<TransformationWeightMap> transformationWeight_;
   //! Map of global index of destination element and corresponding global index of other elements in the same grid
   //! By default, one index of an element corresponds to all index of remaining element in the grid. So it's empty
-  std::vector<std::map<int, std::vector<int> > > transformationPosition_;
+  std::vector<TransformationPositionMap> transformationPosition_;
 
   //! Id of auxillary inputs which help doing transformation dynamically