Ignore:
Timestamp:
06/09/16 11:33:08 (5 years ago)
Author:
mhnguyen
Message:

Optimizing codes: Change the way to compute distribution of grid

+) Instead of using DHT on grid, we make use of it only for elements of grid
+) Make change to calculation of server distribution.

Test
+) On Curie
+) Two times faster than the precedent commit.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • XIOS/trunk/src/node/grid.cpp

    r857 r865  
    3232      , transformations_(0), isTransformed_(false) 
    3333      , axisPositionInGrid_(), positionDimensionDistributed_(1), hasDomainAxisBaseRef_(false) 
    34       , gridSrc_(), hasTransform_(false), order_() 
     34      , gridSrc_(), hasTransform_(false), order_(), globalIndexOnServer_() 
    3535   { 
    3636     setVirtualDomainGroup(); 
     
    4747      , transformations_(0), isTransformed_(false) 
    4848      , axisPositionInGrid_(), positionDimensionDistributed_(1), hasDomainAxisBaseRef_(false) 
    49       , gridSrc_(), hasTransform_(false), order_() 
     49      , gridSrc_(), hasTransform_(false), order_(), globalIndexOnServer_() 
    5050   { 
    5151     setVirtualDomainGroup(); 
     
    409409   //--------------------------------------------------------------- 
    410410 
     411   /*! 
     412     Compute the global index of grid to send to server as well as the connected server of the current client. 
     413     First of all, from the local data on each element of grid, we can calculate their local index which also allows us to know 
     414     their global index. We can have a map of global index of grid and local index that each client holds 
     415     Then, each client holds a piece of information about the distribution of servers, which permits to compute the connected server(s) 
     416     of the current client. 
     417   */ 
    411418   void CGrid::computeIndex(void) 
    412419   { 
     
    434441 
    435442     // Compute mapping between client and server 
    436      size_t globalSizeIndex = 1, indexBegin, indexEnd; 
    437      int range, clientSize = client->clientSize; 
    438      for (int i = 0; i < globalDim_.size(); ++i) globalSizeIndex *= globalDim_[i]; 
    439      indexBegin = 0; 
    440      for (int i = 0; i < clientSize; ++i) 
    441      { 
    442        range = globalSizeIndex / clientSize; 
    443        if (i < (globalSizeIndex%clientSize)) ++range; 
    444        if (i == client->clientRank) break; 
    445        indexBegin += range; 
    446      } 
    447      indexEnd = indexBegin + range - 1; 
    448  
    449      // Then compute distribution on server side 
     443     std::vector<boost::unordered_map<size_t,std::vector<int> > > indexServerOnElement; 
    450444     CServerDistributionDescription serverDistributionDescription(globalDim_, client->serverSize); 
    451      serverDistributionDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), positionDimensionDistributed_); 
    452  
    453      // Finally, compute index mapping between client(s) and server(s) 
    454      clientServerMap_ = new CClientServerMappingDistributed(serverDistributionDescription.getGlobalIndexRange(), 
    455                                                             client->intraComm, 
    456                                                             clientDistribution_->isDataDistributed()); 
    457  
    458      clientServerMap_->computeServerIndexMapping(clientDistribution_->getGlobalIndex()); 
    459      const CClientServerMapping::GlobalIndexMap& globalIndexOnServer = clientServerMap_->getGlobalIndexOnServer(); 
     445     serverDistributionDescription.computeServerGlobalByElement(indexServerOnElement, 
     446                                                                client->clientRank, 
     447                                                                client->clientSize, 
     448                                                                axis_domain_order, 
     449                                                                positionDimensionDistributed_); 
     450     computeIndexByElement(indexServerOnElement, globalIndexOnServer_); 
    460451 
    461452     const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer(); 
    462453     CDistributionClient::GlobalLocalDataMap::const_iterator iteGlobalLocalIndexMap = globalLocalIndexSendToServer.end(), itGlobalLocalIndexMap; 
    463454     CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itbGlobalMap, itGlobalMap; 
    464      itGlobalMap  = itbGlobalMap = globalIndexOnServer.begin(); 
    465      iteGlobalMap = globalIndexOnServer.end(); 
     455     itGlobalMap  = itbGlobalMap = globalIndexOnServer_.begin(); 
     456     iteGlobalMap = globalIndexOnServer_.end(); 
    466457 
    467458     for (; itGlobalMap != iteGlobalMap; ++itGlobalMap) 
     
    491482   } 
    492483 
     484   /*! 
     485      Compute the global of (client) grid to send to server with the global index of each element of grid 
     486      Each element of grid has its own global index associated to a groups of server. We only search for the global index of each element whose 
     487      server is the same, then calculate the global index of grid. This way can reduce so much the time for executing DHT, which only needs to run 
     488      on each element whose size is much smaller than one of whole grid. 
     489      \param [in] indexServerOnElement global index of each element and the rank of server associated with these index 
     490      \param [out] globalIndexOnServer global index of grid and its corresponding rank of server. 
     491   */ 
     492   void CGrid::computeIndexByElement(const std::vector<boost::unordered_map<size_t,std::vector<int> > >& indexServerOnElement, 
     493                                     CClientServerMapping::GlobalIndexMap& globalIndexOnServer) 
     494   { 
     495     CContext* context = CContext::getCurrent(); 
     496     CContextClient* client = context->client; 
     497     int serverSize = client->serverSize; 
     498     std::vector<CDomain*> domList = getDomains(); 
     499     std::vector<CAxis*> axisList = getAxis(); 
     500 
     501     // Some pre-calculations of global index on each element of current grid. 
     502     int nbElement = axis_domain_order.numElements(); 
     503     std::vector<CArray<size_t,1> > globalIndexElement(nbElement); 
     504     int domainIdx = 0, axisIdx = 0; 
     505     std::vector<size_t> elementNGlobal(nbElement); 
     506     elementNGlobal[0] = 1; 
     507     size_t globalSize = 1; 
     508     for (int idx = 0; idx < nbElement; ++idx) 
     509     { 
     510       elementNGlobal[idx] = globalSize; 
     511       size_t elementSize; 
     512       size_t elementGlobalSize = 1; 
     513       if (axis_domain_order(idx)) 
     514       { 
     515         elementSize = domList[domainIdx]->i_index.numElements(); 
     516         globalIndexElement[idx].resize(elementSize); 
     517         for (int jdx = 0; jdx < elementSize; ++jdx) 
     518         { 
     519           globalIndexElement[idx](jdx) = (domList[domainIdx]->i_index)(jdx) + domList[domainIdx]->ni_glo * (domList[domainIdx]->j_index)(jdx); 
     520         } 
     521         elementGlobalSize = domList[domainIdx]->ni_glo.getValue() * domList[domainIdx]->nj_glo.getValue(); 
     522         ++domainIdx; 
     523       } 
     524       else 
     525       { 
     526         elementSize = axisList[axisIdx]->index.numElements(); 
     527         globalIndexElement[idx].resize(elementSize); 
     528         for (int jdx = 0; jdx < elementSize; ++jdx) 
     529         { 
     530           globalIndexElement[idx](jdx) = (axisList[axisIdx]->index)(jdx); 
     531         } 
     532         elementGlobalSize = axisList[axisIdx]->n_glo.getValue(); 
     533         ++axisIdx; 
     534       } 
     535       globalSize *= elementGlobalSize; 
     536     } 
     537 
     538     std::vector<std::vector<bool> > elementOnServer(nbElement, std::vector<bool>(serverSize, false)); 
     539     std::vector<boost::unordered_map<int,std::vector<size_t> > > globalElementIndexOnServer(nbElement); 
     540     for (int idx = 0; idx < nbElement; ++idx) 
     541     { 
     542       std::vector<int> nbIndexOnServer(serverSize,0); 
     543       const boost::unordered_map<size_t,std::vector<int> >& indexServerElement = indexServerOnElement[idx]; 
     544       const CArray<size_t,1>& globalIndexElementOnClient = globalIndexElement[idx]; 
     545       CClientClientDHTInt clientClientDHT(indexServerElement, client->intraComm); 
     546       clientClientDHT.computeIndexInfoMapping(globalIndexElementOnClient); 
     547       const CClientClientDHTInt::Index2VectorInfoTypeMap& globalIndexElementOnServerMap = clientClientDHT.getInfoIndexMap(); 
     548       CClientClientDHTInt::Index2VectorInfoTypeMap::const_iterator itb = globalIndexElementOnServerMap.begin(), 
     549                                                                    ite = globalIndexElementOnServerMap.end(), it; 
     550       for (it = itb; it != ite; ++it) 
     551       { 
     552         const std::vector<int>& tmp = it->second; 
     553         for (int i = 0; i < tmp.size(); ++i) 
     554         { 
     555           ++nbIndexOnServer[tmp[i]]; 
     556         } 
     557       } 
     558 
     559       for (int i = 0; i < serverSize; ++i) 
     560       { 
     561         if (0 != nbIndexOnServer[i]) 
     562         { 
     563           globalElementIndexOnServer[idx][i].resize(nbIndexOnServer[i]); 
     564           nbIndexOnServer[i] = 0; 
     565           elementOnServer[idx][i] = true; 
     566         } 
     567       } 
     568 
     569       for (it = itb; it != ite; ++it) 
     570       { 
     571         const std::vector<int>& tmp = it->second; 
     572         for (int i = 0; i < tmp.size(); ++i) 
     573         { 
     574           globalElementIndexOnServer[idx][tmp[i]][nbIndexOnServer[tmp[i]]] = it->first; 
     575           ++nbIndexOnServer[tmp[i]]; 
     576         } 
     577       } 
     578     } 
     579 
     580    // Determine server which contain global source index 
     581    std::vector<bool> intersectedProc(serverSize, true); 
     582    for (int idx = 0; idx < nbElement; ++idx) 
     583    { 
     584      std::transform(elementOnServer[idx].begin(), elementOnServer[idx].end(), 
     585                     intersectedProc.begin(), intersectedProc.begin(), 
     586                     std::logical_and<bool>()); 
     587    } 
     588 
     589    std::vector<int> srcRank; 
     590    for (int idx = 0; idx < serverSize; ++idx) 
     591    { 
     592      if (intersectedProc[idx]) srcRank.push_back(idx); 
     593    } 
     594 
     595    // Compute the global index of grid from global index of each element. 
     596    for (int i = 0; i < srcRank.size(); ++i) 
     597    { 
     598      size_t ssize = 1; 
     599      int rankSrc = srcRank[i]; 
     600      std::vector<std::vector<size_t>* > globalIndexOfElementTmp(nbElement); 
     601      std::vector<size_t> currentIndex(nbElement,0); 
     602      for (int idx = 0; idx < nbElement; ++idx) 
     603      { 
     604        ssize *= (globalElementIndexOnServer[idx][rankSrc]).size(); 
     605        globalIndexOfElementTmp[idx] = &(globalElementIndexOnServer[idx][rankSrc]); 
     606      } 
     607      globalIndexOnServer[rankSrc].resize(ssize); 
     608 
     609      std::vector<int> idxLoop(nbElement,0); 
     610      int innnerLoopSize = (globalIndexOfElementTmp[0])->size(); 
     611      size_t idx = 0; 
     612      while (idx < ssize) 
     613      { 
     614        for (int ind = 0; ind < nbElement; ++ind) 
     615        { 
     616          if (idxLoop[ind] == (globalIndexOfElementTmp[ind])->size()) 
     617          { 
     618            idxLoop[ind] = 0; 
     619            ++idxLoop[ind+1]; 
     620          } 
     621 
     622          currentIndex[ind] = (*(globalIndexOfElementTmp[ind]))[idxLoop[ind]]; 
     623        } 
     624 
     625        for (int ind = 0; ind < innnerLoopSize; ++ind) 
     626        { 
     627          currentIndex[0] = (*globalIndexOfElementTmp[0])[ind]; 
     628          size_t globalSrcIndex = 0; 
     629          for (int idxElement = 0; idxElement < nbElement; ++idxElement) 
     630          { 
     631            globalSrcIndex += currentIndex[idxElement] * elementNGlobal[idxElement]; 
     632          } 
     633          globalIndexOnServer[rankSrc][idx] = globalSrcIndex; 
     634          ++idx; 
     635          ++idxLoop[0]; 
     636        } 
     637      } 
     638    } 
     639   } 
    493640   //---------------------------------------------------------------- 
    494641 
     
    783930    list<CMessage> listMsg; 
    784931    list<CArray<size_t,1> > listOutIndex; 
    785     const CClientServerMapping::GlobalIndexMap& globalIndexOnServer = clientServerMap_->getGlobalIndexOnServer(); 
     932//    const CClientServerMapping::GlobalIndexMap& globalIndexOnServer = clientServerMap_->getGlobalIndexOnServer(); 
     933//    const CClientServerMapping::GlobalIndexMap& globalIndexOnServer = clientServerMap_->getGlobalIndexOnServer(); 
    786934    const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer(); 
    787935    CDistributionClient::GlobalLocalDataMap::const_iterator itIndex = globalLocalIndexSendToServer.begin(), 
     
    820968    { 
    821969      CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itGlobalMap; 
    822       itGlobalMap = globalIndexOnServer.begin(); 
    823       iteGlobalMap = globalIndexOnServer.end(); 
     970      itGlobalMap = globalIndexOnServer_.begin(); 
     971      iteGlobalMap = globalIndexOnServer_.end(); 
    824972 
    825973      std::map<int,std::vector<int> >localIndexTmp; 
Note: See TracChangeset for help on using the changeset viewer.