Ignore:
Timestamp:
08/04/17 16:03:34 (7 years ago)
Author:
mhnguyen
Message:

Making some changes to allow pools with different number of server

+) Associate context client to each grid distribution (This should be changed in the future)
+) Correct some buffer size estimation
+) Clean some redundant code and add comments

File:
1 edited

Legend:

Unmodified
Added
Removed

  • XIOS/dev/XIOS_DEV_CMIP6/src/node/domain.cpp

    r1232 r1236  
    1818#include "server_distribution_description.hpp" 
    1919#include "client_server_mapping_distributed.hpp" 
    20 #include "zoom_domain.hpp" 
    21 #include "interpolate_domain.hpp" 
    22 #include "generate_rectilinear_domain.hpp" 
    2320 
    2421#include <algorithm> 
    … …  
    3027   CDomain::CDomain(void) 
    3128      : CObjectTemplate<CDomain>(), CDomainAttributes() 
    32       , isChecked(false), relFiles(), isClientChecked(false), nbConnectedClients_(), indSrv_(), connectedServerRank_(), connectedServerZoomRank_() 
     29      , isChecked(false), relFiles(), isClientChecked(false), nbSenders(), indSrv_(), connectedServerRank_() 
    3330      , hasBounds(false), hasArea(false), isCompressible_(false), isUnstructed_(false) 
    3431      , isClientAfterTransformationChecked(false), hasLonLat(false) 
    … …  
    4138   CDomain::CDomain(const StdString & id) 
    4239      : CObjectTemplate<CDomain>(id), CDomainAttributes() 
    43       , isChecked(false), relFiles(), isClientChecked(false), nbConnectedClients_(), indSrv_(), connectedServerRank_(), connectedServerZoomRank_() 
     40      , isChecked(false), relFiles(), isClientChecked(false), nbSenders(), indSrv_(), connectedServerRank_()  
    4441      , hasBounds(false), hasArea(false), isCompressible_(false), isUnstructed_(false) 
    4542      , isClientAfterTransformationChecked(false), hasLonLat(false) 
    … …  
    119116    * \return A map associating the server rank with its minimum buffer size. 
    120117    */ 
    121    std::map<int, StdSize> CDomain::getAttributesBufferSize() 
     118   std::map<int, StdSize> CDomain::getAttributesBufferSize(CContextClient* client) 
    122119   { 
    123120     CContext* context = CContext::getCurrent(); 
    124121     // For now the assumption is that secondary server pools consist of the same number of procs. 
    125122     // CHANGE the line below if the assumption changes. 
    126      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client; 
     123     // CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client; 
    127124 
    128125     std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes(); 
    … …  
    141138     } 
    142139 
    143      boost::unordered_map<int, vector<size_t> >::const_iterator itIndexEnd = indSrv_.end(); 
    144      std::map<int, std::vector<int> >::const_iterator itWrittenIndexEnd = indWrittenSrv_.end(); 
    145      for (size_t k = 0; k < connectedServerRank_.size(); ++k) 
    146      { 
    147        int rank = connectedServerRank_[k]; 
    148        boost::unordered_map<int, std::vector<size_t> >::const_iterator it = indSrv_.find(rank); 
     140     boost::unordered_map<int, vector<size_t> >::const_iterator itIndexEnd = indSrv_[client].end(); 
     141     // std::map<int, std::vector<int> >::const_iterator itWrittenIndexEnd = indWrittenSrv_.end(); 
     142     for (size_t k = 0; k < connectedServerRank_[client].size(); ++k) 
     143     { 
     144       int rank = connectedServerRank_[client][k]; 
     145       boost::unordered_map<int, std::vector<size_t> >::const_iterator it = indSrv_[client].find(rank); 
    149146       size_t idxCount = (it != itIndexEnd) ? it->second.size() : 0; 
    150147 
    151148       // size estimation for sendIndex (and sendArea which is always smaller or equal) 
    152149       size_t sizeIndexEvent = 2 * sizeof(size_t) + 2 * CArray<int,1>::size(idxCount); 
    153        if (isCompressible_) 
    154        { 
    155          std::map<int, std::vector<int> >::const_iterator itWritten = indWrittenSrv_.find(rank); 
    156          size_t writtenIdxCount = (itWritten != itWrittenIndexEnd) ? itWritten->second.size() : 0; 
    157          sizeIndexEvent += CArray<int,1>::size(writtenIdxCount); 
    158        } 
     150       // if (isCompressible_) 
     151       // { 
     152       //   std::map<int, std::vector<int> >::const_iterator itWritten = indWrittenSrv_.find(rank); 
     153       //   size_t writtenIdxCount = (itWritten != itWrittenIndexEnd) ? itWritten->second.size() : 0; 
     154       //   sizeIndexEvent += CArray<int,1>::size(writtenIdxCount); 
     155       // } 
    159156 
    160157       // size estimation for sendLonLat 
    … …  
    761758   } 
    762759 
     760   /* 
     761     General check of the domain to verify its mandatory attributes 
     762   */ 
    763763   void CDomain::checkDomain(void) 
    764764   { 
    … …  
    16861686  { 
    16871687    CContext* context=CContext::getCurrent() ; 
    1688  
    1689     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1; 
     1688     
     1689    // This line should be changed soon. 
    16901690    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1; 
     1691 
     1692    nbSenders.clear(); 
     1693    connectedServerRank_.clear(); 
     1694 
    16911695    for (int p = 0; p < nbSrvPools; ++p) 
    16921696    { 
    … …  
    18301834      CClientServerMapping::GlobalIndexMap::const_iterator it  = globalIndexDomainOnServer.begin(), 
    18311835                                                           ite = globalIndexDomainOnServer.end(); 
    1832       indSrv_.swap(globalIndexDomainOnServer); 
    1833       connectedServerRank_.clear(); 
    1834       for (it = indSrv_.begin(); it != ite; ++it)  
    1835         connectedServerRank_.push_back(it->first); 
     1836      indSrv_[client].swap(globalIndexDomainOnServer); 
     1837      connectedServerRank_[client].clear(); 
     1838      for (it = indSrv_[client].begin(); it != ite; ++it)  
     1839        connectedServerRank_[client].push_back(it->first); 
    18361840 
    18371841      for (std::list<int>::const_iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it) 
    1838         connectedServerRank_.push_back(*it); 
     1842        connectedServerRank_[client].push_back(*it); 
    18391843 
    18401844       // Even if a client has no index, it must connect to at least one server and  
    18411845       // send an "empty" data to this server 
    1842        if (connectedServerRank_.empty()) 
    1843         connectedServerRank_.push_back(client->clientRank % client->serverSize); 
    1844  
    1845       nbConnectedClients_ = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_); 
    1846  
    1847      //  clientServerMap->computeServerIndexMapping(globalIndexDomainZoom); 
    1848      //  CClientServerMapping::GlobalIndexMap& globalIndexDomainZoomOnServer = clientServerMap->getGlobalIndexOnServer(); 
    1849      //  indZoomSrv_.swap(globalIndexDomainZoomOnServer); 
    1850        
    1851      // for (it = indZoomSrv_.begin(); it != indZoomSrv_.end(); ++it) 
    1852      //   connectedServerZoomRank_.push_back(it->first); 
    1853  
    1854      //  nbConnectedClientsZoom_ = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerZoomRank_); 
     1846       if (connectedServerRank_[client].empty()) 
     1847        connectedServerRank_[client].push_back(client->clientRank % client->serverSize); 
     1848 
     1849      nbSenders[client] = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[client]); 
    18551850 
    18561851      delete clientServerMap; 
    … …  
    19601955  { 
    19611956    sendDistributionAttributes(); 
    1962     sendIndex();    
    1963     // sendIndexZoom();  
     1957    sendIndex();        
    19641958    sendMask(); 
    19651959    sendLonLat(); 
    … …  
    19891983 
    19901984      boost::unordered_map<int, vector<size_t> >::const_iterator itIndex, iteIndex; 
    1991       iteIndex = indSrv_.end(); 
    1992       for (int k = 0; k < connectedServerRank_.size(); ++k) 
     1985      iteIndex = indSrv_[client].end(); 
     1986      for (int k = 0; k < connectedServerRank_[client].size(); ++k) 
    19931987      { 
    19941988        int nbIndGlob = 0; 
    1995         int rank = connectedServerRank_[k]; 
    1996         itIndex = indSrv_.find(rank); 
     1989        int rank = connectedServerRank_[client][k]; 
     1990        itIndex = indSrv_[client].find(rank); 
    19971991        if (iteIndex != itIndex) 
    19981992          nbIndGlob = itIndex->second.size(); 
    … …  
    20112005        list_msgsIndex.back() << list_indGlob.back(); //list_indi.back() << list_indj.back(); 
    20122006        
    2013         eventIndex.push(rank, nbConnectedClients_[rank], list_msgsIndex.back()); 
     2007        eventIndex.push(rank, nbSenders[client][rank], list_msgsIndex.back()); 
    20142008      } 
    20152009 
    … …  
    20172011    } 
    20182012  } 
    2019  
    2020   /*! 
    2021     Send global index and zoom index from client to connected client(s) 
    2022     zoom index can be smaller than global index. 
    2023     This function can be used in the future??? 
    2024   */ 
    2025   // void CDomain::sendIndexZoom() 
    2026   // { 
    2027   //   int ns, n, i, j, ind, nv, idx; 
    2028   //   CContext* context = CContext::getCurrent(); 
    2029  
    2030   //   // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1; 
    2031   //   int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1; 
    2032   //   for (int p = 0; p < nbSrvPools; ++p) 
    2033   //   { 
    2034   //     CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client; 
    2035   //     CEventClient eventIndexZoom(getType(), EVENT_ID_INDEX_ZOOM); 
    2036  
    2037   //     list<CMessage> list_msgsIndex; 
    2038   //     list<CArray<int,1> > list_indZoom; 
    2039  
    2040   //     boost::unordered_map<int, vector<size_t> >::const_iterator itZoom, iteZoom; 
    2041   //     iteZoom = indZoomSrv_.end(); 
    2042   //     for (int k = 0; k < connectedServerZoomRank_.size(); ++k) 
    2043   //     { 
    2044   //       int nbIndGlob = 0; 
    2045   //       int rank = connectedServerZoomRank_[k]; 
    2046   //       int nbIndZoom = 0; 
    2047   //       itZoom = indZoomSrv_.find(rank); 
    2048   //       if (iteZoom != itZoom) 
    2049   //         nbIndZoom = itZoom->second.size(); 
    2050          
    2051   //       list_indZoom.push_back(CArray<int,1>(nbIndZoom)); 
    2052   //       CArray<int,1>& indZoom = list_indZoom.back(); 
    2053   //       for (n = 0; n < nbIndZoom; ++n) 
    2054   //       { 
    2055   //         indZoom(n) = static_cast<int>(itZoom->second[n]); 
    2056   //       } 
    2057  
    2058   //       list_msgsIndex.push_back(CMessage()); 
    2059   //       list_msgsIndex.back() << this->getId(); // enum ne fonctionne pour les message => ToFix         
    2060   //       list_msgsIndex.back() << list_indZoom.back() << doZoomByIndex_; //list_indi.back() << list_indj.back       
    2061  
    2062   //       eventIndexZoom.push(rank, nbConnectedClientsZoom_[rank], list_msgsIndex.back()); 
    2063   //     } 
    2064  
    2065   //     client->sendEvent(eventIndexZoom); 
    2066   //   } 
    2067   // } 
    20682013 
    20692014  /*! 
    … …  
    21442089 
    21452090      boost::unordered_map<int, vector<size_t> >::const_iterator it, iteMap; 
    2146       iteMap = indSrv_.end(); 
    2147       for (int k = 0; k < connectedServerRank_.size(); ++k) 
     2091      iteMap = indSrv_[client].end(); 
     2092      for (int k = 0; k < connectedServerRank_[client].size(); ++k) 
    21482093      { 
    21492094        int nbData = 0; 
    2150         int rank = connectedServerRank_[k]; 
    2151         it = indSrv_.find(rank); 
     2095        int rank = connectedServerRank_[client][k]; 
     2096        it = indSrv_[client].find(rank); 
    21522097        if (iteMap != it) 
    21532098          nbData = it->second.size(); 
    … …  
    21632108        list_msgsMask.push_back(CMessage()); 
    21642109        list_msgsMask.back() << this->getId() << list_mask.back(); 
    2165         eventMask.push(rank, nbConnectedClients_[rank], list_msgsMask.back()); 
     2110        eventMask.push(rank, nbSenders[client][rank], list_msgsMask.back()); 
    21662111      } 
    21672112      client->sendEvent(eventMask); 
    … …  
    21922137 
    21932138      boost::unordered_map<int, vector<size_t> >::const_iterator it, iteMap; 
    2194       iteMap = indSrv_.end(); 
    2195       for (int k = 0; k < connectedServerRank_.size(); ++k) 
     2139      iteMap = indSrv_[client].end(); 
     2140      for (int k = 0; k < connectedServerRank_[client].size(); ++k) 
    21962141      { 
    21972142        int nbData = 0; 
    2198         int rank = connectedServerRank_[k]; 
    2199         it = indSrv_.find(rank); 
     2143        int rank = connectedServerRank_[client][k]; 
     2144        it = indSrv_[client].find(rank); 
    22002145        if (iteMap != it) 
    22012146          nbData = it->second.size(); 
    … …  
    22122157        list_msgsArea.back() << this->getId() << hasArea; 
    22132158        list_msgsArea.back() << list_area.back(); 
    2214         eventArea.push(rank, nbConnectedClients_[rank], list_msgsArea.back()); 
     2159        eventArea.push(rank, nbSenders[client][rank], list_msgsArea.back()); 
    22152160      } 
    22162161      client->sendEvent(eventArea); 
    … …  
    22452190 
    22462191      boost::unordered_map<int, vector<size_t> >::const_iterator it, iteMap; 
    2247       iteMap = indSrv_.end(); 
    2248       for (int k = 0; k < connectedServerRank_.size(); ++k) 
     2192      iteMap = indSrv_[client].end(); 
     2193      for (int k = 0; k < connectedServerRank_[client].size(); ++k) 
    22492194      { 
    22502195        int nbData = 0; 
    2251         int rank = connectedServerRank_[k]; 
    2252         it = indSrv_.find(rank); 
     2196        int rank = connectedServerRank_[client][k]; 
     2197        it = indSrv_[client].find(rank); 
    22532198        if (iteMap != it) 
    22542199          nbData = it->second.size(); 
    … …  
    23072252        } 
    23082253 
    2309         eventLon.push(rank, nbConnectedClients_[rank], list_msgsLon.back()); 
    2310         eventLat.push(rank, nbConnectedClients_[rank], list_msgsLat.back()); 
     2254        eventLon.push(rank, nbSenders[client][rank], list_msgsLon.back()); 
     2255        eventLat.push(rank, nbSenders[client][rank], list_msgsLat.back()); 
    23112256      } 
    23122257      client->sendEvent(eventLon); 
    … …  
    23632308 
    23642309      boost::unordered_map<int, vector<size_t> >::const_iterator it, iteMap; 
    2365       iteMap = indSrv_.end(); 
    2366       for (int k = 0; k < connectedServerRank_.size(); ++k) 
     2310      iteMap = indSrv_[client].end(); 
     2311      for (int k = 0; k < connectedServerRank_[client].size(); ++k) 
    23672312      { 
    23682313        int nbData = 0; 
    2369         int rank = connectedServerRank_[k]; 
    2370         it = indSrv_.find(rank); 
     2314        int rank = connectedServerRank_[client][k]; 
     2315        it = indSrv_[client].find(rank); 
    23712316        if (iteMap != it) 
    23722317          nbData = it->second.size(); 
    … …  
    23862331        list_msgsDataIndex.back() << this->getId(); 
    23872332        list_msgsDataIndex.back() << list_data_i_index.back() << list_data_j_index.back(); 
    2388         eventDataIndex.push(rank, nbConnectedClients_[rank], list_msgsDataIndex.back()); 
     2333        eventDataIndex.push(rank, nbSenders[client][rank], list_msgsDataIndex.back()); 
    23892334      } 
    23902335      client->sendEvent(eventDataIndex); 
    … …  
    24052350        case EVENT_ID_INDEX: 
    24062351          recvIndex(event); 
    2407           return true; 
    2408           break; 
    2409         case EVENT_ID_INDEX_ZOOM: 
    2410           recvIndexZoom(event); 
    24112352          return true; 
    24122353          break; 
    … …  
    25082449 
    25092450  /*! 
    2510     Receive index event from clients(s) 
    2511     \param[in] event event contain info about rank and associated index 
    2512   */ 
    2513   void CDomain::recvIndexZoom(CEventServer& event) 
    2514   { 
    2515     string domainId; 
    2516     std::map<int, CBufferIn*> rankBuffers; 
    2517  
    2518     list<CEventServer::SSubEvent>::iterator it; 
    2519     for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it) 
    2520     {       
    2521       CBufferIn* buffer = it->buffer; 
    2522       *buffer >> domainId; 
    2523       rankBuffers[it->rank] = buffer;         
    2524     } 
    2525     get(domainId)->recvIndexZoom(rankBuffers); 
    2526   } 
    2527  
    2528   /*! 
    2529     Receive index information from client(s) 
    2530     \param[in] rankBuffers rank of sending client and the corresponding receive buffer   
    2531   */ 
    2532   void CDomain::recvIndexZoom(std::map<int, CBufferIn*>& rankBuffers) 
    2533   { 
    2534     int nbReceived = rankBuffers.size(), i, ind, index, type_int; 
    2535     recvClientZoomRanks_.resize(nbReceived);     
    2536     int ni_zoom_tmp, ibegin_zoom_tmp, nj_zoom_tmp, jbegin_zoom_tmp; 
    2537  
    2538     std::map<int, CBufferIn*>::iterator it = rankBuffers.begin(), ite = rankBuffers.end(); 
    2539     ind = 0; 
    2540     for (ind = 0; it != ite; ++it, ++ind) 
    2541     {        
    2542        recvClientZoomRanks_[ind] = it->first; 
    2543        CBufferIn& buffer = *(it->second); 
    2544        buffer >> indGlobZoom_[it->first] >> doZoomByIndex_;         
    2545     } 
    2546  
    2547     int nbZoomInd = 0; 
    2548     for (i = 0; i < nbReceived; ++i) 
    2549     { 
    2550       nbZoomInd += indGlobZoom_[recvClientZoomRanks_[i]].numElements(); 
    2551     } 
    2552  
    2553     if (doZoomByIndex_) 
    2554     { 
    2555       zoom_i_index.resize(nbZoomInd); 
    2556       zoom_j_index.resize(nbZoomInd); 
    2557        
    2558       nbZoomInd = 0; 
    2559       for (i = 0; i < nbReceived; ++i) 
    2560       { 
    2561         CArray<int,1>& tmp = indGlobZoom_[recvClientRanks_[i]]; 
    2562         for (ind = 0; ind < tmp.numElements(); ++ind) 
    2563         { 
    2564            index = tmp(ind); 
    2565            zoom_i_index(nbZoomInd) = index % ni_glo; 
    2566            zoom_j_index(nbZoomInd) = index / ni_glo; 
    2567            ++nbZoomInd; 
    2568         }  
    2569       }       
    2570     } 
    2571     else  
    2572     { 
    2573     } 
    2574   } 
    2575  
    2576   /*! 
    25772451    Receive attributes event from clients(s) 
    25782452    \param[in] event event contain info about rank and associated attributes 
Note: See TracChangeset for help on using the changeset viewer.