Changeset 1243


Ignore:
Timestamp:
08/28/17 12:15:49 (3 years ago)
Author:
oabramkina
Message:

Modifications allowing secondary-server pools of different size. It's controlled by a new parameter number_pools_server2.
Tests: complete, toy, ipsl.

This functionality is to be used in future. For now it is disabled and XIOS works as before with one process assigned per pool.

Location:
XIOS/dev/XIOS_DEV_CMIP6/src
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • XIOS/dev/XIOS_DEV_CMIP6/src/client.cpp

    r1233 r1243  
    9696          } 
    9797 
    98 //          myColor=colors[hashClient]; 
    99           myColor=leaders[hashClient] ; 
     98          myColor=colors[hashClient]; 
    10099          MPI_Comm_split(CXios::globalComm,myColor,rank_,&intraComm) ; 
    101100 
     
    275274 
    276275    /*! 
    277     * Return rank in model intraComm 
     276    * Return global rank without oasis and current rank in model intraComm in case of oasis 
    278277    */ 
    279278   int CClient::getRank() 
  • XIOS/dev/XIOS_DEV_CMIP6/src/client.hpp

    r1158 r1243  
    2222        static MPI_Comm& getInterComm(); 
    2323 
    24         //! Get rank of the current process 
     24        //! Get global rank without oasis and current rank in model intraComm in case of oasis 
    2525        static int getRank(); 
    2626 
  • XIOS/dev/XIOS_DEV_CMIP6/src/cxios.cpp

    r1234 r1243  
    2828  bool CXios::usingServer2 = false; 
    2929  int CXios::ratioServer2 = 50; 
     30  int CXios::nbPoolsServer2 = 1; 
    3031  double CXios::bufferSizeFactor = 1.0; 
    3132  const double CXios::defaultBufferSizeFactor = 1.0; 
     
    5556    usingServer2=getin<bool>("using_server2",false) ; 
    5657    ratioServer2=getin<int>("ratio_server2",50); 
     58    nbPoolsServer2=getin<int>("number_pools_server2",1); 
    5759    info.setLevel(getin<int>("info_level",0)) ; 
    5860    report.setLevel(getin<int>("info_level",50)); 
     
    197199    else 
    198200    { 
    199       // If using two server levels, first merge globalRegistry of each server pool into one registry on the first pool 
     201      // If using two server levels: 
     202      // (1) merge registries on each pool 
     203      // (2) send merged registries to the first pool 
     204      // (3) merge received registries on the first pool 
    200205      if (CServer::serverLevel == 2) 
    201206      { 
     
    205210        MPI_Comm_rank(globalComm, &rankGlobal); 
    206211 
     212        // Merge registries defined on each pools 
     213        CRegistry globalRegistrySndServers (CServer::intraComm); 
     214 
    207215        // All pools (except the first): send globalRegistry to the first pool 
    208         if (rankGlobal != firstPoolGlobalRank) 
     216        for (int i=1; i<secondaryServerGlobalRanks.size(); i++) 
    209217        { 
    210           int registrySize = globalRegistry->size(); 
    211           MPI_Send(&registrySize,1,MPI_LONG,firstPoolGlobalRank,15,CXios::globalComm) ; 
    212           CBufferOut buffer(registrySize) ; 
    213           globalRegistry->toBuffer(buffer) ; 
    214           MPI_Send(buffer.start(),registrySize,MPI_CHAR,firstPoolGlobalRank,15,CXios::globalComm) ; 
     218          if (rankGlobal == secondaryServerGlobalRanks[i]) 
     219          { 
     220            globalRegistrySndServers.mergeRegistry(*globalRegistry) ; 
     221            int registrySize = globalRegistrySndServers.size(); 
     222            MPI_Send(&registrySize,1,MPI_LONG,firstPoolGlobalRank,15,CXios::globalComm) ; 
     223            CBufferOut buffer(registrySize) ; 
     224            globalRegistrySndServers.toBuffer(buffer) ; 
     225            MPI_Send(buffer.start(),registrySize,MPI_CHAR,firstPoolGlobalRank,15,CXios::globalComm) ; 
     226          } 
    215227        } 
    216228 
    217229        // First pool: receive globalRegistry of all secondary server pools, merge and write the resultant registry 
    218         else 
     230        if (rankGlobal == firstPoolGlobalRank) 
    219231        { 
    220232          MPI_Status status; 
    221233          char* recvBuff; 
    222234 
    223           CRegistry globalRegistrySndServers (CServer::intraComm); 
    224235          globalRegistrySndServers.mergeRegistry(*globalRegistry) ; 
    225236 
  • XIOS/dev/XIOS_DEV_CMIP6/src/cxios.hpp

    r1234 r1243  
    4040 
    4141     static bool printLogs2Files; //!< Printing out logs into files 
    42      static bool usingOasis ; //!< Using Oasis 
    43      static bool usingServer ; //!< Using server (server mode) 
    44      static bool usingServer2 ; //!< Using secondary server (server mode). IMPORTANT: Use this variable ONLY in CServer::initialize(). 
    45      static int ratioServer2 ;  //!< Percentage of server processors dedicated to secondary server 
     42     static bool usingOasis ;     //!< Using Oasis 
     43     static bool usingServer ;    //!< Using server (server mode) 
     44     static bool usingServer2 ;   //!< Using secondary server (server mode). IMPORTANT: Use this variable ONLY in CServer::initialize(). 
     45     static int ratioServer2 ;    //!< Percentage of server processors dedicated to secondary server 
     46     static int nbPoolsServer2 ;  //!< Number of pools created on the secondary server 
    4647     static double bufferSizeFactor; //!< Factor used to tune the buffer size 
    4748     static const double defaultBufferSizeFactor; //!< Default factor value 
  • XIOS/dev/XIOS_DEV_CMIP6/src/node/grid.hpp

    r1236 r1243  
    203203         CArray<int, 1> storeIndex_client; 
    204204 
    205          std::map<CContextClient*, map<int, CArray<int, 1> > > storeIndex_toSrv; // Same grid but can be sent to several pools 
     205/** Map containing indexes that will be sent in sendIndex(). In future: change the key to pair<distrType, serverSize> (?) */ 
     206         std::map<CContextClient*, map<int, CArray<int, 1> > > storeIndex_toSrv; 
     207 
    206208         map<int, CArray<int, 1> > storeIndex_fromSrv; // Support, for now, reading with level-1 server 
     209 
     210 
    207211         std::map<CContextClient*, std::map<int,int> > nbSenders, nbReadSenders; 
    208212 
     
    215219         map<int, CArray<size_t, 1> > outLocalIndexStoreOnClient;  
    216220 
    217          // If a client wants to write out data, it'll use this index. 
    218          // A client receives global index of data from other clients (via recvIndex), 
    219          // then does remapping these index into local index to WRITE into a file 
    220          CArray<size_t,1> localIndexToWriteOnClient, localIndexToWriteOnServer; 
     221/** Indexes calculated based on server distribution (serverDistribution_). They are used for writing data into a file. */ 
     222         CArray<size_t,1> localIndexToWriteOnServer; 
     223 
     224/** Indexes calculated based on client distribution (clientDistribution_). They are not used at all. 
     225    They should be the same as localIndexToWriteOnServer and potentially can be used as an additional check.*/ 
     226         CArray<size_t,1> localIndexToWriteOnClient; 
    221227 
    222228         CArray<size_t,1> indexFromClients; 
     
    286292        bool isAxisListSet, isDomListSet, isScalarListSet; 
    287293 
     294/** Distribution calculated in computeClientIndex() based on the knowledge of the entire grid */ 
    288295        CDistributionClient* clientDistribution_; 
     296 
     297/** Distribution calculated upon receiving indexes */ 
    289298        CDistributionServer* serverDistribution_; 
     299 
    290300        CClientServerMapping* clientServerMap_; 
    291301        size_t writtenDataSize_; 
    292302        int numberWrittenIndexes_, totalNumberWrittenIndexes_, offsetWrittenIndexes_; 
     303 
     304/** Map storing ranks of connected servers. In future: change the key to the server size (?) */ 
     305        std::map<CContextClient*, std::vector<int> > connectedServerRank_; 
     306 
     307/** Map storing data size that will be sent to connected servers. In future: change the key to the server size (?) */ 
    293308        std::map<CContextClient*, std::map<int,size_t> > connectedDataSize_; 
    294         std::map<CContextClient*, std::vector<int> > connectedServerRank_; 
     309 
    295310        bool isDataDistributed_;         
    296311         //! True if and only if the data defined on the grid can be outputted in a compressed way 
     
    305320        std::map<CGrid*, std::pair<bool,StdString> > gridSrc_; 
    306321        bool hasTransform_; 
     322 
     323/** Map storing data size that will be sent to connected servers. In future: change the key to the server size (?) */ 
    307324        std::map<CContextClient*, CClientServerMapping::GlobalIndexMap> globalIndexOnServer_; 
    308             // List order of axis and domain in a grid, if there is a domain, it will take value 1 (true), axis 0 (false) 
     325 
     326/** List order of axis and domain in a grid, if there is a domain, it will take value 1 (true), axis 0 (false) */ 
    309327        std::vector<int> order_; 
     328 
    310329   }; // class CGrid 
    311330 
  • XIOS/dev/XIOS_DEV_CMIP6/src/server.cpp

    r1234 r1243  
    110110          else 
    111111          { 
     112            int firstSndSrvRank = srvRanks.size()*(100.-CXios::ratioServer2)/100. ; 
     113            int poolLeader = firstSndSrvRank; 
     114//*********** (1) Comment out the line below to set one process per pool 
     115//            sndServerGlobalRanks.push_back(srvRanks[poolLeader]); 
     116            int nbPools = CXios::nbPoolsServer2; 
     117            if ( nbPools > reqNbProc || nbPools < 1) 
     118            { 
     119              error(0)<<"WARNING: void CServer::initialize(void)"<<endl 
     120                  << "It is impossible to allocate the requested number of pools = "<<nbPools 
     121                  <<" on the secondary server. It will be set so that there is one process per pool."<<endl; 
     122              nbPools = reqNbProc; 
     123            } 
     124            int remainder = ((int) (srvRanks.size()*CXios::ratioServer2/100.)) % nbPools; 
     125            int procsPerPool = ((int) (srvRanks.size()*CXios::ratioServer2/100.)) / nbPools; 
    112126            for (i=0; i<srvRanks.size(); i++) 
    113127            { 
    114               if (i >= srvRanks.size()*(100.-CXios::ratioServer2)/100.) 
     128              if (i >= firstSndSrvRank) 
    115129              { 
     130                if (rank_ == srvRanks[i]) 
     131                { 
     132                  serverLevel=2; 
     133                } 
     134                poolLeader += procsPerPool; 
     135                if (remainder != 0) 
     136                { 
     137                  ++poolLeader; 
     138                  --remainder; 
     139                } 
     140//*********** (2) Comment out the two lines below to set one process per pool 
     141//                if (poolLeader < srvRanks.size()) 
     142//                  sndServerGlobalRanks.push_back(srvRanks[poolLeader]); 
     143//*********** (3) Uncomment the line below to set one process per pool 
    116144                sndServerGlobalRanks.push_back(srvRanks[i]); 
    117                 if (rank_ == srvRanks[i]) serverLevel=2; 
    118145              } 
    119146              else 
     
    122149              } 
    123150            } 
    124           } 
    125         } 
     151            if (serverLevel==2) 
     152            { 
     153              info(50)<<"The number of secondary server pools is "<< sndServerGlobalRanks.size() <<endl ; 
     154              for (i=0; i<sndServerGlobalRanks.size(); i++) 
     155              { 
     156                if (rank_>= sndServerGlobalRanks[i]) 
     157                { 
     158                  if ( i == sndServerGlobalRanks.size()-1) 
     159                  { 
     160                    myColor = colors.size() + sndServerGlobalRanks[i]; 
     161                  } 
     162                  else if (rank_< sndServerGlobalRanks[i+1]) 
     163                  { 
     164                    myColor = colors.size() + sndServerGlobalRanks[i]; 
     165                    break; 
     166                  } 
     167                } 
     168              } 
     169            } 
     170          } 
     171        } 
     172 
    126173        // (2) Create intraComm 
    127         myColor = (serverLevel == 2) ? rank_ : colors[hashServer]; 
     174        if (serverLevel != 2) myColor=colors[hashServer]; 
    128175        MPI_Comm_split(CXios::globalComm, myColor, rank_, &intraComm) ; 
    129176 
     
    232279          else 
    233280          { 
     281            int firstSndSrvRank = size*(100.-CXios::ratioServer2)/100. ; 
     282            int poolLeader = firstSndSrvRank; 
     283//*********** (1) Comment out the line below to set one process per pool 
     284//            sndServerGlobalRanks.push_back(srvGlobalRanks[poolLeader]); 
     285            int nbPools = CXios::nbPoolsServer2; 
     286            if ( nbPools > reqNbProc || nbPools < 1) 
     287            { 
     288              error(0)<<"WARNING: void CServer::initialize(void)"<<endl 
     289                  << "It is impossible to allocate the requested number of pools = "<<nbPools 
     290                  <<" on the secondary server. It will be set so that there is one process per pool."<<endl; 
     291              nbPools = reqNbProc; 
     292            } 
     293            int remainder = ((int) (size*CXios::ratioServer2/100.)) % nbPools; 
     294            int procsPerPool = ((int) (size*CXios::ratioServer2/100.)) / nbPools; 
    234295            for (int i=0; i<size; i++) 
    235296            { 
    236               if (i >= size*(100.-CXios::ratioServer2)/100.) 
     297              if (i >= firstSndSrvRank) 
    237298              { 
     299                if (globalRank == srvGlobalRanks[i]) 
     300                { 
     301                  serverLevel=2; 
     302                } 
     303                poolLeader += procsPerPool; 
     304                if (remainder != 0) 
     305                { 
     306                  ++poolLeader; 
     307                  --remainder; 
     308                } 
     309//*********** (2) Comment out the two lines below to set one process per pool 
     310//                if (poolLeader < size) 
     311//                  sndServerGlobalRanks.push_back(srvGlobalRanks[poolLeader]); 
     312//*********** (3) Uncomment the line below to set one process per pool 
    238313                sndServerGlobalRanks.push_back(srvGlobalRanks[i]); 
    239                 if (globalRank == srvGlobalRanks[i]) serverLevel=2; 
    240314              } 
    241315              else 
     
    244318              } 
    245319            } 
    246             myColor = (serverLevel == 2) ? globalRank : 0; 
     320            if (serverLevel==2) 
     321            { 
     322              info(50)<<"The number of secondary server pools is "<< sndServerGlobalRanks.size() <<endl ; 
     323              for (int i=0; i<sndServerGlobalRanks.size(); i++) 
     324              { 
     325                if (globalRank>= sndServerGlobalRanks[i]) 
     326                { 
     327                  if (i == sndServerGlobalRanks.size()-1) 
     328                  { 
     329                    myColor = sndServerGlobalRanks[i]; 
     330                  } 
     331                  else if (globalRank< sndServerGlobalRanks[i+1]) 
     332                  { 
     333                    myColor = sndServerGlobalRanks[i]; 
     334                    break; 
     335                  } 
     336                } 
     337              } 
     338            } 
     339            if (serverLevel != 2) myColor=0; 
    247340            MPI_Comm_split(localComm, myColor, rank_, &intraComm) ; 
    248341          } 
     
    270363 
    271364//      (3) Create interComms between primary and secondary servers 
    272         MPI_Comm_size(intraComm,&size) ; 
     365        int intraCommSize, intraCommRank ; 
     366        MPI_Comm_size(intraComm,&intraCommSize) ; 
     367        MPI_Comm_rank(intraComm, &intraCommRank) ; 
     368 
    273369        if (serverLevel == 1) 
    274370        { 
     
    276372          { 
    277373            int srvSndLeader = sndServerGlobalRanks[i]; 
    278             info(50)<<"intercommCreate::client (server level 1) "<<globalRank<<" intraCommSize : "<<size 
    279                 <<" intraCommRank :"<<rank_<<"  clientLeader "<< srvSndLeader<<endl ; 
     374            info(50)<<"intercommCreate::client (server level 1) "<<globalRank<<" intraCommSize : "<<intraCommSize 
     375                <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< srvSndLeader<<endl ; 
    280376            MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvSndLeader, 0, &newComm) ; 
    281377            interCommRight.push_back(newComm) ; 
     
    284380        else if (serverLevel == 2) 
    285381        { 
    286           info(50)<<"intercommCreate::server "<<rank_<<" intraCommSize : "<<size 
    287                    <<" intraCommRank :"<<rank_<<"  clientLeader "<< srvGlobalRanks[0] <<endl ; 
     382          info(50)<<"intercommCreate::server "<<globalRank<<" intraCommSize : "<<intraCommSize 
     383                   <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< srvGlobalRanks[0] <<endl ; 
    288384          MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvGlobalRanks[0], 0, &newComm) ; 
    289385          interCommLeft.push_back(newComm) ; 
  • XIOS/dev/XIOS_DEV_CMIP6/src/server.hpp

    r1180 r1243  
    4646 
    4747      public: 
    48         //! Get rank of the current process 
     48        //! Get rank of the current process in the intraComm 
    4949        static int getRank(); 
    5050 
     
    6767 
    6868      private: 
    69         static vector<int> sndServerGlobalRanks;  //!< Global ranks of secondary server processes 
    70         static int rank_;                   //!< If (!oasis) global rank, else rank in the intraComm returned by oasis 
    71         static int nbContexts;              //!< Number of contexts registered by server 
     69        static vector<int> sndServerGlobalRanks;  //!< Global ranks of pool leaders on the secondary server 
     70        static int rank_;                         //!< If (!oasis) global rank, else rank in the intraComm returned by oasis 
     71        static int nbContexts;                    //!< Number of contexts registered by server 
    7272        static StdOFStream m_infoStream; 
    7373        static StdOFStream m_errorStream; 
Note: See TracChangeset for help on using the changeset viewer.