Changeset 1243
- Timestamp:
- 08/28/17 12:15:49 (7 years ago)
- Location:
- XIOS/dev/XIOS_DEV_CMIP6/src
- Files:
-
- 7 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/dev/XIOS_DEV_CMIP6/src/client.cpp
r1233 r1243 96 96 } 97 97 98 // myColor=colors[hashClient]; 99 myColor=leaders[hashClient] ; 98 myColor=colors[hashClient]; 100 99 MPI_Comm_split(CXios::globalComm,myColor,rank_,&intraComm) ; 101 100 … … 275 274 276 275 /*! 277 * Return rank in model intraComm276 * Return global rank without oasis and current rank in model intraComm in case of oasis 278 277 */ 279 278 int CClient::getRank() -
XIOS/dev/XIOS_DEV_CMIP6/src/client.hpp
r1158 r1243 22 22 static MPI_Comm& getInterComm(); 23 23 24 //! Get rank of the current process24 //! Get global rank without oasis and current rank in model intraComm in case of oasis 25 25 static int getRank(); 26 26 -
XIOS/dev/XIOS_DEV_CMIP6/src/cxios.cpp
r1234 r1243 28 28 bool CXios::usingServer2 = false; 29 29 int CXios::ratioServer2 = 50; 30 int CXios::nbPoolsServer2 = 1; 30 31 double CXios::bufferSizeFactor = 1.0; 31 32 const double CXios::defaultBufferSizeFactor = 1.0; … … 55 56 usingServer2=getin<bool>("using_server2",false) ; 56 57 ratioServer2=getin<int>("ratio_server2",50); 58 nbPoolsServer2=getin<int>("number_pools_server2",1); 57 59 info.setLevel(getin<int>("info_level",0)) ; 58 60 report.setLevel(getin<int>("info_level",50)); … … 197 199 else 198 200 { 199 // If using two server levels, first merge globalRegistry of each server pool into one registry on the first pool 201 // If using two server levels: 202 // (1) merge registries on each pool 203 // (2) send merged registries to the first pool 204 // (3) merge received registries on the first pool 200 205 if (CServer::serverLevel == 2) 201 206 { … … 205 210 MPI_Comm_rank(globalComm, &rankGlobal); 206 211 212 // Merge registries defined on each pools 213 CRegistry globalRegistrySndServers (CServer::intraComm); 214 207 215 // All pools (except the first): send globalRegistry to the first pool 208 if (rankGlobal != firstPoolGlobalRank)216 for (int i=1; i<secondaryServerGlobalRanks.size(); i++) 209 217 { 210 int registrySize = globalRegistry->size(); 211 MPI_Send(®istrySize,1,MPI_LONG,firstPoolGlobalRank,15,CXios::globalComm) ; 212 CBufferOut buffer(registrySize) ; 213 globalRegistry->toBuffer(buffer) ; 214 MPI_Send(buffer.start(),registrySize,MPI_CHAR,firstPoolGlobalRank,15,CXios::globalComm) ; 218 if (rankGlobal == secondaryServerGlobalRanks[i]) 219 { 220 globalRegistrySndServers.mergeRegistry(*globalRegistry) ; 221 int registrySize = globalRegistrySndServers.size(); 222 MPI_Send(®istrySize,1,MPI_LONG,firstPoolGlobalRank,15,CXios::globalComm) ; 223 CBufferOut buffer(registrySize) ; 224 globalRegistrySndServers.toBuffer(buffer) ; 225 MPI_Send(buffer.start(),registrySize,MPI_CHAR,firstPoolGlobalRank,15,CXios::globalComm) ; 226 } 215 227 } 216 228 217 229 // First pool: receive globalRegistry of all secondary server pools, merge and write the resultant registry 218 else230 if (rankGlobal == firstPoolGlobalRank) 219 231 { 220 232 MPI_Status status; 221 233 char* recvBuff; 222 234 223 CRegistry globalRegistrySndServers (CServer::intraComm);224 235 globalRegistrySndServers.mergeRegistry(*globalRegistry) ; 225 236 -
XIOS/dev/XIOS_DEV_CMIP6/src/cxios.hpp
r1234 r1243 40 40 41 41 static bool printLogs2Files; //!< Printing out logs into files 42 static bool usingOasis ; //!< Using Oasis 43 static bool usingServer ; //!< Using server (server mode) 44 static bool usingServer2 ; //!< Using secondary server (server mode). IMPORTANT: Use this variable ONLY in CServer::initialize(). 45 static int ratioServer2 ; //!< Percentage of server processors dedicated to secondary server 42 static bool usingOasis ; //!< Using Oasis 43 static bool usingServer ; //!< Using server (server mode) 44 static bool usingServer2 ; //!< Using secondary server (server mode). IMPORTANT: Use this variable ONLY in CServer::initialize(). 45 static int ratioServer2 ; //!< Percentage of server processors dedicated to secondary server 46 static int nbPoolsServer2 ; //!< Number of pools created on the secondary server 46 47 static double bufferSizeFactor; //!< Factor used to tune the buffer size 47 48 static const double defaultBufferSizeFactor; //!< Default factor value -
XIOS/dev/XIOS_DEV_CMIP6/src/node/grid.hpp
r1236 r1243 203 203 CArray<int, 1> storeIndex_client; 204 204 205 std::map<CContextClient*, map<int, CArray<int, 1> > > storeIndex_toSrv; // Same grid but can be sent to several pools 205 /** Map containing indexes that will be sent in sendIndex(). In future: change the key to pair<distrType, serverSize> (?) */ 206 std::map<CContextClient*, map<int, CArray<int, 1> > > storeIndex_toSrv; 207 206 208 map<int, CArray<int, 1> > storeIndex_fromSrv; // Support, for now, reading with level-1 server 209 210 207 211 std::map<CContextClient*, std::map<int,int> > nbSenders, nbReadSenders; 208 212 … … 215 219 map<int, CArray<size_t, 1> > outLocalIndexStoreOnClient; 216 220 217 // If a client wants to write out data, it'll use this index. 218 // A client receives global index of data from other clients (via recvIndex), 219 // then does remapping these index into local index to WRITE into a file 220 CArray<size_t,1> localIndexToWriteOnClient, localIndexToWriteOnServer; 221 /** Indexes calculated based on server distribution (serverDistribution_). They are used for writing data into a file. */ 222 CArray<size_t,1> localIndexToWriteOnServer; 223 224 /** Indexes calculated based on client distribution (clientDistribution_). They are not used at all. 225 They should be the same as localIndexToWriteOnServer and potentially can be used as an additional check.*/ 226 CArray<size_t,1> localIndexToWriteOnClient; 221 227 222 228 CArray<size_t,1> indexFromClients; … … 286 292 bool isAxisListSet, isDomListSet, isScalarListSet; 287 293 294 /** Distribution calculated in computeClientIndex() based on the knowledge of the entire grid */ 288 295 CDistributionClient* clientDistribution_; 296 297 /** Distribution calculated upon receiving indexes */ 289 298 CDistributionServer* serverDistribution_; 299 290 300 CClientServerMapping* clientServerMap_; 291 301 size_t writtenDataSize_; 292 302 int numberWrittenIndexes_, totalNumberWrittenIndexes_, offsetWrittenIndexes_; 303 304 /** Map storing ranks of connected servers. In future: change the key to the server size (?) */ 305 std::map<CContextClient*, std::vector<int> > connectedServerRank_; 306 307 /** Map storing data size that will be sent to connected servers. In future: change the key to the server size (?) */ 293 308 std::map<CContextClient*, std::map<int,size_t> > connectedDataSize_; 294 std::map<CContextClient*, std::vector<int> > connectedServerRank_; 309 295 310 bool isDataDistributed_; 296 311 //! True if and only if the data defined on the grid can be outputted in a compressed way … … 305 320 std::map<CGrid*, std::pair<bool,StdString> > gridSrc_; 306 321 bool hasTransform_; 322 323 /** Map storing data size that will be sent to connected servers. In future: change the key to the server size (?) */ 307 324 std::map<CContextClient*, CClientServerMapping::GlobalIndexMap> globalIndexOnServer_; 308 // List order of axis and domain in a grid, if there is a domain, it will take value 1 (true), axis 0 (false) 325 326 /** List order of axis and domain in a grid, if there is a domain, it will take value 1 (true), axis 0 (false) */ 309 327 std::vector<int> order_; 328 310 329 }; // class CGrid 311 330 -
XIOS/dev/XIOS_DEV_CMIP6/src/server.cpp
r1234 r1243 110 110 else 111 111 { 112 int firstSndSrvRank = srvRanks.size()*(100.-CXios::ratioServer2)/100. ; 113 int poolLeader = firstSndSrvRank; 114 //*********** (1) Comment out the line below to set one process per pool 115 // sndServerGlobalRanks.push_back(srvRanks[poolLeader]); 116 int nbPools = CXios::nbPoolsServer2; 117 if ( nbPools > reqNbProc || nbPools < 1) 118 { 119 error(0)<<"WARNING: void CServer::initialize(void)"<<endl 120 << "It is impossible to allocate the requested number of pools = "<<nbPools 121 <<" on the secondary server. It will be set so that there is one process per pool."<<endl; 122 nbPools = reqNbProc; 123 } 124 int remainder = ((int) (srvRanks.size()*CXios::ratioServer2/100.)) % nbPools; 125 int procsPerPool = ((int) (srvRanks.size()*CXios::ratioServer2/100.)) / nbPools; 112 126 for (i=0; i<srvRanks.size(); i++) 113 127 { 114 if (i >= srvRanks.size()*(100.-CXios::ratioServer2)/100.)128 if (i >= firstSndSrvRank) 115 129 { 130 if (rank_ == srvRanks[i]) 131 { 132 serverLevel=2; 133 } 134 poolLeader += procsPerPool; 135 if (remainder != 0) 136 { 137 ++poolLeader; 138 --remainder; 139 } 140 //*********** (2) Comment out the two lines below to set one process per pool 141 // if (poolLeader < srvRanks.size()) 142 // sndServerGlobalRanks.push_back(srvRanks[poolLeader]); 143 //*********** (3) Uncomment the line below to set one process per pool 116 144 sndServerGlobalRanks.push_back(srvRanks[i]); 117 if (rank_ == srvRanks[i]) serverLevel=2;118 145 } 119 146 else … … 122 149 } 123 150 } 124 } 125 } 151 if (serverLevel==2) 152 { 153 info(50)<<"The number of secondary server pools is "<< sndServerGlobalRanks.size() <<endl ; 154 for (i=0; i<sndServerGlobalRanks.size(); i++) 155 { 156 if (rank_>= sndServerGlobalRanks[i]) 157 { 158 if ( i == sndServerGlobalRanks.size()-1) 159 { 160 myColor = colors.size() + sndServerGlobalRanks[i]; 161 } 162 else if (rank_< sndServerGlobalRanks[i+1]) 163 { 164 myColor = colors.size() + sndServerGlobalRanks[i]; 165 break; 166 } 167 } 168 } 169 } 170 } 171 } 172 126 173 // (2) Create intraComm 127 myColor = (serverLevel == 2) ? rank_ :colors[hashServer];174 if (serverLevel != 2) myColor=colors[hashServer]; 128 175 MPI_Comm_split(CXios::globalComm, myColor, rank_, &intraComm) ; 129 176 … … 232 279 else 233 280 { 281 int firstSndSrvRank = size*(100.-CXios::ratioServer2)/100. ; 282 int poolLeader = firstSndSrvRank; 283 //*********** (1) Comment out the line below to set one process per pool 284 // sndServerGlobalRanks.push_back(srvGlobalRanks[poolLeader]); 285 int nbPools = CXios::nbPoolsServer2; 286 if ( nbPools > reqNbProc || nbPools < 1) 287 { 288 error(0)<<"WARNING: void CServer::initialize(void)"<<endl 289 << "It is impossible to allocate the requested number of pools = "<<nbPools 290 <<" on the secondary server. It will be set so that there is one process per pool."<<endl; 291 nbPools = reqNbProc; 292 } 293 int remainder = ((int) (size*CXios::ratioServer2/100.)) % nbPools; 294 int procsPerPool = ((int) (size*CXios::ratioServer2/100.)) / nbPools; 234 295 for (int i=0; i<size; i++) 235 296 { 236 if (i >= size*(100.-CXios::ratioServer2)/100.)297 if (i >= firstSndSrvRank) 237 298 { 299 if (globalRank == srvGlobalRanks[i]) 300 { 301 serverLevel=2; 302 } 303 poolLeader += procsPerPool; 304 if (remainder != 0) 305 { 306 ++poolLeader; 307 --remainder; 308 } 309 //*********** (2) Comment out the two lines below to set one process per pool 310 // if (poolLeader < size) 311 // sndServerGlobalRanks.push_back(srvGlobalRanks[poolLeader]); 312 //*********** (3) Uncomment the line below to set one process per pool 238 313 sndServerGlobalRanks.push_back(srvGlobalRanks[i]); 239 if (globalRank == srvGlobalRanks[i]) serverLevel=2;240 314 } 241 315 else … … 244 318 } 245 319 } 246 myColor = (serverLevel == 2) ? globalRank : 0; 320 if (serverLevel==2) 321 { 322 info(50)<<"The number of secondary server pools is "<< sndServerGlobalRanks.size() <<endl ; 323 for (int i=0; i<sndServerGlobalRanks.size(); i++) 324 { 325 if (globalRank>= sndServerGlobalRanks[i]) 326 { 327 if (i == sndServerGlobalRanks.size()-1) 328 { 329 myColor = sndServerGlobalRanks[i]; 330 } 331 else if (globalRank< sndServerGlobalRanks[i+1]) 332 { 333 myColor = sndServerGlobalRanks[i]; 334 break; 335 } 336 } 337 } 338 } 339 if (serverLevel != 2) myColor=0; 247 340 MPI_Comm_split(localComm, myColor, rank_, &intraComm) ; 248 341 } … … 270 363 271 364 // (3) Create interComms between primary and secondary servers 272 MPI_Comm_size(intraComm,&size) ; 365 int intraCommSize, intraCommRank ; 366 MPI_Comm_size(intraComm,&intraCommSize) ; 367 MPI_Comm_rank(intraComm, &intraCommRank) ; 368 273 369 if (serverLevel == 1) 274 370 { … … 276 372 { 277 373 int srvSndLeader = sndServerGlobalRanks[i]; 278 info(50)<<"intercommCreate::client (server level 1) "<<globalRank<<" intraCommSize : "<< size279 <<" intraCommRank :"<< rank_<<" clientLeader "<< srvSndLeader<<endl ;374 info(50)<<"intercommCreate::client (server level 1) "<<globalRank<<" intraCommSize : "<<intraCommSize 375 <<" intraCommRank :"<<intraCommRank<<" clientLeader "<< srvSndLeader<<endl ; 280 376 MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvSndLeader, 0, &newComm) ; 281 377 interCommRight.push_back(newComm) ; … … 284 380 else if (serverLevel == 2) 285 381 { 286 info(50)<<"intercommCreate::server "<< rank_<<" intraCommSize : "<<size287 <<" intraCommRank :"<< rank_<<" clientLeader "<< srvGlobalRanks[0] <<endl ;382 info(50)<<"intercommCreate::server "<<globalRank<<" intraCommSize : "<<intraCommSize 383 <<" intraCommRank :"<<intraCommRank<<" clientLeader "<< srvGlobalRanks[0] <<endl ; 288 384 MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvGlobalRanks[0], 0, &newComm) ; 289 385 interCommLeft.push_back(newComm) ; -
XIOS/dev/XIOS_DEV_CMIP6/src/server.hpp
r1180 r1243 46 46 47 47 public: 48 //! Get rank of the current process 48 //! Get rank of the current process in the intraComm 49 49 static int getRank(); 50 50 … … 67 67 68 68 private: 69 static vector<int> sndServerGlobalRanks; //!< Global ranks of secondary server processes70 static int rank_; //!< If (!oasis) global rank, else rank in the intraComm returned by oasis71 static int nbContexts; //!< Number of contexts registered by server69 static vector<int> sndServerGlobalRanks; //!< Global ranks of pool leaders on the secondary server 70 static int rank_; //!< If (!oasis) global rank, else rank in the intraComm returned by oasis 71 static int nbContexts; //!< Number of contexts registered by server 72 72 static StdOFStream m_infoStream; 73 73 static StdOFStream m_errorStream;
Note: See TracChangeset
for help on using the changeset viewer.