- Timestamp:
- 07/12/21 20:02:30 (3 years ago)
- Location:
- XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/grid_remote_connector.cpp
r1999 r2179 1 1 #include "grid_remote_connector.hpp" 2 2 #include "client_client_dht_template.hpp" 3 #include "leader_process.hpp" 3 4 #include "mpi.hpp" 4 5 … … 7 8 namespace xios 8 9 { 9 10 /** 11 * \brief class constructor. 12 * \param srcView List of sources views. 13 * \param dstView List of remotes views. 14 * \param localComm Local MPI communicator 15 * \param remoteSize Size of the remote communicator 16 */ 10 17 CGridRemoteConnector::CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, MPI_Comm localComm, int remoteSize) 11 18 : srcView_(srcView), dstView_(dstView), localComm_(localComm), remoteSize_(remoteSize) 12 19 {} 13 20 21 /** 22 * \brief class constructor. 23 * \param srcView List of sources views. 24 * \param dstView List of remotes views. 25 * \param localComm Local MPI communicator 26 * \param remoteSize Size of the remote communicator 27 */ 14 28 CGridRemoteConnector::CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CLocalView*>& dstView, MPI_Comm localComm, int remoteSize) 15 29 : srcView_(srcView), localComm_(localComm), remoteSize_(remoteSize) … … 18 32 } 19 33 34 35 /** 36 * \brief Compute if each view composing the source grid and the remote grid is distributed or not. 37 * Result is stored on internal attributes \b isSrcViewDistributed_ and \b isDstViewDistributed_. 38 * \detail To compute this, a hash is computed for each array on indices. The hash must permutable, i.e. 39 * the order of the list of global indices doesn't influence the value of the hash. So simply a sum of 40 * hash of each indices is used for the whole array. After, the computed hash are compared with each other 41 * ranks of \b localComm_ MPI communicator using an MPI_ALLReduce. If, for each ranks, the hash is the same 42 * then the view is not distributed 43 */ 44 void CGridRemoteConnector::computeViewDistribution(void) 45 { 46 HashXIOS<size_t> hashGlobalIndex; // hash function-object 47 48 int nDst = dstView_.size() ; 49 vector<size_t> hashRank(remoteSize_) ; 50 isDstViewDistributed_.resize(nDst) ; 51 52 for(int i=0; i<nDst; i++) 53 { 54 map<int,CArray<size_t,1>> globalIndexView ; 55 dstView_[i]->getGlobalIndexView(globalIndexView) ; 56 hashRank.assign(remoteSize_,0) ; // everybody ranks to 0 except rank of the remote view I have 57 // that would be assign to my local hash 58 for(auto& it : globalIndexView) 59 { 60 int rank=it.first ; 61 CArray<size_t,1>& globalIndex = it.second ; 62 size_t globalIndexSize = globalIndex.numElements(); 63 size_t hashValue=0 ; 64 for(size_t ind=0;ind<globalIndexSize;ind++) hashValue += hashGlobalIndex(globalIndex(ind)) ; 65 hashRank[rank] += hashValue ; 66 } 67 // sum all the hash for every process of the local comm. The reduce is on the size of remote view (remoteSize_) 68 // after that for each rank of the remote view, we get the hash 69 MPI_Allreduce(MPI_IN_PLACE, hashRank.data(), remoteSize_, MPI_SIZE_T, MPI_SUM, localComm_) ; 70 size_t value = hashRank[0] ; 71 isDstViewDistributed_[i]=false ; 72 for(int j=0 ; j<remoteSize_ ; j++) 73 if (value != hashRank[j]) 74 { 75 isDstViewDistributed_[i]=true ; 76 break ; 77 } 78 } 79 80 int nSrc = srcView_.size() ; 81 int commSize,commRank ; 82 MPI_Comm_size(localComm_,&commSize) ; 83 MPI_Comm_rank(localComm_,&commRank) ; 84 hashRank.resize(commSize,0) ; 85 isSrcViewDistributed_.resize(nSrc) ; 86 87 for(int i=0; i<nSrc; i++) 88 { 89 CArray<size_t,1> globalIndex ; 90 srcView_[i]->getGlobalIndexView(globalIndex) ; 91 hashRank.assign(commSize,0) ; // 0 for everybody except my rank 92 size_t globalIndexSize = globalIndex.numElements() ; 93 size_t hashValue=0 ; 94 for(size_t ind=0;ind<globalIndexSize;ind++) hashValue += hashGlobalIndex(globalIndex(ind)) ; 95 hashRank[commRank] += hashValue ; 96 97 // Same method than for remote view 98 MPI_Allreduce(MPI_IN_PLACE, hashRank.data(), commSize, MPI_SIZE_T, MPI_SUM, localComm_) ; 99 size_t value = hashRank[0] ; 100 isSrcViewDistributed_[i]=false ; 101 for(int j=0 ; j<commSize ; j++) 102 if (value != hashRank[j]) 103 { 104 isSrcViewDistributed_[i]=true ; 105 break ; 106 } 107 } 108 109 } 110 111 /** 112 * \brief Compute the connector, i.e. compute the \b elements_ attribute. 113 * \detail In order to achive better optimisation, 114 * we distingute the case when the grid is not distributed on source grid (\bcomputeSrcNonDistributed), 115 * or the remote grid (\b computeDstNonDistributed), or the both (\b computeSrcDstNonDistributed). 116 * Otherwise the generic method is called computeGenericMethod. Note that in the case, if one element view 117 * is not distributed on the source and on the remote grid, then we can used the tensorial product 118 * property to computing it independently using \b computeSrcDstNonDistributed method. 119 * After that, we call the \b removeRedondantRanks method to supress blocks of data that can be sent 120 * redondantly the the remote servers 121 */ 20 122 void CGridRemoteConnector::computeConnector(void) 21 123 { 22 computeGenericMethod() ; 23 } 24 25 void CGridRemoteConnector::computeGenericMethod(void) 124 computeViewDistribution() ; 125 vector<CLocalView*> srcView ; 126 vector<CDistributedView*> dstView ; 127 vector<int> indElements ; 128 elements_.resize(srcView_.size()) ; 129 130 bool srcViewsNonDistributed=true ; 131 for(int i=0;i<srcView_.size();i++) srcViewsNonDistributed &= !isSrcViewDistributed_[i] ; 132 133 bool dstViewsNonDistributed=true ; 134 for(int i=0;i<dstView_.size();i++) dstViewsNonDistributed &= !isDstViewDistributed_[i] ; 135 136 if (srcViewsNonDistributed) 137 { 138 int commRank, commSize ; 139 MPI_Comm_rank(localComm_,&commRank) ; 140 MPI_Comm_size(localComm_,&commSize) ; 141 list<int> remoteRanks; 142 list<int> notUsed ; 143 map<int,bool> ranks ; 144 computeLeaderProcess(commRank, commSize, remoteSize_, remoteRanks, notUsed) ; 145 for(int rank : remoteRanks) ranks[rank]=true ; 146 147 for(int i=0; i<srcView_.size(); i++) 148 { 149 if (isDstViewDistributed_[i]) computeSrcNonDistributed(i) ; 150 else computeSrcDstNonDistributed(i, ranks) ; 151 } 152 } 153 else if (dstViewsNonDistributed) 154 { 155 map<int,bool> ranks ; 156 for(int i=0;i<remoteSize_;i++) ranks[i]=true ; 157 for(int i=0; i<srcView_.size(); i++) 158 { 159 if (isSrcViewDistributed_[i]) computeDstNonDistributed(i,ranks) ; 160 else computeSrcDstNonDistributed(i,ranks) ; 161 } 162 } 163 else 164 { 165 for(int i=0;i<srcView_.size();i++) 166 if (isSrcViewDistributed_[i] || isDstViewDistributed_[i]) 167 { 168 srcView.push_back(srcView_[i]) ; 169 dstView.push_back(dstView_[i]) ; 170 indElements.push_back(i) ; 171 } 172 173 computeGenericMethod(srcView, dstView, indElements) ; 174 175 map<int,bool> ranks ; 176 for(auto& it : elements_[indElements[0]]) 177 { 178 if (it.second.numElements()==0) ranks[it.first] = false ; 179 else ranks[it.first] = true ; 180 } 181 182 for(int i=0;i<srcView_.size();i++) 183 if (!isSrcViewDistributed_[i] && !isDstViewDistributed_[i]) computeSrcDstNonDistributed(i, ranks) ; 184 } 185 186 removeRedondantRanks() ; 187 } 188 189 190 /** 191 * \brief Compute the connector for the element \b i when the source view is not distributed. 192 * After the call element_[i] is defined. 193 * \param i Indice of the element composing the source grid. 194 */ 195 196 void CGridRemoteConnector::computeSrcNonDistributed(int i) 197 { 198 auto& element = elements_[i] ; 199 map<int,CArray<size_t,1>> globalIndexView ; 200 dstView_[i]->getGlobalIndexView(globalIndexView) ; 201 202 CClientClientDHTTemplate<int>::Index2InfoTypeMap dataInfo; 203 204 for(auto& it : globalIndexView) 205 { 206 auto& globalIndex=it.second ; 207 for(size_t ind : globalIndex) dataInfo[ind]=it.first ; 208 } 209 210 // First we feed the distributed hash map with key (remote global index) 211 // associated with the value of the remote rank 212 CClientClientDHTTemplate<int> DHT(dataInfo, localComm_) ; 213 // after we feed the DHT with the local global indices of the source view 214 215 int commRank, commSize ; 216 MPI_Comm_rank(localComm_,&commRank) ; 217 MPI_Comm_size(localComm_,&commSize) ; 218 CArray<size_t,1> srcIndex ; 219 // like the source view is not distributed, then only the rank 0 need to feed the DHT 220 if (commRank==0) srcView_[i]->getGlobalIndexView(srcIndex) ; 221 222 // compute the mapping 223 DHT.computeIndexInfoMapping(srcIndex) ; 224 auto& returnInfo = DHT.getInfoIndexMap() ; 225 226 // returnInfo contains now the map for each global indices to send to a list of remote rank 227 // only for the rank=0 because it is the one to feed the DHT 228 // so it need to send the list to each server leader i.e. the local process that handle specifically one or more 229 // servers 230 231 // rankIndGlo : rankIndGlo[rank][indGlo] : list of indice to send the the remote server of rank "rank" 232 vector<vector<size_t>> rankIndGlo(remoteSize_) ; 233 if (commRank==0) 234 for(auto& it1 : returnInfo) 235 for(auto& it2 : it1.second) rankIndGlo[it2].push_back(it1.first) ; 236 237 238 vector<MPI_Request> requests ; 239 240 if (commRank==0) 241 { 242 requests.resize(remoteSize_) ; 243 for(int i=0 ; i<remoteSize_;i++) 244 { 245 // ok send only the global indices for a server to the "server leader" 246 int rank = getLeaderRank(commSize, remoteSize_, i) ; 247 MPI_Isend(rankIndGlo[i].data(), rankIndGlo[i].size(), MPI_SIZE_T, rank, i ,localComm_, &requests[i]) ; 248 } 249 } 250 251 list<int> remoteRanks; 252 list<int> notUsed ; 253 // I am a server leader of which remote ranks ? 254 computeLeaderProcess(commRank, commSize, remoteSize_, remoteRanks, notUsed) ; 255 256 for(auto remoteRank : remoteRanks) 257 { 258 MPI_Status status ; 259 int size ; 260 MPI_Probe(0,remoteRank,localComm_, &status); 261 MPI_Get_count(&status, MPI_SIZE_T, &size) ; 262 elements_[i][remoteRank].resize(size) ; 263 // for each remote ranks receive the global indices from proc 0 264 MPI_Recv(elements_[i][remoteRank].dataFirst(),size, MPI_SIZE_T,0,remoteRank, localComm_,&status) ; 265 } 266 267 if (commRank==0) 268 { 269 vector<MPI_Status> status(remoteSize_) ; 270 // asynchronous for sender, wait for completion 271 MPI_Waitall(remoteSize_, requests.data(), status.data()) ; 272 } 273 } 274 275 /** 276 * \brief Compute the remote connector for the element \b i when the remote view is not distributed. 277 * After the call, element_[i] is defined. 278 * \param i Indice of the element composing the remote grid. 279 * \param ranks The list of rank for which the local proc is in charge to compute the connector 280 * (if leader server for exemple). if ranks[rank] == false the corresponding elements_ 281 * is set to void array (no data to sent) just in order to notify corresponding remote server 282 * that the call is collective with each other one 283 */ 284 void CGridRemoteConnector::computeDstNonDistributed(int i, map<int,bool>& ranks) 285 { 286 auto& element = elements_[i] ; 287 map<int,CArray<size_t,1>> globalIndexView ; 288 dstView_[i]->getGlobalIndexView(globalIndexView) ; 289 290 291 CClientClientDHTTemplate<int>::Index2InfoTypeMap dataInfo; 292 293 // First we feed the distributed hash map with key (remote global index) 294 // associated with the value of the remote rank 295 for(auto& it : globalIndexView) 296 if (it.first==0) // since the remote view is not distributed, insert only the remote rank 0 297 { 298 auto& globalIndex=it.second ; 299 for(size_t ind : globalIndex) dataInfo[ind]=0 ; // associated the the rank 0 300 } 301 302 CClientClientDHTTemplate<int> DHT(dataInfo, localComm_) ; 303 // after we feed the DHT with the local global indices of the source view 304 305 CArray<size_t,1> srcIndex ; 306 srcView_[i]->getGlobalIndexView(srcIndex) ; 307 DHT.computeIndexInfoMapping(srcIndex) ; 308 auto& returnInfo = DHT.getInfoIndexMap() ; 309 310 // returnInfo contains now the map for each global indices to send to a list of remote rank 311 // now construct the element_ list of global indices for each rank in my list except if the erray must be empty 312 for (auto& rank : ranks) 313 { 314 if (rank.second) // non empty array => for rank that have not any data to be received 315 { 316 int size=0 ; 317 for(auto& it : returnInfo) if (!it.second.empty()) size++ ; 318 auto& array = element[rank.first] ; 319 array.resize(size) ; 320 size=0 ; 321 for(auto& it : returnInfo) 322 if (!it.second.empty()) 323 { 324 array(size)=it.first ; 325 size++ ; 326 } 327 } 328 else element[rank.first] = CArray<size_t,1>(0) ; // empty array => for rank that have not any data to be received 329 } 330 } 331 332 /** 333 * \brief Compute the remote connector for the element \b i when the source and the remote view are not distributed. 334 * After the call, element_[i] is defined. 335 * \param i Indice of the element composing the remote grid. 336 * \param ranks The list of rank for which the local proc is in charge to compute the connector 337 * (if leader server for exemple). if ranks[rank] == false the corresponding elements_ 338 * is set to void array (no data to sent) just in order to notify corresponding remote server 339 * that the call is collective with each other one 340 */ 341 342 void CGridRemoteConnector::computeSrcDstNonDistributed(int i, map<int,bool>& ranks) 343 { 344 auto& element = elements_[i] ; 345 map<int,CArray<size_t,1>> globalIndexView ; 346 dstView_[i]->getGlobalIndexView(globalIndexView) ; 347 348 349 CClientClientDHTTemplate<int>::Index2InfoTypeMap dataInfo; 350 // First we feed the distributed hash map with key (remote global index) 351 // associated with the value of the remote rank 352 353 for(auto& it : globalIndexView) 354 if (it.first==0) // insert only the remote rank 0 since the remote view is not distributed 355 { 356 auto& globalIndex=it.second ; 357 for(size_t ind : globalIndex) dataInfo[ind]=0 ; // associated the the rank 0 358 } 359 360 CClientClientDHTTemplate<int> DHT(dataInfo, localComm_) ; 361 // after we feed the DHT with the local global indices of the source view 362 363 int commRank, commSize ; 364 MPI_Comm_rank(localComm_,&commRank) ; 365 MPI_Comm_size(localComm_,&commSize) ; 366 CArray<size_t,1> srcIndex ; 367 368 // like the source view is not distributed, then only the rank 0 need to feed the DHT 369 if (commRank==0) srcView_[i]->getGlobalIndexView(srcIndex) ; 370 DHT.computeIndexInfoMapping(srcIndex) ; 371 auto& returnInfo = DHT.getInfoIndexMap() ; 372 373 vector<size_t> indGlo ; 374 if (commRank==0) 375 for(auto& it1 : returnInfo) 376 for(auto& it2 : it1.second) indGlo.push_back(it1.first) ; 377 378 // now local rank 0 know which indices to seed to remote rank 0, but all the server 379 // must receive the same information. So only the leader rank will sent this. 380 // So local rank 0 must broadcast the information to all leader. 381 // for this we create a new communicator composed of local process that must send data 382 // to a remote rank, data are broadcasted, and element_[i] is construction for each remote 383 // rank in charge 384 int color=0 ; 385 if (ranks.empty()) color=0 ; 386 else color=1 ; 387 if (commRank==0) color=1 ; 388 MPI_Comm newComm ; 389 MPI_Comm_split(localComm_, color, commRank, &newComm) ; 390 if (color==1) 391 { 392 // ok, I am part of the process that must send something to one or more remote server 393 // so I get the list of global indices from rank 0 394 int dataSize ; 395 if (commRank==0) dataSize=indGlo.size() ; 396 MPI_Bcast(&dataSize,1,MPI_INT, 0, newComm) ; 397 indGlo.resize(dataSize) ; 398 MPI_Bcast(indGlo.data(),dataSize,MPI_SIZE_T,0,newComm) ; 399 } 400 MPI_Comm_free(&newComm) ; 401 402 // construct element_[i] from indGlo 403 for(auto& rank : ranks) 404 { 405 if (rank.second) 406 { 407 int dataSize=indGlo.size(); 408 auto& element = elements_[i][rank.first] ; 409 element.resize(dataSize) ; 410 for(int i=0;i<dataSize; i++) element(i)=indGlo[i] ; 411 } 412 else element[rank.first] = CArray<size_t,1>(0) ; 413 } 414 415 } 416 417 418 /** 419 * \brief Generic method the compute the grid remote connector. Only distributed elements are specifed in the source view and remote view. 420 * Connector for non distributed elements are computed separatly to improve performance and memory consumption. After the call, 421 * \b elements_ is defined. 422 * \param srcView List of the source views composing the grid, without non distributed views 423 * \param dstView List of the remote views composing the grid, without non distributed views 424 * \param indElements Index of the view making the correspondance between all views and views distributed (that are in input) 425 */ 426 void CGridRemoteConnector::computeGenericMethod(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, vector<int>& indElements) 26 427 { 27 428 // generic method, every element can be distributed 28 int nDst = dstView _.size() ;429 int nDst = dstView.size() ; 29 430 vector<size_t> dstSliceSize(nDst) ; 30 431 dstSliceSize[0] = 1 ; 31 for(int i=1; i<nDst; i++) dstSliceSize[i] = dstView _[i-1]->getGlobalSize()*dstSliceSize[i-1] ;432 for(int i=1; i<nDst; i++) dstSliceSize[i] = dstView[i-1]->getGlobalSize()*dstSliceSize[i-1] ; 32 433 33 434 CClientClientDHTTemplate<int>::Index2VectorInfoTypeMap dataInfo ; 34 35 435 CClientClientDHTTemplate<size_t>::Index2VectorInfoTypeMap info ; // info map 436 437 // first, we need to feed the DHT with the global index of the remote server 438 // for that : 439 // First the first element insert the in a DHT with key as the rank and value the list of global index associated 440 // Then get the previously stored index associate with the remote rank I am in charge and reinsert the global index 441 // corresponding to the position of the element in the remote view suing tensorial product 442 // finaly we get only the list of remote global index I am in charge for the whole remote grid 443 36 444 for(int pos=0; pos<nDst; pos++) 37 445 { 38 446 size_t sliceSize=dstSliceSize[pos] ; 39 447 map<int,CArray<size_t,1>> globalIndexView ; 40 dstView _[pos]->getGlobalIndexView(globalIndexView) ;448 dstView[pos]->getGlobalIndexView(globalIndexView) ; 41 449 42 450 CClientClientDHTTemplate<size_t>::Index2VectorInfoTypeMap lastInfo(info) ; … … 80 488 } 81 489 490 // we feed the DHT with the remote global index 82 491 CClientClientDHTTemplate<int> dataRanks(dataInfo, localComm_) ; 83 492 84 493 // generate list of global index for src view 85 int nSrc = srcView _.size() ;494 int nSrc = srcView.size() ; 86 495 vector<size_t> srcSliceSize(nSrc) ; 87 496 88 497 srcSliceSize[0] = 1 ; 89 for(int i=1; i<nSrc; i++) srcSliceSize[i] = srcView _[i-1]->getGlobalSize()*srcSliceSize[i-1] ;498 for(int i=1; i<nSrc; i++) srcSliceSize[i] = srcView[i-1]->getGlobalSize()*srcSliceSize[i-1] ; 90 499 91 500 vector<size_t> srcGlobalIndex ; 92 501 size_t sliceIndex=0 ; 93 srcView_[nSrc-1]->getGlobalIndex(srcGlobalIndex, sliceIndex, srcSliceSize.data(), srcView_.data(), nSrc-1) ; 94 502 srcView[nSrc-1]->getGlobalIndex(srcGlobalIndex, sliceIndex, srcSliceSize.data(), srcView.data(), nSrc-1) ; 503 // now we have the global index of the source grid in srcGlobalIndex 504 // we feed the DHT with the src global index (if we have) 95 505 if (srcGlobalIndex.size()>0) 96 506 { … … 104 514 } 105 515 const auto& returnInfo = dataRanks.getInfoIndexMap() ; 516 // returnInfo contains now the map for each global indices to send to a list of remote rank 517 // but we want to use the tensorial product property to get the same information using only global 518 // index of element view. So the idea is to reverse the information : for a global index of the grid 519 // to send to the remote server, what is the global index of each element composing the grid ? 106 520 107 521 vector<map<int, set<size_t>>> elements(nSrc) ; // internal representation of elements composing the grid … … 120 534 } 121 535 122 elements_.resize(nSrc) ;536 // elements_.resize(nSrc) ; 123 537 for(int i=0 ; i<nSrc; i++) 124 538 { … … 128 542 int rank=rankInd.first ; 129 543 set<size_t>& indGlo = rankInd.second ; 130 CArray<size_t,1>& indGloArray = elements_[i ][rank] ;544 CArray<size_t,1>& indGloArray = elements_[indElements[i]][rank] ; 131 545 indGloArray.resize(indGlo.size()) ; 132 546 int j=0 ; … … 140 554 // that it receive global index with no data (0-size) 141 555 vector<int> ranks(remoteSize_,0) ; 142 for(auto& it : elements_[ 0]) ranks[it.first] = 1 ;556 for(auto& it : elements_[indElements[0]]) ranks[it.first] = 1 ; 143 557 MPI_Allreduce(MPI_IN_PLACE, ranks.data(), remoteSize_, MPI_INT, MPI_SUM, localComm_) ; 144 558 int commRank, commSize ; … … 149 563 if (ranks[i]==0) 150 564 { 151 if (pos%commSize==commRank) for(auto& element : elements_) element[i] = CArray<size_t,1>(0) ; 565 if (pos%commSize==commRank) 566 for(int j=0 ; j<nSrc; j++) elements_[indElements[j]][i] = CArray<size_t,1>(0) ; 152 567 pos++ ; 153 568 } 154 569 } 155 570 156 571 /** 572 * \brief Once the connector is computed (compute \b elements_), redondant data can be send to the server. 573 * The goal of this method is to make a hash of each block of indice that determine wich data to send to a 574 * of a specific server rank using a hash method. So data to send to a rank is associated to a hash. 575 * After we compare hash between local rank and remove redondant data corresponding to the same hash. 576 */ 577 void CGridRemoteConnector::removeRedondantRanks(void) 578 { 579 int commRank ; 580 MPI_Comm_rank(localComm_,&commRank) ; 581 582 set<int> ranks; 583 for(auto& element : elements_) 584 for(auto& it : element) ranks.insert(it.first) ; 585 586 for(auto& element : elements_) 587 for(auto& it : element) 588 if (ranks.count(it.first)==0) ERROR("void CGridRemoteConnector::removeRedondantRanks(void)",<<"number of ranks in elements is not coherent between each element") ; 589 590 HashXIOS<size_t> hashGlobalIndex; 591 592 map<int,size_t> hashRanks ; 593 for(auto& element : elements_) 594 for(auto& it : element) 595 { 596 auto& globalIndex=it.second ; 597 int rank=it.first ; 598 size_t hash ; 599 hash=0 ; 600 for(int i=0; i<globalIndex.numElements(); i++) hash+=hashGlobalIndex(globalIndex(i)) ; 601 if (hashRanks.count(rank)==0) hashRanks[rank]=hash ; 602 else hashRanks[rank]=hashGlobalIndex.hashCombine(hashRanks[rank],hash) ; 603 } 604 // a hash is now computed for data block I will sent to the server. 605 606 CClientClientDHTTemplate<int>::Index2InfoTypeMap info ; 607 608 map<size_t,int> hashRank ; 609 HashXIOS<int> hashGlobalIndexRank; 610 for(auto& it : hashRanks) 611 { 612 it.second = hashGlobalIndexRank.hashCombine(it.first,it.second) ; 613 info[it.second]=commRank ; 614 hashRank[it.second]=it.first ; 615 } 616 617 // we feed a DHT map with key : hash, value : myrank 618 CClientClientDHTTemplate<int> dataHash(info, localComm_) ; 619 CArray<size_t,1> hashList(hashRank.size()) ; 620 621 int i=0 ; 622 for(auto& it : hashRank) { hashList(i)=it.first ; i++; } 623 624 // now who are the ranks that have the same hash : feed the DHT with my list of hash 625 dataHash.computeIndexInfoMapping(hashList) ; 626 auto& hashRankList = dataHash.getInfoIndexMap() ; 627 628 629 for(auto& it : hashRankList) 630 { 631 size_t hash = it.first ; 632 auto& ranks = it.second ; 633 634 bool first=true ; 635 // only the process with the lowest rank get in charge of sendinf data to remote server 636 for(int rank : ranks) if (commRank>rank) first=false ; 637 if (!first) 638 { 639 int rankToRemove=hashRank[hash] ; 640 for(auto& element : elements_) element.erase(rankToRemove) ; 641 } 642 } 643 644 645 } 157 646 } -
XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/grid_remote_connector.hpp
r1999 r2179 20 20 CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, MPI_Comm localComm, int remoteSize) ; 21 21 CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CLocalView*>& dstView, MPI_Comm localComm, int remoteSize) ; 22 void computeViewDistribution(void) ; 22 23 void computeConnector(void) ; 23 void computeGenericMethod(void) ; 24 void computeGenericMethod(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, vector<int>& indElements) ; 25 void computeSrcDstNonDistributed(int i, map<int,bool>& ranks) ; 26 void computeDstNonDistributed(int i, map<int,bool>& ranks) ; 27 void computeSrcNonDistributed(int i) ; 28 void removeRedondantRanks(void) ; 24 29 std::map<int, CArray<size_t,1>>& getDistributedGlobalIndex(int pos) { return elements_[pos] ;} 25 30 26 31 private: 32 33 /** 34 * Source views composing the source grid. The vector store an internal copy of pointer elements composing the grid. 35 * It is feed at construction time 36 */ 37 vector<CLocalView*> srcView_ ; 38 39 /** 40 * Destination views composing the source grid. The vector store an internal copy of pointer elements composing the grid 41 * It is feed at construction time 42 */ 43 vector<CDistributedView*> dstView_ ; 44 45 /** 46 * The list of global indices to send to each rank of the remote view (servers). The vector store the information for each element, and the map 47 * specify a list of global indices to send to each rank of the remote view. 48 * size of element_[] -> number of elements composing the grids (source/destination) 49 * element[i][rank] == CArray<size_t,1> -> list of global indices to send to the remote process of rank \b rank for the view \b i 50 * The is computed when calling computeConnector internal methods 51 * map can be returned trough the public accessor : getDistributedGlobalIndex(int) 52 */ 27 53 vector<map<int, CArray<size_t,1>>> elements_ ; 28 vector<CLocalView*> srcView_ ; 29 vector<CDistributedView*> dstView_ ; 54 55 /** 56 /* internal copy of the local communicator (client workflow). Feed at construction time. 57 */ 30 58 MPI_Comm localComm_ ; 59 60 /** 61 /* size of the remote communicator (== nb of servers). Feed at consctruction time 62 */ 31 63 int remoteSize_ ; 32 64 65 /** 66 /* for each view composing the source grid, the vector store the information about the distribution of the element, i.e. 67 * if each ranks of the local view has exactly the same global indices than each other. This is computed when calling 68 * \b computeViewDistribution method. 69 */ 70 vector<bool> isSrcViewDistributed_ ; 71 72 /** 73 /* for each view composing the destination grid, the vector store the information about the distribution of the element, i.e. 74 * if each ranks of the remote view has exactly the same global indices than each other. This is computed when calling 75 * \b computeViewDistribution method. 76 */ 77 vector<bool> isDstViewDistributed_ ; 78 79 33 80 } ; 34 81
Note: See TracChangeset
for help on using the changeset viewer.