Changeset 862 for XIOS/trunk/src/transformation/grid_transformation.cpp
- Timestamp:
- 06/09/16 11:32:27 (8 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/trunk/src/transformation/grid_transformation.cpp
r858 r862 94 94 if (false == (gridDestination_->axis_domain_order)(i)) 95 95 { 96 axisPositionInGrid.push_back(idx); 97 ++idx; 96 axisPositionInGrid.push_back(i); 97 // axisPositionInGrid.push_back(idx); 98 // ++idx; 98 99 } 99 100 else 100 101 { 101 ++idx; 102 domPositionInGrid.push_back(idx); 103 ++idx; 102 domPositionInGrid.push_back(i); 103 // ++idx; 104 // domPositionInGrid.push_back(idx); 105 // ++idx; 104 106 } 105 107 } … … 120 122 if (false == (gridDestination_->axis_domain_order)(i)) 121 123 { 122 initializeAxisAlgorithms(idx); 123 ++idx; 124 initializeAxisAlgorithms(i); 125 // initializeAxisAlgorithms(idx); 126 // ++idx; 124 127 } 125 128 else 126 129 { 127 ++idx; 128 initializeDomainAlgorithms(idx); 129 ++idx; 130 initializeDomainAlgorithms(i); 131 // ++idx; 132 // initializeDomainAlgorithms(idx); 133 // ++idx; 130 134 } 131 135 } … … 356 360 -) Make current grid destination become grid source in the next transformation 357 361 */ 362 //void CGridTransformation::computeAll(const std::vector<CArray<double,1>* >& dataAuxInputs, Time timeStamp) 363 //{ 364 // if (nbAlgos_ < 1) return; 365 // if (!auxInputs_.empty() && !dynamicalTransformation_) { dynamicalTransformation_ = true; return; } 366 // if (dynamicalTransformation_) 367 // { 368 // if (timeStamp_.insert(timeStamp).second) 369 // DestinationIndexMap().swap(currentGridIndexToOriginalGridIndex_); // Reset map 370 // else 371 // return; 372 // } 373 // 374 // CContext* context = CContext::getCurrent(); 375 // CContextClient* client = context->client; 376 // 377 // ListAlgoType::const_iterator itb = listAlgos_.begin(), 378 // ite = listAlgos_.end(), it; 379 // 380 // CGenericAlgorithmTransformation* algo = 0; 381 // int nbAgloTransformation = 0; // Only count for executed transformation. Generate domain is a special one, not executed in the list 382 // for (it = itb; it != ite; ++it) 383 // { 384 // int elementPositionInGrid = it->first; 385 // ETranformationType transType = (it->second).first; 386 // int transformationOrder = (it->second).second; 387 // DestinationIndexMap globaIndexWeightFromDestToSource; 388 // 389 // // First of all, select an algorithm 390 // if (!dynamicalTransformation_ || (algoTransformation_.size() < listAlgos_.size())) 391 // { 392 // selectAlgo(elementPositionInGrid, transType, transformationOrder, algoTypes_[std::distance(itb, it)]); 393 // algo = algoTransformation_.back(); 394 // } 395 // else 396 // algo = algoTransformation_[std::distance(itb, it)]; 397 // 398 // if (0 != algo) // Only registered transformation can be executed 399 // { 400 // algo->computeIndexSourceMapping(dataAuxInputs); 401 // 402 // // Recalculate the distribution of grid destination 403 // CDistributionClient distributionClientDest(client->clientRank, gridDestination_); 404 // const CDistributionClient::GlobalLocalDataMap& globalLocalIndexGridDestSendToServer = distributionClientDest.getGlobalLocalDataSendToServer(); 405 // 406 // // ComputeTransformation of global index of each element 407 // std::vector<int> gridDestinationDimensionSize = gridDestination_->getGlobalDimension(); 408 // std::vector<int> gridSrcDimensionSize = gridSource_->getGlobalDimension(); 409 // int elementPosition = it->first; 410 // algo->computeGlobalSourceIndex(elementPosition, 411 // gridDestinationDimensionSize, 412 // gridSrcDimensionSize, 413 // globalLocalIndexGridDestSendToServer, 414 // globaIndexWeightFromDestToSource); 415 // 416 // // Compute transformation of global indexes among grids 417 // computeTransformationMapping(globaIndexWeightFromDestToSource); 418 // 419 // // Update number of local index on each transformation 420 // nbLocalIndexOnGridDest_.push_back(globalLocalIndexGridDestSendToServer.size()); 421 // 422 // if (1 < nbAlgos_) 423 // { 424 // // Now grid destination becomes grid source in a new transformation 425 // if (nbAgloTransformation != (nbAlgos_-1)) setUpGrid(elementPositionInGrid, transType, nbAgloTransformation); 426 // } 427 // ++nbAgloTransformation; 428 // } 429 // } 430 //} 431 358 432 void CGridTransformation::computeAll(const std::vector<CArray<double,1>* >& dataAuxInputs, Time timeStamp) 359 433 { … … 386 460 ETranformationType transType = (it->second).first; 387 461 int transformationOrder = (it->second).second; 388 DestinationIndexMap globaIndexWeightFromDestToSource;462 SourceDestinationIndexMap globaIndexWeightFromSrcToDst; 389 463 390 464 // First of all, select an algorithm … … 401 475 algo->computeIndexSourceMapping(dataAuxInputs); 402 476 403 // Recalculate the distribution of grid destination404 CDistributionClient distributionClientDest(client->clientRank, gridDestination_);405 const CDistributionClient::GlobalLocalDataMap& globalLocalIndexGridDestSendToServer = distributionClientDest.getGlobalLocalDataSendToServer();406 407 477 // ComputeTransformation of global index of each element 408 std::vector<int> gridDestinationDimensionSize = gridDestination_->getGlobalDimension();409 std::vector<int> gridSrcDimensionSize = gridSource_->getGlobalDimension();410 478 int elementPosition = it->first; 411 479 algo->computeGlobalSourceIndex(elementPosition, 412 gridDestinationDimensionSize, 413 gridSrcDimensionSize, 414 globalLocalIndexGridDestSendToServer, 415 globaIndexWeightFromDestToSource); 480 gridSource_, 481 gridDestination_, 482 globaIndexWeightFromSrcToDst); 416 483 417 484 // Compute transformation of global indexes among grids 418 computeTransformationMapping(globaIndexWeightFromDestToSource); 419 420 // Update number of local index on each transformation 421 nbLocalIndexOnGridDest_.push_back(globalLocalIndexGridDestSendToServer.size()); 485 computeTransformationMapping(globaIndexWeightFromSrcToDst); 422 486 423 487 if (1 < nbAlgos_) … … 435 499 \param [in] globalIndexWeightFromDestToSource global index mapping between grid destination and grid source 436 500 */ 437 void CGridTransformation::computeTransformationMapping(const DestinationIndexMap& globalIndexWeightFromDestToSource)501 void CGridTransformation::computeTransformationMapping(const SourceDestinationIndexMap& globaIndexWeightFromSrcToDst) 438 502 { 439 503 CContext* context = CContext::getCurrent(); 440 504 CContextClient* client = context->client; 441 442 CTransformationMapping transformationMap(gridDestination_, gridSource_); 443 444 transformationMap.computeTransformationMapping(globalIndexWeightFromDestToSource); 445 446 const CTransformationMapping::ReceivedIndexMap& globalIndexToReceive = transformationMap.getGlobalIndexReceivedOnGridDestMapping(); 447 CTransformationMapping::ReceivedIndexMap::const_iterator itbMapRecv, itMapRecv, iteMapRecv; 448 itbMapRecv = globalIndexToReceive.begin(); 449 iteMapRecv = globalIndexToReceive.end(); 505 int nbClient = client->clientSize; 506 int clientRank = client->clientRank; 507 508 // Recalculate the distribution of grid destination 509 CDistributionClient distributionClientDest(client->clientRank, gridDestination_); 510 CDistributionClient::GlobalLocalDataMap& globalLocalIndexGridDestSendToServer = distributionClientDest.getGlobalLocalDataSendToServer(); 511 // Update number of local index on each transformation 512 nbLocalIndexOnGridDest_.push_back(globalLocalIndexGridDestSendToServer.size()); 513 514 // Find out number of index sent from grid source and number of index received on grid destination 515 SourceDestinationIndexMap::const_iterator itbIndex = globaIndexWeightFromSrcToDst.begin(), 516 iteIndex = globaIndexWeightFromSrcToDst.end(), itIndex; 517 typedef boost::unordered_map<size_t, std::vector<std::pair<size_t,double> > > SendIndexMap; 518 std::map<int,int> sendRankSizeMap,recvRankSizeMap; 519 int connectedClient = globaIndexWeightFromSrcToDst.size(); 520 int* recvCount=new int[nbClient]; 521 int* displ=new int[nbClient]; 522 int* sendRankBuff=new int[connectedClient]; 523 int* sendSizeBuff=new int[connectedClient]; 524 int n = 0; 525 for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex, ++n) 526 { 527 sendRankBuff[n] = itIndex->first; 528 const SendIndexMap& sendIndexMap = itIndex->second; 529 SendIndexMap::const_iterator itbSend = sendIndexMap.begin(), iteSend = sendIndexMap.end(), itSend; 530 int sendSize = 0; 531 for (itSend = itbSend; itSend != iteSend; ++itSend) 532 { 533 sendSize += itSend->second.size(); 534 } 535 sendSizeBuff[n] = sendSize; 536 sendRankSizeMap[itIndex->first] = sendSize; 537 } 538 MPI_Allgather(&connectedClient,1,MPI_INT,recvCount,1,MPI_INT,client->intraComm); 539 540 displ[0]=0 ; 541 for(int n=1;n<nbClient;n++) displ[n]=displ[n-1]+recvCount[n-1]; 542 int recvSize=displ[nbClient-1]+recvCount[nbClient-1]; 543 int* recvRankBuff=new int[recvSize]; 544 int* recvSizeBuff=new int[recvSize]; 545 MPI_Allgatherv(sendRankBuff,connectedClient,MPI_INT,recvRankBuff,recvCount,displ,MPI_INT,client->intraComm); 546 MPI_Allgatherv(sendSizeBuff,connectedClient,MPI_INT,recvSizeBuff,recvCount,displ,MPI_INT,client->intraComm); 547 for (int i = 0; i < nbClient; ++i) 548 { 549 int currentPos = displ[i]; 550 for (int j = 0; j < recvCount[i]; ++j) 551 if (recvRankBuff[currentPos+j] == clientRank) 552 { 553 recvRankSizeMap[i] = recvSizeBuff[currentPos+j]; 554 } 555 } 556 557 558 559 // Sending global index of grid source to corresponding process as well as the corresponding mask 560 std::vector<MPI_Request> requests; 561 std::vector<MPI_Status> status; 562 boost::unordered_map<int, unsigned char* > recvMaskDst; 563 boost::unordered_map<int, unsigned long* > recvGlobalIndexSrc; 564 for (std::map<int,int>::const_iterator itRecv = recvRankSizeMap.begin(); itRecv != recvRankSizeMap.end(); ++itRecv) 565 { 566 int recvRank = itRecv->first; 567 int recvSize = itRecv->second; 568 recvMaskDst[recvRank] = new unsigned char [recvSize]; 569 recvGlobalIndexSrc[recvRank] = new unsigned long [recvSize]; 570 571 requests.push_back(MPI_Request()); 572 MPI_Irecv(recvGlobalIndexSrc[recvRank], recvSize, MPI_UNSIGNED_LONG, recvRank, 46, client->intraComm, &requests.back()); 573 requests.push_back(MPI_Request()); 574 MPI_Irecv(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 47, client->intraComm, &requests.back()); 575 } 576 577 boost::unordered_map<int, CArray<size_t,1> > globalIndexDst; 578 boost::unordered_map<int, CArray<double,1> > weightDst; 579 boost::unordered_map<int, unsigned char* > sendMaskDst; 580 boost::unordered_map<int, unsigned long* > sendGlobalIndexSrc; 581 for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex) 582 { 583 int sendRank = itIndex->first; 584 int sendSize = sendRankSizeMap[sendRank]; 585 const SendIndexMap& sendIndexMap = itIndex->second; 586 SendIndexMap::const_iterator itbSend = sendIndexMap.begin(), iteSend = sendIndexMap.end(), itSend; 587 globalIndexDst[sendRank].resize(sendSize); 588 weightDst[sendRank].resize(sendSize); 589 sendMaskDst[sendRank] = new unsigned char [sendSize]; 590 sendGlobalIndexSrc[sendRank] = new unsigned long [sendSize]; 591 int countIndex = 0; 592 for (itSend = itbSend; itSend != iteSend; ++itSend) 593 { 594 const std::vector<std::pair<size_t,double> >& dstWeight = itSend->second; 595 for (int idx = 0; idx < dstWeight.size(); ++idx) 596 { 597 globalIndexDst[sendRank](countIndex) = dstWeight[idx].first; 598 weightDst[sendRank](countIndex) = dstWeight[idx].second; 599 if (0 < globalLocalIndexGridDestSendToServer.count(dstWeight[idx].first)) 600 sendMaskDst[sendRank][countIndex] = 1; 601 else 602 sendMaskDst[sendRank][countIndex] = 0; 603 sendGlobalIndexSrc[sendRank][countIndex] = itSend->first; 604 ++countIndex; 605 } 606 } 607 608 // Send global index source and mask 609 requests.push_back(MPI_Request()); 610 MPI_Isend(sendGlobalIndexSrc[sendRank], sendSize, MPI_UNSIGNED_LONG, sendRank, 46, client->intraComm, &requests.back()); 611 requests.push_back(MPI_Request()); 612 MPI_Isend(sendMaskDst[sendRank], sendSize, MPI_UNSIGNED_CHAR, sendRank, 47, client->intraComm, &requests.back()); 613 } 614 615 status.resize(requests.size()); 616 MPI_Waitall(requests.size(), &requests[0], &status[0]); 617 618 // Okie, now use the mask to identify which index source we need to send, then also signal the destination which masked index we will return 619 std::vector<MPI_Request>().swap(requests); 620 std::vector<MPI_Status>().swap(status); 621 // Okie, on destination side, we will wait for information of masked index of source 622 for (std::map<int,int>::const_iterator itSend = sendRankSizeMap.begin(); itSend != sendRankSizeMap.end(); ++itSend) 623 { 624 int recvRank = itSend->first; 625 int recvSize = itSend->second; 626 627 requests.push_back(MPI_Request()); 628 MPI_Irecv(sendMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests.back()); 629 } 630 631 // Ok, now we fill in local index of grid source (we even count for masked index) 632 CDistributionClient distributionClientSrc(client->clientRank, gridSource_); 633 CDistributionClient::GlobalLocalDataMap& globalLocalIndexGridSrcSendToServer = distributionClientSrc.getGlobalLocalDataSendToServer(); 634 localIndexToSendFromGridSource_.push_back(SendingIndexGridSourceMap()); 635 SendingIndexGridSourceMap& tmpSend = localIndexToSendFromGridSource_.back(); 636 for (std::map<int,int>::const_iterator itRecv = recvRankSizeMap.begin(); itRecv != recvRankSizeMap.end(); ++itRecv) 637 { 638 int recvRank = itRecv->first; 639 int recvSize = itRecv->second; 640 unsigned char* recvMask = recvMaskDst[recvRank]; 641 unsigned long* recvIndexSrc = recvGlobalIndexSrc[recvRank]; 642 int realSendSize = 0; 643 for (int idx = 0; idx < recvSize; ++idx) 644 { 645 if (0 != (*(recvMask+idx))) // OKie, now we have a demand from non-masked index destination 646 if (0 < globalLocalIndexGridSrcSendToServer.count(*(recvIndexSrc+idx))) // check whether index source is masked 647 ++realSendSize; 648 else // inform the destination that this index is masked 649 *(recvMask+idx) = 0; 650 } 651 652 tmpSend[recvRank].resize(realSendSize); 653 realSendSize = 0; 654 for (int idx = 0; idx < recvSize; ++idx) 655 { 656 if (0 != (*(recvMask+idx))) // OKie, now we have a demand from non-masked index destination 657 { 658 tmpSend[recvRank](realSendSize) = globalLocalIndexGridSrcSendToServer[*(recvIndexSrc+idx)]; 659 ++realSendSize; 660 } 661 } 662 663 // Okie, now inform the destination which source index are masked 664 requests.push_back(MPI_Request()); 665 MPI_Isend(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests.back()); 666 } 667 status.resize(requests.size()); 668 MPI_Waitall(requests.size(), &requests[0], &status[0]); 669 670 // Cool, now we can fill in local index of grid destination (counted for masked index) 450 671 localIndexToReceiveOnGridDest_.push_back(RecvIndexGridDestinationMap()); 451 672 RecvIndexGridDestinationMap& recvTmp = localIndexToReceiveOnGridDest_.back(); 452 for (itMapRecv = itbMapRecv; itMapRecv != iteMapRecv; ++itMapRecv) 453 { 454 int sourceRank = itMapRecv->first; 455 int numGlobalIndex = (itMapRecv->second).size(); 456 recvTmp[sourceRank].resize(numGlobalIndex); 457 for (int i = 0; i < numGlobalIndex; ++i) 458 { 459 recvTmp[sourceRank][i] = make_pair((itMapRecv->second)[i].localIndex,(itMapRecv->second)[i].weight); 460 } 461 } 462 463 // Find out local index on grid source (to send) 464 const CTransformationMapping::SentIndexMap& globalIndexToSend = transformationMap.getGlobalIndexSendToGridDestMapping(); 465 CTransformationMapping::SentIndexMap::const_iterator itbMap, itMap, iteMap; 466 itbMap = globalIndexToSend.begin(); 467 iteMap = globalIndexToSend.end(); 468 localIndexToSendFromGridSource_.push_back(SendingIndexGridSourceMap()); 469 SendingIndexGridSourceMap& tmpSend = localIndexToSendFromGridSource_.back(); 470 for (itMap = itbMap; itMap != iteMap; ++itMap) 471 { 472 int destRank = itMap->first; 473 int vecSize = itMap->second.size(); 474 tmpSend[destRank].resize(vecSize); 475 for (int idx = 0; idx < vecSize; ++idx) 476 { 477 tmpSend[destRank](idx) = itMap->second[idx].first; 478 } 479 } 480 } 673 for (std::map<int,int>::const_iterator itSend = sendRankSizeMap.begin(); itSend != sendRankSizeMap.end(); ++itSend) 674 { 675 int recvRank = itSend->first; 676 int recvSize = itSend->second; 677 unsigned char* recvMask = sendMaskDst[recvRank]; 678 679 CArray<size_t,1>& recvIndexDst = globalIndexDst[recvRank]; 680 CArray<double,1>& recvWeightDst = weightDst[recvRank]; 681 int realRecvSize = 0; 682 for (int idx = 0; idx < recvSize; ++idx) 683 { 684 if (0 != *(recvMask+idx)) // OKie, now we have a non-masked index destination 685 ++realRecvSize; 686 } 687 688 int localIndexDst; 689 recvTmp[recvRank].resize(realRecvSize); 690 realRecvSize = 0; 691 for (int idx = 0; idx < recvSize; ++idx) 692 { 693 if (0 != *(recvMask+idx)) // OKie, now we have a demand from non-masked index destination 694 { 695 recvTmp[recvRank][realRecvSize].first = globalLocalIndexGridDestSendToServer[recvIndexDst(idx)]; 696 recvTmp[recvRank][realRecvSize].second = recvWeightDst(idx); 697 ++realRecvSize; 698 } 699 } 700 } 701 702 delete [] recvCount; 703 delete [] displ; 704 delete [] sendRankBuff; 705 delete [] recvRankBuff; 706 delete [] sendSizeBuff; 707 delete [] recvSizeBuff; 708 709 boost::unordered_map<int, unsigned char* >::const_iterator itChar; 710 for (itChar = sendMaskDst.begin(); itChar != sendMaskDst.end(); ++itChar) 711 delete [] itChar->second; 712 for (itChar = recvMaskDst.begin(); itChar != recvMaskDst.end(); ++itChar) 713 delete [] itChar->second; 714 boost::unordered_map<int, unsigned long* >::const_iterator itLong; 715 for (itLong = sendGlobalIndexSrc.begin(); itLong != sendGlobalIndexSrc.end(); ++itLong) 716 delete [] itLong->second; 717 for (itLong = recvGlobalIndexSrc.begin(); itLong != recvGlobalIndexSrc.end(); ++itLong) 718 delete [] itLong->second; 719 720 } 721 722 ///*! 723 // Compute exchange index between grid source and grid destination 724 // \param [in] globalIndexWeightFromDestToSource global index mapping between grid destination and grid source 725 //*/ 726 //void CGridTransformation::computeTransformationMapping(const DestinationIndexMap& globalIndexWeightFromDestToSource) 727 //{ 728 // CContext* context = CContext::getCurrent(); 729 // CContextClient* client = context->client; 730 // 731 // CTransformationMapping transformationMap(gridDestination_, gridSource_); 732 // 733 // transformationMap.computeTransformationMapping(globalIndexWeightFromDestToSource); 734 // 735 // const CTransformationMapping::ReceivedIndexMap& globalIndexToReceive = transformationMap.getGlobalIndexReceivedOnGridDestMapping(); 736 // CTransformationMapping::ReceivedIndexMap::const_iterator itbMapRecv, itMapRecv, iteMapRecv; 737 // itbMapRecv = globalIndexToReceive.begin(); 738 // iteMapRecv = globalIndexToReceive.end(); 739 // localIndexToReceiveOnGridDest_.push_back(RecvIndexGridDestinationMap()); 740 // RecvIndexGridDestinationMap& recvTmp = localIndexToReceiveOnGridDest_.back(); 741 // for (itMapRecv = itbMapRecv; itMapRecv != iteMapRecv; ++itMapRecv) 742 // { 743 // int sourceRank = itMapRecv->first; 744 // int numGlobalIndex = (itMapRecv->second).size(); 745 // recvTmp[sourceRank].resize(numGlobalIndex); 746 // for (int i = 0; i < numGlobalIndex; ++i) 747 // { 748 // recvTmp[sourceRank][i] = make_pair((itMapRecv->second)[i].localIndex,(itMapRecv->second)[i].weight); 749 // } 750 // } 751 // 752 // // Find out local index on grid source (to send) 753 // const CTransformationMapping::SentIndexMap& globalIndexToSend = transformationMap.getGlobalIndexSendToGridDestMapping(); 754 // CTransformationMapping::SentIndexMap::const_iterator itbMap, itMap, iteMap; 755 // itbMap = globalIndexToSend.begin(); 756 // iteMap = globalIndexToSend.end(); 757 // localIndexToSendFromGridSource_.push_back(SendingIndexGridSourceMap()); 758 // SendingIndexGridSourceMap& tmpSend = localIndexToSendFromGridSource_.back(); 759 // for (itMap = itbMap; itMap != iteMap; ++itMap) 760 // { 761 // int destRank = itMap->first; 762 // int vecSize = itMap->second.size(); 763 // tmpSend[destRank].resize(vecSize); 764 // for (int idx = 0; idx < vecSize; ++idx) 765 // { 766 // tmpSend[destRank](idx) = itMap->second[idx].first; 767 // } 768 // } 769 //} 481 770 482 771 bool CGridTransformation::isSpecialTransformation(ETranformationType transType)
Note: See TracChangeset
for help on using the changeset viewer.