Changeset 1099 for XIOS/dev/dev_olga/src/node/grid.cpp
- Timestamp:
- 04/14/17 16:53:56 (7 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/dev/dev_olga/src/node/grid.cpp
r1077 r1099 96 96 * 97 97 * \return A map associating the server rank with its minimum buffer size. 98 * TODO: Refactor code 98 99 */ 99 100 std::map<int, StdSize> CGrid::getAttributesBufferSize() … … 102 103 103 104 // The grid indexes require a similar size as the actual data 104 std::map<int, StdSize> dataSizes = getDataBufferSize(); 105 std::map<int, StdSize>::iterator it, itE = dataSizes.end(); 106 for (it = dataSizes.begin(); it != itE; ++it) 105 std::vector<std::map<int, StdSize> > dataSizes = getDataBufferSize(); 106 for (size_t i = 0; i < dataSizes.size(); ++i) 107 107 { 108 it->second += 2 * sizeof(bool); 109 if (it->second > attributesSizes[it->first]) 110 attributesSizes[it->first] = it->second; 108 std::map<int, StdSize>::iterator it, itE = dataSizes[i].end(); 109 for (it = dataSizes[i].begin(); it != itE; ++it) 110 { 111 it->second += 2 * sizeof(bool); 112 if (it->second > attributesSizes[it->first]) 113 attributesSizes[it->first] = it->second; 114 } 111 115 } 112 116 117 std::map<int, StdSize>::iterator it, itE; 113 118 // Account for the axis attributes 114 119 std::vector<CAxis*> axisList = getAxis(); 115 120 for (size_t i = 0; i < axisList.size(); ++i) 116 121 { 117 std::map<int, StdSize> axisAttBuffSize = axisList[i]->getAttributesBufferSize(); 122 std::map<int, StdSize> axisAttBuffSize = axisList[i]->getAttributesBufferSize(); 118 123 for (it = axisAttBuffSize.begin(), itE = axisAttBuffSize.end(); it != itE; ++it) 119 124 { … … 144 149 * \return A map associating the server rank with its minimum buffer size. 145 150 */ 146 std::map<int, StdSize> CGrid::getDataBufferSize(const std::string& id /*= ""*/) 147 { 148 std::map<int, StdSize> dataSizes; 151 std::vector<std::map<int, StdSize> > CGrid::getDataBufferSize(const std::string& id /*= ""*/) 152 { 149 153 // The record index is sometimes sent along with the data but we always 150 154 // include it in the size calculation for the sake of simplicity 151 155 const size_t extraSize = CEventClient::headerSize + (id.empty() ? getId() : id).size() + 2 * sizeof(size_t); 152 153 std::map<int, size_t>::const_iterator itEnd = connectedDataSize_.end(); 154 for (size_t k = 0; k < connectedServerRank_.size(); ++k) 156 CContext* context = CContext::getCurrent(); 157 int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1; 158 std::vector<std::map<int, StdSize> > dataSizes(nbSrvPools); 159 for (int p = 0; p < nbSrvPools; ++p) 155 160 { 156 int rank = connectedServerRank_[k]; 157 std::map<int, size_t>::const_iterator it = connectedDataSize_.find(rank); 158 size_t count = (it != itEnd) ? it->second : 0; 159 160 dataSizes.insert(std::make_pair(rank, extraSize + CArray<double,1>::size(count))); 161 std::map<int, size_t>::const_iterator itEnd = connectedDataSize_[p].end(); 162 for (size_t k = 0; k < connectedServerRank_[p].size(); ++k) // TODO: Should change connectedServerRank_[0] to something more general 163 { 164 int rank = connectedServerRank_[p][k]; 165 std::map<int, size_t>::const_iterator it = connectedDataSize_[0].find(rank); 166 size_t count = (it != itEnd) ? it->second : 0; 167 168 dataSizes[p].insert(std::make_pair(rank, extraSize + CArray<double,1>::size(count))); 169 } 161 170 } 162 171 … … 276 285 int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1; 277 286 nbSrvPools = 1; 278 for (int i = 0; i < nbSrvPools; ++i)287 for (int p = 0; p < nbSrvPools; ++p) 279 288 { 280 289 if (isScalarGrid()) … … 304 313 if (this->isChecked) return; 305 314 this->checkAttributesAfterTransformation(); 306 this->checkMask(); 315 316 // TODO: Transfer grid attributes 317 if (!context->hasClient && context->hasServer) this->createMask(); 307 318 this->computeIndex(); 308 319 … … 506 517 // It works only for the same number of procs on secondary pools 507 518 int nbSrvPools = 1; 508 509 for (int i = 0; i < nbSrvPools; ++i) 519 for (int p = 0; p < nbSrvPools; ++p) 510 520 { 511 CContextClient* client = (context->hasServer) ? (context->hasClient ? context->clientPrimServer[ i] : context->client) : context->client;521 CContextClient* client = (context->hasServer) ? (context->hasClient ? context->clientPrimServer[p] : context->client) : context->client; 512 522 // CContextClient* client = (context->hasServer) ? context->clientPrimServer[i] : context->client; 513 523 // CContextServer* server = (context->hasServer) ? context->server : 0 ; … … 517 527 // First of all, compute distribution on client side 518 528 if (0 != serverDistribution_) 529 { 519 530 clientDistribution_ = new CDistributionClient(rank, this, serverDistribution_->getGlobalLocalIndex()); 531 storeIndex_client.resize(serverDistribution_->getGridSize()); 532 int nbStoreIndex = storeIndex_client.numElements(); 533 for (int idx = 0; idx < nbStoreIndex; ++idx) storeIndex_client(idx) = idx; 534 } 520 535 else 536 { 521 537 clientDistribution_ = new CDistributionClient(rank, this); 522 523 // Get local data index on client 524 storeIndex_client.resize(clientDistribution_->getLocalDataIndexOnClient().size()); 525 int nbStoreIndex = storeIndex_client.numElements(); 526 for (int idx = 0; idx < nbStoreIndex; ++idx) storeIndex_client(idx) = (clientDistribution_->getLocalDataIndexOnClient())[idx]; 538 // Get local data index on client 539 storeIndex_client.resize(clientDistribution_->getLocalDataIndexOnClient().size()); 540 int nbStoreIndex = storeIndex_client.numElements(); 541 for (int idx = 0; idx < nbStoreIndex; ++idx) storeIndex_client(idx) = (clientDistribution_->getLocalDataIndexOnClient())[idx]; 542 } 543 527 544 if (0 == serverDistribution_) 528 545 isDataDistributed_= clientDistribution_->isDataDistributed(); … … 535 552 // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1; 536 553 int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1; 537 // int nbSrvPools = 1; 538 for (int i = 0; i < nbSrvPools; ++i) 554 connectedServerRank_.resize(nbSrvPools); 555 connectedDataSize_.resize(nbSrvPools); 556 nbSenders.resize(nbSrvPools); 557 558 for (int p = 0; p < nbSrvPools; ++p) 539 559 { 540 CContextClient* client = (context->hasServer) ? context->clientPrimServer[ i] : context->client;541 542 connectedServerRank_ .clear();560 CContextClient* client = (context->hasServer) ? context->clientPrimServer[p] : context->client; 561 562 connectedServerRank_[p].clear(); 543 563 544 564 if (!doGridHaveDataDistributed()) … … 550 570 for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank) 551 571 { 552 connectedServerRank_ .push_back(*itRank);553 connectedDataSize_[ *itRank] = ssize;572 connectedServerRank_[p].push_back(*itRank); 573 connectedDataSize_[p][*itRank] = ssize; 554 574 } 555 575 } … … 583 603 if (iteGlobalLocalIndexMap != itGlobalLocalIndexMap) 584 604 { 585 if (connectedDataSize_ .end() == connectedDataSize_.find(serverRank))586 connectedDataSize_[ serverRank] = 1;605 if (connectedDataSize_[p].end() == connectedDataSize_[p].find(serverRank)) 606 connectedDataSize_[p][serverRank] = 1; 587 607 else 588 ++connectedDataSize_[ serverRank];608 ++connectedDataSize_[p][serverRank]; 589 609 } 590 610 } … … 592 612 593 613 for (itGlobalMap = itbGlobalMap; itGlobalMap != iteGlobalMap; ++itGlobalMap) { 594 connectedServerRank_ .push_back(itGlobalMap->first);614 connectedServerRank_[p].push_back(itGlobalMap->first); 595 615 } 596 616 597 nbSenders = clientServerMap_->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_);617 nbSenders[p] = clientServerMap_->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[p]); 598 618 } 599 619 } … … 626 646 // isDataDistributed_= clientDistribution_->isDataDistributed(); 627 647 648 628 649 // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1; 629 650 … … 633 654 computeConnectedClients(); 634 655 } 635 636 656 637 657 // connectedServerRank_.clear(); … … 708 728 int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1; 709 729 nbSrvPools = 1; 710 for (int i = 0; i < nbSrvPools; ++i)730 for (int p = 0; p < nbSrvPools; ++p) 711 731 { 712 CContextClient* client = context->hasServer ? context->clientPrimServer[ i] : context->client;732 CContextClient* client = context->hasServer ? context->clientPrimServer[p] : context->client; 713 733 int serverSize = client->serverSize; 714 734 std::vector<CDomain*> domList = getDomains(); … … 1139 1159 // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1; 1140 1160 int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1; 1141 for (int i = 0; i < nbSrvPools; ++i) 1142 { 1143 CContextClient* client = context->hasServer ? context->clientPrimServer[i] : context->client; 1161 connectedServerRank_.resize(nbSrvPools); 1162 connectedDataSize_.resize(nbSrvPools); 1163 nbSenders.resize(nbSrvPools); 1164 1165 for (int p = 0; p < nbSrvPools; ++p) 1166 { 1167 CContextClient* client = context->hasServer ? context->clientPrimServer[p] : context->client; 1144 1168 1145 1169 storeIndex_client.resize(1); 1146 1170 storeIndex_client(0) = 0; 1147 1171 1148 connectedServerRank_ .clear();1172 connectedServerRank_[p].clear(); 1149 1173 1150 1174 if (0 == client->clientRank) … … 1152 1176 for (int rank = 0; rank < client->serverSize; ++rank) 1153 1177 { 1154 connectedServerRank_ .push_back(rank);1155 connectedDataSize_[ rank] = 1;1156 nbSenders[ rank] = 1;1178 connectedServerRank_[p].push_back(rank); 1179 connectedDataSize_[p][rank] = 1; 1180 nbSenders[p][rank] = 1; 1157 1181 } 1158 1182 } … … 1200 1224 // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1; 1201 1225 int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1; 1202 for (int i = 0; i < nbSrvPools; ++i)1203 { 1204 CContextClient* client = context->hasServer ? context->clientPrimServer[ i] : context->client;1226 for (int p = 0; p < nbSrvPools; ++p) 1227 { 1228 CContextClient* client = context->hasServer ? context->clientPrimServer[p] : context->client; 1205 1229 1206 1230 CEventClient event(getType(), EVENT_ID_INDEX); … … 1257 1281 { 1258 1282 CContext* context = CContext::getCurrent(); 1259 // CContextClient* client = context->client;1260 // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;1261 1283 int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1; 1262 1284 for (int p = 0; p < nbSrvPools; ++p) … … 1368 1390 } 1369 1391 1370 for (int ns = 0; ns < connectedServerRank_ .size(); ++ns)1392 for (int ns = 0; ns < connectedServerRank_[p].size(); ++ns) 1371 1393 { 1372 rank = connectedServerRank_[ ns];1394 rank = connectedServerRank_[p][ns]; 1373 1395 int nb = 0; 1374 1396 if (globalIndexTmp.end() != globalIndexTmp.find(rank)) … … 1391 1413 listMsg.back() << getId() << isDataDistributed_ << isCompressible_ << listOutIndex.back(); 1392 1414 1393 event.push(rank, nbSenders[ rank], listMsg.back());1415 event.push(rank, nbSenders[p][rank], listMsg.back()); 1394 1416 } 1395 1417 … … 1419 1441 { 1420 1442 CContext* context = CContext::getCurrent(); 1443 1421 1444 int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1; 1422 1445 nbSrvPools = 1; 1446 // connectedServerRank_.resize(nbSrvPools); 1447 // nbSenders.resize(nbSrvPools); 1423 1448 for (int p = 0; p < nbSrvPools; ++p) 1424 1449 { … … 1428 1453 // CContextClient* client = (context->hasServer) ? context->client : context->clientPrimServer[p]; 1429 1454 numberWrittenIndexes_ = totalNumberWrittenIndexes_ = offsetWrittenIndexes_ = 0; 1430 connectedServerRank_= ranks;1455 //connectedServerRank_[p] = ranks; 1431 1456 1432 1457 for (int n = 0; n < ranks.size(); n++) … … 1465 1490 if (2 == axis_domain_order(i)) //domain 1466 1491 { 1467 // nZoomBegin[indexMap[i]] = domainList[domainId]->zoom_ibegin_srv;1468 // nZoomSize[indexMap[i]] = domainList[domainId]->zoom_ni_srv;1469 //nZoomBeginGlobal[indexMap[i]] = domainList[domainId]->global_zoom_ibegin;1492 nZoomBegin[indexMap[i]] = domainList[domainId]->zoom_ibegin; 1493 nZoomSize[indexMap[i]] = domainList[domainId]->zoom_ni; 1494 nZoomBeginGlobal[indexMap[i]] = domainList[domainId]->global_zoom_ibegin; 1470 1495 // zoomIndex.push_back(domainList[domainId]->zoom_i_index); 1471 //nGlob[indexMap[i]] = domainList[domainId]->ni_glo;1472 1473 // nZoomBegin[indexMap[i] + 1] = domainList[domainId]->zoom_jbegin_srv;1474 // nZoomSize[indexMap[i] + 1] = domainList[domainId]->zoom_nj_srv;1475 //nZoomBeginGlobal[indexMap[i] + 1] = domainList[domainId]->global_zoom_jbegin;1496 nGlob[indexMap[i]] = domainList[domainId]->ni_glo; 1497 1498 nZoomBegin[indexMap[i] + 1] = domainList[domainId]->zoom_jbegin; 1499 nZoomSize[indexMap[i] + 1] = domainList[domainId]->zoom_nj; 1500 nZoomBeginGlobal[indexMap[i] + 1] = domainList[domainId]->global_zoom_jbegin; 1476 1501 // zoomIndex.push_back(domainList[domainId]->zoom_j_index); 1477 //nGlob[indexMap[i] + 1] = domainList[domainId]->nj_glo;1478 1479 int nbZoom = domainList[domainId]->zoom_i_index.numElements();1480 zoomIndex.push_back(CArray<int,1>(nbZoom));1481 CArray<int,1>& zoomDomain = zoomIndex.back();1482 for (int ind = 0; ind < nbZoom; ++ind)1483 {1484 zoomDomain(ind) = domainList[domainId]->zoom_i_index(ind) + domainList[domainId]->zoom_j_index(ind) * domainList[domainId]->ni_glo;1485 }1486 1487 globalSize *= domainList[domainId]->ni_glo * domainList[domainId]->nj_glo;1502 nGlob[indexMap[i] + 1] = domainList[domainId]->nj_glo; 1503 1504 // int nbZoom = domainList[domainId]->zoom_i_index.numElements(); 1505 // zoomIndex.push_back(CArray<int,1>(nbZoom)); 1506 // CArray<int,1>& zoomDomain = zoomIndex.back(); 1507 // for (int ind = 0; ind < nbZoom; ++ind) 1508 // { 1509 // zoomDomain(ind) = domainList[domainId]->zoom_i_index(ind) + domainList[domainId]->zoom_j_index(ind) * domainList[domainId]->ni_glo; 1510 // } 1511 1512 // globalSize *= domainList[domainId]->ni_glo * domainList[domainId]->nj_glo; 1488 1513 ++domainId; 1489 1514 } 1490 1515 else if (1 == axis_domain_order(i)) // axis 1491 1516 { 1492 // nZoomBegin[indexMap[i]] = axisList[axisId]->zoom_begin_srv;1493 // nZoomSize[indexMap[i]] = axisList[axisId]->zoom_size_srv;1494 //nZoomBeginGlobal[indexMap[i]] = axisList[axisId]->global_zoom_begin;1495 zoomIndex.push_back(axisList[axisId]->zoom_index);1496 //nGlob[indexMap[i]] = axisList[axisId]->n_glo;1497 globalSize *= axisList[axisId]->n_glo;1517 nZoomBegin[indexMap[i]] = axisList[axisId]->zoom_begin; 1518 nZoomSize[indexMap[i]] = axisList[axisId]->zoom_n; 1519 nZoomBeginGlobal[indexMap[i]] = axisList[axisId]->global_zoom_begin; 1520 // zoomIndex.push_back(axisList[axisId]->zoom_index); 1521 nGlob[indexMap[i]] = axisList[axisId]->n_glo; 1522 // globalSize *= axisList[axisId]->n_glo; 1498 1523 ++axisId; 1499 1524 } 1500 1525 else // scalar 1501 1526 { 1502 CArray<int,1> zoomScalar(1);1503 zoomScalar(0) = 0;1504 //nZoomBegin[indexMap[i]] = 0;1505 //nZoomSize[indexMap[i]] = 1;1506 //nZoomBeginGlobal[indexMap[i]] = 0;1507 zoomIndex.push_back(zoomScalar);1508 //nGlob[indexMap[i]] = 1;1527 // CArray<int,1> zoomScalar(1); 1528 // zoomScalar(0) = 0; 1529 nZoomBegin[indexMap[i]] = 0; 1530 nZoomSize[indexMap[i]] = 1; 1531 nZoomBeginGlobal[indexMap[i]] = 0; 1532 // zoomIndex.push_back(zoomScalar); 1533 nGlob[indexMap[i]] = 1; 1509 1534 ++scalarId; 1510 1535 } … … 1512 1537 dataSize = 1; 1513 1538 1514 //for (int i = 0; i < nZoomSize.size(); ++i)1515 //dataSize *= nZoomSize[i];1516 //serverDistribution_ = new CDistributionServer(server->intraCommRank, nZoomBegin, nZoomSize,1517 //nZoomBeginGlobal, nGlob);1518 for (int i = 0; i < zoomIndex.size(); ++i)1519 {1520 dataSize *= zoomIndex[i].numElements();1521 }1522 serverDistribution_ = new CDistributionServer(server->intraCommRank, zoomIndex, nGlobElement);1539 for (int i = 0; i < nZoomSize.size(); ++i) 1540 dataSize *= nZoomSize[i]; 1541 serverDistribution_ = new CDistributionServer(server->intraCommRank, nZoomBegin, nZoomSize, 1542 nZoomBeginGlobal, nGlob); 1543 // for (int i = 0; i < zoomIndex.size(); ++i) 1544 // { 1545 // dataSize *= zoomIndex[i].numElements(); 1546 // } 1547 // serverDistribution_ = new CDistributionServer(server->intraCommRank, zoomIndex, nGlobElement); 1523 1548 } 1524 1549 1525 1550 CArray<size_t,1> outIndex; 1526 1551 buffer >> outIndex; 1552 serverDistribution_->computeLocalIndex(outIndex); 1553 1527 1554 if (isDataDistributed_) 1528 serverDistribution_->computeLocalIndex(outIndex);1555 {} 1529 1556 else 1530 1557 { … … 1533 1560 // THE PROBLEM HERE IS THAT DATA CAN BE NONDISTRIBUTED ON CLIENT AND DISTRIBUTED ON SERVER 1534 1561 // BELOW IS THE TEMPORARY FIX only for a single type of element (domain, asix, scalar) 1535 dataSize = serverDistribution_->getGlobalIndexEachDimension()[0].numElements(); 1536 outIndex.resize(dataSize); 1537 outIndex = serverDistribution_->getGlobalIndexEachDimension()[0]; 1562 dataSize = serverDistribution_->getGridSize(); 1563 // dataSize = serverDistribution_->getGlobalIndexEachDimension()[0].numElements(); 1564 // outIndex.resize(dataSize); 1565 // outIndex = serverDistribution_->getGlobalIndexEachDimension()[0]; 1538 1566 1539 1567 } … … 1541 1569 1542 1570 outIndexFromClient.insert(std::make_pair(rank, outIndex)); 1543 connectedDataSize_[rank] = outIndex.numElements();1571 // connectedDataSize_[p][rank] = outIndex.numElements(); 1544 1572 numberWrittenIndexes_ += outIndex.numElements(); 1545 1573 } 1546 1574 1547 int sizeData = 0;1548 for (map<int, CArray<size_t, 1> >::iterator it = outIndexFromClient.begin(); it != outIndexFromClient.end(); ++it)1549 {1550 sizeData += it->second.numElements();1551 }1552 indexFromClients.resize(sizeData);1553 sizeData = 0;1554 for (map<int, CArray<size_t, 1> >::iterator it = outIndexFromClient.begin(); it != outIndexFromClient.end(); ++it)1555 {1556 CArray<size_t, 1>& tmp0 = it->second;1557 CArray<size_t, 1> tmp1 = indexFromClients(Range(sizeData, sizeData + tmp0.numElements() - 1));1558 tmp1 = tmp0;1559 sizeData += tmp0.numElements();1560 }1575 // int sizeData = 0; 1576 // for (map<int, CArray<size_t, 1> >::iterator it = outIndexFromClient.begin(); it != outIndexFromClient.end(); ++it) 1577 // { 1578 // sizeData += it->second.numElements(); 1579 // } 1580 // indexFromClients.resize(sizeData); 1581 // sizeData = 0; 1582 // for (map<int, CArray<size_t, 1> >::iterator it = outIndexFromClient.begin(); it != outIndexFromClient.end(); ++it) 1583 // { 1584 // CArray<size_t, 1>& tmp0 = it->second; 1585 // CArray<size_t, 1> tmp1 = indexFromClients(Range(sizeData, sizeData + tmp0.numElements() - 1)); 1586 // tmp1 = tmp0; 1587 // sizeData += tmp0.numElements(); 1588 // } 1561 1589 1562 1590 // if (isScalarGrid()) return; … … 1571 1599 totalNumberWrittenIndexes_ = numberWrittenIndexes_; 1572 1600 1573 nbSenders= CClientServerMappingDistributed::computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, ranks);1601 // nbSenders[p] = CClientServerMappingDistributed::computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, ranks); 1574 1602 } 1575 1603 }
Note: See TracChangeset
for help on using the changeset viewer.