Changeset 2034
- Timestamp:
- 02/15/21 21:14:02 (4 years ago)
- Location:
- XIOS/dev/dev_oa/src
- Files:
-
- 9 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/dev/dev_oa/src/config/domain_attribute.conf
r1965 r2034 66 66 /* LOCAL */ 67 67 DECLARE_ATTRIBUTE(int, ntiles, false) 68 DECLARE_ATTRIBUTE(bool, tile_only, false) 68 69 DECLARE_ARRAY(int, 1, tile_ni, false) 69 70 DECLARE_ARRAY(int, 1, tile_nj, false) -
XIOS/dev/dev_oa/src/distribution_client.cpp
r1637 r2034 11 11 namespace xios { 12 12 13 CDistributionClient::CDistributionClient(int rank, CGrid* grid )13 CDistributionClient::CDistributionClient(int rank, CGrid* grid, bool isTiled) 14 14 : CDistribution(rank, 0) 15 15 , axisDomainOrder_() … … 24 24 , elementNLocal_(), elementNGlobal_() 25 25 { 26 readDistributionInfo(grid );26 readDistributionInfo(grid, isTiled); 27 27 createGlobalIndex(); 28 28 } … … 50 50 \param [in] grid Grid to read 51 51 */ 52 void CDistributionClient::readDistributionInfo(CGrid* grid )52 void CDistributionClient::readDistributionInfo(CGrid* grid, bool isTiled) 53 53 { 54 54 std::vector<CDomain*> domList = grid->getDomains(); … … 57 57 CArray<int,1> axisDomainOrder = grid->axis_domain_order; 58 58 59 readDistributionInfo(domList, axisList, scalarList, axisDomainOrder );59 readDistributionInfo(domList, axisList, scalarList, axisDomainOrder, isTiled); 60 60 61 61 // Then check mask of grid … … 102 102 \param [in] scalarList List of scalar of grid 103 103 \param [in] axisDomainOrder order of axis and domain inside a grid. 2 if domain, 1 if axis and zero if scalar 104 // \param [in] gridMask Mask of grid, for now, keep it 3 dimension, but it needs changing 104 \param [in] isTiled If true, domain data attributes should be ignored 105 105 */ 106 106 void CDistributionClient::readDistributionInfo(const std::vector<CDomain*>& domList, 107 107 const std::vector<CAxis*>& axisList, 108 108 const std::vector<CScalar*>& scalarList, 109 const CArray<int,1>& axisDomainOrder) 109 const CArray<int,1>& axisDomainOrder, 110 bool isTiled) 110 111 { 111 112 domainNum_ = domList.size(); … … 177 178 nBeginGlobal_.at(indexMap_[idx]+1) = domList[domIndex]->jbegin; 178 179 179 dataBegin_.at(indexMap_[idx]+1) = domList[domIndex]->data_jbegin.getValue();180 dataIndex_.at(indexMap_[idx]+1).reference(domList[domIndex]->data_j_index);181 infoIndex_.at(indexMap_[idx]+1).reference(domList[domIndex]->j_index);182 183 180 // On the i axis 184 181 nLocal_.at(indexMap_[idx]) = domList[domIndex]->ni.getValue(); … … 187 184 nBeginGlobal_.at(indexMap_[idx]) = domList[domIndex]->ibegin; 188 185 189 dataBegin_.at(indexMap_[idx]) = domList[domIndex]->data_ibegin.getValue(); 190 dataIndex_.at(indexMap_[idx]).reference(domList[domIndex]->data_i_index); 191 infoIndex_.at(indexMap_[idx]).reference(domList[domIndex]->i_index); 192 193 dataNIndex_.at(idx) = domList[domIndex]->data_i_index.numElements(); 194 dataDims_.at(idx) = domList[domIndex]->data_dim.getValue(); 186 if (isTiled) 187 // Ignore all data attributes, if defined, for tiled domains 188 { 189 dataBegin_.at(indexMap_[idx]+1) = 0; 190 dataBegin_.at(indexMap_[idx]) = 0; 191 192 // Fill dataIndex_ and infoIndex_ 193 CArray<int,1>& infoIndexI = infoIndex_.at(indexMap_[idx]); 194 CArray<int,1>& infoIndexJ = infoIndex_.at(indexMap_[idx]+1); 195 CArray<int,1>& dataIndexI = dataIndex_.at(indexMap_[idx]); 196 CArray<int,1>& dataIndexJ = dataIndex_.at(indexMap_[idx]+1); 197 domList[domIndex]->computeCompressionTiled(dataIndexI, dataIndexJ, infoIndexI, infoIndexJ); 198 199 } 200 else 201 { 202 // On the j axis 203 dataBegin_.at(indexMap_[idx]+1) = domList[domIndex]->data_jbegin.getValue(); 204 dataIndex_.at(indexMap_[idx]+1).reference(domList[domIndex]->data_j_index); 205 infoIndex_.at(indexMap_[idx]+1).reference(domList[domIndex]->j_index); 206 207 // On the i axis 208 dataBegin_.at(indexMap_[idx]) = domList[domIndex]->data_ibegin.getValue(); 209 dataIndex_.at(indexMap_[idx]).reference(domList[domIndex]->data_i_index); 210 infoIndex_.at(indexMap_[idx]).reference(domList[domIndex]->i_index); 211 212 } 213 214 dataNIndex_.at(idx) = isTiled ? (domList[domIndex]->ni*domList[domIndex]->nj) : domList[domIndex]->data_i_index.numElements(); 215 dataDims_.at(idx) = isTiled ? 1 : domList[domIndex]->data_dim.getValue(); 195 216 196 217 isDataDistributed_ |= domList[domIndex]->isDistributed(); -
XIOS/dev/dev_oa/src/distribution_client.hpp
r1637 r2034 34 34 public: 35 35 /** Default constructor */ 36 CDistributionClient(int rank, CGrid* grid );36 CDistributionClient(int rank, CGrid* grid, bool isTiled = false); 37 37 38 38 void createGlobalIndexSendToServer(); … … 61 61 protected: 62 62 void createGlobalIndex(); 63 void readDistributionInfo(CGrid* grid );63 void readDistributionInfo(CGrid* grid, bool isTiled); 64 64 void readDistributionInfo(const std::vector<CDomain*>& domList, 65 65 const std::vector<CAxis*>& axisList, 66 66 const std::vector<CScalar*>& scalarList, 67 const CArray<int,1>& axisDomainOrder); 67 const CArray<int,1>& axisDomainOrder, 68 bool isTiled); 68 69 private: 69 70 //! Create local index of a domain -
XIOS/dev/dev_oa/src/filter/source_filter.cpp
r1967 r2034 61 61 { 62 62 const double nanValue = std::numeric_limits<double>::quiet_NaN(); 63 storedTileData.resize(grid->storeIndex_client.numElements()); 63 // storedTileData.resize(grid->storeIndex_client.numElements()); 64 storedTileData.resize(grid->getDataSize()); 64 65 storedTileData = nanValue; 65 66 } … … 69 70 { 70 71 // Data entering workflow will be exactly of size ni*nj for a grid 2d or ni*nj*n for a grid 3d 71 streamData(date, storedTileData );72 streamData(date, storedTileData, true); 72 73 ntiles = 0; 73 74 } … … 75 76 76 77 template <int N> 77 void CSourceFilter::streamData(CDate date, const CArray<double, N>& data )78 void CSourceFilter::streamData(CDate date, const CArray<double, N>& data, bool isTiled) 78 79 { 79 80 date = date + offset; // this is a temporary solution, it should be part of a proper temporal filter … … 94 95 { 95 96 if (mask) 96 grid->maskField(data, packet->data); 97 if (isTiled) 98 grid->maskField(data, packet->data, isTiled); 99 else 100 grid->maskField(data, packet->data); 97 101 else 98 102 grid->inputField(data, packet->data); … … 117 121 } 118 122 119 template void CSourceFilter::streamData<1>(CDate date, const CArray<double, 1>& data );120 template void CSourceFilter::streamData<2>(CDate date, const CArray<double, 2>& data );121 template void CSourceFilter::streamData<3>(CDate date, const CArray<double, 3>& data );122 template void CSourceFilter::streamData<4>(CDate date, const CArray<double, 4>& data );123 template void CSourceFilter::streamData<5>(CDate date, const CArray<double, 5>& data );124 template void CSourceFilter::streamData<6>(CDate date, const CArray<double, 6>& data );125 template void CSourceFilter::streamData<7>(CDate date, const CArray<double, 7>& data );123 template void CSourceFilter::streamData<1>(CDate date, const CArray<double, 1>& data, bool isTiled); 124 template void CSourceFilter::streamData<2>(CDate date, const CArray<double, 2>& data, bool isTiled); 125 template void CSourceFilter::streamData<3>(CDate date, const CArray<double, 3>& data, bool isTiled); 126 template void CSourceFilter::streamData<4>(CDate date, const CArray<double, 4>& data, bool isTiled); 127 template void CSourceFilter::streamData<5>(CDate date, const CArray<double, 5>& data, bool isTiled); 128 template void CSourceFilter::streamData<6>(CDate date, const CArray<double, 6>& data, bool isTiled); 129 template void CSourceFilter::streamData<7>(CDate date, const CArray<double, 7>& data, bool isTiled); 126 130 127 131 template void CSourceFilter::streamTile<1>(CDate date, const CArray<double, 1>& data, int ntile); -
XIOS/dev/dev_oa/src/filter/source_filter.hpp
r1966 r2034 46 46 */ 47 47 template <int N> 48 void streamData(CDate date, const CArray<double, N>& data );48 void streamData(CDate date, const CArray<double, N>& data, bool isTiled = false); 49 49 50 50 template <int N> -
XIOS/dev/dev_oa/src/node/domain.cpp
r1966 r2034 35 35 , clients(), hasLatInReadFile_(false), hasBoundsLatInReadFile_(false) 36 36 , hasLonInReadFile_(false), hasBoundsLonInReadFile_(false) 37 , isTiled_(false) 37 , isTiled_(false), isTiledOnly_(false) 38 38 { 39 39 } … … 49 49 , clients(), hasLatInReadFile_(false), hasBoundsLatInReadFile_(false) 50 50 , hasLonInReadFile_(false), hasBoundsLonInReadFile_(false) 51 , isTiled_(false) 51 , isTiled_(false), isTiledOnly_(false) 52 52 { 53 53 } … … 255 255 { 256 256 return isTiled_; 257 } 258 CATCH 259 260 bool CDomain::isTiledOnly(void) const 261 TRY 262 { 263 return isTiledOnly_; 257 264 } 258 265 CATCH … … 1285 1292 } 1286 1293 } 1294 } 1295 CATCH_DUMP_ATTR 1296 1297 //---------------------------------------------------------------- 1298 1299 /*! 1300 * For tiled domains, data_i/j_index should not take into 1301 * account parameters defining data (data_ni/nj, data_i/jbegin...) 1302 * \param [out] dataIndexI 1303 * \param [out] dataIndexJ 1304 * \param [out] infoIndexI 1305 * \param [out] infoIndexJ 1306 */ 1307 1308 void CDomain::computeCompressionTiled(CArray<int,1>& dataIndexI, CArray<int,1>& dataIndexJ, 1309 CArray<int,1>& infoIndexI, CArray<int,1>& infoIndexJ) 1310 TRY 1311 { 1312 const int dsize = ni * nj; 1313 dataIndexI.resize(dsize); 1314 dataIndexJ.resize(dsize); 1315 1316 dataIndexJ = 0; 1317 for (int k = 0; k < ni; ++k) 1318 dataIndexI(k) = k; 1319 1320 infoIndexI.resize(ni*nj); 1321 for (int j = 0; j < nj; ++j) 1322 for (int i = 0; i < ni; ++i) infoIndexI(i+j*ni) = i+ibegin; 1323 1324 infoIndexJ.resize(ni*nj); 1325 for (int j = 0; j < nj; ++j) 1326 for (int i = 0; i < ni; ++i) infoIndexJ(i+j*ni) = j+jbegin; 1287 1327 } 1288 1328 CATCH_DUMP_ATTR … … 1752 1792 TRY 1753 1793 { 1754 if (!ntiles.isEmpty() && ntiles.getValue() >1) isTiled_ = true; 1794 if (!ntiles.isEmpty() && ntiles.getValue() >=1) isTiled_ = true; 1795 if (!tile_only.isEmpty() && tile_only.getValue() == true) { 1796 isTiled_ = true; 1797 isTiledOnly_ = true; 1798 } 1799 1755 1800 if (isTiled_) 1756 1801 { -
XIOS/dev/dev_oa/src/node/domain.hpp
r1966 r2034 106 106 107 107 bool isTiled(void) const; 108 bool isTiledOnly(void) const; 108 109 int getTileId(int i, int j) const; 109 110 int getTileDataISize(int tileId) const; 110 111 int getTileDataJSize(int tileId) const; 112 void computeCompressionTiled(CArray<int,1>& dataIndexI, CArray<int,1>& dataIndexJ, 113 CArray<int,1>& infoIndexI, CArray<int,1>& infoIndexJ); 111 114 112 115 std::vector<int> getNbGlob(); … … 228 231 std::map<int, std::vector<int> > connectedServerRank_; 229 232 230 233 //! True if and only if the data defined on the domain can be outputted in a compressed way 231 234 bool isCompressible_; 232 235 bool isRedistributed_; … … 235 238 std::unordered_map<size_t,size_t> globalLocalIndexMap_; 236 239 240 //! True if tiled data is defined on the domain 237 241 bool isTiled_; 242 //! True if ONLY tiled data is defined on the domain 243 bool isTiledOnly_; 238 244 239 245 private: -
XIOS/dev/dev_oa/src/node/grid.cpp
r1966 r2034 40 40 , clients() 41 41 , nTiles_(0) 42 , isTiled_(false) 42 , isTiled_(false), isTiledOnly_(false) 43 43 , storeTileIndex() 44 44 { … … 64 64 , clients() 65 65 , nTiles_(0) 66 , isTiled_(false) 66 , isTiled_(false), isTiledOnly_(false) 67 67 , storeTileIndex() 68 68 { … … 638 638 else domListP[i]->checkAttributesOnClient(); 639 639 if (domListP[i]->isTiled()) this->isTiled_ = true; 640 if (domListP[i]->isTiledOnly()) this->isTiledOnly_ = true; 640 641 } 641 642 } … … 802 803 outLocalIndexStoreOnClient.insert(make_pair(rank, CArray<size_t,1>(globalIndex.numElements()))); 803 804 CArray<size_t,1>& localIndex = outLocalIndexStoreOnClient[rank]; 805 size_t nbIndex = 0; 806 807 // Keep this code for this moment but it should be removed (or moved to DEBUG) to improve performance 808 for (size_t idx = 0; idx < globalIndex.numElements(); ++idx) 809 { 810 if (itGloe != globalDataIndex.find(globalIndex(idx))) 811 { 812 ++nbIndex; 813 } 814 } 815 816 if (doGridHaveDataDistributed(client) && (nbIndex != localIndex.numElements())) 817 ERROR("void CGrid::computeClientIndex()", 818 << "Number of local index on client is different from number of received global index" 819 << "Rank of sent client " << rank <<"." 820 << "Number of local index " << nbIndex << ". " 821 << "Number of received global index " << localIndex.numElements() << "."); 822 823 nbIndex = 0; 824 for (size_t idx = 0; idx < globalIndex.numElements(); ++idx) 825 { 826 if (itGloe != globalDataIndex.find(globalIndex(idx))) 827 { 828 localIndex(idx) = globalDataIndex[globalIndex(idx)]; 829 } 830 } 831 } 832 } 833 } 834 CATCH_DUMP_ATTR 835 836 //--------------------------------------------------------------- 837 838 /* 839 Compute the global index and its local index taking account mask and data index. 840 These global indexes will be used to compute the connection of this client (sender) to its servers (receivers) 841 (via function computeConnectedClient) 842 These global indexes also correspond to data sent to servers (if any) 843 */ 844 void CGrid::computeClientIndexTiled() 845 TRY 846 { 847 CContext* context = CContext::getCurrent(); 848 849 CContextClient* client = context->client; 850 int rank = client->clientRank; 851 852 clientDistributionTiled_ = new CDistributionClient(rank, this, true); 853 // Get local data index on client 854 int nbStoreIndex = clientDistributionTiled_->getLocalDataIndexOnClient().size(); 855 int nbStoreGridMask = clientDistributionTiled_->getLocalMaskIndexOnClient().size(); 856 // nbStoreGridMask = nbStoreIndex if grid mask is defined, and 0 otherwise 857 storeIndexTiled_client.resize(nbStoreIndex); 858 storeMaskTiled_client.resize(nbStoreGridMask); 859 for (int idx = 0; idx < nbStoreIndex; ++idx) storeIndexTiled_client(idx) = (clientDistributionTiled_->getLocalDataIndexOnClient())[idx]; 860 for (int idx = 0; idx < nbStoreGridMask; ++idx) storeMaskTiled_client(idx) = (clientDistributionTiled_->getLocalMaskIndexOnClient())[idx]; 861 862 if (0 == serverDistribution_) isDataDistributed_= clientDistributionTiled_->isDataDistributed(); 863 else 864 { 865 // Mapping global index received from clients to the storeIndex_client 866 CDistributionClient::GlobalLocalDataMap& globalDataIndex = clientDistributionTiled_->getGlobalDataIndexOnClient(); 867 CDistributionClient::GlobalLocalDataMap::const_iterator itGloe = globalDataIndex.end(); 868 map<int, CArray<size_t, 1> >::iterator itb = outGlobalIndexFromClientTiled.begin(), 869 ite = outGlobalIndexFromClientTiled.end(), it; 870 871 for (it = itb; it != ite; ++it) 872 { 873 int rank = it->first; 874 CArray<size_t,1>& globalIndex = outGlobalIndexFromClientTiled[rank]; 875 outLocalIndexStoreOnClientTiled.insert(make_pair(rank, CArray<size_t,1>(globalIndex.numElements()))); 876 CArray<size_t,1>& localIndex = outLocalIndexStoreOnClientTiled[rank]; 804 877 size_t nbIndex = 0; 805 878 … … 978 1051 else 979 1052 { 980 computeClientIndex(); 1053 if (this->isTiled_) 1054 { 1055 computeClientIndexTiled(); 1056 if (!this->isTiledOnly_) 1057 computeClientIndex(); 1058 } 1059 else 1060 computeClientIndex(); 1061 981 1062 if (this->isTiled_) computeTileIndex(); 982 1063 if (context->hasClient) … … 1543 1624 CATCH 1544 1625 1545 void CGrid::maskField_arr(const double* const data, CArray<double, 1>& stored) const 1546 TRY 1547 { 1548 const StdSize size = storeIndex_client.numElements(); 1626 void CGrid::maskField_arr(const double* const data, CArray<double, 1>& stored, bool isTiled) const 1627 TRY 1628 { 1629 const CArray<int, 1>& storeIndex_clientP = isTiled ? storeIndexTiled_client : storeIndex_client; 1630 const CArray<bool, 1>& storeMask_clientP = isTiled ? storeMaskTiled_client : storeMask_client; 1631 const StdSize size = storeIndex_clientP.numElements(); 1549 1632 stored.resize(size); 1550 1633 const double nanValue = std::numeric_limits<double>::quiet_NaN(); 1551 1634 1552 if (storeMask_client .numElements() != 0)1553 for(StdSize i = 0; i < size; i++) stored(i) = (storeMask_client (i)) ? data[storeIndex_client(i)] : nanValue;1635 if (storeMask_clientP.numElements() != 0) 1636 for(StdSize i = 0; i < size; i++) stored(i) = (storeMask_clientP(i)) ? data[storeIndex_clientP(i)] : nanValue; 1554 1637 else 1555 for(StdSize i = 0; i < size; i++) stored(i) = data[storeIndex_client (i)];1638 for(StdSize i = 0; i < size; i++) stored(i) = data[storeIndex_clientP(i)]; 1556 1639 } 1557 1640 CATCH … … 2513 2596 CATCH_DUMP_ATTR 2514 2597 2598 bool CGrid::isTiled(void) const 2599 TRY 2600 { 2601 return isTiled_; 2602 } 2603 CATCH 2604 2605 bool CGrid::isTiledOnly(void) const 2606 TRY 2607 { 2608 return isTiledOnly_; 2609 } 2610 CATCH 2611 2515 2612 bool CGrid::isTransformed() 2516 2613 TRY -
XIOS/dev/dev_oa/src/node/grid.hpp
r1966 r2034 99 99 void inputField(const CArray<double,n>& field, CArray<double,1>& stored) const; 100 100 template <int n> 101 void maskField(const CArray<double,n>& field, CArray<double,1>& stored ) const;101 void maskField(const CArray<double,n>& field, CArray<double,1>& stored, bool isTiled = false) const; 102 102 template <int n> 103 103 void outputField(const CArray<double,1>& stored, CArray<double,n>& field) const; … … 211 211 public: 212 212 CArray<int, 1> storeIndex_client; 213 CArray<int, 1> storeIndexTiled_client; 213 214 CArray<bool, 1> storeMask_client; 215 CArray<bool, 1> storeMaskTiled_client; 214 216 215 217 /** Map containing indexes that will be sent in sendIndex(). */ … … 229 231 /** Map storing received indexes. Key = sender rank, value = index array. */ 230 232 map<int, CArray<size_t, 1> > outGlobalIndexFromClient; 233 234 /** Map storing received indexes. Key = sender rank, value = index array for tiled domains */ 235 map<int, CArray<size_t, 1> > outGlobalIndexFromClientTiled; 231 236 232 237 // Manh Ha's comment: " A client receives global index from other clients (via recvIndex) … … 238 243 * The map is created in CGrid::computeClientIndex and filled upon receiving data in CField::recvUpdateData() */ 239 244 map<int, CArray<size_t, 1> > outLocalIndexStoreOnClient; 245 246 /** Map storing received data. Key = sender rank, value = data array. 247 * The map is created in CGrid::computeClientIndex and filled upon receiving data in CField::recvUpdateData() */ 248 map<int, CArray<size_t, 1> > outLocalIndexStoreOnClientTiled; 249 240 250 241 251 /** Indexes calculated based on server-like distribution. … … 263 273 264 274 int getNTiles(); 275 bool isTiled(void) const; 276 bool isTiledOnly(void) const; 265 277 266 278 private: … … 280 292 void restoreField_arr(const CArray<double, 1>& stored, double* const data) const; 281 293 void uncompressField_arr(const double* const data, CArray<double, 1>& outData) const; 282 void maskField_arr(const double* const data, CArray<double, 1>& stored ) const;294 void maskField_arr(const double* const data, CArray<double, 1>& stored, bool isTiled = false) const; 283 295 void copyTile_arr(const double* const tileData, CArray<double, 1>& stored, int tileId); 284 296 … … 308 320 309 321 void computeClientIndex(); 322 void computeClientIndexTiled(); 310 323 void computeConnectedClients(); 311 324 void computeClientIndexScalarGrid(); … … 325 338 326 339 int nTiles_; 340 /** True if tiled data is defined on the grid */ 327 341 bool isTiled_; 342 /** True if ONLY tiled data is defined on the grid */ 343 bool isTiledOnly_; 344 328 345 /** Vector containing local domain indexes for each tile */ 329 346 std::vector<CArray<int,1> > storeTileIndex; … … 337 354 /** Client-like distribution calculated based on the knowledge of the entire grid */ 338 355 CDistributionClient* clientDistribution_; 356 357 /** Client-like distribution calculated based on the knowledge of the entire grid in case of a tiled domain */ 358 CDistributionClient* clientDistributionTiled_; 359 339 360 340 361 /** Server-like distribution calculated upon receiving indexes */ … … 409 430 410 431 template <int n> 411 void CGrid::maskField(const CArray<double,n>& field, CArray<double,1>& stored ) const432 void CGrid::maskField(const CArray<double,n>& field, CArray<double,1>& stored, bool isTiled) const 412 433 { 413 434 //#ifdef __XIOS_DEBUG … … 419 440 << "Grid = " << this->getId()) 420 441 //#endif 421 this->maskField_arr(field.dataFirst(), stored); 422 } 442 this->maskField_arr(field.dataFirst(), stored, isTiled); 443 } 444 445 // template <int n> 446 // void CGrid::maskFieldTiled(const CArray<double,n>& field, CArray<double,1>& stored) const 447 // { 448 ////#ifdef __XIOS_DEBUG 449 // if (this->getDataSize() != field.numElements()) 450 // ERROR("void CGrid::maskField(const CArray<double,n>& field, CArray<double,1>& stored) const", 451 // << "[ Awaiting data of size = " << this->getDataSize() << ", " 452 // << "Received data size = " << field.numElements() << " ] " 453 // << "The data array does not have the right size! " 454 // << "Grid = " << this->getId()) 455 ////#endif 456 // this->maskField_arr(field.dataFirst(), stored, true); 457 // } 423 458 424 459 template <int n>
Note: See TracChangeset
for help on using the changeset viewer.