source: XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution_client.cpp @ 1878

Last change on this file since 1878 was 1869, checked in by ymipsl, 4 years ago

Some update...

YM
File size: 19.8 KB
Line 
1/*!
2   \file distribution_client.cpp
3   \author Ha NGUYEN
4   \since 13 Jan 2015
5   \date 09 Mars 2015
6
7   \brief Index distribution on client side.
8 */
9#include "distribution_client.hpp"
10
11namespace xios {
12
13CDistributionClient::CDistributionClient(int rank, CGrid* grid)
14   : CDistribution(rank, 0)
15   , axisDomainOrder_()
16   , nLocal_(), nGlob_(), nBeginLocal_(), nBeginGlobal_()
17   , dataNIndex_(), dataDims_(), dataBegin_(), dataIndex_()
18   , gridMask_(), indexMap_()
19   , isDataDistributed_(true), axisNum_(0), domainNum_(0)
20   , localDataIndex_(), localMaskIndex_()
21   , globalLocalDataSendToServerMap_()
22   , infoIndex_(), isComputed_(false)
23   , elementLocalIndex_(), elementGlobalIndex_(), elementIndexData_()
24   , elementNLocal_(), elementNGlobal_()
25{
26  readDistributionInfo(grid);
27  createGlobalIndex();
28}
29
30CDistributionClient::~CDistributionClient()
31{ /* Nothing to do */ }
32
33void CDistributionClient::partialClear()
34{
35  GlobalLocalMap void1 ;
36  GlobalLocalMap void2 ;
37  std::vector<int> void3 ;
38  std::vector<bool> void4 ;
39
40  globalLocalDataSendToServerMap_.swap(void1) ;
41  globalDataIndex_.swap(void2) ;
42  localDataIndex_.swap(void3);
43  localMaskIndex_.swap(void4) ;
44}
45
46/*!
47  Read information of a grid to generate distribution.
48  Every grid is composed of several axis or/and domain(s). Their information are processed
49stored and used to calculate index distribution between client and server
50  \param [in] grid Grid to read
51*/
52void CDistributionClient::readDistributionInfo(CGrid* grid)
53{
54  std::vector<CDomain*> domList = grid->getDomains();
55  std::vector<CAxis*> axisList = grid->getAxis();
56  std::vector<CScalar*> scalarList = grid->getScalars();
57  CArray<int,1> axisDomainOrder = grid->axis_domain_order;
58
59  readDistributionInfo(domList, axisList, scalarList, axisDomainOrder);
60
61  // Then check mask of grid
62  int gridDim = domList.size() * 2 + axisList.size();
63  switch (gridDim) {
64    case 0:
65      gridMask_.resize(1);
66      gridMask_(0) = true;
67      break;
68    case 1:
69      if (!grid->mask_1d.isEmpty()) readGridMaskInfo(grid->mask_1d);
70      break;
71    case 2:
72      if (!grid->mask_2d.isEmpty()) readGridMaskInfo(grid->mask_2d);
73      break;
74    case 3:
75      if (!grid->mask_3d.isEmpty()) readGridMaskInfo(grid->mask_3d);
76      break;
77    case 4:
78      if (!grid->mask_4d.isEmpty()) readGridMaskInfo(grid->mask_4d);
79      break;
80    case 5:
81      if (!grid->mask_5d.isEmpty()) readGridMaskInfo(grid->mask_5d);
82      break;
83    case 6:
84      if (!grid->mask_6d.isEmpty()) readGridMaskInfo(grid->mask_6d);
85      break;
86    case 7:
87      if (!grid->mask_7d.isEmpty()) readGridMaskInfo(grid->mask_7d);
88      break;
89    default:
90      break;
91  }
92}
93
94/*!
95  Read information from domain(s) and axis to generate distribution.
96  All information related to domain, e.g ibegin, jbegin, ni, nj, ni_glo, nj_glo
97as well as related to axis, e.g dataNIndex, dataIndex will be stored to compute
98the distribution between clients and servers. Till now, every data structure of domain has been kept
99like before, e.g: data_n_index to make sure a compability, however, it should be changed?
100  \param [in] domList List of domains of grid
101  \param [in] axisList List of axis of grid
102  \param [in] scalarList List of scalar of grid
103  \param [in] axisDomainOrder order of axis and domain inside a grid. 2 if domain, 1 if axis and zero if scalar
104//  \param [in] gridMask Mask of grid, for now, keep it 3 dimension, but it needs changing
105*/
106void CDistributionClient::readDistributionInfo(const std::vector<CDomain*>& domList,
107                                               const std::vector<CAxis*>& axisList,
108                                               const std::vector<CScalar*>& scalarList,
109                                               const CArray<int,1>& axisDomainOrder)
110{
111  domainNum_ = domList.size();
112  axisNum_   = axisList.size();
113  numElement_ = axisDomainOrder.numElements(); // Number of element, e.x: Axis, Domain
114
115  axisDomainOrder_.resize(numElement_);
116  axisDomainOrder_ = axisDomainOrder;
117
118  // Because domain and axis can be in any order (axis1, domain1, axis2, axis3, )
119  // their position should be specified. In axisDomainOrder, domain == true, axis == false
120  int idx = 0;
121  indexMap_.resize(numElement_);
122  this->dims_ = numElement_;
123  for (int i = 0; i < numElement_; ++i)
124  {
125    indexMap_[i] = idx;
126    if (2 == axisDomainOrder(i))
127    {
128      ++(this->dims_);
129      idx += 2;
130    }
131    else ++idx;
132  }
133
134  // Size of each dimension (local and global)
135  nLocal_.resize(this->dims_);
136  nGlob_.resize(this->dims_);
137  nBeginLocal_.resize(this->dims_,0);
138  nBeginGlobal_.resize(this->dims_,0);
139
140  // Data_n_index of domain or axis (For now, axis uses its size as data_n_index
141  dataNIndex_.resize(numElement_);
142  dataDims_.resize(numElement_);
143  dataBegin_.resize(this->dims_);
144
145  // Data_*_index of each dimension
146  dataIndex_.resize(this->dims_);
147  infoIndex_.resize(this->dims_);
148
149  // A trick to determine position of each domain in domainList
150  int domIndex = 0, axisIndex = 0, scalarIndex = 0;
151  idx = 0;
152
153  elementLocalIndex_.resize(numElement_);
154  elementGlobalIndex_.resize(numElement_);
155  elementIndexData_.resize(numElement_);
156  elementNLocal_.resize(numElement_);
157  elementNGlobal_.resize(numElement_);
158  elementNLocal_[0] = 1;
159  elementNGlobal_[0] = 1;
160  size_t localSize = 1, globalSize = 1;
161
162  isDataDistributed_ = false;
163  // Update all the vectors above
164  for (idx = 0; idx < numElement_; ++idx)
165  {
166    int eleDim = axisDomainOrder(idx);
167    elementNLocal_[idx] = localSize;
168    elementNGlobal_[idx] = globalSize;
169
170    // If this is a domain
171    if (2 == eleDim)
172    {
173      // On the j axis
174      nLocal_.at(indexMap_[idx]+1) = domList[domIndex]->nj.getValue();
175      nGlob_.at(indexMap_[idx]+1)  = domList[domIndex]->nj_glo.getValue();
176      nBeginLocal_.at(indexMap_[idx]+1) = 0;
177      nBeginGlobal_.at(indexMap_[idx]+1) = domList[domIndex]->jbegin;
178
179      dataBegin_.at(indexMap_[idx]+1) = domList[domIndex]->data_jbegin.getValue();
180      dataIndex_.at(indexMap_[idx]+1).reference(domList[domIndex]->data_j_index);
181      infoIndex_.at(indexMap_[idx]+1).reference(domList[domIndex]->j_index);
182
183      // On the i axis
184      nLocal_.at(indexMap_[idx]) = domList[domIndex]->ni.getValue();
185      nGlob_.at(indexMap_[idx]) = domList[domIndex]->ni_glo.getValue();
186      nBeginLocal_.at(indexMap_[idx]) = 0;
187      nBeginGlobal_.at(indexMap_[idx]) = domList[domIndex]->ibegin;
188
189      dataBegin_.at(indexMap_[idx]) = domList[domIndex]->data_ibegin.getValue();
190      dataIndex_.at(indexMap_[idx]).reference(domList[domIndex]->data_i_index);
191      infoIndex_.at(indexMap_[idx]).reference(domList[domIndex]->i_index);
192
193      dataNIndex_.at(idx) = domList[domIndex]->data_i_index.numElements();
194      dataDims_.at(idx) = domList[domIndex]->data_dim.getValue();
195
196      isDataDistributed_ |= domList[domIndex]->isDistributed();
197
198      localSize *= nLocal_.at(indexMap_[idx]+1)* nLocal_.at(indexMap_[idx]);
199      globalSize *= nGlob_.at(indexMap_[idx]+1)* nGlob_.at(indexMap_[idx]);
200      ++domIndex;
201    }
202    else if (1 == eleDim)// So it's an axis
203    {
204      nLocal_.at(indexMap_[idx]) = axisList[axisIndex]->n.getValue();
205      nGlob_.at(indexMap_[idx]) = axisList[axisIndex]->n_glo.getValue();
206      nBeginLocal_.at(indexMap_[idx]) = 0;
207      nBeginGlobal_.at(indexMap_[idx]) = axisList[axisIndex]->begin.getValue();
208
209      dataBegin_.at(indexMap_[idx]) = axisList[axisIndex]->data_begin.getValue();
210      dataIndex_.at(indexMap_[idx]).reference(axisList[axisIndex]->data_index);
211      infoIndex_.at(indexMap_[idx]).reference(axisList[axisIndex]->index);
212      dataNIndex_.at(idx) = axisList[axisIndex]->data_index.numElements();
213      dataDims_.at(idx) = 1;
214
215      isDataDistributed_ |= axisList[axisIndex]->isDistributed();
216
217      localSize *= nLocal_.at(indexMap_[idx]);
218      globalSize *= nGlob_.at(indexMap_[idx]);
219
220      ++axisIndex;
221    }
222    else // scalar
223    {
224      nLocal_.at(indexMap_[idx]) = 1;
225      nGlob_.at(indexMap_[idx]) = 1;
226      nBeginLocal_.at(indexMap_[idx]) = 0;
227      nBeginGlobal_.at(indexMap_[idx]) = 1;
228
229      dataBegin_.at(indexMap_[idx]) = 0;
230      dataIndex_.at(indexMap_[idx]).resize(1); dataIndex_.at(indexMap_[idx])(0) = 0;
231      infoIndex_.at(indexMap_[idx]).resize(1); infoIndex_.at(indexMap_[idx])(0) = 0;
232      dataNIndex_.at(idx) = 1;
233      dataDims_.at(idx) = 1;
234
235      isDataDistributed_ |= false;
236
237      localSize *= nLocal_.at(indexMap_[idx]);
238      globalSize *= nGlob_.at(indexMap_[idx]);
239
240      ++scalarIndex;
241    }
242  }
243}
244
245/*!
246  Create local index of domain(s).
247  A domain can have data index which even contains the "ghost" points. Very often, these
248data surround the true data. In order to send correct data to server,
249a client need to know index of the true data.
250*/
251void CDistributionClient::createLocalDomainDataIndex()
252{
253  int idxDomain = 0;
254  for (int i = 0; i < axisDomainOrder_.numElements(); ++i)
255  {
256    if (2 == axisDomainOrder_(i))
257    {
258      elementIndexData_[i].resize(dataNIndex_[i]);
259      elementIndexData_[i] = false;
260      int iIdx, jIdx = 0, count = 0, localIndex;
261      for (int j = 0; j < dataNIndex_[i]; ++j)
262      {
263        iIdx = getDomainIndex((dataIndex_[indexMap_[i]])(j), (dataIndex_[indexMap_[i]+1])(j),
264                              dataBegin_[indexMap_[i]], dataBegin_[indexMap_[i]+1],
265                              dataDims_[i], nLocal_[indexMap_[i]], jIdx);
266
267        if ((iIdx >= nBeginLocal_[indexMap_[i]]) && (iIdx < nLocal_[indexMap_[i]]) &&
268           (jIdx >= nBeginLocal_[indexMap_[i]+1]) && (jIdx < nLocal_[indexMap_[i]+1]))
269        {
270          ++count;
271          elementIndexData_[i](j) = true;
272        }
273      }
274
275      elementLocalIndex_[i].resize(count);
276      elementGlobalIndex_[i].resize(count);
277      count = 0;
278      CArray<bool,1>& tmpIndexElementData = elementIndexData_[i];
279      CArray<int,1>& tmpLocalElementIndex = elementLocalIndex_[i];
280      CArray<size_t,1>& tmpGlobalElementIndex = elementGlobalIndex_[i];
281      for (int j = 0; j < dataNIndex_[i]; ++j)
282      {
283        if (tmpIndexElementData(j))
284        {
285          iIdx = getDomainIndex((dataIndex_[indexMap_[i]])(j), (dataIndex_[indexMap_[i]+1])(j),
286                                dataBegin_[indexMap_[i]], dataBegin_[indexMap_[i]+1],
287                                dataDims_[i], nLocal_[indexMap_[i]], jIdx);
288          localIndex = tmpLocalElementIndex(count) = iIdx + jIdx * nLocal_[indexMap_[i]];
289          tmpGlobalElementIndex(count) = (infoIndex_[indexMap_[i]])(localIndex) + ((infoIndex_[indexMap_[i]+1])(localIndex))*nGlob_[indexMap_[i]];
290          ++count;
291        }
292      }
293      ++idxDomain;
294    }
295  }
296}
297
298/*!
299  Create local index of axis.
300*/
301void CDistributionClient::createLocalAxisDataIndex()
302{
303  int idxAxis = 0;
304  for (int i = 0; i < axisDomainOrder_.numElements(); ++i)
305  {
306    if (1 == axisDomainOrder_(i))
307    {
308      elementIndexData_[i].resize(dataNIndex_[i]);
309      elementIndexData_[i] = false;
310      int iIdx = 0, count = 0;
311      for (int j = 0; j < dataNIndex_[i]; ++j)
312      {
313        iIdx = getAxisIndex((dataIndex_[indexMap_[i]])(j), dataBegin_[indexMap_[i]], nLocal_[indexMap_[i]]);
314        if ((iIdx >= nBeginLocal_[indexMap_[i]]) &&
315           (iIdx < nLocal_[indexMap_[i]]) )//&& (axisMasks_[idxAxis](iIdx)))
316        {
317          ++count;
318          elementIndexData_[i](j) = true;
319        }
320      }
321
322      elementLocalIndex_[i].resize(count);
323      elementGlobalIndex_[i].resize(count);
324      count = 0;
325      CArray<bool,1>& tmpIndexElementData = elementIndexData_[i];
326      CArray<int,1>& tmpLocalElementIndex = elementLocalIndex_[i];
327      CArray<size_t,1>& tmpGlobalElementIndex = elementGlobalIndex_[i];
328      for (int j = 0; j < dataNIndex_[i]; ++j)
329      {
330        if (tmpIndexElementData(j))
331        {
332          iIdx = tmpLocalElementIndex(count) = getAxisIndex((dataIndex_[indexMap_[i]])(j), dataBegin_[indexMap_[i]], nLocal_[indexMap_[i]]);
333          tmpGlobalElementIndex(count) = (infoIndex_[indexMap_[i]])(iIdx);
334          ++count;
335        }
336      }
337      ++idxAxis;
338    }
339  }
340}
341
342/*!
343  Create local index of scalar.
344*/
345void CDistributionClient::createLocalScalarDataIndex()
346{
347  int idxAxis = 0;
348  for (int i = 0; i < axisDomainOrder_.numElements(); ++i)
349  {
350    if (0 == axisDomainOrder_(i))
351    {
352      elementIndexData_[i].resize(dataNIndex_[i]);
353      elementIndexData_[i] = true;
354      int count = 1;
355
356      elementLocalIndex_[i].resize(count);
357      elementLocalIndex_[i] = 0;
358      elementGlobalIndex_[i].resize(count);
359      elementGlobalIndex_[i] = 0;
360    }
361  }
362}
363
364/*!
365   Create global index on client
366   In order to do the mapping between client-server, each client creates its own
367global index of sending data. This global index is then used to calculate to which server
368the client needs to send it data as well as which part of data belongs to the server.
369So as to make clients and server coherent in order of index, global index is calculated by
370take into account of C-convention, the rightmost dimension varies faster.
371*/
372void CDistributionClient::createGlobalIndexSendToServer()
373{
374  if (isComputed_) return;
375  isComputed_ = true;
376  createLocalDomainDataIndex();
377  createLocalAxisDataIndex();
378  createLocalScalarDataIndex();
379
380  int idxDomain = 0, idxAxis = 0;
381  std::vector<int> eachElementSize(numElement_);
382
383  // Precompute size of the loop
384  for (int i = 0; i < numElement_; ++i)
385  {
386    eachElementSize[i] = elementLocalIndex_[i].numElements();
387  }
388
389  //   Compute size of the global index on client
390  std::vector<StdSize> idxLoop(numElement_,0);
391  std::vector<StdSize> currentIndex(numElement_,0);
392  std::vector<StdSize> currentGlobalIndex(numElement_,0);
393  int innerLoopSize = eachElementSize[0];
394  size_t idx = 0, indexLocalDataOnClientCount = 0;
395  size_t ssize = 1;
396
397  for (int i = 0; i < numElement_; ++i) ssize *= eachElementSize[i];
398  localDataSize_=ssize ;
399
400  localDataIndex_.resize(ssize);
401  if (!gridMask_.isEmpty()) localMaskIndex_.resize(ssize);
402  localMaskedDataIndex_.resize(ssize);
403  globalDataIndex_.rehash(std::ceil(ssize/globalDataIndex_.max_load_factor()));
404  globalLocalDataSendToServerMap_.rehash(std::ceil(ssize/globalLocalDataSendToServerMap_.max_load_factor()));
405
406
407  // We need to loop with data index
408  idxLoop.assign(numElement_,0);
409  idx = indexLocalDataOnClientCount = 0;
410  ssize = 1; for (int i = 0; i < numElement_; ++i) ssize *= dataNIndex_[i];
411  innerLoopSize = dataNIndex_[0];
412  int countLocalData = 0;
413  std::vector<int> correctIndexOfElement(numElement_,0);
414  bool isOuterIndexCorrect = true;
415  while (idx < ssize)
416  {
417    for (int i = 0; i < numElement_-1; ++i)
418    {
419      if (idxLoop[i] == dataNIndex_[i])
420      {
421        idxLoop[i] = 0;
422        correctIndexOfElement[i] = 0;
423        ++idxLoop[i+1];
424        if (isOuterIndexCorrect) ++correctIndexOfElement[i+1];
425      }
426    }
427
428    // Depending the inner-most element axis or domain,
429    // The outer loop index begins correspondingly at one (1) or zero (0)
430    bool isIndexElementDataCorrect = true;
431    for (int i = 1; i < numElement_; ++i)
432    {
433      if (elementIndexData_[i](idxLoop[i]))
434      {
435        currentIndex[i] = elementLocalIndex_[i](correctIndexOfElement[i]);
436        currentGlobalIndex[i] = elementGlobalIndex_[i](correctIndexOfElement[i]);
437        isIndexElementDataCorrect &= true;
438      }
439      else isIndexElementDataCorrect = false;
440    }
441
442    isOuterIndexCorrect = isIndexElementDataCorrect;
443
444    if (isOuterIndexCorrect)
445    {
446      // Inner most index
447      int correctIndexInnerElement = 0;
448      for (int i = 0; i < innerLoopSize; ++i)
449      {
450        bool isCurrentIndexDataCorrect = isOuterIndexCorrect;
451        if (elementIndexData_[0](i))
452        {
453          currentIndex[0] = elementLocalIndex_[0](correctIndexInnerElement);
454          currentGlobalIndex[0] = elementGlobalIndex_[0](correctIndexInnerElement);
455          isCurrentIndexDataCorrect &= true;
456          ++correctIndexInnerElement;
457        }
458        else isCurrentIndexDataCorrect = false;
459
460        if (isCurrentIndexDataCorrect)
461        {
462          bool maskTmp = true;
463          bool maskGridTmp = true;
464          size_t globalIndex = 0;
465          for (int k = 0; k < numElement_; ++k)
466          {
467            globalIndex += (currentGlobalIndex[k])*elementNGlobal_[k];
468          }
469          globalDataIndex_[globalIndex] = indexLocalDataOnClientCount;
470          localDataIndex_[indexLocalDataOnClientCount] = countLocalData;
471          globalLocalDataSendToServerMap_[globalIndex] = indexLocalDataOnClientCount;
472          localMaskedDataIndex_[indexLocalDataOnClientCount] = indexLocalDataOnClientCount;
473
474          // Grid mask: unmasked values will be replaces by NaN and then all values will be sent
475          if (!gridMask_.isEmpty())
476          {
477            int gridMaskIndex = 0;
478            for (int k = 0; k < this->numElement_; ++k)
479            {
480              gridMaskIndex += (currentIndex[k])*elementNLocal_[k];
481            }
482            maskGridTmp =  gridMask_(gridMaskIndex);
483            if (maskGridTmp)
484              localMaskIndex_[indexLocalDataOnClientCount] = true;
485            else
486              localMaskIndex_[indexLocalDataOnClientCount] = false;
487          }
488
489          ++indexLocalDataOnClientCount;
490
491        }
492        ++countLocalData;
493        correctIndexOfElement[0] = correctIndexInnerElement;;
494      }
495    }
496    else countLocalData+=innerLoopSize ;
497
498    idxLoop[0] += innerLoopSize;
499    idx += innerLoopSize;
500  }
501}
502
503void CDistributionClient::createGlobalIndex()
504{
505}
506
507/*!
508  Retrieve index i and index j of a domain from its data index
509  Data contains not only true data, which are sent to servers, but also ghost data, which
510very often play a role of border of each local data, so does data index. Because data of a domain
511can be one dimension, or two dimensions, there is a need to convert data index to domain index
512  \param [in] dataIIndex index of i data
513  \param [in] dataJIndex index of j data
514  \param [in] dataIBegin index begin of i data
515  \param [in] dataJBegin index begin of j data
516  \param [in] dataDim dimension of data (1 or 2)
517  \param [in] ni local size ni of domain
518  \param [out] j j index of domain
519  \return i index of domain
520*/
521int CDistributionClient::getDomainIndex(const int& dataIIndex, const int& dataJIndex,
522                                        const int& dataIBegin, const int& dataJBegin,
523                                        const int& dataDim, const int& ni, int& j)
524{
525  int i;
526  int tempI = dataIIndex + dataIBegin,
527      tempJ = (dataJIndex + dataJBegin);
528  if (ni == 0)
529  {
530    i = -1;
531    j = -1;
532    return i;
533  }
534  if ((tempI < 0) || (tempJ < 0))
535  {
536    i = -1;
537    j = -1;
538    return i;
539  }
540  else
541  {
542    i = (dataDim == 1) ? (tempI) % ni : (tempI) ;
543    j = (dataDim == 1) ? (tempI) / ni : (tempJ) ;
544  }
545  return i;
546}
547
548/*!
549  Retrieve index of an axis from its data index
550  \param [in] dataIndex index of data
551  \param [in] dataBegin index begin of data
552  \param [in] ni local size of axis
553  \return index of domain
554*/
555int CDistributionClient::getAxisIndex(const int& dataIndex, const int& dataBegin, const int& ni)
556{
557  if (ni == 0)
558  {
559    return -1;
560  }
561  int tempI = dataIndex;
562  if ((tempI < 0) || (tempI > ni))
563    return -1;
564  else
565    return tempI;
566}
567
568/*!
569  Return global local data mapping of client
570*/
571CDistributionClient::GlobalLocalDataMap& CDistributionClient::getGlobalLocalDataSendToServer()
572{
573  if (!isComputed_) createGlobalIndexSendToServer();
574  return globalLocalDataSendToServerMap_;
575}
576
577CDistributionClient::GlobalLocalDataMap& CDistributionClient::getGlobalDataIndexOnClient()
578{
579  if (!isComputed_) createGlobalIndexSendToServer();
580  return globalDataIndex_;
581}
582
583/*!
584  Return local data index of client
585*/
586const std::vector<int>& CDistributionClient::getLocalDataIndexOnClient()
587{
588  if (!isComputed_) createGlobalIndexSendToServer();
589  return localDataIndex_;
590}
591
592/*!
593  Return local mask index of client
594*/
595const std::vector<bool>& CDistributionClient::getLocalMaskIndexOnClient()
596{
597  if (!isComputed_) createGlobalIndexSendToServer();
598  return localMaskIndex_;
599}
600
601/*!
602  Return local mask index of client
603*/
604const std::vector<int>& CDistributionClient::getLocalMaskedDataIndexOnClient()
605{
606  if (!isComputed_) createGlobalIndexSendToServer();
607  return localMaskedDataIndex_;
608}
609
610} // namespace xios
Note: See TracBrowser for help on using the repository browser.