source: XIOS/trunk/src/distribution_client.cpp @ 861

Last change on this file since 861 was 861, checked in by mhnguyen, 5 years ago

Improving the performance of index distribution calculation

+) Preccompute all index by element

Test
+) On Curie
+) All tests pass

File size: 35.8 KB
Line 
1/*!
2   \file distribution_client.cpp
3   \author Ha NGUYEN
4   \since 13 Jan 2015
5   \date 09 Mars 2015
6
7   \brief Index distribution on client side.
8 */
9#include "distribution_client.hpp"
10
11namespace xios {
12
13CDistributionClient::CDistributionClient(int rank, int dims, const CArray<size_t,1>& globalIndex)
14   : CDistribution(rank, dims, globalIndex)
15   , axisDomainOrder_()
16   , nLocal_(), nGlob_(), nBeginLocal_(), nBeginGlobal_(),nZoomBegin_(), nZoomEnd_()
17   , dataNIndex_(), dataDims_(), dataBegin_(), dataIndex_(), domainMasks_(), axisMasks_()
18   , gridMask_(), indexMap_()
19   , isDataDistributed_(true), axisNum_(0), domainNum_(0)
20   , localDataIndex_(), localMaskIndex_()
21   , globalLocalDataSendToServerMap_()
22   , infoIndex_(), isComputed_(false)
23   , elementLocalIndex_(), elementGlobalIndex_(), elementIndexData_()
24   , elementZoomMask_(), elementNLocal_(), elementNGlobal_()
25{
26}
27
28CDistributionClient::CDistributionClient(int rank, CGrid* grid)
29   : CDistribution(rank, 0)
30   , axisDomainOrder_()
31   , nLocal_(), nGlob_(), nBeginLocal_(), nBeginGlobal_(),nZoomBegin_(), nZoomEnd_()
32   , dataNIndex_(), dataDims_(), dataBegin_(), dataIndex_(), domainMasks_(), axisMasks_()
33   , gridMask_(), indexMap_()
34   , isDataDistributed_(true), axisNum_(0), domainNum_(0)
35   , localDataIndex_(), localMaskIndex_()
36   , globalLocalDataSendToServerMap_()
37   , infoIndex_(), isComputed_(false)
38   , elementLocalIndex_(), elementGlobalIndex_(), elementIndexData_()
39   , elementZoomMask_(), elementNLocal_(), elementNGlobal_()
40{
41  readDistributionInfo(grid);
42  createGlobalIndex();
43}
44
45CDistributionClient::~CDistributionClient()
46{ /* Nothing to do */ }
47
48/*!
49  Read information of a grid to generate distribution.
50  Every grid is composed of several axis or/and domain(s). Their information are processed
51stored and used to calculate index distribution between client and server
52  \param [in] grid Grid to read
53*/
54void CDistributionClient::readDistributionInfo(CGrid* grid)
55{
56  std::vector<CDomain*> domList = grid->getDomains();
57  std::vector<CAxis*> axisList = grid->getAxis();
58  CArray<bool,1> axisDomainOrder = grid->axis_domain_order;
59
60  std::vector<CDomain*>::iterator itbDom, iteDom, itDom;
61  std::vector<CAxis*>::iterator itbAxis, iteAxis, itAxis;
62
63  itbDom  = itDom  = domList.begin();  iteDom  = domList.end();
64  itbAxis = itAxis = axisList.begin(); iteAxis = axisList.end();
65
66  readDistributionInfo(domList, axisList, axisDomainOrder);
67
68  // Then check mask of grid
69  int gridDim = domList.size() * 2 + axisList.size();
70  grid->checkMask();
71  switch (gridDim) {
72    case 1:
73      readGridMaskInfo(grid->mask_1d);
74      break;
75    case 2:
76      readGridMaskInfo(grid->mask_2d);
77      break;
78    case 3:
79      readGridMaskInfo(grid->mask_3d);
80      break;
81    default:
82      break;
83  }
84}
85
86/*!
87  Read information from domain(s) and axis to generate distribution.
88  All information related to domain, e.g ibegin, jbegin, ni, nj, ni_glo, nj_glo
89as well as related to axis, e.g dataNIndex, dataIndex will be stored to compute
90the distribution between clients and servers. Till now, every data structure of domain has been kept
91like before, e.g: data_n_index to make sure a compability, however, it should be changed?
92  \param [in] domList List of domains of grid
93  \param [in] axisList List of axis of grid
94  \param [in] axisDomainOrder order of axis and domain inside a grid. True if domain, false if axis
95//  \param [in] gridMask Mask of grid, for now, keep it 3 dimension, but it needs changing
96*/
97void CDistributionClient::readDistributionInfo(const std::vector<CDomain*>& domList,
98                                               const std::vector<CAxis*>& axisList,
99                                               const CArray<bool,1>& axisDomainOrder)
100{
101  domainNum_ = domList.size();
102  axisNum_   = axisList.size();
103  numElement_ = axisDomainOrder.numElements(); // Number of element, e.x: Axis, Domain
104
105  axisDomainOrder_.resize(numElement_);
106  axisDomainOrder_ = axisDomainOrder;
107
108  // Each domain or axis has its mask, of course
109  domainMasks_.resize(domainNum_);
110  for (int i = 0; i < domainNum_;++i)
111  {
112    domainMasks_[i].resize(domList[i]->mask_1d.numElements());
113    domainMasks_[i] = domList[i]->mask_1d;
114  }
115
116  axisMasks_.resize(axisNum_);
117  for (int i = 0; i < axisNum_; ++i)
118  {
119    axisMasks_[i].resize(axisList[i]->mask.numElements());
120    axisMasks_[i] = axisList[i]->mask;
121  }
122
123  // Because domain and axis can be in any order (axis1, domain1, axis2, axis3, )
124  // their position should be specified. In axisDomainOrder, domain == true, axis == false
125  int idx = 0;
126  indexMap_.resize(numElement_);
127  this->dims_ = numElement_;
128  for (int i = 0; i < numElement_; ++i)
129  {
130    indexMap_[i] = idx;
131    if (true == axisDomainOrder(i))
132    {
133      ++(this->dims_);
134      idx += 2;
135    }
136    else ++idx;
137  }
138
139  // Size of each dimension (local and global)
140  nLocal_.resize(this->dims_);
141  nGlob_.resize(this->dims_);
142  nBeginLocal_.resize(this->dims_,0);
143  nBeginGlobal_.resize(this->dims_,0);
144  nZoomBegin_.resize(this->dims_);
145  nZoomEnd_.resize(this->dims_);
146
147  // Data_n_index of domain or axis (For now, axis uses its size as data_n_index
148  dataNIndex_.resize(numElement_);
149  dataDims_.resize(numElement_);
150  dataBegin_.resize(this->dims_);
151
152  // Data_*_index of each dimension
153  dataIndex_.resize(this->dims_);
154  infoIndex_.resize(this->dims_);
155
156  // A trick to determine position of each domain in domainList
157  int domIndex = 0, axisIndex = 0;
158  idx = 0;
159
160  elementLocalIndex_.resize(numElement_);
161  elementGlobalIndex_.resize(numElement_);
162  elementIndexData_.resize(numElement_);
163  elementZoomMask_.resize(numElement_);
164  elementNLocal_.resize(numElement_);
165  elementNGlobal_.resize(numElement_);
166  elementNLocal_[0] = 1;
167  elementNGlobal_[0] = 1;
168  size_t localSize = 1, globalSize = 1;
169
170  isDataDistributed_ = false;
171  // Update all the vectors above
172  for (idx = 0; idx < numElement_; ++idx)
173  {
174    bool isDomain = axisDomainOrder(idx);
175    elementNLocal_[idx] = localSize;
176    elementNGlobal_[idx] = globalSize;
177
178    // If this is a domain
179    if (isDomain)
180    {
181      // On the j axis
182      nLocal_.at(indexMap_[idx]+1) = domList[domIndex]->nj.getValue();
183      nGlob_.at(indexMap_[idx]+1)  = domList[domIndex]->nj_glo.getValue();
184      nBeginLocal_.at(indexMap_[idx]+1) = 0;
185      nBeginGlobal_.at(indexMap_[idx]+1) = domList[domIndex]->jbegin;
186      nZoomBegin_.at((indexMap_[idx]+1)) = domList[domIndex]->global_zoom_jbegin;
187      nZoomEnd_.at((indexMap_[idx]+1))   = domList[domIndex]->global_zoom_jbegin + domList[domIndex]->global_zoom_nj-1;
188
189      dataBegin_.at(indexMap_[idx]+1) = domList[domIndex]->data_jbegin.getValue(); //(2 == domList[domIndex]->data_dim) ? domList[domIndex]->data_jbegin.getValue() : -1;
190      dataIndex_.at(indexMap_[idx]+1) = &(domList[domIndex]->data_j_index);
191      infoIndex_.at(indexMap_[idx]+1) = &(domList[domIndex]->j_index);
192
193      // On the i axis
194      nLocal_.at(indexMap_[idx]) = domList[domIndex]->ni.getValue();
195      nGlob_.at(indexMap_[idx]) = domList[domIndex]->ni_glo.getValue();
196      nBeginLocal_.at(indexMap_[idx]) = 0;
197      nBeginGlobal_.at(indexMap_[idx]) = domList[domIndex]->ibegin;
198      nZoomBegin_.at((indexMap_[idx])) = domList[domIndex]->global_zoom_ibegin;
199      nZoomEnd_.at((indexMap_[idx]))   = domList[domIndex]->global_zoom_ibegin + domList[domIndex]->global_zoom_ni-1;
200
201      dataBegin_.at(indexMap_[idx]) = domList[domIndex]->data_ibegin.getValue();
202      dataIndex_.at(indexMap_[idx]) = &(domList[domIndex]->data_i_index);
203      infoIndex_.at(indexMap_[idx]) = &(domList[domIndex]->i_index);
204
205      dataNIndex_.at(idx) = domList[domIndex]->data_i_index.numElements();
206      dataDims_.at(idx) = domList[domIndex]->data_dim.getValue();
207
208      isDataDistributed_ |= domList[domIndex]->isDistributed();
209
210      localSize *= nLocal_.at(indexMap_[idx]+1)* nLocal_.at(indexMap_[idx]);
211      globalSize *= nGlob_.at(indexMap_[idx]+1)* nGlob_.at(indexMap_[idx]);
212      ++domIndex;
213    }
214    else // So it's an axis
215    {
216      nLocal_.at(indexMap_[idx]) = axisList[axisIndex]->n.getValue();
217      nGlob_.at(indexMap_[idx]) = axisList[axisIndex]->n_glo.getValue();
218      nBeginLocal_.at(indexMap_[idx]) = 0;
219      nBeginGlobal_.at(indexMap_[idx]) = axisList[axisIndex]->begin.getValue();
220      nZoomBegin_.at((indexMap_[idx])) = axisList[axisIndex]->global_zoom_begin;
221      nZoomEnd_.at((indexMap_[idx])) = axisList[axisIndex]->global_zoom_begin + axisList[axisIndex]->global_zoom_n-1;
222
223      dataBegin_.at(indexMap_[idx]) = axisList[axisIndex]->data_begin.getValue();
224      dataIndex_.at(indexMap_[idx]) = &(axisList[axisIndex]->data_index);
225      infoIndex_.at(indexMap_[idx]) = &(axisList[axisIndex]->index);
226      dataNIndex_.at(idx) = axisList[axisIndex]->data_index.numElements();
227      dataDims_.at(idx) = 1;
228
229      isDataDistributed_ |= axisList[axisIndex]->isDistributed();
230
231      localSize *= nLocal_.at(indexMap_[idx]);
232      globalSize *= nGlob_.at(indexMap_[idx]);
233
234      ++axisIndex;
235    }
236  }
237}
238
239/*!
240  Create local index of domain(s).
241  A domain can have data index which even contains the "ghost" points. Very often, these
242data surround the true data. In order to send correct data to server,
243a client need to know index of the true data.
244*/
245void CDistributionClient::createLocalDomainDataIndex()
246{
247  int idxDomain = 0;
248  for (int i = 0; i < axisDomainOrder_.numElements(); ++i)
249  {
250    if (axisDomainOrder_(i))
251    {
252      elementIndexData_[i].resize(dataNIndex_[i]);
253      elementIndexData_[i] = false;
254      int iIdx, jIdx = 0, count = 0, localIndex;
255      for (int j = 0; j < dataNIndex_[i]; ++j)
256      {
257        iIdx = getDomainIndex((*dataIndex_[indexMap_[i]])(j), (*dataIndex_[indexMap_[i]+1])(j),
258                              dataBegin_[indexMap_[i]], dataBegin_[indexMap_[i]+1],
259                              dataDims_[i], nLocal_[indexMap_[i]], jIdx);
260
261        if ((iIdx >= nBeginLocal_[indexMap_[i]]) && (iIdx < nLocal_[indexMap_[i]]) &&
262           (jIdx >= nBeginLocal_[indexMap_[i]+1]) && (jIdx < nLocal_[indexMap_[i]+1]) &&
263           (domainMasks_[idxDomain](iIdx + jIdx*nLocal_[indexMap_[i]])))
264        {
265          ++count;
266          elementIndexData_[i](j) = true;
267        }
268      }
269
270      elementLocalIndex_[i].resize(count);
271      elementGlobalIndex_[i].resize(count);
272      elementZoomMask_[i].resize(count);
273      elementZoomMask_[i] = false;
274      count = 0;
275      CArray<bool,1>& tmpIndexElementData = elementIndexData_[i];
276      CArray<bool,1>& tmpZoomMaskElement = elementZoomMask_[i];
277      CArray<int,1>& tmpLocalElementIndex = elementLocalIndex_[i];
278      CArray<size_t,1>& tmpGlobalElementIndex = elementGlobalIndex_[i];
279      for (int j = 0; j < dataNIndex_[i]; ++j)
280      {
281        if (tmpIndexElementData(j))
282        {
283          iIdx = getDomainIndex((*dataIndex_[indexMap_[i]])(j), (*dataIndex_[indexMap_[i]+1])(j),
284                                dataBegin_[indexMap_[i]], dataBegin_[indexMap_[i]+1],
285                                dataDims_[i], nLocal_[indexMap_[i]], jIdx);
286          localIndex = tmpLocalElementIndex(count) = iIdx + jIdx * nLocal_[indexMap_[i]];
287          tmpGlobalElementIndex(count) = (*infoIndex_[indexMap_[i]])(localIndex) + ((*infoIndex_[indexMap_[i]+1])(localIndex))*nGlob_[indexMap_[i]];
288          if ((((*infoIndex_[indexMap_[i]])(localIndex)) <= nZoomEnd_[indexMap_[i]])
289             && (nZoomBegin_[indexMap_[i]] <= ((*infoIndex_[indexMap_[i]])(localIndex)))
290             && (((*infoIndex_[indexMap_[i]+1])(localIndex)) <= nZoomEnd_[indexMap_[i]+1])
291             && (nZoomBegin_[indexMap_[i]+1] <= ((*infoIndex_[indexMap_[i]+1])(localIndex))))
292          {
293            tmpZoomMaskElement(count) = true;
294          }
295          ++count;
296        }
297      }
298      ++idxDomain;
299    }
300  }
301}
302
303/*!
304  Create local index of axis.
305*/
306void CDistributionClient::createLocalAxisDataIndex()
307{
308  int idxAxis = 0;
309  for (int i = 0; i < axisDomainOrder_.numElements(); ++i)
310  {
311    if (!axisDomainOrder_(i))
312    {
313      elementIndexData_[i].resize(dataNIndex_[i]);
314      elementIndexData_[i] = false;
315      int iIdx = 0, count = 0, localIndex = 0;
316      for (int j = 0; j < dataNIndex_[i]; ++j)
317      {
318        iIdx = getAxisIndex((*dataIndex_[indexMap_[i]])(j), dataBegin_[indexMap_[i]], nLocal_[indexMap_[i]]);
319        if ((iIdx >= nBeginLocal_[indexMap_[i]]) &&
320           (iIdx < nLocal_[indexMap_[i]]) && (axisMasks_[idxAxis](iIdx)))
321        {
322          ++count;
323          elementIndexData_[i](j) = true;
324        }
325      }
326
327      elementLocalIndex_[i].resize(count);
328      elementGlobalIndex_[i].resize(count);
329      elementZoomMask_[i].resize(count);
330      elementZoomMask_[i] = false;
331      count = 0;
332      CArray<bool,1>& tmpIndexElementData = elementIndexData_[i];
333      CArray<bool,1>& tmpZoomMaskElement = elementZoomMask_[i];
334      CArray<int,1>& tmpLocalElementIndex = elementLocalIndex_[i];
335      CArray<size_t,1>& tmpGlobalElementIndex = elementGlobalIndex_[i];
336      for (int j = 0; j < dataNIndex_[i]; ++j)
337      {
338        if (tmpIndexElementData(j))
339        {
340          iIdx = tmpLocalElementIndex(count) = getAxisIndex((*dataIndex_[indexMap_[i]])(j), dataBegin_[indexMap_[i]], nLocal_[indexMap_[i]]);
341          tmpGlobalElementIndex(count) = (*infoIndex_[indexMap_[i]])(iIdx);
342          if ((((*infoIndex_[indexMap_[i]])(iIdx)) <= nZoomEnd_[indexMap_[i]])
343             && (nZoomBegin_[indexMap_[i]] <= ((*infoIndex_[indexMap_[i]])(iIdx))))
344          {
345            tmpZoomMaskElement(count) = true;
346          }
347          ++count;
348        }
349      }
350      ++idxAxis;
351    }
352  }
353}
354
355/*!
356   Create global index on client
357   In order to do the mapping between client-server, each client creates its own
358global index of sending data. This global index is then used to calculate to which server
359the client needs to send it data as well as which part of data belongs to the server.
360So as to make clients and server coherent in order of index, global index is calculated by
361take into account of C-convention, the rightmost dimension varies faster.
362*/
363void CDistributionClient::createGlobalIndexSendToServer()
364{
365  if (isComputed_) return;
366  isComputed_ = true;
367  createLocalDomainDataIndex();
368  createLocalAxisDataIndex();
369
370  int idxDomain = 0, idxAxis = 0;
371  std::vector<int> eachElementSize(numElement_);
372
373  // Precompute size of the loop
374  for (int i = 0; i < numElement_; ++i)
375  {
376    eachElementSize[i] = elementLocalIndex_[i].numElements();
377  }
378
379  //   Compute size of the global index on client
380  std::vector<StdSize> idxLoop(numElement_,0);
381  std::vector<StdSize> currentIndex(numElement_,0);
382  std::vector<StdSize> currentGlobalIndex(numElement_,0);
383  int innerLoopSize = eachElementSize[0];
384  size_t idx = 0, indexLocalDataOnClientCount = 0, indexSend2ServerCount = 0;
385  size_t ssize = 1;
386  for (int i = 0; i < numElement_; ++i) ssize *= eachElementSize[i];
387  while (idx < ssize)
388  {
389    for (int i = 0; i < numElement_-1; ++i)
390    {
391      if (idxLoop[i] == eachElementSize[i])
392      {
393        idxLoop[i] = 0;
394        ++idxLoop[i+1];
395      }
396    }
397
398    // Find out outer index
399    // Depending the inner-most element is axis or domain,
400    // The outer loop index begins correspondingly at one (1) or zero (0)
401    for (int i = 1; i < numElement_; ++i)
402    {
403      currentIndex[i] = elementLocalIndex_[i](idxLoop[i]);
404    }
405
406    // Inner most index
407    for (int i = 0; i < innerLoopSize; ++i)
408    {
409      int gridMaskIndex = 0;
410      for (int k = 0; k < this->numElement_; ++k)
411      {
412        gridMaskIndex += (currentIndex[k])*elementNLocal_[k];
413      }
414
415      if (gridMask_(gridMaskIndex))
416      {
417        ++indexLocalDataOnClientCount;
418        bool isIndexOnServer = true;
419
420        for (int idxElement = 0; idxElement < this->numElement_; ++idxElement)
421        {
422          isIndexOnServer = isIndexOnServer && elementZoomMask_[idxElement](idxLoop[idxElement]);
423        }
424        if (isIndexOnServer) ++indexSend2ServerCount;
425      }
426    }
427    idxLoop[0] += innerLoopSize;
428    idx += innerLoopSize;
429  }
430
431  // Now allocate these arrays
432  localDataIndex_.resize(indexLocalDataOnClientCount);
433  localMaskIndex_.resize(indexSend2ServerCount);
434  globalLocalDataSendToServerMap_.rehash(std::ceil(indexSend2ServerCount/globalLocalDataSendToServerMap_.max_load_factor())); //globalLocalDataSendToServerMap_.reserve(indexSend2ServerCount);
435
436  // We need to loop with data index
437  idxLoop.assign(numElement_,0);
438  idx = indexLocalDataOnClientCount = indexSend2ServerCount = 0;
439  ssize = 1; for (int i = 0; i < numElement_; ++i) ssize *= dataNIndex_[i];
440  innerLoopSize = dataNIndex_[0];
441  int countLocalData = 0;
442  std::vector<int> correctOuterIndex(numElement_,0);
443  bool isOuterIndexCorrect = true;
444  while (idx < ssize)
445  {
446    for (int i = 0; i < numElement_-1; ++i)
447    {
448      if (idxLoop[i] == dataNIndex_[i])
449      {
450        idxLoop[i] = 0;
451        correctOuterIndex[i] = 0;
452        ++idxLoop[i+1];
453        if (isOuterIndexCorrect) ++correctOuterIndex[i+1];
454      }
455    }
456
457    // Depending the inner-most element axis or domain,
458    // The outer loop index begins correspondingly at one (1) or zero (0)
459    bool isIndexElementDataCorrect = true;
460    for (int i = 1; i < numElement_; ++i)
461    {
462      if (elementIndexData_[i](idxLoop[i]))
463      {
464        currentIndex[i] = elementLocalIndex_[i](correctOuterIndex[i]);
465        currentGlobalIndex[i] = elementGlobalIndex_[i](correctOuterIndex[i]);
466        isIndexElementDataCorrect &= true;
467      }
468      else isIndexElementDataCorrect = false;
469    }
470
471    isOuterIndexCorrect = isIndexElementDataCorrect;
472
473    // Inner most index
474    int correctIndexElement = 0;
475    for (int i = 0; i < innerLoopSize; ++i)
476    {
477      bool isCurrentIndexDataCorrect = isOuterIndexCorrect;
478      if (elementIndexData_[0](i))
479      {
480        currentIndex[0] = elementLocalIndex_[0](correctIndexElement);
481        currentGlobalIndex[0] = elementGlobalIndex_[0](correctIndexElement);
482        isCurrentIndexDataCorrect &= true;
483        ++correctIndexElement;
484      }
485      else isCurrentIndexDataCorrect = false;
486
487      if (isCurrentIndexDataCorrect)
488      {
489        int gridMaskIndex = 0; //currentIndex[0];
490        for (int k = 0; k < this->numElement_; ++k)
491        {
492          gridMaskIndex += (currentIndex[k])*elementNLocal_[k];
493        }
494
495        if (gridMask_(gridMaskIndex))
496        {
497          localDataIndex_[indexLocalDataOnClientCount] = countLocalData;
498          bool isIndexOnServer = true;
499          for (int idxElement = 0; idxElement < this->numElement_; ++idxElement)
500          {
501            isIndexOnServer = isIndexOnServer && elementZoomMask_[idxElement](idxLoop[idxElement]);
502          }
503
504          if (isIndexOnServer)
505          {
506            size_t globalIndex = 0; //currentGlobalIndex[0];
507            for (int k = 0; k < numElement_; ++k)
508            {
509              globalIndex += (currentGlobalIndex[k])*elementNGlobal_[k];
510            }
511            globalLocalDataSendToServerMap_[globalIndex] = indexLocalDataOnClientCount;
512            localMaskIndex_[indexSend2ServerCount] = gridMaskIndex;
513            ++indexSend2ServerCount;
514          }
515          ++indexLocalDataOnClientCount;
516        }
517      }
518      ++countLocalData;
519    }
520    idxLoop[0] += innerLoopSize;
521    idx += innerLoopSize;
522  }
523}
524
525///*!
526//  Create local index of domain(s).
527//  A domain can have data index which even contains the "ghost" points. Very often, these
528//data surround the true data. In order to send correct data to server,
529//a client need to know index of the true data.
530//*/
531//void CDistributionClient::createLocalDomainDataIndex()
532//{
533//  int numDomain = 0;
534//  for (int i = 0; i < axisDomainOrder_.numElements(); ++i)
535//    if (axisDomainOrder_(i)) ++numDomain;
536//
537//  localDomainIndex_.resize(numDomain*2);
538//  indexDomainData_.resize(numDomain);
539//
540//  int idxDomain = 0;
541//  for (int i = 0; i < axisDomainOrder_.numElements(); ++i)
542//  {
543//    if (axisDomainOrder_(i))
544//    {
545//      int iIdx, jIdx = 0, count = 0;
546//      indexDomainData_[idxDomain].resize(dataNIndex_[i], false);
547//      for (int j = 0; j < dataNIndex_[i]; ++j)
548//      {
549//        iIdx = getDomainIndex((*dataIndex_[indexMap_[i]])(j), (*dataIndex_[indexMap_[i]+1])(j),
550//                              dataBegin_[indexMap_[i]], dataBegin_[indexMap_[i]+1],
551//                              dataDims_[i], nLocal_[indexMap_[i]], jIdx);
552//
553//        if ((iIdx >= nBeginLocal_[indexMap_[i]]) && (iIdx < nLocal_[indexMap_[i]]) &&
554//           (jIdx >= nBeginLocal_[indexMap_[i]+1]) && (jIdx < nLocal_[indexMap_[i]+1]) &&
555//           (domainMasks_[idxDomain](iIdx + jIdx*nLocal_[indexMap_[i]])))
556//        {
557//          (localDomainIndex_[idxDomain]).push_back(iIdx);
558//          (localDomainIndex_[idxDomain*2+1]).push_back(jIdx);
559//          indexDomainData_[idxDomain][j] = true;
560//        }
561//      }
562//      ++idxDomain;
563//    }
564//  }
565//}
566
567///*!
568//  Create local index of axis.
569//*/
570//void CDistributionClient::createLocalAxisDataIndex()
571//{
572//  int numAxis = 0;
573//  for (int i = 0; i < axisDomainOrder_.numElements(); ++i)
574//    if (!axisDomainOrder_(i)) ++numAxis;
575//
576//  localAxisIndex_.resize(numAxis);
577//  indexAxisData_.resize(numAxis);
578//
579//  int idxAxis = 0;
580//  for (int i = 0; i < axisDomainOrder_.numElements(); ++i)
581//  {
582//    if (!axisDomainOrder_(i))
583//    {
584//      int iIdx = 0;
585//      indexAxisData_[idxAxis].resize(dataNIndex_[i], false);
586//      for (int j = 0; j < dataNIndex_[i]; ++j)
587//      {
588//        iIdx = getAxisIndex((*dataIndex_[indexMap_[i]])(j), dataBegin_[indexMap_[i]], nLocal_[indexMap_[i]]);
589//        if ((iIdx >= nBeginLocal_[indexMap_[i]]) &&
590//           (iIdx < nLocal_[indexMap_[i]]) && (axisMasks_[idxAxis](iIdx)))
591//        {
592//          localAxisIndex_[idxAxis].push_back(iIdx);
593//          indexAxisData_[idxAxis][j] = true;
594//        }
595//      }
596//      ++idxAxis;
597//    }
598//  }
599//}
600
601void CDistributionClient::createGlobalIndex()
602{
603  size_t ssize = 1, idx = 0;
604  for (int i = 0; i < this->dims_; ++i)
605  ssize *= nLocal_[i];
606
607  this->globalIndex_.resize(ssize);
608  std::vector<int> idxLoop(this->numElement_,0);
609  int innnerLoopSize = (*infoIndex_[0]).numElements();
610  while (idx < ssize)
611  {
612    for (int i = 0; i < this->numElement_; ++i)
613    {
614      if (idxLoop[i] == (*infoIndex_[indexMap_[i]]).numElements())
615      {
616        idxLoop[i] = 0;
617        ++idxLoop[i+1];
618      }
619    }
620
621    for (int i = 0; i < innnerLoopSize; ++i)
622    {
623      size_t globalIndex = (*infoIndex_[0])(idxLoop[0]);
624      size_t mulDim = 1;
625      for (int idxElement = 0; idxElement < this->numElement_; ++idxElement)
626      {
627        if (axisDomainOrder_(idxElement))
628        {
629          int jb = (0 == idxElement) ? 1 : 0;
630          for (int j = jb; j <= 1; ++j)
631          {
632            mulDim *= nGlob_[indexMap_[idxElement]+j-1];
633            globalIndex += ((*infoIndex_[indexMap_[idxElement]+j])(idxLoop[idxElement]))*mulDim;
634          }
635        }
636        else
637        {
638          if (0 != idxElement)
639          {
640            mulDim *= nGlob_[indexMap_[idxElement]-1];
641            globalIndex += ((*infoIndex_[indexMap_[idxElement]])(idxLoop[idxElement]))*mulDim;
642          }
643        }
644      }
645
646      this->globalIndex_(idx) = globalIndex;
647      ++idxLoop[0];
648      ++idx;
649    }
650  }
651
652}
653
654
655///*!
656//   Create global index on client
657//   In order to do the mapping between client-server, each client creates its own
658//global index of sending data. This global index is then used to calculate to which server
659//the client needs to send it data as well as which part of data belongs to the server.
660//So as to make clients and server coherent in order of index, global index is calculated by
661//take into account of C-convention, the rightmost dimension varies faster.
662//*/
663//void CDistributionClient::createGlobalIndexSendToServer()
664//{
665//  if (isComputed_) return;
666//  isComputed_ = true;
667//  createLocalDomainDataIndex();
668//  createLocalAxisDataIndex();
669//
670//  int idxDomain = 0, idxAxis = 0;
671//  std::vector<int> eachElementSize(numElement_);
672//
673//  // Precompute size of the loop
674//  for (int i = 0; i < numElement_; ++i)
675//  {
676//    if(axisDomainOrder_(i))
677//    {
678//      eachElementSize[i] = localDomainIndex_[idxDomain].size();
679//      idxDomain += 2;
680//    }
681//    else
682//    {
683//      eachElementSize[i] = localAxisIndex_[idxAxis].size();
684//      ++idxAxis;
685//    }
686//  }
687//
688//  //   Compute size of the global index on client
689//  std::vector<StdSize> idxLoop(numElement_,0);
690//  std::vector<StdSize> currentIndex(this->dims_,0);
691//  int innerLoopSize = eachElementSize[0];
692//  size_t idx = 0, indexLocalDataOnClientCount = 0, indexSend2ServerCount = 0;
693//  size_t ssize = 1;
694//  for (int i = 0; i < numElement_; ++i) ssize *= eachElementSize[i];
695//  while (idx < ssize)
696//  {
697//    for (int i = 0; i < numElement_-1; ++i)
698//    {
699//      if (idxLoop[i] == eachElementSize[i])
700//      {
701//        idxLoop[i] = 0;
702//        ++idxLoop[i+1];
703//      }
704//    }
705//
706//    // Find out outer index
707//    // Depending the inner-most element is axis or domain,
708//    // The outer loop index begins correspondingly at one (1) or zero (0)
709//    idxDomain = idxAxis = 0;
710//    if (axisDomainOrder_(0)) ++idxDomain;
711//    else ++idxAxis;
712//    for (int i = 1; i < numElement_; ++i)
713//    {
714//      if (axisDomainOrder_(i))
715//      {
716//        currentIndex[indexMap_[i]]   = localDomainIndex_[idxDomain][idxLoop[i]];
717//        currentIndex[indexMap_[i]+1] = localDomainIndex_[idxDomain+1][idxLoop[i]];
718//        idxDomain += 2;
719//      }
720//      else
721//      {
722//        currentIndex[indexMap_[i]]   = localAxisIndex_[idxAxis][idxLoop[i]];
723//        ++idxAxis;
724//      }
725//    }
726//
727//    // Inner most index
728//    idxDomain = idxAxis = 0;
729//    for (int i = 0; i < innerLoopSize; ++i)
730//    {
731//      if (axisDomainOrder_(0))
732//      {
733//        currentIndex[0] = localDomainIndex_[idxDomain][i];
734//        currentIndex[1] = localDomainIndex_[idxDomain+1][i];
735//      }
736//      else currentIndex[0]   = localAxisIndex_[idxAxis][i];
737//
738//      StdSize gridMaskIndex = currentIndex[0];
739//      int mulDimMask = 1;
740//      for (int k = 1; k < this->dims_; ++k)
741//      {
742//        mulDimMask *= nLocal_[k-1];
743//        gridMaskIndex += (currentIndex[k])*mulDimMask;
744//      }
745//
746//      if (gridMask_(gridMaskIndex))
747//      {
748//        ++indexLocalDataOnClientCount;
749//        bool isIndexOnServer = true;
750//
751//        for (int idxElement = 0; idxElement < this->numElement_; ++idxElement)
752//        {
753//          int actualIdx = 0;
754//          if (axisDomainOrder_(idxElement))
755//          {
756//            actualIdx = currentIndex[indexMap_[idxElement]]+currentIndex[indexMap_[idxElement]+1]*nLocal_[indexMap_[idxElement]];
757//            isIndexOnServer = isIndexOnServer && (((*infoIndex_[indexMap_[idxElement]])(actualIdx)) <= nZoomEnd_[indexMap_[idxElement]])
758//                                              && (nZoomBegin_[indexMap_[idxElement]] <= ((*infoIndex_[indexMap_[idxElement]])(actualIdx)))
759//                                              && (((*infoIndex_[indexMap_[idxElement]+1])(actualIdx)) <= nZoomEnd_[indexMap_[idxElement]+1])
760//                                              && (nZoomBegin_[indexMap_[idxElement]+1] <= ((*infoIndex_[indexMap_[idxElement]+1])(actualIdx)));
761//          }
762//          else
763//          {
764//            isIndexOnServer = isIndexOnServer && (((*infoIndex_[indexMap_[idxElement]])(currentIndex[indexMap_[idxElement]])) <= nZoomEnd_[indexMap_[idxElement]])
765//                                              && (nZoomBegin_[indexMap_[idxElement]] <= ((*infoIndex_[indexMap_[idxElement]])(currentIndex[indexMap_[idxElement]])));
766//          }
767//        }
768//        if (isIndexOnServer) ++indexSend2ServerCount;
769//      }
770//
771//    }
772//    idxLoop[0] += innerLoopSize;
773//    idx += innerLoopSize;
774//  }
775//
776//  // Now allocate these arrays
777//  localDataIndex_.resize(indexLocalDataOnClientCount);
778//  localMaskIndex_.resize(indexSend2ServerCount);
779//  globalLocalDataSendToServerMap_.rehash(std::ceil(indexSend2ServerCount/globalLocalDataSendToServerMap_.max_load_factor())); //globalLocalDataSendToServerMap_.reserve(indexSend2ServerCount);
780//
781//  // We need to loop with data index
782//  idxLoop.assign(numElement_,0);
783//  idx = indexLocalDataOnClientCount = indexSend2ServerCount = 0;
784//  ssize = 1; for (int i = 0; i < numElement_; ++i) ssize *= dataNIndex_[i];
785//  innerLoopSize = dataNIndex_[0];
786//  int countLocalData = 0;
787//  std::vector<int> correctOuterIndex(numElement_,0);
788//  bool isOuterIndexCorrect = true;
789//  while (idx < ssize)
790//  {
791//    for (int i = 0; i < numElement_-1; ++i)
792//    {
793//      if (idxLoop[i] == dataNIndex_[i])
794//      {
795//        idxLoop[i] = 0;
796//        correctOuterIndex[i] = 0;
797//        ++idxLoop[i+1];
798//        if (isOuterIndexCorrect) ++correctOuterIndex[i+1];
799//      }
800//    }
801//
802//    // Depending the inner-most element axis or domain,
803//    // The outer loop index begins correspondingly at one (1) or zero (0)
804//    idxDomain = idxAxis = 0;
805//    if (axisDomainOrder_(0)) ++idxDomain;
806//    else ++idxAxis;
807//    bool isIndexDomainDataCorrect = true;
808//    bool isIndexAxisDataCorrect = true;
809//
810//    for (int i = 1; i < numElement_; ++i)
811//    {
812//      if (axisDomainOrder_(i))
813//      {
814//        if (indexDomainData_[idxDomain][idxLoop[i]])
815//        {
816//          currentIndex[indexMap_[i]]   = localDomainIndex_[idxDomain][correctOuterIndex[i]];
817//          currentIndex[indexMap_[i]+1] = localDomainIndex_[idxDomain*2+1][correctOuterIndex[i]];
818//          isIndexDomainDataCorrect &= true;
819//        }
820//        else isIndexDomainDataCorrect = false;
821//        ++idxDomain;
822//      }
823//      else
824//      {
825//        if (indexAxisData_[idxAxis][idxLoop[i]])
826//        {
827//          currentIndex[indexMap_[i]]   = localAxisIndex_[idxAxis][correctOuterIndex[i]];
828//          isIndexAxisDataCorrect &= true;
829//        }
830//        else isIndexAxisDataCorrect = false;
831//        ++idxAxis;
832//      }
833//    }
834//
835//    isOuterIndexCorrect = (isIndexAxisDataCorrect) && (isIndexDomainDataCorrect);
836//
837//    // Inner most index
838//    idxDomain = idxAxis = 0;
839//    int correctIndexDomain = 0, correctIndexAxis = 0;
840//    for (int i = 0; i < innerLoopSize; ++i)
841//    {
842//      bool isCurrentIndexDomainDataCorrect = isIndexDomainDataCorrect;
843//      bool isCurrentIndexAxisDataCorrect = isIndexAxisDataCorrect;
844//
845//      if (axisDomainOrder_(0))
846//      {
847//        if (indexDomainData_[idxDomain][i])
848//        {
849//          currentIndex[0] = localDomainIndex_[idxDomain][correctIndexDomain];
850//          currentIndex[1] = localDomainIndex_[idxDomain+1][correctIndexDomain];
851//          isCurrentIndexDomainDataCorrect &= true;
852//          ++correctIndexDomain;
853//        }
854//        else isCurrentIndexDomainDataCorrect = false;
855//      }
856//      else
857//      {
858//        if (indexAxisData_[idxAxis][i])
859//        {
860//          currentIndex[0] = localAxisIndex_[idxAxis][correctIndexAxis];
861//          isCurrentIndexAxisDataCorrect &= true;
862//          ++correctIndexAxis;
863//        }
864//        else isCurrentIndexAxisDataCorrect = false;
865//      }
866//
867//      int gridMaskIndex = currentIndex[0];
868//      int mulDimMask = 1;
869//      for (int k = 1; k < this->dims_; ++k)
870//      {
871//        mulDimMask *= nLocal_[k-1];
872//        gridMaskIndex += (currentIndex[k])*mulDimMask;
873//      }
874//
875//      if (isCurrentIndexDomainDataCorrect &&
876//          isCurrentIndexAxisDataCorrect &&
877//          gridMask_(gridMaskIndex))
878//      {
879//        localDataIndex_[indexLocalDataOnClientCount] = countLocalData;
880//        bool isIndexOnServer = true;
881//        for (int idxElement = 0; idxElement < this->numElement_; ++idxElement)
882//        {
883//          int actualIdx = 0;
884//          if (axisDomainOrder_(idxElement))
885//          {
886//            actualIdx = currentIndex[indexMap_[idxElement]]+currentIndex[indexMap_[idxElement]+1]*nLocal_[indexMap_[idxElement]];
887//            isIndexOnServer = isIndexOnServer && (((*infoIndex_[indexMap_[idxElement]])(actualIdx)) <= nZoomEnd_[indexMap_[idxElement]])
888//                                              && (nZoomBegin_[indexMap_[idxElement]] <= ((*infoIndex_[indexMap_[idxElement]])(actualIdx)))
889//                                              && (((*infoIndex_[indexMap_[idxElement]+1])(actualIdx)) <= nZoomEnd_[indexMap_[idxElement]+1])
890//                                              && (nZoomBegin_[indexMap_[idxElement]+1] <= ((*infoIndex_[indexMap_[idxElement]+1])(actualIdx)));
891//          }
892//          else
893//          {
894//            isIndexOnServer = isIndexOnServer && (((*infoIndex_[indexMap_[idxElement]])(currentIndex[indexMap_[idxElement]])) <= nZoomEnd_[indexMap_[idxElement]])
895//                                              && (nZoomBegin_[indexMap_[idxElement]] <= ((*infoIndex_[indexMap_[idxElement]])(currentIndex[indexMap_[idxElement]])));
896//          }
897//        }
898//
899//        if (isIndexOnServer)
900//        {
901//          int actualIdx = (axisDomainOrder_(0)) ? currentIndex[0]+currentIndex[1]*nLocal_[0]
902//                                                : currentIndex[0];
903//          size_t globalIndex = (*infoIndex_[0])(actualIdx); //idxLoop[0] + nBeginGlobal_[0];
904//          size_t mulDim = 1;
905//          for (int idxElement = 0; idxElement < this->numElement_; ++idxElement)
906//          {
907//            if (axisDomainOrder_(idxElement))
908//            {
909//              actualIdx = currentIndex[indexMap_[idxElement]]+currentIndex[indexMap_[idxElement]+1]*nLocal_[indexMap_[idxElement]];
910//              int jb = (0 == idxElement) ? 1 : 0;
911//              for (int j = jb; j <= 1; ++j)
912//              {
913//                mulDim *= nGlob_[indexMap_[idxElement]+j-1];
914//                globalIndex += ((*infoIndex_[indexMap_[idxElement]+j])(actualIdx))*mulDim;
915//              }
916//            }
917//            else
918//            {
919//              if (0 != idxElement)
920//              {
921//                mulDim *= nGlob_[indexMap_[idxElement]-1];
922//                globalIndex += ((*infoIndex_[indexMap_[idxElement]])(currentIndex[indexMap_[idxElement]]))*mulDim;
923//              }
924//            }
925//          }
926//          globalLocalDataSendToServerMap_[globalIndex] = indexLocalDataOnClientCount;
927//          localMaskIndex_[indexSend2ServerCount] = gridMaskIndex;
928//          ++indexSend2ServerCount;
929//        }
930//        ++indexLocalDataOnClientCount;
931//      }
932//      ++countLocalData;
933//    }
934//    idxLoop[0] += innerLoopSize;
935//    idx += innerLoopSize;
936//  }
937//}
938
939/*!
940  Retrieve index i and index j of a domain from its data index
941  Data contains not only true data, which are sent to servers, but also ghost data, which
942very often play a role of border of each local data, so does data index. Because data of a domain
943can be one dimension, or two dimensions, there is a need to convert data index to domain index
944  \param [in] dataIIndex index of i data
945  \param [in] dataJIndex index of j data
946  \param [in] dataIBegin index begin of i data
947  \param [in] dataJBegin index begin of j data
948  \param [in] dataDim dimension of data (1 or 2)
949  \param [in] ni local size ni of domain
950  \param [out] j j index of domain
951  \return i index of domain
952*/
953int CDistributionClient::getDomainIndex(const int& dataIIndex, const int& dataJIndex,
954                                        const int& dataIBegin, const int& dataJBegin,
955                                        const int& dataDim, const int& ni, int& j)
956{
957  int tempI = dataIIndex + dataIBegin,
958      tempJ = (dataJIndex + dataJBegin);
959  int i = (dataDim == 1) ? (tempI) % ni
960                         : (tempI) ;
961  j = (dataDim == 1) ? (tempI) / ni
962                     : (tempJ) ;
963
964  return i;
965}
966
967/*!
968  Retrieve index of an axis from its data index
969  \param [in] dataIndex index of data
970  \param [in] dataBegin index begin of data
971  \param [in] ni local size of axis
972  \return index of domain
973*/
974int CDistributionClient::getAxisIndex(const int& dataIndex, const int& dataBegin, const int& ni)
975{
976   int tempI = dataIndex + dataBegin;
977   return ((tempI)%ni);
978}
979
980/*!
981  Return global local data mapping of client
982*/
983CDistributionClient::GlobalLocalDataMap& CDistributionClient::getGlobalLocalDataSendToServer()
984{
985  if (!isComputed_) createGlobalIndexSendToServer();
986  return globalLocalDataSendToServerMap_;
987}
988
989/*!
990  Return local data index of client
991*/
992const std::vector<int>& CDistributionClient::getLocalDataIndexOnClient()
993{
994  if (!isComputed_) createGlobalIndexSendToServer();
995  return localDataIndex_;
996}
997
998/*!
999  Return local mask index of client
1000*/
1001const std::vector<int>& CDistributionClient::getLocalMaskIndexOnClient()
1002{
1003  if (!isComputed_) createGlobalIndexSendToServer();
1004  return localMaskIndex_;
1005}
1006
1007} // namespace xios
Note: See TracBrowser for help on using the repository browser.