Changeset 1642 for XIOS/dev/branch_openmp/src/distribution_client.cpp

Timestamp:

01/23/19 10:31:44 (5 years ago)

Author:

yushan

Message:

dev on ADA. add flag switch _usingEP/_usingMPI

File:

• XIOS/dev/branch_openmp/src/distribution_client.cpp (modified) (25 diffs)

XIOS/dev/branch_openmp/src/distribution_client.cpp

@@ -14 +14 @@
    : CDistribution(rank, 0)
    , axisDomainOrder_()
-   , nLocal_(), nGlob_(), nBeginLocal_(), nBeginGlobal_(),nZoomBegin_(), nZoomEnd_()
-   , dataNIndex_(), dataDims_(), dataBegin_(), dataIndex_(), domainMasks_(), axisMasks_()
+   , nLocal_(), nGlob_(), nBeginLocal_(), nBeginGlobal_()
+   , dataNIndex_(), dataDims_(), dataBegin_(), dataIndex_()
    , gridMask_(), indexMap_()
    , isDataDistributed_(true), axisNum_(0), domainNum_(0)
@@ -22 +22 @@
    , infoIndex_(), isComputed_(false)
    , elementLocalIndex_(), elementGlobalIndex_(), elementIndexData_()
-   , elementZoomMask_(), elementNLocal_(), elementNGlobal_()
+   , elementNLocal_(), elementNGlobal_()
 {
   readDistributionInfo(grid);
@@ -36 +36 @@
   GlobalLocalMap void2 ;
   std::vector<int> void3 ;
-  std::vector<int> void4 ;
+  std::vector<bool> void4 ;
 
   globalLocalDataSendToServerMap_.swap(void1) ;
@@ -61 +61 @@
   // Then check mask of grid
   int gridDim = domList.size() * 2 + axisList.size();
-  grid->checkMask();
   switch (gridDim) {
     case 0:
@@ -68 +67 @@
       break;
     case 1:
-      readGridMaskInfo(grid->mask_1d);
+      if (!grid->mask_1d.isEmpty()) readGridMaskInfo(grid->mask_1d);
       break;
     case 2:
-      readGridMaskInfo(grid->mask_2d);
+      if (!grid->mask_2d.isEmpty()) readGridMaskInfo(grid->mask_2d);
       break;
     case 3:
-      readGridMaskInfo(grid->mask_3d);
+      if (!grid->mask_3d.isEmpty()) readGridMaskInfo(grid->mask_3d);
       break;
     case 4:
-      readGridMaskInfo(grid->mask_4d);
+      if (!grid->mask_4d.isEmpty()) readGridMaskInfo(grid->mask_4d);
       break;
     case 5:
-      readGridMaskInfo(grid->mask_5d);
+      if (!grid->mask_5d.isEmpty()) readGridMaskInfo(grid->mask_5d);
       break;
     case 6:
-      readGridMaskInfo(grid->mask_6d);
+      if (!grid->mask_6d.isEmpty()) readGridMaskInfo(grid->mask_6d);
       break;
     case 7:
-      readGridMaskInfo(grid->mask_7d);
+      if (!grid->mask_7d.isEmpty()) readGridMaskInfo(grid->mask_7d);
       break;
     default:
@@ -117 +116 @@
   axisDomainOrder_ = axisDomainOrder;
 
-  // Each domain or axis has its mask, of course
-  domainMasks_.resize(domainNum_);
-  for (int i = 0; i < domainNum_;++i)
-  {
-    domainMasks_[i].resize(domList[i]->domainMask.numElements());
-    domainMasks_[i] = domList[i]->domainMask;
-  }
-
-  axisMasks_.resize(axisNum_);
-  for (int i = 0; i < axisNum_; ++i)
-  {
-    axisMasks_[i].resize(axisList[i]->mask.numElements());
-    axisMasks_[i] = axisList[i]->mask;
-  }
-
   // Because domain and axis can be in any order (axis1, domain1, axis2, axis3, )
   // their position should be specified. In axisDomainOrder, domain == true, axis == false
@@ -153 +137 @@
   nBeginLocal_.resize(this->dims_,0);
   nBeginGlobal_.resize(this->dims_,0);
-  nZoomBegin_.resize(this->dims_);
-  nZoomEnd_.resize(this->dims_);
 
   // Data_n_index of domain or axis (For now, axis uses its size as data_n_index
@@ -172 +154 @@
   elementGlobalIndex_.resize(numElement_);
   elementIndexData_.resize(numElement_);
-  elementZoomMask_.resize(numElement_);
   elementNLocal_.resize(numElement_);
   elementNGlobal_.resize(numElement_);
@@ -195 +176 @@
       nBeginLocal_.at(indexMap_[idx]+1) = 0;
       nBeginGlobal_.at(indexMap_[idx]+1) = domList[domIndex]->jbegin;
-      nZoomBegin_.at((indexMap_[idx]+1)) = 0; //domList[domIndex]->global_zoom_jbegin;
-      nZoomEnd_.at((indexMap_[idx]+1))   = domList[domIndex]->nj_glo.getValue()- 1; //domList[domIndex]->global_zoom_jbegin + domList[domIndex]->global_zoom_nj-1;
 
       dataBegin_.at(indexMap_[idx]+1) = domList[domIndex]->data_jbegin.getValue();
@@ -207 +186 @@
       nBeginLocal_.at(indexMap_[idx]) = 0;
       nBeginGlobal_.at(indexMap_[idx]) = domList[domIndex]->ibegin;
-      nZoomBegin_.at((indexMap_[idx])) = 0; // domList[domIndex]->global_zoom_ibegin;
-      nZoomEnd_.at((indexMap_[idx]))   = domList[domIndex]->ni_glo.getValue() -1; //domList[domIndex]->global_zoom_ibegin + domList[domIndex]->global_zoom_ni-1;
 
       dataBegin_.at(indexMap_[idx]) = domList[domIndex]->data_ibegin.getValue();
@@ -229 +206 @@
       nBeginLocal_.at(indexMap_[idx]) = 0;
       nBeginGlobal_.at(indexMap_[idx]) = axisList[axisIndex]->begin.getValue();
-      nZoomBegin_.at((indexMap_[idx])) = 0; //axisList[axisIndex]->global_zoom_begin;
-      nZoomEnd_.at((indexMap_[idx])) = axisList[axisIndex]->n_glo.getValue() - 1; //axisList[axisIndex]->global_zoom_begin + axisList[axisIndex]->global_zoom_n-1;
 
       dataBegin_.at(indexMap_[idx]) = axisList[axisIndex]->data_begin.getValue();
@@ -251 +226 @@
       nBeginLocal_.at(indexMap_[idx]) = 0;
       nBeginGlobal_.at(indexMap_[idx]) = 1;
-      nZoomBegin_.at((indexMap_[idx])) = 0;
-      nZoomEnd_.at((indexMap_[idx])) = 0;
 
       dataBegin_.at(indexMap_[idx]) = 0;
@@ -293 +266 @@
 
         if ((iIdx >= nBeginLocal_[indexMap_[i]]) && (iIdx < nLocal_[indexMap_[i]]) &&
-           (jIdx >= nBeginLocal_[indexMap_[i]+1]) && (jIdx < nLocal_[indexMap_[i]+1]) &&
-           (domainMasks_[idxDomain](iIdx + jIdx*nLocal_[indexMap_[i]])))
+           (jIdx >= nBeginLocal_[indexMap_[i]+1]) && (jIdx < nLocal_[indexMap_[i]+1]))
         {
           ++count;
@@ -303 +275 @@
       elementLocalIndex_[i].resize(count);
       elementGlobalIndex_[i].resize(count);
-      elementZoomMask_[i].resize(count);
-      elementZoomMask_[i] = false;
       count = 0;
       CArray<bool,1>& tmpIndexElementData = elementIndexData_[i];
-      CArray<bool,1>& tmpZoomMaskElement = elementZoomMask_[i];
       CArray<int,1>& tmpLocalElementIndex = elementLocalIndex_[i];
       CArray<size_t,1>& tmpGlobalElementIndex = elementGlobalIndex_[i];
@@ -319 +288 @@
           localIndex = tmpLocalElementIndex(count) = iIdx + jIdx * nLocal_[indexMap_[i]];
           tmpGlobalElementIndex(count) = (infoIndex_[indexMap_[i]])(localIndex) + ((infoIndex_[indexMap_[i]+1])(localIndex))*nGlob_[indexMap_[i]];
-          if ((((infoIndex_[indexMap_[i]])(localIndex)) <= nZoomEnd_[indexMap_[i]])
-             && (nZoomBegin_[indexMap_[i]] <= ((infoIndex_[indexMap_[i]])(localIndex)))
-             && (((infoIndex_[indexMap_[i]+1])(localIndex)) <= nZoomEnd_[indexMap_[i]+1])
-             && (nZoomBegin_[indexMap_[i]+1] <= ((infoIndex_[indexMap_[i]+1])(localIndex))))
-          {
-            tmpZoomMaskElement(count) = true;
-          }
           ++count;
         }
@@ -346 +308 @@
       elementIndexData_[i].resize(dataNIndex_[i]);
       elementIndexData_[i] = false;
-      int iIdx = 0, count = 0, localIndex = 0;
+      int iIdx = 0, count = 0;
       for (int j = 0; j < dataNIndex_[i]; ++j)
       {
         iIdx = getAxisIndex((dataIndex_[indexMap_[i]])(j), dataBegin_[indexMap_[i]], nLocal_[indexMap_[i]]);
         if ((iIdx >= nBeginLocal_[indexMap_[i]]) &&
-           (iIdx < nLocal_[indexMap_[i]]) && (axisMasks_[idxAxis](iIdx)))
+           (iIdx < nLocal_[indexMap_[i]]) )//&& (axisMasks_[idxAxis](iIdx)))
         {
           ++count;
@@ -360 +322 @@
       elementLocalIndex_[i].resize(count);
       elementGlobalIndex_[i].resize(count);
-      elementZoomMask_[i].resize(count);
-      elementZoomMask_[i] = false;
       count = 0;
       CArray<bool,1>& tmpIndexElementData = elementIndexData_[i];
-      CArray<bool,1>& tmpZoomMaskElement = elementZoomMask_[i];
       CArray<int,1>& tmpLocalElementIndex = elementLocalIndex_[i];
       CArray<size_t,1>& tmpGlobalElementIndex = elementGlobalIndex_[i];
@@ -373 +332 @@
           iIdx = tmpLocalElementIndex(count) = getAxisIndex((dataIndex_[indexMap_[i]])(j), dataBegin_[indexMap_[i]], nLocal_[indexMap_[i]]);
           tmpGlobalElementIndex(count) = (infoIndex_[indexMap_[i]])(iIdx);
-          if ((((infoIndex_[indexMap_[i]])(iIdx)) <= nZoomEnd_[indexMap_[i]])
-             && (nZoomBegin_[indexMap_[i]] <= ((infoIndex_[indexMap_[i]])(iIdx))))
-          {
-            tmpZoomMaskElement(count) = true;
-          }
           ++count;
         }
@@ -404 +358 @@
       elementGlobalIndex_[i].resize(count);
       elementGlobalIndex_[i] = 0;
-      elementZoomMask_[i].resize(count);
-      elementZoomMask_[i] = true;
     }
   }
@@ -440 +392 @@
   std::vector<StdSize> currentGlobalIndex(numElement_,0);
   int innerLoopSize = eachElementSize[0];
-  size_t idx = 0, indexLocalDataOnClientCount = 0, indexSend2ServerCount = 0;
+  size_t idx = 0, indexLocalDataOnClientCount = 0;
   size_t ssize = 1;
+
   for (int i = 0; i < numElement_; ++i) ssize *= eachElementSize[i];
-  while (idx < ssize)
-  {
-    for (int i = 0; i < numElement_-1; ++i)
-    {
-      if (idxLoop[i] == eachElementSize[i])
-      {
-        idxLoop[i] = 0;
-        ++idxLoop[i+1];
-      }
-    }
-
-    // Find out outer index
-    // Depending the inner-most element is axis or domain,
-    // The outer loop index begins correspondingly at one (1) or zero (0)
-    for (int i = 1; i < numElement_; ++i)
-    {
-      currentIndex[i] = elementLocalIndex_[i](idxLoop[i]);
-    }
-
-    // Inner most index
-    for (int i = 0; i < innerLoopSize; ++i)
-    {
-      int gridMaskIndex = 0;
-      currentIndex[0] = elementLocalIndex_[0](i);
-      for (int k = 0; k < this->numElement_; ++k)
-      {
-        gridMaskIndex += (currentIndex[k])*elementNLocal_[k];
-      }
-
-      if (gridMask_(gridMaskIndex))
-      {
-        ++indexLocalDataOnClientCount;
-        bool isIndexOnServer = true;
-
-        for (int idxElement = 0; idxElement < this->numElement_; ++idxElement)
-        {
-          isIndexOnServer = isIndexOnServer && elementZoomMask_[idxElement](idxLoop[idxElement]);
-        }
-        if (isIndexOnServer) ++indexSend2ServerCount;
-      }
-    }
-    idxLoop[0] += innerLoopSize;
-    idx += innerLoopSize;
-  }
-
-  // Now allocate these arrays
-  localDataIndex_.resize(indexLocalDataOnClientCount);
-  localMaskIndex_.resize(indexSend2ServerCount);
-  localMaskedDataIndex_.resize(indexSend2ServerCount);
-  globalDataIndex_.rehash(std::ceil(indexLocalDataOnClientCount/globalDataIndex_.max_load_factor())); //globalLocalDataSendToServerMap_.reserve(indexSend2ServerCount);
-  globalLocalDataSendToServerMap_.rehash(std::ceil(indexSend2ServerCount/globalLocalDataSendToServerMap_.max_load_factor())); //globalLocalDataSendToServerMap_.reserve(indexSend2ServerCount);
+
+  localDataIndex_.resize(ssize);
+  if (!gridMask_.isEmpty()) localMaskIndex_.resize(ssize);
+  localMaskedDataIndex_.resize(ssize);
+  globalDataIndex_.rehash(std::ceil(ssize/globalDataIndex_.max_load_factor()));
+  globalLocalDataSendToServerMap_.rehash(std::ceil(ssize/globalLocalDataSendToServerMap_.max_load_factor()));
+
 
   // We need to loop with data index
   idxLoop.assign(numElement_,0);
-  idx = indexLocalDataOnClientCount = indexSend2ServerCount = 0;
+  idx = indexLocalDataOnClientCount = 0;
   ssize = 1; for (int i = 0; i < numElement_; ++i) ssize *= dataNIndex_[i];
   innerLoopSize = dataNIndex_[0];
@@ -550 +459 @@
         if (isCurrentIndexDataCorrect)
         {
-          int gridMaskIndex = 0;
-          for (int k = 0; k < this->numElement_; ++k)
+          bool maskTmp = true;
+          bool maskGridTmp = true;
+          size_t globalIndex = 0;
+          for (int k = 0; k < numElement_; ++k)
           {
-            gridMaskIndex += (currentIndex[k])*elementNLocal_[k];
+            globalIndex += (currentGlobalIndex[k])*elementNGlobal_[k];
           }
-
-          if (gridMask_(gridMaskIndex))
+          globalDataIndex_[globalIndex] = indexLocalDataOnClientCount;
+          localDataIndex_[indexLocalDataOnClientCount] = countLocalData;
+          globalLocalDataSendToServerMap_[globalIndex] = indexLocalDataOnClientCount;
+          localMaskedDataIndex_[indexLocalDataOnClientCount] = indexLocalDataOnClientCount;
+
+          // Grid mask: unmasked values will be replaces by NaN and then all values will be sent
+          if (!gridMask_.isEmpty())
           {
-            size_t globalIndex = 0;
-            for (int k = 0; k < numElement_; ++k)
+            int gridMaskIndex = 0;
+            for (int k = 0; k < this->numElement_; ++k)
             {
-              globalIndex += (currentGlobalIndex[k])*elementNGlobal_[k];
+              gridMaskIndex += (currentIndex[k])*elementNLocal_[k];
             }
-            globalDataIndex_[globalIndex] = indexLocalDataOnClientCount;
-            localDataIndex_[indexLocalDataOnClientCount] = countLocalData;
-            bool isIndexOnServer = true;
-            for (int idxElement = 0; idxElement < this->numElement_; ++idxElement)
-            {
-              isIndexOnServer = isIndexOnServer && elementZoomMask_[idxElement](correctIndexOfElement[idxElement]);
-            }
-
-            if (isIndexOnServer)
-            {
-              globalLocalDataSendToServerMap_[globalIndex] = indexLocalDataOnClientCount;
-              localMaskIndex_[indexSend2ServerCount] = gridMaskIndex;
-              localMaskedDataIndex_[indexSend2ServerCount] = indexLocalDataOnClientCount;
-              ++indexSend2ServerCount;
-            }
-            ++indexLocalDataOnClientCount;
+            maskGridTmp =  gridMask_(gridMaskIndex);
+            if (maskGridTmp)
+              localMaskIndex_[indexLocalDataOnClientCount] = true;
+            else
+              localMaskIndex_[indexLocalDataOnClientCount] = false;
           }
+
+          ++indexLocalDataOnClientCount;
+
         }
         ++countLocalData;
@@ -585 +493 @@
       }
     }
+    else countLocalData+=innerLoopSize ;
+
     idxLoop[0] += innerLoopSize;
     idx += innerLoopSize;
@@ -612 +522 @@
                                         const int& dataDim, const int& ni, int& j)
 {
+  int i;
   int tempI = dataIIndex + dataIBegin,
       tempJ = (dataJIndex + dataJBegin);
-  int i = (dataDim == 1) ? (tempI) % ni
-                         : (tempI) ;
-  j = (dataDim == 1) ? (tempI) / ni
-                     : (tempJ) ;
-
+  if (ni == 0)
+  {
+    i = -1;
+    j = -1;
+    return i;
+  }
+  if ((tempI < 0) || (tempJ < 0))
+  {
+    i = -1;
+    j = -1;
+    return i;
+  }
+  else
+  {
+    i = (dataDim == 1) ? (tempI) % ni : (tempI) ;
+    j = (dataDim == 1) ? (tempI) / ni : (tempJ) ;
+  }
   return i;
 }
@@ -631 +554 @@
 int CDistributionClient::getAxisIndex(const int& dataIndex, const int& dataBegin, const int& ni)
 {
-   int tempI = dataIndex + dataBegin;
-   return ((tempI)%ni);
+  if (ni == 0)
+  {
+    return -1;
+  }
+  int tempI = dataIndex;
+  if ((tempI < 0) || (tempI > ni))
+    return -1;
+  else
+    return tempI;
 }
 
@@ -662 +592 @@
   Return local mask index of client
 */
-const std::vector<int>& CDistributionClient::getLocalMaskIndexOnClient()
+const std::vector<bool>& CDistributionClient::getLocalMaskIndexOnClient()
 {
   if (!isComputed_) createGlobalIndexSendToServer();