Changeset 1637 for XIOS/trunk/src/node

Timestamp:

01/14/19 13:33:48 (5 years ago)

Author:

oabramkina

Message:

Merging dev to trunk. Major changes:

(1) Grid mask is applied in the source filter of clients: unmasked values are replaced by NaN. It is not reconstructed any more by servers.

(2) Domain/axis mask has been incorporated into data index, with only data index sent to servers.

Location:

XIOS/trunk/src/node

Files:

• axis.cpp (modified) (27 diffs)
• axis.hpp (modified) (1 diff)
• domain.cpp (modified) (16 diffs)
• domain.hpp (modified) (5 diffs)
• field.cpp (modified) (7 diffs)
• field.hpp (modified) (1 diff)
• grid.cpp (modified) (6 diffs)
• grid.hpp (modified) (7 diffs)

XIOS/trunk/src/node/axis.cpp

@@ -274 +274 @@
    TRY
    {
-      if (this->n_glo.isEmpty())
+     CContext* context=CContext::getCurrent();
+
+     if (this->n_glo.isEmpty())
         ERROR("CAxis::checkAttributes(void)",
               << "[ id = '" << getId() << "' , context = '" << CObjectFactory::GetCurrentContextId() << "' ] "
@@ -314 +316 @@
       }
 
-      // Remove this check because it doesn't make sense in case of a hole or overlapping axes
       if (!this->value.isEmpty())
       {
-//        StdSize true_size = value.numElements();
-//        if (this->n.getValue() != true_size)
-//          ERROR("CAxis::checkAttributes(void)",
-//                << "[ id = '" << getId() << "' , context = '" << CObjectFactory::GetCurrentContextId() << "' ] "
-//                << "The axis is wrongly defined, attribute 'value' has a different size (" << true_size << ") than the one defined by the \'size\' attribute (" << n.getValue() << ").");
+        // Avoid this check at writing because it fails in case of a hole
+        if (context->hasClient)
+        {
+          StdSize true_size = value.numElements();
+          if (this->n.getValue() != true_size)
+            ERROR("CAxis::checkAttributes(void)",
+                << "[ id = '" << getId() << "' , context = '" << CObjectFactory::GetCurrentContextId() << "' ] "
+                << "The axis is wrongly defined, attribute 'value' has a different size (" << true_size
+                << ") than the one defined by the \'size\' attribute (" << n.getValue() << ").");
+        }
         this->hasValue = true;
       }
@@ -327 +333 @@
       this->checkBounds();
 
-      CContext* context=CContext::getCurrent();
       if (context->hasClient)
       {
+        this->checkMask();
         this->checkData();
-        this->checkMask();
         this->checkLabel();
       }
@@ -338 +343 @@
 
    /*!
-      Check the validity of data and fill in values if any.
+      Check the validity of data, fill in values if any, and apply mask.
    */
    void CAxis::checkData()
@@ -359 +364 @@
       {
         data_index.resize(data_n);
-        for (int i = 0; i < data_n; ++i) data_index(i) = i;
-      }
+        for (int i = 0; i < data_n; ++i)
+        {
+          if ((i+data_begin) >= 0 && (i+data_begin<n))
+          {
+            if (mask(i+data_begin))
+              data_index(i) = i+data_begin;
+            else
+              data_index(i) = -1;
+          }
+          else
+            data_index(i) = -1;
+        }
+      }
+      else
+      {
+        if (data_index.numElements() != data_n)
+        {
+          ERROR("CAxis::checkData(void)",
+                << "[ id = " << this->getId() << " , context = '" << CObjectFactory::GetCurrentContextId() << " ] "
+                << "The size of data_index = "<< data_index.numElements() << "is not equal to the data size data_n = " << data_n.getValue() << ").");
+        }
+        for (int i = 0; i < data_n; ++i)
+        {
+          if ((i+data_begin) >= 0 && (i+data_begin<n) && !mask(i+data_begin))
+            data_index(i) = -1;
+        }
+      }
+
    }
    CATCH_DUMP_ATTR
@@ -377 +408 @@
       if (!mask.isEmpty())
       {
-         if (mask.extent(0) != n)
-           ERROR("CAxis::checkMask(void)",
-                 << "[ id = " << this->getId() << " , context = '" << CObjectFactory::GetCurrentContextId() << " ] "
-                 << "The mask does not have the same size as the local domain." << std::endl
-                 << "Local size is " << n.getValue() << "." << std::endl
-                 << "Mask size is " << mask.extent(0) << ".");
-      }
-      else // (mask.isEmpty())
-      { // If no mask was defined, we create a default one without any masked point.
-         mask.resize(n);
-         for (int i = 0; i < n; ++i)
-         {
-           mask(i) = true;
-         }
+        if (mask.extent(0) != n)
+        {
+          ERROR("CAxis::checkMask(void)",
+              << "[ id = " << this->getId() << " , context = '" << CObjectFactory::GetCurrentContextId() << " ] "
+              << "The mask does not have the same size as the local domain." << std::endl
+              << "Local size is " << n.getValue() << "." << std::endl
+              << "Mask size is " << mask.extent(0) << ".");
+        }
+      }
+      else
+      {
+        mask.resize(n);
+        mask = true;
       }
    }
@@ -598 +628 @@
 
         // Calculate the compressed index if any
-        std::set<int> writtenInd;
-        if (isCompressible_)
-        {
-          for (int idx = 0; idx < data_index.numElements(); ++idx)
-          {
-            int ind = CDistributionClient::getAxisIndex(data_index(idx), data_begin, ni);
-
-            if (ind >= 0 && ind < ni && mask(ind))
-            {
-              ind += ibegin;
-              writtenInd.insert(ind);
-            }
-          }
-        }
+//        std::set<int> writtenInd;
+//        if (isCompressible_)
+//        {
+//          for (int idx = 0; idx < data_index.numElements(); ++idx)
+//          {
+//            int ind = CDistributionClient::getAxisIndex(data_index(idx), data_begin, ni);
+//
+//            if (ind >= 0 && ind < ni && mask(ind))
+//            {
+//              ind += ibegin;
+//              writtenInd.insert(ind);
+//            }
+//          }
+//        }
 
         // Compute the global index of the current client (process) hold
@@ -680 +710 @@
           connectedServerRank_[nbServer].push_back(client->clientRank % client->serverSize);
 
-         nbSenders[nbServer] = CClientServerMapping::computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[nbServer]);
+        nbSenders[nbServer] = CClientServerMapping::computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[nbServer]);
 
         delete clientServerMap;
@@ -718 +748 @@
     CArray<size_t,1>::const_iterator itSrvb = writtenGlobalIndex.begin(),
                                      itSrve = writtenGlobalIndex.end(), itSrv;  
+
+    localIndexToWriteOnServer.resize(writtenGlobalIndex.numElements());
+    nbWritten = 0;
     for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
     {
@@ -723 +756 @@
       if (ite != globalLocalIndexMap_.find(indGlo))
       {
-        ++nbWritten;
-      }
-    }
-
-    localIndexToWriteOnServer.resize(writtenGlobalIndex.numElements());
-//      localIndexToWriteOnServer.resize(nbWritten);
-
-    nbWritten = 0;
-    for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
-    {
-      indGlo = *itSrv;
-      if (ite != globalLocalIndexMap_.find(indGlo))
-      {
         localIndexToWriteOnServer(nbWritten) = globalLocalIndexMap_[indGlo];
-        ++nbWritten;
-      }
-    }
+      }
+      else
+      {
+        localIndexToWriteOnServer(nbWritten) = -1;
+      }
+      ++nbWritten;
+    }
+
   }
   CATCH_DUMP_ATTR
@@ -780 +805 @@
         }                 
       }
+//
+//      nbWritten = 0;
+//      for (int idx = 0; idx < data_index.numElements(); ++idx)
+//      {
+//        if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
+//        {
+//          ++nbWritten;
+//        }
+//      }
+//
+//      compressedIndexToWriteOnServer[writtenCommSize].resize(nbWritten);
+//      nbWritten = 0;
+//      for (int idx = 0; idx < data_index.numElements(); ++idx)
+//      {
+//        if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
+//        {
+//          compressedIndexToWriteOnServer[writtenCommSize](nbWritten) = localGlobalIndexMap[data_index(idx)];
+//          ++nbWritten;
+//        }
+//      }
 
       nbWritten = 0;
@@ -1027 +1072 @@
 
   /*
-    Send attributes of axis from a group of client to other group of clients/servers 
-    on supposing that these attributes are distributed among the clients of the sending group
-    In the future, if new attributes are added, they should also be processed in this function
+    Send axis attributes from a group of clients to another group of clients/servers
+    supposing that these attributes are distributed among the clients of the sending group
+    In future, if new attributes are added, they should also be processed in this function
   */
   void CAxis::sendDistributedAttributes(void)
   TRY
   {
-    int ns, n, i, j, ind, nv, idx;
+    int ind, idx;
     std::list<CContextClient*>::iterator it;
 
@@ -1046 +1091 @@
       list<CMessage> listData;
       list<CArray<int,1> > list_indi, list_dataInd;
-      list<CArray<bool,1> > list_mask;
       list<CArray<double,1> > list_val;
       list<CArray<double,2> > list_bounds;
       list<CArray<string,1> > list_label;
 
+      // Cut off the ghost points
       int nbIndex = index.numElements();
       CArray<int,1> dataIndex(nbIndex);
@@ -1057 +1102 @@
       {
         if (0 <= data_index(idx) && data_index(idx) < nbIndex)
-          dataIndex(idx) = 1;
+          dataIndex(data_index(idx)) = 1;
       }
 
@@ -1064 +1109 @@
       for (int k = 0; k < connectedServerRank_[nbServer].size(); ++k)
       {
-        int nbData = 0;
+        int nbData = 0, nbDataCount = 0;
         int rank = connectedServerRank_[nbServer][k];
         it = indSrv_[nbServer].find(rank);
@@ -1071 +1116 @@
 
         list_indi.push_back(CArray<int,1>(nbData));
-        list_dataInd.push_back(CArray<int,1>(nbData));        
-        list_mask.push_back(CArray<bool,1>(nbData));
+        list_dataInd.push_back(CArray<int,1>(nbData));
 
         if (hasValue)
@@ -1084 +1128 @@
 
         CArray<int,1>& indi = list_indi.back();
-        CArray<int,1>& dataIndi = list_dataInd.back();        
-        CArray<bool,1>& maskIndi = list_mask.back();
-
-        for (n = 0; n < nbData; ++n)
+        CArray<int,1>& dataIndi = list_dataInd.back();
+        dataIndi = -1;
+
+        for (int n = 0; n < nbData; ++n)
         {
           idx = static_cast<int>(it->second[n]);
@@ -1094 +1138 @@
           ind = globalLocalIndexMap_[idx];
           dataIndi(n) = dataIndex(ind);
-          maskIndi(n) = mask(ind);
 
           if (hasValue)
@@ -1118 +1161 @@
         listData.push_back(CMessage());
         listData.back() << this->getId()
-                        << list_indi.back() << list_dataInd.back() << list_mask.back();
+                        << list_indi.back() << list_dataInd.back();
 
         listData.back() << hasValue;
@@ -1173 +1216 @@
     int nbReceived = ranks.size(), idx, ind, gloInd, locInd;
     vector<CArray<int,1> > vec_indi(nbReceived), vec_dataInd(nbReceived);
-    vector<CArray<bool,1> > vec_mask(nbReceived);
     vector<CArray<double,1> > vec_val(nbReceived);
     vector<CArray<double,2> > vec_bounds(nbReceived);
@@ -1183 +1225 @@
       buffer >> vec_indi[idx];
       buffer >> vec_dataInd[idx];      
-      buffer >> vec_mask[idx];
 
       buffer >> hasValue;
@@ -1220 +1261 @@
          if (0 == globalLocalIndexMap_.count(gloInd))
          {
-           index(nbIndLoc) = gloInd % n_glo;
-           globalLocalIndexMap_[gloInd] = nbIndLoc;
+           index(nbIndexGlob) = gloInd % n_glo;
+           globalLocalIndexMap_[gloInd] = nbIndexGlob;
            ++nbIndexGlob;
          } 
@@ -1234 +1275 @@
     CArray<int,1> nonCompressedData(nbData);
     nonCompressedData = -1;   
-    mask.resize(nbData);
+    // Mask is incorporated into data_index and is not sent/received anymore
+    mask.resize(0);
     if (hasValue)
       value.resize(nbData);
@@ -1247 +1289 @@
       CArray<int,1>& indi = vec_indi[idx];
       CArray<int,1>& dataIndi = vec_dataInd[idx];
-      CArray<bool,1>& maskIndi = vec_mask[idx];
       int nb = indi.numElements();
       for (int n = 0; n < nb; ++n)
@@ -1255 +1296 @@
         nonCompressedData(locInd) = (-1 == nonCompressedData(locInd)) ? dataIndi(n) : nonCompressedData(locInd);
 
-        if (!mask(locInd)) // Only rewrite mask if it's not true
-          mask(locInd) = maskIndi(n);
-        
         if (hasValue)
           value(locInd) = vec_val[idx](n);
@@ -1272 +1310 @@
     }
     
-    int nbCompressedData = 0; 
+    int nbCompressedData = 0;
     for (idx = 0; idx < nonCompressedData.numElements(); ++idx)
     {
       if (0 <= nonCompressedData(idx))
-        ++nbCompressedData;        
+        ++nbCompressedData;
     }
 
@@ -1286 +1324 @@
       {
         data_index(nbCompressedData) = idx % n;
-        ++nbCompressedData;        
+        ++nbCompressedData;
       }
     }
 
     data_begin.setValue(0);
+    data_n.setValue(data_index.numElements());
   }
   CATCH_DUMP_ATTR

XIOS/trunk/src/node/axis.hpp

@@ -126 +126 @@
         bool hasLabel;
 
-        CArray<size_t,1> localIndexToWriteOnServer;        
+        CArray<int,1> localIndexToWriteOnServer;
 
       private:

XIOS/trunk/src/node/domain.cpp

@@ -179 +179 @@
        // size estimation for sendIndex (and sendArea which is always smaller or equal)
        size_t sizeIndexEvent = 2 * sizeof(size_t) + 2 * CArray<int,1>::size(idxCount);
-       // if (isCompressible_)
-       // {
-       //   std::map<int, std::vector<int> >::const_iterator itWritten = indWrittenSrv_.find(rank);
-       //   size_t writtenIdxCount = (itWritten != itWrittenIndexEnd) ? itWritten->second.size() : 0;
-       //   sizeIndexEvent += CArray<int,1>::size(writtenIdxCount);
-       // }
 
        // size estimation for sendLonLat
@@ -1139 +1133 @@
    TRY
    {
+     int i,j,ind;
       if (!data_i_index.isEmpty())
       {
@@ -1159 +1154 @@
                    << "'data_j_index' must be defined when 'data_i_index' is set and 'data_dim' is 2.");
           }
+          for (int k=0; k<data_i_index.numElements(); ++k)
+          {
+            i = data_i_index(k)+data_ibegin ;
+            j = data_j_index(k)+data_jbegin ;
+            if (i>=0 && i<ni && j>=0 && j<nj)
+            {
+              ind=j*ni+i ;
+              if (!domainMask(ind))
+              {
+                data_i_index(k) = -1;
+                data_j_index(k) = -1;
+              }
+            }
+            else
+            {
+              data_i_index(k) = -1;
+              data_j_index(k) = -1;
+            }
+          }
         }
         else // (1 == data_dim)
@@ -1165 +1179 @@
           {
             data_j_index.resize(data_ni);
-            for (int j = 0; j < data_ni; ++j) data_j_index(j) = 0;
+            data_j_index = 0;
+          }
+          for (int k=0; k<data_i_index.numElements(); ++k)
+          {
+            i=data_i_index(k)+data_ibegin ;
+            if (i>=0 && i < domainMask.size())
+            {
+              if (!domainMask(i)) data_i_index(k) = -1;
+            }
+            else
+              data_i_index(k) = -1;
+
+            if (!domainMask(i)) data_i_index(k) = -1;
           }
         }
@@ -1180 +1206 @@
           data_i_index.resize(data_ni);
           data_j_index.resize(data_ni);
-
-          for (int i = 0; i < data_ni; ++i)
+          data_j_index = 0;
+
+          for (int k = 0; k < data_ni; ++k)
           {
-            data_i_index(i) = i;
-            data_j_index(i) = 0;
+            i=k+data_ibegin ;
+            if (i>=0 && i < domainMask.size())
+            {
+              if (domainMask(i))
+                data_i_index(k) = k;
+              else
+                data_i_index(k) = -1;
+            }
+            else
+              data_i_index(k) = -1;
           }
         }
@@ -1193 +1228 @@
           data_j_index.resize(dsize);
 
-          for(int count = 0, j = 0; j < data_nj; ++j)
+          for(int count = 0, kj = 0; kj < data_nj; ++kj)
           {
-            for(int i = 0; i < data_ni; ++i, ++count)
+            for(int ki = 0; ki < data_ni; ++ki, ++count)
             {
-              data_i_index(count) = i;
-              data_j_index(count) = j;
+              i = ki + data_ibegin;
+              j = kj + data_jbegin;
+              ind=j*ni+i ;
+              if (i>=0 && i<ni && j>=0 && j<nj)
+              {
+                if (domainMask(ind))
+                {
+                  data_i_index(count) = ki;
+                  data_j_index(count) = kj;
+                }
+                else
+                {
+                  data_i_index(count) = -1;
+                  data_j_index(count) = -1;
+                }
+              }
+              else
+              {
+                data_i_index(count) = -1;
+                data_j_index(count) = -1;
+              }
             }
           }
@@ -1872 +1926 @@
             connectedServerRank_[nbServer].push_back(client->clientRank % client->serverSize);
 
+          // Now check if all servers have data to receive. If not, master client will send empty data.
+          // This ensures that all servers will participate in collective calls upon receiving even if they have no date to receive.
+          std::vector<int> counts (clientSize);
+          std::vector<int> displs (clientSize);
+          displs[0] = 0;
+          int localCount = connectedServerRank_[nbServer].size() ;
+          MPI_Gather(&localCount, 1, MPI_INT, &counts[0], 1, MPI_INT, 0, client->intraComm) ;
+          for (int i = 0; i < clientSize-1; ++i)
+          {
+            displs[i+1] = displs[i] + counts[i];
+          }
+          std::vector<int> allConnectedServers(displs[clientSize-1]+counts[clientSize-1]);
+          MPI_Gatherv(&(connectedServerRank_[nbServer])[0], localCount, MPI_INT, &allConnectedServers[0], &counts[0], &displs[0], MPI_INT, 0, client->intraComm);
+
+          if ((allConnectedServers.size() != nbServer) && (rank == 0))
+          {
+            std::vector<bool> isSrvConnected (nbServer, false);
+            for (int i = 0; i < allConnectedServers.size(); ++i) isSrvConnected[allConnectedServers[i]] = true;
+            for (int i = 0; i < nbServer; ++i)
+            {
+              if (!isSrvConnected[i]) connectedServerRank_[nbServer].push_back(i);
+            }
+          }
           nbSenders[nbServer] = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[nbServer]);
           delete clientServerMap;
@@ -1908 +1985 @@
                                        itSrve = writtenGlobalIndex.end(), itSrv;
 
-//      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
-//      {
-//        indGlo = *itSrv;
-//        if (ite != globalLocalIndexMap_.find(indGlo))
-//        {
-//          ++nbWritten;
-//        }
-//      }
-
-//      localIndexToWriteOnServer.resize(nbWritten);
       localIndexToWriteOnServer.resize(writtenGlobalIndex.numElements());
-
       nbWritten = 0;
       for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
@@ -1927 +1993 @@
         {
           localIndexToWriteOnServer(nbWritten) = globalLocalIndexMap_[indGlo];
-          ++nbWritten;
         }
         else
         {
-          localIndexToWriteOnServer(nbWritten) = 0;
-          ++nbWritten;
-        }
-      }
-      
-      // if (isCompressible())
-      // {
-      //   nbWritten = 0;
-      //   std::unordered_map<size_t,size_t> localGlobalIndexMap;
-      //   for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
-      //   {
-      //     indGlo = *itSrv;
-      //     if (ite != globalLocalIndexMap_.find(indGlo))
-      //     {
-      //       localGlobalIndexMap[localIndexToWriteOnServer(nbWritten)] = indGlo;
-      //       ++nbWritten;
-      //     }                 
-      //   }
-
-      //   nbWritten = 0;
-      //   for (int idx = 0; idx < data_i_index.numElements(); ++idx)
-      //   {
-      //     if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_i_index(idx)))
-      //     {
-      //       ++nbWritten;
-      //     }
-      //   }
-
-      //   compressedIndexToWriteOnServer.resize(nbWritten);
-      //   nbWritten = 0;
-      //   for (int idx = 0; idx < data_i_index.numElements(); ++idx)
-      //   {
-      //     if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_i_index(idx)))
-      //     {
-      //       compressedIndexToWriteOnServer(nbWritten) = localGlobalIndexMap[data_i_index(idx)];
-      //       ++nbWritten;
-      //     }
-      //   }
-
-      //   numberWrittenIndexes_ = nbWritten;
-      //   if (isDistributed())
-      //   {            
-      //     MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
-      //     MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
-      //     offsetWrittenIndexes_ -= numberWrittenIndexes_;
-      //   }
-      //   else
-      //     totalNumberWrittenIndexes_ = numberWrittenIndexes_;
-      // }      
+          localIndexToWriteOnServer(nbWritten) = -1;
+        }
+        ++nbWritten;
+      }
    }
    CATCH_DUMP_ATTR
@@ -2063 +2083 @@
     sendDistributionAttributes();
     sendIndex();       
-    sendMask();
     sendLonLat();
     sendArea();    
@@ -2174 +2193 @@
 
   /*!
-    Send mask index from client to connected(s) clients    
-  */
-  void CDomain::sendMask()
-  TRY
-  {
-    int ns, n, i, j, ind, nv, idx;
-    std::list<CContextClient*>::iterator it;
-    for (it=clients.begin(); it!=clients.end(); ++it)
-    {
-      CContextClient* client = *it;
-      int serverSize = client->serverSize;
-
-      // send area for each connected server
-      CEventClient eventMask(getType(), EVENT_ID_MASK);
-
-      list<CMessage> list_msgsMask;
-      list<CArray<bool,1> > list_mask;
-
-      std::unordered_map<int, vector<size_t> >::const_iterator it, iteMap;
-      iteMap = indSrv_[serverSize].end();
-      for (int k = 0; k < connectedServerRank_[serverSize].size(); ++k)
-      {
-        int nbData = 0;
-        int rank = connectedServerRank_[serverSize][k];
-        it = indSrv_[serverSize].find(rank);
-        if (iteMap != it)
-          nbData = it->second.size();
-        list_mask.push_back(CArray<bool,1>(nbData));
-
-        const std::vector<size_t>& temp = it->second;
-        for (n = 0; n < nbData; ++n)
-        {
-          idx = static_cast<int>(it->second[n]);
-          list_mask.back()(n) = domainMask(globalLocalIndexMap_[idx]);
-        }
-
-        list_msgsMask.push_back(CMessage());
-        list_msgsMask.back() << this->getId() << list_mask.back();
-        eventMask.push(rank, nbSenders[serverSize][rank], list_msgsMask.back());
-      }
-      client->sendEvent(eventMask);
-    }
-  }
-  CATCH_DUMP_ATTR
-
-  /*!
     Send area from client to connected client(s)
   */
@@ -2458 +2431 @@
         case EVENT_ID_INDEX:
           recvIndex(event);
-          return true;
-          break;
-        case EVENT_ID_MASK:
-          recvMask(event);
           return true;
           break;
@@ -2555 +2524 @@
            jIndex = (jIndex < 0) ? 0 : jIndex;
            nbIndLoc = iIndex + ni * jIndex;
-           if (nbIndLoc < nbIndexGlobMax)
-           {
-             i_index(nbIndLoc) = index % ni_glo;
-             j_index(nbIndLoc) = index / ni_glo;
-             globalLocalIndexMap_[index] = nbIndLoc;  
-             ++nbIndGlob;
-           }
-           // i_index(nbIndGlob) = index % ni_glo;
-           // j_index(nbIndGlob) = index / ni_glo;
-           // globalLocalIndexMap_[index] = nbIndGlob;  
-           // ++nbIndGlob;
+           i_index(nbIndGlob) = index % ni_glo;
+           j_index(nbIndGlob) = index / ni_glo;
+           globalLocalIndexMap_[index] = nbIndGlob;
+           ++nbIndGlob;
          } 
       } 
@@ -2580 +2542 @@
       j_index.resizeAndPreserve(nbIndGlob);
     }
+
+    domainMask.resize(0); // Mask is not defined anymore on servers
   }
   CATCH
@@ -2619 +2583 @@
 
   }
-  CATCH_DUMP_ATTR
-
-  /*!
-    Receive area event from clients(s)
-    \param[in] event event contain info about rank and associated area
-  */
-  void CDomain::recvMask(CEventServer& event)
-  TRY
-  {
-    string domainId;
-    std::map<int, CBufferIn*> rankBuffers;
-
-    list<CEventServer::SSubEvent>::iterator it;
-    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
-    {      
-      CBufferIn* buffer = it->buffer;
-      *buffer >> domainId;
-      rankBuffers[it->rank] = buffer;     
-    }
-    get(domainId)->recvMask(rankBuffers);
-  }
-  CATCH
-
-  /*!
-    Receive mask information from client(s)
-    \param[in] rankBuffers rank of sending client and the corresponding receive buffer  
-  */
-  void CDomain::recvMask(std::map<int, CBufferIn*>& rankBuffers)
-  TRY
-  {
-    int nbReceived = rankBuffers.size(), i, ind, index, lInd;
-    if (nbReceived != recvClientRanks_.size())
-      ERROR("void CDomain::recvMask(std::map<int, CBufferIn*>& rankBuffers)",
-           << "The number of sending clients is not correct."
-           << "Expected number: " << recvClientRanks_.size() << " but received " << nbReceived);
-
-    vector<CArray<bool,1> > recvMaskValue(nbReceived);      
-    for (i = 0; i < recvClientRanks_.size(); ++i)
-    {
-      int rank = recvClientRanks_[i];
-      CBufferIn& buffer = *(rankBuffers[rank]);      
-      buffer >> recvMaskValue[i];
-    }
-
-    int nbMaskInd = 0;
-    for (i = 0; i < nbReceived; ++i)
-    {
-      nbMaskInd += recvMaskValue[i].numElements();
-    }
-  
-    if (nbMaskInd != globalLocalIndexMap_.size())
-      info (0) << "If domain " << this->getDomainOutputName() <<" does not have overlapped regions between processes "
-               << "something must be wrong with mask index "<< std::endl;
-
-    nbMaskInd = globalLocalIndexMap_.size();
-    mask_1d.resize(nbMaskInd);
-    domainMask.resize(nbMaskInd);
-    mask_1d = false;
-    
-    for (i = 0; i < nbReceived; ++i)
-    {
-      CArray<int,1>& tmpInd = indGlob_[recvClientRanks_[i]];
-      CArray<bool,1>& tmp = recvMaskValue[i];
-      for (ind = 0; ind < tmp.numElements(); ++ind)
-      {
-        lInd = globalLocalIndexMap_[size_t(tmpInd(ind))];
-        if (!mask_1d(lInd)) // Only rewrite mask_1d if it's not true
-          mask_1d(lInd) = tmp(ind);
-      }
-    }
-    domainMask=mask_1d ;
-  }
-  CATCH_DUMP_ATTR
-
+ CATCH_DUMP_ATTR
   /*!
     Receive longitude event from clients(s)
@@ -3046 +2937 @@
          dataIIndex(lInd) = (-1 == dataIIndex(lInd)) ? tmpI(ind) : dataIIndex(lInd); // Only fill in dataIndex if there is no data
          dataJIndex(lInd) = (-1 == dataJIndex(lInd)) ? tmpJ(ind) : dataJIndex(lInd);  
-
-         if (!domainMask(lInd))   // Include mask info into data index on the RECEIVE getServerDimensionSizes
-         {
-           dataIIndex(lInd) = dataJIndex(lInd) = -1;
-         }
       } 
     }

XIOS/trunk/src/node/domain.hpp

@@ -49 +49 @@
          {
            EVENT_ID_INDEX, EVENT_ID_LON, EVENT_ID_LAT, 
-           EVENT_ID_AREA, EVENT_ID_MASK,
+           EVENT_ID_AREA,
            EVENT_ID_DATA_INDEX, EVENT_ID_SERVER_ATTRIBUT
          } ;
@@ -142 +142 @@
          CArray<double, 1> areavalue;
 
-         CArray<size_t,1> localIndexToWriteOnServer;         
+         CArray<int,1> localIndexToWriteOnServer;
 
          CArray<bool, 1> domainMask; // mask_1d, mask_2d -> domainMask
@@ -175 +175 @@
          void sendIndex();
          void sendDistributionAttributes();
-         void sendMask();
          void sendArea();
          void sendLonLat();         
@@ -186 +185 @@
          static void recvDistributionAttributes(CEventServer& event);
          static void recvIndex(CEventServer& event);
-         static void recvMask(CEventServer& event);         
          static void recvLon(CEventServer& event);
          static void recvLat(CEventServer& event);
@@ -193 +191 @@
          void recvDistributionAttributes(CBufferIn& buffer);                  
          void recvIndex(std::map<int, CBufferIn*>& rankBuffers);         
-         void recvMask(std::map<int, CBufferIn*>& rankBuffers);
          void recvLon(std::map<int, CBufferIn*>& rankBuffers);
          void recvLat(std::map<int, CBufferIn*>& rankBuffers);

XIOS/trunk/src/node/field.cpp

@@ -1122 +1122 @@
      {
         if (!instantDataFilter)
-          instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid,true));
+          instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, false));
 
 
@@ -1138 +1138 @@
      {
        if (!instantDataFilter)
-         instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true));
+         instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, false, false));
 
              // If the field data is to be read by the client or/and written to a file
@@ -1184 +1184 @@
          {
            checkTimeAttributes();
-           instantDataFilter = serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, freq_offset, true,
+           instantDataFilter = serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, false, freq_offset, true,
                                                                                                        detectMissingValues, defaultValue));
          }
@@ -1190 +1190 @@
          {
             if (check_if_active.isEmpty()) check_if_active = false; 
-            instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, false, NoneDu, false,
+            instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, false, true, NoneDu, false,
                                                                                                         detectMissingValues, defaultValue));
          }
@@ -1275 +1275 @@
          {
            checkTimeAttributes();
-           serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, freq_offset, true,
+           serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, false, freq_offset, true,
                                                                                    detectMissingValues, defaultValue));
          }
@@ -1292 +1292 @@
          {
            if (check_if_active.isEmpty()) check_if_active = false;
-           clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, NoneDu, false,
+           clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, true, NoneDu, false,
                                                                                    detectMissingValues, defaultValue));
          }
@@ -1887 +1887 @@
    CATCH
 
+   bool CField::hasGridMask(void) const
+   TRY
+   {
+     return (this->grid->hasMask());
+   }
+   CATCH
+
    DEFINE_REF_FUNC(Field,field)
 } // namespace xios

XIOS/trunk/src/node/field.hpp

@@ -209 +209 @@
         bool hasExpression(void) const;
 
+        bool hasGridMask(void) const;
+
       public:
          /// Propriétés privées ///

XIOS/trunk/src/node/grid.cpp

@@ -364 +364 @@
    }
    CATCH_DUMP_ATTR
+   bool CGrid::hasMask() const
+   TRY
+   {
+     return (!mask_1d.isEmpty() || !mask_2d.isEmpty() || !mask_3d.isEmpty() ||
+             !mask_4d.isEmpty() || !mask_5d.isEmpty() || !mask_6d.isEmpty() || !mask_7d.isEmpty());
+   }
+   CATCH
 
    /*
@@ -453 +460 @@
    CATCH_DUMP_ATTR
 
-/*!
-  A grid can have multiple dimension, so can its mask in the form of multi-dimension array.
-It's not a good idea to store all multi-dimension arrays corresponding to each mask.
-One of the ways is to convert this array into 1-dimension one and every process is taken place on it.
-  \param [in] multi-dimension array grid mask
-*/
-
-  void CGrid::getLocalMask(CArray<bool,1>& localMask)
-  TRY
-  {
-      std::vector<CDomain*> domainP = this->getDomains();
-      std::vector<CAxis*> axisP = this->getAxis();
-      int dim = domainP.size() * 2 + axisP.size();
-
-      switch (dim)
-      {
-        case 0:
-          getLocalMask(mask_0d, localMask);
-          break;
-        case 1:
-          getLocalMask(mask_1d, localMask);
-          break;
-        case 2:
-          getLocalMask(mask_2d, localMask);
-          break;
-        case 3:
-          getLocalMask(mask_3d, localMask);
-          break;
-        case 4:
-          getLocalMask(mask_4d, localMask);
-          break;
-        case 5:
-          getLocalMask(mask_5d, localMask);
-          break;
-        case 6:
-          getLocalMask(mask_6d, localMask);
-          break;
-        case 7:
-          getLocalMask(mask_7d, localMask);
-          break;
-        default:
-          break;
-      }
-  }
-  CATCH_DUMP_ATTR
-      
    /*
      Modify value of mask in a certain index
@@ -736 +697 @@
      CContext* context = CContext::getCurrent();
 
-     CContextClient* client = context->client;  // Here it's not important which contextClient to recuperate
+     CContextClient* client = context->client;
      int rank = client->clientRank;
 
      clientDistribution_ = new CDistributionClient(rank, this);
      // Get local data index on client
-     storeIndex_client.resize(clientDistribution_->getLocalDataIndexOnClient().size());
-     int nbStoreIndex = storeIndex_client.numElements();
+     int nbStoreIndex = clientDistribution_->getLocalDataIndexOnClient().size();
+     int nbStoreGridMask = clientDistribution_->getLocalMaskIndexOnClient().size();
+     // nbStoreGridMask = nbStoreIndex if grid mask is defined, and 0 otherwise
+     storeIndex_client.resize(nbStoreIndex);
+     storeMask_client.resize(nbStoreGridMask);
      for (int idx = 0; idx < nbStoreIndex; ++idx) storeIndex_client(idx) = (clientDistribution_->getLocalDataIndexOnClient())[idx];
+     for (int idx = 0; idx < nbStoreGridMask; ++idx) storeMask_client(idx) = (clientDistribution_->getLocalMaskIndexOnClient())[idx];
 
      if (0 == serverDistribution_) isDataDistributed_= clientDistribution_->isDataDistributed();
@@ -884 +849 @@
          if (connectedServerRank_[receiverSize].empty())
           connectedServerRank_[receiverSize].push_back(client->clientRank % client->serverSize);
+
+         // Now check if all servers have data to receive. If not, master client will send empty data.
+         // This ensures that all servers will participate in collective calls upon receiving even if they have no date to receive.
+         std::vector<int> counts (client->clientSize);
+         std::vector<int> displs (client->clientSize);
+         displs[0] = 0;
+         int localCount = connectedServerRank_[receiverSize].size() ;
+         MPI_Gather(&localCount, 1, MPI_INT, &counts[0], 1, MPI_INT, 0, client->intraComm) ;
+         for (int i = 0; i < client->clientSize-1; ++i)
+         {
+           displs[i+1] = displs[i] + counts[i];
+         }
+         std::vector<int> allConnectedServers(displs[client->clientSize-1]+counts[client->clientSize-1]);
+         MPI_Gatherv(&(connectedServerRank_[receiverSize])[0], localCount, MPI_INT, &allConnectedServers[0], &counts[0], &displs[0], MPI_INT, 0, client->intraComm);
+
+         if ((allConnectedServers.size() != receiverSize) && (client->clientRank == 0))
+         {
+           std::vector<bool> isSrvConnected (receiverSize, false);
+           for (int i = 0; i < allConnectedServers.size(); ++i) isSrvConnected[allConnectedServers[i]] = true;
+           for (int i = 0; i < receiverSize; ++i)
+           {
+             if (!isSrvConnected[i]) connectedServerRank_[receiverSize].push_back(i);
+           }
+         }
 
          nbSenders[receiverSize] = clientServerMap_->computeConnectedClients(receiverSize, client->clientSize, client->intraComm, connectedServerRank_[receiverSize]);
@@ -1369 +1358 @@
    CATCH
 
+   void CGrid::maskField_arr(const double* const data, CArray<double, 1>& stored) const
+   {
+      const StdSize size = storeIndex_client.numElements();
+      stored.resize(size);
+      const double nanValue = std::numeric_limits<double>::quiet_NaN();
+
+      if (storeMask_client.numElements() != 0)
+        for(StdSize i = 0; i < size; i++) stored(i) = (storeMask_client(i)) ? data[storeIndex_client(i)] : nanValue;
+      else
+        for(StdSize i = 0; i < size; i++) stored(i) = data[storeIndex_client(i)];
+   }
+
    void CGrid::uncompressField_arr(const double* const data, CArray<double, 1>& out) const
    TRY
@@ -1836 +1837 @@
           nGlob.push_back(1);  
         }
-
-        modifyMaskSize(nSize, false);
-
-        // These below codes are reserved for future
-        CDistributionServer srvDist(server->intraCommRank, nBegin, nSize, nBeginGlobal, nGlob); 
-        map<int, CArray<size_t, 1> >::iterator itb = outGlobalIndexFromClient.begin(),
-                                               ite = outGlobalIndexFromClient.end(), it;  
-        const CDistributionServer::GlobalLocalMap&  globalLocalMask = srvDist.getGlobalLocalIndex();
-        CDistributionServer::GlobalLocalMap::const_iterator itSrv;
-        size_t nb = 0;
-        for (it = itb; it != ite; ++it)
-        {
-          CArray<size_t,1>& globalInd = it->second;
-          for (size_t idx = 0; idx < globalInd.numElements(); ++idx)
-          {
-            if (globalLocalMask.end() != globalLocalMask.find(globalInd(idx))) ++nb;
-          }
-        }
-        
-        CArray<int,1> indexToModify(nb);
-        nb = 0;    
-        for (it = itb; it != ite; ++it)
-        {
-          CArray<size_t,1>& globalInd = it->second;
-          for (size_t idx = 0; idx < globalInd.numElements(); ++idx)
-          {
-            itSrv = globalLocalMask.find(globalInd(idx));
-            if (globalLocalMask.end() != itSrv) 
-            {
-              indexToModify(nb) = itSrv->second;
-              ++nb;
-            }
-          }
-        }
-
-        modifyMask(indexToModify, true);
       }
 

XIOS/trunk/src/node/grid.hpp

@@ -94 +94 @@
          template <int n>
          void inputField(const CArray<double,n>& field, CArray<double,1>& stored) const;
+         template <int n>
+         void maskField(const CArray<double,n>& field, CArray<double,1>& stored) const;
          template <int n>
          void outputField(const CArray<double,1>& stored, CArray<double,n>& field) const;  
@@ -203 +205 @@
          bool hasTransform();
          size_t getGlobalWrittenSize(void) ;
-         void getLocalMask(CArray<bool,1>& localMask) ;
-         template<int N>
-         void getLocalMask(const CArray<bool,N>& gridMask, CArray<bool,1>& localMask) ;
       public:
          CArray<int, 1> storeIndex_client;
+         CArray<bool, 1> storeMask_client;
 
 /** Map containing indexes that will be sent in sendIndex(). */
@@ -247 +247 @@
          CArray<size_t,1> indexFromClients;
 
+         bool hasMask(void) const;
          void checkMask(void);
          void createMask(void);
@@ -273 +274 @@
         void restoreField_arr(const CArray<double, 1>& stored, double* const data) const;
         void uncompressField_arr(const double* const data, CArray<double, 1>& outData) const;
+        void maskField_arr(const double* const data, CArray<double, 1>& stored) const;
 
         void setVirtualDomainGroup(CDomainGroup* newVDomainGroup);
@@ -387 +389 @@
 
    template <int n>
+   void CGrid::maskField(const CArray<double,n>& field, CArray<double,1>& stored) const
+   {
+//#ifdef __XIOS_DEBUG
+      if (this->getDataSize() != field.numElements())
+         ERROR("void CGrid::inputField(const  CArray<double,n>& field, CArray<double,1>& stored) const",
+                << "[ Awaiting data of size = " << this->getDataSize() << ", "
+                << "Received data size = "      << field.numElements() << " ] "
+                << "The data array does not have the right size! "
+                << "Grid = " << this->getId())
+//#endif
+      this->maskField_arr(field.dataFirst(), stored);
+   }
+
+   template <int n>
    void CGrid::outputField(const CArray<double,1>& stored, CArray<double,n>& field) const
    TRY
@@ -424 +440 @@
    TRY
    {
-     if (!gridMask.isEmpty() || createMask)
-     {
-       int idx = 0;
-       int numElement = axisDomainOrder.numElements();
-       int dim = domainMasks.size() * 2 + axisMasks.size();
-       std::vector<CDomain*> domainP = this->getDomains();
-       std::vector<CAxis*> axisP = this->getAxis();
-
-       std::vector<int> idxLoop(dim,0), indexMap(numElement), eachDimSize(dim);
-       std::vector<int> currentIndex(dim);
-       int idxDomain = 0, idxAxis = 0;
+     int idx = 0;
+     int numElement = axisDomainOrder.numElements();
+     int dim = domainMasks.size() * 2 + axisMasks.size();
+     std::vector<CDomain*> domainP = this->getDomains();
+     std::vector<CAxis*> axisP = this->getAxis();
+
+     std::vector<int> idxLoop(dim,0), indexMap(numElement), eachDimSize(dim);
+     std::vector<int> currentIndex(dim);
+     int idxDomain = 0, idxAxis = 0;
+    for (int i = 0; i < numElement; ++i)
+    {
+      indexMap[i] = idx;
+      if (2 == axisDomainOrder(i)) {
+          eachDimSize[indexMap[i]]   = domainP[idxDomain]->ni;
+          eachDimSize[indexMap[i]+1] = domainP[idxDomain]->nj;
+          idx += 2; ++idxDomain;
+      }
+      else if (1 == axisDomainOrder(i)) {
+//        eachDimSize[indexMap[i]] = axisMasks[idxAxis]->numElements();
+        eachDimSize[indexMap[i]] = axisP[idxAxis]->n;
+        ++idx; ++idxAxis;
+      }
+      else {};
+    }
+
+    if (!gridMask.isEmpty() && !createMask)
+    {
+      for (int i = 0; i < dim; ++i)
+      {
+        if (gridMask.extent(i) != eachDimSize[i])
+          ERROR("CGrid::checkMask(void)",
+                << "The mask has one dimension whose size is different from the one of the local grid." << std::endl
+                << "Local size of dimension " << i << " is " << eachDimSize[i] << "." << std::endl
+                << "Mask size for dimension " << i << " is " << gridMask.extent(i) << "." << std::endl
+                << "Grid = " << this->getId())
+      }
+    }
+    else {
+        CArrayBoolTraits<CArray<bool,N> >::resizeArray(gridMask,eachDimSize);
+        gridMask = true;
+    }
+
+    int ssize = gridMask.numElements();
+    idx = 0;
+    while (idx < ssize)
+    {
+      for (int i = 0; i < dim-1; ++i)
+      {
+        if (idxLoop[i] == eachDimSize[i])
+        {
+          idxLoop[i] = 0;
+          ++idxLoop[i+1];
+        }
+      }
+
+      // Find out outer index
+      idxDomain = idxAxis = 0;
+      bool maskValue = true;
       for (int i = 0; i < numElement; ++i)
       {
-        indexMap[i] = idx;
-        if (2 == axisDomainOrder(i)) {
-            eachDimSize[indexMap[i]]   = domainP[idxDomain]->ni;
-            eachDimSize[indexMap[i]+1] = domainP[idxDomain]->nj;
-            idx += 2; ++idxDomain;
+        if (2 == axisDomainOrder(i))
+        {
+          int idxTmp = idxLoop[indexMap[i]] + idxLoop[indexMap[i]+1] * eachDimSize[indexMap[i]];
+          if (idxTmp < (*domainMasks[idxDomain]).numElements())
+            maskValue = maskValue && (*domainMasks[idxDomain])(idxTmp);
+          else
+            maskValue = false;
+          ++idxDomain;
         }
-        else if (1 == axisDomainOrder(i)) {
-  //        eachDimSize[indexMap[i]] = axisMasks[idxAxis]->numElements();
-          eachDimSize[indexMap[i]] = axisP[idxAxis]->n;
-          ++idx; ++idxAxis;
+        else if (1 == axisDomainOrder(i))
+        {
+          int idxTmp = idxLoop[indexMap[i]];
+          if (idxTmp < (*axisMasks[idxAxis]).numElements())
+            maskValue = maskValue && (*axisMasks[idxAxis])(idxTmp);
+          else
+            maskValue = false;
+
+          ++idxAxis;
         }
-        else {};
-      }
-
-//      if (!gridMask.isEmpty() && !createMask)
-      if (!createMask)
+      }
+
+      int maskIndex = idxLoop[0];
+      int mulDim = 1;
+      for (int k = 1; k < dim; ++k)
       {
-        for (int i = 0; i < dim; ++i)
-        {
-          if (gridMask.extent(i) != eachDimSize[i])
-            ERROR("CGrid::checkMask(void)",
-                  << "The mask has one dimension whose size is different from the one of the local grid." << std::endl
-                  << "Local size of dimension " << i << " is " << eachDimSize[i] << "." << std::endl
-                  << "Mask size for dimension " << i << " is " << gridMask.extent(i) << "." << std::endl
-                  << "Grid = " << this->getId())
-        }
-      }
-      else {
-          CArrayBoolTraits<CArray<bool,N> >::resizeArray(gridMask,eachDimSize);
-          gridMask = true;
-      }
-
-      int ssize = gridMask.numElements();
-      idx = 0;
-      while (idx < ssize)
-      {
-        for (int i = 0; i < dim-1; ++i)
-        {
-          if (idxLoop[i] == eachDimSize[i])
-          {
-            idxLoop[i] = 0;
-            ++idxLoop[i+1];
-          }
-        }
-
-        // Find out outer index
-        idxDomain = idxAxis = 0;
-        bool maskValue = true;
-        for (int i = 0; i < numElement; ++i)
-        {
-          if (2 == axisDomainOrder(i))
-          {
-            int idxTmp = idxLoop[indexMap[i]] + idxLoop[indexMap[i]+1] * eachDimSize[indexMap[i]];
-            if (idxTmp < (*domainMasks[idxDomain]).numElements())
-              maskValue = maskValue && (*domainMasks[idxDomain])(idxTmp);
-            else
-              maskValue = false;
-            ++idxDomain;
-          }
-          else if (1 == axisDomainOrder(i))
-          {
-            int idxTmp = idxLoop[indexMap[i]];
-            if (idxTmp < (*axisMasks[idxAxis]).numElements())
-              maskValue = maskValue && (*axisMasks[idxAxis])(idxTmp);
-            else
-              maskValue = false;
-
-            ++idxAxis;
-          }
-        }
-
-        int maskIndex = idxLoop[0];
-        int mulDim = 1;
-        for (int k = 1; k < dim; ++k)
-        {
-          mulDim *= eachDimSize[k-1];
-          maskIndex += idxLoop[k]*mulDim;
-        }
-        *(gridMask.dataFirst()+maskIndex) &= maskValue;
-
-        ++idxLoop[0];
-        ++idx;
-      }
-     }
+        mulDim *= eachDimSize[k-1];
+        maskIndex += idxLoop[k]*mulDim;
+      }
+      *(gridMask.dataFirst()+maskIndex) &= maskValue;
+
+      ++idxLoop[0];
+      ++idx;
+    }
    }
    CATCH_DUMP_ATTR
@@ -565 +577 @@
    ///--------------------------------------------------------------
 
-  /*!
-    A grid can have multiple dimension, so can its mask in the form of multi-dimension array.
-  It's not a good idea to store all multi-dimension arrays corresponding to each mask.
-  One of the ways is to convert this array into 1-dimension one and every process is taken place on it.
-    \param [in] multi-dimension array grid mask
-  */
-  template<int N>
-  void CGrid::getLocalMask(const CArray<bool,N>& gridMask, CArray<bool,1>& localMask)
-  TRY
-  {
-    if (gridMask.isEmpty()) return ;
-    int dim = gridMask.dimensions();
-    std::vector<int> dimensionSizes(dim);
-    for (int i = 0; i < dim; ++i) dimensionSizes[i] = gridMask.extent(i);
-
-    std::vector<int> idxLoop(dim,0);
-    int ssize = gridMask.numElements(), idx = 0;
-    localMask.resize(ssize);
-    while (idx < ssize)
-    {
-      for (int i = 0; i < dim-1; ++i)
-      {
-        if (idxLoop[i] == dimensionSizes[i])
-        {
-          idxLoop[i] = 0;
-          ++idxLoop[i+1];
-        }
-      }
-
-      int maskIndex = idxLoop[0];
-      int mulDim = 1;
-      for (int k = 1; k < dim; ++k)
-      {
-        mulDim *= dimensionSizes[k-1];
-        maskIndex += idxLoop[k]*mulDim;
-      }
-      localMask(maskIndex) = *(gridMask.dataFirst()+maskIndex);
-
-      ++idxLoop[0];
-      ++idx;
-    }
-  }
-   CATCH_DUMP_ATTR
+
 
    // Declare/Define CGridGroup and CGridDefinition