Changeset 1235 for XIOS

Timestamp:

08/03/17 15:50:40 (7 years ago)

Author:

mhnguyen

Message:

Fixing a bug on writting axis label

+) Axis label is correctly processed before being written
+) Do some code cleaning and add some comments

Test
+) On Curie
+) OK

Location:

XIOS/dev/XIOS_DEV_CMIP6/src

Files:

• io/nc4_data_output.cpp (modified) (7 diffs)
• node/axis.cpp (modified) (26 diffs)
• node/axis.hpp (modified) (7 diffs)
• node/field.cpp (modified) (2 diffs)
• node/grid.cpp (modified) (7 diffs)
• node/grid.hpp (modified) (1 diff)
• server_distribution_description.cpp (modified) (1 diff)
• server_distribution_description.hpp (modified) (1 diff)

XIOS/dev/XIOS_DEV_CMIP6/src/io/nc4_data_output.cpp

@@ -1136 +1136 @@
         int zoom_size  = (MULTI_FILE == SuperClass::type) ? axis->zoom_n
                                                           : axis->global_zoom_n;
-        int zoom_begin = (MULTI_FILE == SuperClass::type) ? axis->zoom_begin
-                                                          : axis->global_zoom_begin;
+
+        int zoom_count = axis->zoom_n;                                                 
+        int zoom_begin = axis->zoom_begin;
+        // int zoom_begin = (MULTI_FILE == SuperClass::type) ? axis->global_zoom_begin 
+        //                                                   : axis->zoom_begin;
 
         if ((0 == axis->zoom_n) && (MULTI_FILE == SuperClass::type)) return;
@@ -1148 +1151 @@
         nc_type typePrec ;
         if (axis->prec.isEmpty()) typePrec =  NC_FLOAT ;
-        else if (axis->prec==4)  typePrec =  NC_FLOAT ;
+        else if (axis->prec==4)   typePrec =  NC_FLOAT ;
         else if (axis->prec==8)   typePrec =  NC_DOUBLE ;
          
@@ -1198 +1201 @@
             for (int i = 0; i < nbWritten; i++) axis_value(i) = axis->value(indexToWrite(i));
             CArray<double,2> axis_bounds;
+            CArray<string,1> axis_label;
+            if (!axis->label.isEmpty())
+            {
+              axis_label.resize(indexToWrite.numElements());
+              for (int i = 0; i < nbWritten; i++) axis_label(i) = axis->label(indexToWrite(i));
+            }
 
             switch (SuperClass::type)
@@ -1203 +1212 @@
               case MULTI_FILE:
               {
-                SuperClassWriter::writeData(axis_value, axisid, isCollective, 0);
+                if (axis->label.isEmpty())
+                  SuperClassWriter::writeData(axis_value, axisid, isCollective, 0);
 
                 if (!axis->bounds.isEmpty() && axis->label.isEmpty())
@@ -1209 +1219 @@
 
                 // Need to check after
-                if (! axis->label.isEmpty())  SuperClassWriter::writeData(axis->label_srv, axisid, isCollective, 0);
+                if (!axis->label.isEmpty()) 
+                  SuperClassWriter::writeData(axis_label, axisid, isCollective, 0);
  
                 SuperClassWriter::definition_start();
@@ -1219 +1230 @@
                 std::vector<StdSize> count(1), countBounds(2) ;
                 start[0] = startBounds[0] = zoom_begin - axis->global_zoom_begin;
-                count[0] = countBounds[0] = zoom_size;
+                count[0] = countBounds[0] = zoom_count; // zoom_size
                 startBounds[1] = 0;
                 countBounds[1] = 2;
-                SuperClassWriter::writeData(axis_value, axisid, isCollective, 0, &start, &count);
-
-                if (!axis->bounds.isEmpty()&& axis->label.isEmpty())
+
+                if (axis->label.isEmpty())
+                  SuperClassWriter::writeData(axis_value, axisid, isCollective, 0, &start, &count);
+
+                if (!axis->bounds.isEmpty() && axis->label.isEmpty())
                 {
                   axis_bounds.resize(2, indexToWrite.numElements());
@@ -1236 +1249 @@
 
                 // Need to check after
-                if (! axis->label.isEmpty())  SuperClassWriter::writeData(axis->label_srv, axisid, isCollective, 0);
+                if (!axis->label.isEmpty())
+                {
+                  std::vector<StdSize> startLabel(2), countLabel(2);
+                  startLabel[0] = start[0]; startLabel[1] = 0;
+                  countLabel[0] = count[0]; countLabel[1] = stringArrayLen;
+                  SuperClassWriter::writeData(axis_label, axisid, isCollective, 0, &startLabel, &countLabel);
+                }
 
                 SuperClassWriter::definition_start();

XIOS/dev/XIOS_DEV_CMIP6/src/node/axis.cpp

@@ -10 +10 @@
 #include "context_server.hpp"
 #include "xios_spl.hpp"
-#include "inverse_axis.hpp"
-#include "zoom_axis.hpp"
-#include "interpolate_axis.hpp"
 #include "server_distribution_description.hpp"
 #include "client_server_mapping_distributed.hpp"
@@ -25 +22 @@
       , CAxisAttributes(), isChecked(false), relFiles(), areClientAttributesChecked_(false)
       , isClientAfterTransformationChecked(false)
-      , hasBounds_(false), isCompressible_(false)
+      , hasBounds(false), isCompressible_(false)
       , numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
       , transformationMap_(), hasValue(false), hasLabel(false)
@@ -36 +33 @@
       , CAxisAttributes(), isChecked(false), relFiles(), areClientAttributesChecked_(false)
       , isClientAfterTransformationChecked(false)
-      , hasBounds_(false), isCompressible_(false)
+      , hasBounds(false), isCompressible_(false)
       , numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
       , transformationMap_(), hasValue(false), hasLabel(false)
@@ -179 +176 @@
 
          size_t sizeValEvent = CArray<double,1>::size(it->second.size());
-         if (hasBounds_)
+         if (hasBounds)
            sizeValEvent += CArray<double,2>::size(2 * it->second.size());
  
@@ -353 +350 @@
                << "Axis size is " << n.getValue() << "." << std::endl
                << "Bounds size is "<< bounds.extent(0) << " x " << bounds.extent(1) << ".");
-       hasBounds_ = true;
+       hasBounds = true;
      }
-     else hasBounds_ = false;
+     else hasBounds = false;
    }
 
@@ -372 +369 @@
   }
 
+  /*!
+    Check whether we can do compressed output
+  */
   void CAxis::checkEligibilityForCompressedOutput()
   {
@@ -378 +378 @@
   }
 
+  /*
+    Check whether we do zooming by indexing
+    return true if do zooming by index
+  */
   bool CAxis::zoomByIndex()
   {
@@ -383 +387 @@
   }
 
-   bool CAxis::dispatchEvent(CEventServer& event)
-   {
-      if (SuperClass::dispatchEvent(event)) return true;
-      else
-      {
-        switch(event.type)
-        {
-           case EVENT_ID_DISTRIBUTION_ATTRIBUTE :
-             recvDistributionAttribute(event);
-             return true;
-             break;
-          case EVENT_ID_NON_DISTRIBUTED_ATTRIBUTES:
-            recvNonDistributedAttributes(event);
+  /*!
+    Dispatch event from the lower communication layer then process event according to its type
+  */
+  bool CAxis::dispatchEvent(CEventServer& event)
+  {
+     if (SuperClass::dispatchEvent(event)) return true;
+     else
+     {
+       switch(event.type)
+       {
+          case EVENT_ID_DISTRIBUTION_ATTRIBUTE :
+            recvDistributionAttribute(event);
             return true;
             break;
-          case EVENT_ID_DISTRIBUTED_ATTRIBUTES:
-            recvDistributedAttributes(event);
-            return true;
-            break;
-           default :
-             ERROR("bool CAxis::dispatchEvent(CEventServer& event)",
-                    << "Unknown Event");
-           return false;
-         }
-      }
-   }
+         case EVENT_ID_NON_DISTRIBUTED_ATTRIBUTES:
+           recvNonDistributedAttributes(event);
+           return true;
+           break;
+         case EVENT_ID_DISTRIBUTED_ATTRIBUTES:
+           recvDistributedAttributes(event);
+           return true;
+           break;
+          default :
+            ERROR("bool CAxis::dispatchEvent(CEventServer& event)",
+                   << "Unknown Event");
+          return false;
+        }
+     }
+  }
 
    /*!
@@ -423 +430 @@
 
    /*
-     The (spatial) transformation sometimes can change attributes of an axis. Therefore, we should recheck them.
+     The (spatial) transformation sometimes can change attributes of an axis (e.g zoom can change mask or generate can change whole attributes)
+     Therefore, we should recheck them.
    */
    void CAxis::checkAttributesOnClientAfterTransformation(const std::vector<int>& globalDim, int orderPositionInGrid,
@@ -432 +440 @@
      if (this->isClientAfterTransformationChecked) return;
      if (context->hasClient)
-     {
-       if (index.numElements() != n_glo.getValue()) computeConnectedServer(globalDim, orderPositionInGrid, distType);
+     {        
+       if ((orderPositionInGrid == CServerDistributionDescription::defaultDistributedDimension(globalDim.size(), distType))
+         || (index.numElements() != n_glo))
+          computeConnectedClients(globalDim, orderPositionInGrid, distType);
      }
 
@@ -439 +449 @@
    }
 
-   // Send all checked attributes to server
+   /*
+     Send all checked attributes to server? (We dont have notion of server any more so client==server)
+     \param [in] globalDim global dimension of grid containing this axis
+     \param [in] orderPositionInGrid the relative order of this axis in the grid (e.g grid composed of domain+axis -> orderPositionInGrid is 2)
+     \param [in] distType distribution type of the server. For now, we only have band distribution.
+
+   */
    void CAxis::sendCheckedAttributes(const std::vector<int>& globalDim, int orderPositionInGrid,
                                      CServerDistributionDescription::ServerDistributionType distType)
@@ -461 +477 @@
                              CServerDistributionDescription::ServerDistributionType distType)
   {
-     if (index.numElements() == n_glo.getValue())
-       sendNonDistributedAttributes();
-     else
+     sendDistributionAttribute(globalDim, orderPositionInGrid, distType);
+
+     // if (index.numElements() == n_glo.getValue())
+     if ((orderPositionInGrid == CServerDistributionDescription::defaultDistributedDimension(globalDim.size(), distType))
+         || (index.numElements() != n_glo))
      {
        sendDistributedAttributes();       
      }
-     sendDistributionAttribute(globalDim, orderPositionInGrid, distType);
-  }
-
-  void CAxis::computeConnectedServer(const std::vector<int>& globalDim, int orderPositionInGrid,
+     else
+     {
+       sendNonDistributedAttributes();    
+     }     
+  }
+
+  /*
+    Compute the connection between group of clients (or clients/servers).
+    (E.g: Suppose we have 2 group of clients in two model: A (client role) connect to B (server role),
+    this function calculate number of clients B connect to one client of A)
+     \param [in] globalDim global dimension of grid containing this axis
+     \param [in] orderPositionInGrid the relative order of this axis in the grid (e.g grid composed of domain+axis -> orderPositionInGrid is 2)
+     \param [in] distType distribution type of the server. For now, we only have band distribution.
+  */
+  void CAxis::computeConnectedClients(const std::vector<int>& globalDim, int orderPositionInGrid,
                                      CServerDistributionDescription::ServerDistributionType distType)
   {
@@ -489 +518 @@
       size_t nbIndex = index.numElements();
 
-      for (size_t idx = 0; idx < nbIndex; ++idx)
-      {
-        globalLocalIndexMap_[index(idx)] = idx;
-      }
+      // First of all, we should compute the mapping of the global index and local index of the current client
+      if (globalLocalIndexMap_.empty())
+      {
+        for (size_t idx = 0; idx < nbIndex; ++idx)
+        {
+          globalLocalIndexMap_[index(idx)] = idx;
+        }
+      }
+
+      // Calculate the compressed index if any
       std::set<int> writtenInd;
       if (isCompressible_)
@@ -509 +544 @@
       }
 
-      CServerDistributionDescription serverDescriptionGlobal(globalDim, nbServer, distType);
-      int distributedDimensionOnServer = serverDescriptionGlobal.getDimensionDistributed();
-      CClientServerMapping::GlobalIndexMap globalIndexAxisOnServer;
-      if (distributedDimensionOnServer == orderPositionInGrid) // So we have distributed axis on client side and also on server side*
-      {
-        std::vector<int> nGlobAxis(1);
-        nGlobAxis[0] = n_glo.getValue();
-
-        size_t globalSizeIndex = 1, indexBegin, indexEnd;
-        for (int i = 0; i < nGlobAxis.size(); ++i) globalSizeIndex *= nGlobAxis[i];
-        indexBegin = 0;
-        if (globalSizeIndex <= clientSize)
-        {
-          indexBegin = rank%globalSizeIndex;
-          indexEnd = indexBegin;
-        }
-        else
-        {
-          for (int i = 0; i < clientSize; ++i)
-          {
-            range = globalSizeIndex / clientSize;
-            if (i < (globalSizeIndex%clientSize)) ++range;
-            if (i == client->clientRank) break;
-            indexBegin += range;
-          }
-          indexEnd = indexBegin + range - 1;
-        }
-
-        CArray<size_t,1> globalIndex(index.numElements());
-        for (size_t idx = 0; idx < globalIndex.numElements(); ++idx)
-          globalIndex(idx) = index(idx);
-
-        CServerDistributionDescription serverDescription(nGlobAxis, nbServer);
-        serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd));
-        CClientServerMapping* clientServerMap = new CClientServerMappingDistributed(serverDescription.getGlobalIndexRange(), client->intraComm);
-        clientServerMap->computeServerIndexMapping(globalIndex);
-        globalIndexAxisOnServer = clientServerMap->getGlobalIndexOnServer();
-        delete clientServerMap;
+      // Compute the global index of the current client (process) hold
+      std::vector<int> nGlobAxis(1);
+      nGlobAxis[0] = n_glo.getValue();
+
+      size_t globalSizeIndex = 1, indexBegin, indexEnd;
+      for (int i = 0; i < nGlobAxis.size(); ++i) globalSizeIndex *= nGlobAxis[i];
+      indexBegin = 0;
+      if (globalSizeIndex <= clientSize)
+      {
+        indexBegin = rank%globalSizeIndex;
+        indexEnd = indexBegin;
       }
       else
       {
-        std::vector<size_t> globalIndexServer(n_glo.getValue());
-        for (size_t idx = 0; idx < n_glo.getValue(); ++idx)
-        {
-          globalIndexServer[idx] = idx;
-        }
-
-        for (int idx = 0; idx < nbServer; ++idx)
-        {
-          globalIndexAxisOnServer[idx] = globalIndexServer;
-        }
-      }
+        for (int i = 0; i < clientSize; ++i)
+        {
+          range = globalSizeIndex / clientSize;
+          if (i < (globalSizeIndex%clientSize)) ++range;
+          if (i == client->clientRank) break;
+          indexBegin += range;
+        }
+        indexEnd = indexBegin + range - 1;
+      }
+
+      CArray<size_t,1> globalIndex(index.numElements());
+      for (size_t idx = 0; idx < globalIndex.numElements(); ++idx)
+        globalIndex(idx) = index(idx);
+
+      // Describe the distribution of server side
+      CServerDistributionDescription serverDescription(nGlobAxis, nbServer);
+      std::vector<int> serverZeroIndex;
+      serverZeroIndex = serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 0);      
+
+      std::list<int> serverZeroIndexLeader;
+      std::list<int> serverZeroIndexNotLeader; 
+      CContextClient::computeLeader(client->clientRank, client->clientSize, serverZeroIndex.size(), serverZeroIndexLeader, serverZeroIndexNotLeader);
+      for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+        *it = serverZeroIndex[*it];
+
+      // Find out the connection between client and server side
+      CClientServerMapping* clientServerMap = new CClientServerMappingDistributed(serverDescription.getGlobalIndexRange(), client->intraComm);
+      clientServerMap->computeServerIndexMapping(globalIndex);
+      CClientServerMapping::GlobalIndexMap& globalIndexAxisOnServer = clientServerMap->getGlobalIndexOnServer();      
+
 
       indSrv_.swap(globalIndexAxisOnServer);
-
       CClientServerMapping::GlobalIndexMap::const_iterator it  = indSrv_.begin(),
                                                            ite = indSrv_.end();
-
       connectedServerRank_.clear();
       for (it = indSrv_.begin(); it != ite; ++it) {
@@ -572 +597 @@
       }
 
+      for (std::list<int>::const_iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+        connectedServerRank_.push_back(*it);
+
+       // Even if a client has no index, it must connect to at least one server and 
+       // send an "empty" data to this server
+       if (connectedServerRank_.empty())
+        connectedServerRank_.push_back(client->clientRank % client->serverSize);
+
       nbConnectedClients_ = CClientServerMapping::computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_);
-    }
-  }
-
-   void CAxis::computeWrittenIndex()
-   {  
-      if (computedWrittenIndex_) return;
-      computedWrittenIndex_ = true;
-
-      CContext* context=CContext::getCurrent();      
-      CContextServer* server = context->server; 
-
-      std::vector<int> nBegin(1), nSize(1), nBeginGlobal(1), nGlob(1);
-      nBegin[0]       = zoom_begin;
-      nSize[0]        = zoom_n;   
-      nBeginGlobal[0] = 0; 
-      nGlob[0]        = n_glo;
-      CDistributionServer srvDist(server->intraCommSize, nBegin, nSize, nBeginGlobal, nGlob); 
-      const CArray<size_t,1>& writtenGlobalIndex  = srvDist.getGlobalIndex();
-
-      size_t nbWritten = 0, indGlo;      
+
+      delete clientServerMap;
+    }
+  }
+
+  /*
+    Compute the index of data to write into file
+    (Different from the previous version, this version of XIOS allows data be written into file (classical role),
+    or transfered to another clients)
+  */
+  void CAxis::computeWrittenIndex()
+  {  
+    if (computedWrittenIndex_) return;
+    computedWrittenIndex_ = true;
+
+    CContext* context=CContext::getCurrent();      
+    CContextServer* server = context->server; 
+
+    // We describe the distribution of client (server) on which data are written
+    std::vector<int> nBegin(1), nSize(1), nBeginGlobal(1), nGlob(1);
+    nBegin[0]       = zoom_begin;
+    nSize[0]        = zoom_n;   
+    nBeginGlobal[0] = 0; 
+    nGlob[0]        = n_glo;
+    CDistributionServer srvDist(server->intraCommSize, nBegin, nSize, nBeginGlobal, nGlob); 
+    const CArray<size_t,1>& writtenGlobalIndex  = srvDist.getGlobalIndex();
+
+    // Because all written data are local on a client, 
+    // we need to compute the local index on the server from its corresponding global index
+    size_t nbWritten = 0, indGlo;      
+    boost::unordered_map<size_t,size_t>::const_iterator itb = globalLocalIndexMap_.begin(),
+                                                        ite = globalLocalIndexMap_.end(), it;          
+    CArray<size_t,1>::const_iterator itSrvb = writtenGlobalIndex.begin(),
+                                     itSrve = writtenGlobalIndex.end(), itSrv;  
+    if (!zoomByIndex())
+    {   
+      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+      {
+        indGlo = *itSrv;
+        if (ite != globalLocalIndexMap_.find(indGlo))
+        {          
+          ++nbWritten;
+        }                 
+      }
+
+      localIndexToWriteOnServer.resize(nbWritten);
+
+      nbWritten = 0;
+      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+      {
+        indGlo = *itSrv;
+        if (ite != globalLocalIndexMap_.find(indGlo))
+        {
+          localIndexToWriteOnServer(nbWritten) = globalLocalIndexMap_[indGlo];
+          ++nbWritten;
+        }                 
+      }
+    }
+    else
+    {
+      nbWritten = 0;
       boost::unordered_map<size_t,size_t>::const_iterator itb = globalLocalIndexMap_.begin(),
-                                                          ite = globalLocalIndexMap_.end(), it;          
-      CArray<size_t,1>::const_iterator itSrvb = writtenGlobalIndex.begin(),
-                                       itSrve = writtenGlobalIndex.end(), itSrv;  
-      if (!zoomByIndex())
-      {   
-        for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
-        {
-          indGlo = *itSrv;
-          if (ite != globalLocalIndexMap_.find(indGlo))
-          {          
-            ++nbWritten;
-          }                 
-        }
-
-        localIndexToWriteOnServer.resize(nbWritten);
-
-        nbWritten = 0;
-        for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
-        {
-          indGlo = *itSrv;
-          if (ite != globalLocalIndexMap_.find(indGlo))
-          {
-            localIndexToWriteOnServer(nbWritten) = globalLocalIndexMap_[indGlo];
-            ++nbWritten;
-          }                 
-        }
+                                                          ite = globalLocalIndexMap_.end(), it;
+      for (int i = 0; i < zoom_index.numElements(); ++i)
+      {
+         if (ite != globalLocalIndexMap_.find(zoom_index(i)))
+          ++nbWritten;
+      }
+
+      localIndexToWriteOnServer.resize(nbWritten);
+
+      nbWritten = 0;
+      for (int i = 0; i < zoom_index.numElements(); ++i)
+      {
+         if (ite != globalLocalIndexMap_.find(zoom_index(i)))
+         {
+           localIndexToWriteOnServer(nbWritten) = globalLocalIndexMap_[zoom_index(i)];
+           ++nbWritten;
+         }
+      }
+    }
+
+    if (isCompressible())
+    {
+      nbWritten = 0;
+      boost::unordered_map<size_t,size_t> localGlobalIndexMap;
+      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+      {
+        indGlo = *itSrv;
+        if (ite != globalLocalIndexMap_.find(indGlo))
+        {
+          localGlobalIndexMap[localIndexToWriteOnServer(nbWritten)] = indGlo;
+          ++nbWritten;
+        }                 
+      }
+
+      nbWritten = 0;
+      for (int idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
+        {
+          ++nbWritten;
+        }
+      }
+
+      compressedIndexToWriteOnServer.resize(nbWritten);
+      nbWritten = 0;
+      for (int idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
+        {
+          compressedIndexToWriteOnServer(nbWritten) = localGlobalIndexMap[data_index(idx)];
+          ++nbWritten;
+        }
+      }
+
+      numberWrittenIndexes_ = nbWritten;
+      if (isDistributed())
+      {
+             
+        MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+        MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+        offsetWrittenIndexes_ -= numberWrittenIndexes_;
       }
       else
-      {
-        nbWritten = 0;
-        boost::unordered_map<size_t,size_t>::const_iterator itb = globalLocalIndexMap_.begin(),
-                                                            ite = globalLocalIndexMap_.end(), it;
-        for (int i = 0; i < zoom_index.numElements(); ++i)
-        {
-           if (ite != globalLocalIndexMap_.find(zoom_index(i)))
-            ++nbWritten;
-        }
-
-        localIndexToWriteOnServer.resize(nbWritten);
-
-        nbWritten = 0;
-        for (int i = 0; i < zoom_index.numElements(); ++i)
-        {
-           if (ite != globalLocalIndexMap_.find(zoom_index(i)))
-           {
-             localIndexToWriteOnServer(nbWritten) = globalLocalIndexMap_[zoom_index(i)];
-             ++nbWritten;
-           }
-        }
-      }
-
-      if (isCompressible())
-      {
-        nbWritten = 0;
-        boost::unordered_map<size_t,size_t> localGlobalIndexMap;
-        for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
-        {
-          indGlo = *itSrv;
-          if (ite != globalLocalIndexMap_.find(indGlo))
-          {
-            localGlobalIndexMap[localIndexToWriteOnServer(nbWritten)] = indGlo;
-            ++nbWritten;
-          }                 
-        }
-
-        nbWritten = 0;
-        for (int idx = 0; idx < data_index.numElements(); ++idx)
-        {
-          if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
-          {
-            ++nbWritten;
-          }
-        }
-
-        compressedIndexToWriteOnServer.resize(nbWritten);
-        nbWritten = 0;
-        for (int idx = 0; idx < data_index.numElements(); ++idx)
-        {
-          if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
-          {
-            compressedIndexToWriteOnServer(nbWritten) = localGlobalIndexMap[data_index(idx)];
-            ++nbWritten;
-          }
-        }
-
-        numberWrittenIndexes_ = nbWritten;
-        if (isDistributed())
-        {
-               
-          MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
-          MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
-          offsetWrittenIndexes_ -= numberWrittenIndexes_;
-        }
-        else
-          totalNumberWrittenIndexes_ = numberWrittenIndexes_;
-      }
-
-   }
-
-
-
+        totalNumberWrittenIndexes_ = numberWrittenIndexes_;
+    }
+  }
+
+
+  /*!
+    Send distribution information from a group of client (client role) to another group of client (server role)
+    The distribution of a group of client (server role) is imposed by the group of client (client role)
+    \param [in] globalDim global dimension of grid containing this axis
+    \param [in] orderPositionInGrid the relative order of this axis in the grid (e.g grid composed of domain+axis -> orderPositionInGrid is 2)
+    \param [in] distType distribution type of the server. For now, we only have band distribution.
+  */
   void CAxis::sendDistributionAttribute(const std::vector<int>& globalDim, int orderPositionInGrid,
                                         CServerDistributionDescription::ServerDistributionType distType)
@@ -712 +760 @@
       std::vector<std::vector<int> > serverIndexBegin = serverDescription.getServerIndexBegin();
       std::vector<std::vector<int> > serverDimensionSizes = serverDescription.getServerDimensionSizes();
-
-      globalDimGrid.resize(globalDim.size());
-      for (int idx = 0; idx < globalDim.size(); ++idx) globalDimGrid(idx) = globalDim[idx];
 
       CEventClient event(getType(),EVENT_ID_DISTRIBUTION_ATTRIBUTE);
@@ -732 +777 @@
           CMessage& msg = msgs.back();
           msg << this->getId();
-          msg << ni << begin << end;
-          // msg << global_zoom_begin.getValue() << global_zoom_n.getValue();
-          msg << isCompressible_;
-          msg << orderPositionInGrid;
-          msg << globalDimGrid;
+          msg << ni << begin << end;          
+          msg << isCompressible_;                    
 
           event.push(*itRank,1,msg);
@@ -746 +788 @@
   }
 
-  void CAxis::sendNonDistributedAttributes()
-  {
-    CContext* context = CContext::getCurrent();
-
-    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
-    for (int p = 0; p < nbSrvPools; ++p)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
-
-      CEventClient event(getType(), EVENT_ID_NON_DISTRIBUTED_ATTRIBUTES);
-      size_t nbIndex = index.numElements();
-      size_t nbDataIndex = 0;
-
-      for (int idx = 0; idx < data_index.numElements(); ++idx)
-      {
-        int ind = data_index(idx);
-        if (ind >= 0 && ind < nbIndex) ++nbDataIndex;
-      }
-
-      CArray<int,1> dataIndex(nbDataIndex);
-      nbDataIndex = 0;
-      for (int idx = 0; idx < data_index.numElements(); ++idx)
-      {
-        int ind = data_index(idx);
-        if (ind >= 0 && ind < nbIndex)
-        {
-          dataIndex(nbDataIndex) = ind;
-          ++nbDataIndex;
-        }
-      }
-
-      if (client->isServerLeader())
-      {
-        std::list<CMessage> msgs;
-
-        const std::list<int>& ranks = client->getRanksServerLeader();
-        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-        {
-          msgs.push_back(CMessage());
-          CMessage& msg = msgs.back();
-          msg << this->getId();
-          msg << index.getValue() << dataIndex << mask.getValue();
-          msg << hasValue;
-          if (hasValue) msg << value.getValue();
-          msg << hasBounds_;
-          if (hasBounds_) msg << bounds.getValue();
-
-          event.push(*itRank, 1, msg);
-        }
-        client->sendEvent(event);
-      }
-      else client->sendEvent(event);
-    }
-  }
-
-  void CAxis::recvNonDistributedAttributes(CEventServer& event)
-  {
-    list<CEventServer::SSubEvent>::iterator it;
-    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
-    {
-      CBufferIn* buffer = it->buffer;
-      string axisId;
-      *buffer >> axisId;
-      get(axisId)->recvNonDistributedAttributes(it->rank, *buffer);
-    }
-  }
-
-  void CAxis::recvNonDistributedAttributes(int rank, CBufferIn& buffer)
-  { 
-    CArray<int,1> tmp_index, tmp_data_index, tmp_zoom_index;
-    CArray<bool,1> tmp_mask;
-    CArray<double,1> tmp_val;
-    CArray<double,2> tmp_bnds;
-
-    buffer >> tmp_index;
-    index.reference(tmp_index);
-    buffer >> tmp_data_index;
-    data_index.reference(tmp_data_index);
-    buffer >> tmp_mask;
-    mask.reference(tmp_mask);
-
-    buffer >> hasValue;
-    if (hasValue)
-    {
-      buffer >> tmp_val;
-      value.reference(tmp_val);
-    }
-
-    buffer >> hasBounds_;
-    if (hasBounds_)
-    {
-      buffer >> tmp_bnds;
-      bounds.reference(tmp_bnds);
-    }
-
-    data_begin.setValue(0);
-    globalLocalIndexMap_.rehash(std::ceil(index.numElements()/globalLocalIndexMap_.max_load_factor()));
-    for (int idx = 0; idx < index.numElements(); ++idx) globalLocalIndexMap_[idx] = index(idx);
-  }
-
-  void CAxis::sendDistributedAttributes(void)
-  {
-    int ns, n, i, j, ind, nv, idx;
-    CContext* context = CContext::getCurrent();
-    
-    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
-    for (int p = 0; p < nbSrvPools; ++p)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
-
-      CEventClient eventData(getType(), EVENT_ID_DISTRIBUTED_ATTRIBUTES);
-
-      list<CMessage> listData;
-      list<CArray<int,1> > list_indi, list_dataInd, list_zoomInd;
-      list<CArray<bool,1> > list_mask;
-      list<CArray<double,1> > list_val;
-      list<CArray<double,2> > list_bounds;
-
-      int nbIndex = index.numElements();
-      CArray<int,1> dataIndex(nbIndex);
-      dataIndex = -1;
-      for (int inx = 0; inx < data_index.numElements(); ++inx)
-      {
-        if (0 <= data_index(inx) && data_index(inx) < nbIndex)
-          dataIndex(inx) = data_index(inx);
-      }
-
-      boost::unordered_map<int, std::vector<size_t> >::const_iterator it, iteMap;
-      iteMap = indSrv_.end();
-      for (int k = 0; k < connectedServerRank_.size(); ++k)
-      {
-        int nbData = 0;
-        int rank = connectedServerRank_[k];
-        int nbSendingClient = nbConnectedClients_[rank];
-        it = indSrv_.find(rank);
-        if (iteMap != it)
-          nbData = it->second.size();
-
-        list_indi.push_back(CArray<int,1>(nbData));
-        list_dataInd.push_back(CArray<int,1>(nbData));        
-        list_mask.push_back(CArray<bool,1>(nbData));
-
-        if (hasValue)
-          list_val.push_back(CArray<double,1>(nbData));
-
-        if (hasBounds_)
-        {
-          list_bounds.push_back(CArray<double,2>(2,nbData));
-        }
-
-        CArray<int,1>& indi = list_indi.back();
-        CArray<int,1>& dataIndi = list_dataInd.back();        
-        CArray<bool,1>& maskIndi = list_mask.back();
-
-        for (n = 0; n < nbData; ++n)
-        {
-          idx = static_cast<int>(it->second[n]);
-          indi(n) = idx;
-
-          ind = globalLocalIndexMap_[idx];
-          dataIndi(n) = dataIndex(ind);
-          maskIndi(n) = mask(ind);
-
-          if (hasValue)
-          {
-            CArray<double,1>& val = list_val.back();
-            val(n) = value(ind);
-          }
-
-          if (hasBounds_)
-          {
-            CArray<double,2>& boundsVal = list_bounds.back();
-            boundsVal(0, n) = bounds(0,n);
-            boundsVal(1, n) = bounds(1,n);
-          }
-        }
-
-        listData.push_back(CMessage());
-        listData.back() << this->getId()
-                        << list_indi.back() << list_dataInd.back() << list_mask.back();
-
-        listData.back() << hasValue;
-        if (hasValue)
-          listData.back() << list_val.back();
-
-        listData.back() << hasBounds_;
-        if (hasBounds_)
-          listData.back() << list_bounds.back();
-
-        eventData.push(rank, nbConnectedClients_[rank], listData.back());
-      }
-
-      client->sendEvent(eventData);
-    }
-  }
-
-  void CAxis::recvDistributedAttributes(CEventServer& event)
-  {
-    string axisId;
-    vector<int> ranks;
-    vector<CBufferIn*> buffers;
-
-    list<CEventServer::SSubEvent>::iterator it;
-    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
-    {
-      ranks.push_back(it->rank);
-      CBufferIn* buffer = it->buffer;
-      *buffer >> axisId;
-      buffers.push_back(buffer);
-    }
-    get(axisId)->recvDistributedAttributes(ranks, buffers);
-  }
-
-
-  void CAxis::recvDistributedAttributes(vector<int>& ranks, vector<CBufferIn*> buffers)
-  {
-    int nbReceived = ranks.size();
-    vector<CArray<int,1> > vec_indi(nbReceived), vec_dataInd(nbReceived), vec_zoomInd(nbReceived);    
-    vector<CArray<bool,1> > vec_mask(nbReceived);
-    vector<CArray<double,1> > vec_val(nbReceived);
-    vector<CArray<double,2> > vec_bounds(nbReceived);
-    
-    for (int idx = 0; idx < nbReceived; ++idx)
-    {      
-      CBufferIn& buffer = *buffers[idx];
-      buffer >> vec_indi[idx];
-      buffer >> vec_dataInd[idx];      
-      buffer >> vec_mask[idx];
-
-      buffer >> hasValue;
-      if (hasValue)
-        buffer >> vec_val[idx];
-
-      buffer >> hasBounds_;
-      if (hasBounds_)
-        buffer >> vec_bounds[idx];
-    }
-
-    int nbData = 0;
-    for (int idx = 0; idx < nbReceived; ++idx)
-    {
-      nbData += vec_indi[idx].numElements();
-    }
-
-    index.resize(nbData);
-    globalLocalIndexMap_.rehash(std::ceil(index.numElements()/globalLocalIndexMap_.max_load_factor()));
-    CArray<int,1> nonCompressedData(nbData);    
-    mask.resize(nbData);
-    if (hasValue)
-      value.resize(nbData);
-    if (hasBounds_)
-      bounds.resize(2,nbData);
-
-    nbData = 0;
-    for (int idx = 0; idx < nbReceived; ++idx)
-    {
-      CArray<int,1>& indi = vec_indi[idx];
-      CArray<int,1>& dataIndi = vec_dataInd[idx];
-      CArray<bool,1>& maskIndi = vec_mask[idx];
-      int nb = indi.numElements();
-      for (int n = 0; n < nb; ++n)
-      {
-        index(nbData) = indi(n);
-        globalLocalIndexMap_[indi(n)] = nbData;
-        nonCompressedData(nbData) = (0 <= dataIndi(n)) ? nbData : -1;
-        mask(nbData) = maskIndi(n);
-        if (hasValue)
-          value(nbData) = vec_val[idx](n);
-        if (hasBounds_)
-        {
-          bounds(0,nbData) = vec_bounds[idx](0,n);
-          bounds(1,nbData) = vec_bounds[idx](1,n);
-        }
-        ++nbData;
-      }
-    }
-
-    int nbIndex = index.numElements();
-    int nbCompressedData = 0; 
-    for (int idx = 0; idx < nonCompressedData.numElements(); ++idx)
-    {
-      if (0 <= nonCompressedData(idx) && nonCompressedData(idx) < nbIndex)
-        ++nbCompressedData;        
-    }
-
-    data_index.resize(nbCompressedData);
-    nbCompressedData = 0;
-    for (int idx = 0; idx < nonCompressedData.numElements(); ++idx)
-    {
-      if (0 <= nonCompressedData(idx) && nonCompressedData(idx) < nbIndex)
-      {
-        data_index(nbCompressedData) = nonCompressedData(idx);
-        ++nbCompressedData;        
-      }
-    }
-    data_begin.setValue(0);
-
-    if (hasLabel)
-    {
-      //label_srv(ind_srv) = labelVal( ind);
-    }
-  }
-
+  /*
+    Receive distribution attribute from another client
+    \param [in] event event containing data of these attributes
+  */
   void CAxis::recvDistributionAttribute(CEventServer& event)
   {
@@ -1057 +800 @@
   }
 
+  /*
+    Receive distribution attribute from another client
+    \param [in] buffer buffer containing data of these attributes
+  */
   void CAxis::recvDistributionAttribute(CBufferIn& buffer)
   {
@@ -1066 +813 @@
     std::vector<int>::iterator itZoomBegin, itZoomEnd, itZoom;
 
-    buffer >> ni_srv >> begin_srv >> end_srv;
-    // buffer >> global_zoom_begin_tmp >> global_zoom_n_tmp;   
-    buffer >> isCompressible_;
-    buffer >> orderPosInGrid;
-    buffer >> globalDimGrid;    
+    buffer >> ni_srv >> begin_srv >> end_srv;    
+    buffer >> isCompressible_;            
 
     // Set up new local size of axis on the receiving clients
@@ -1116 +860 @@
   }
 
+  /*
+    Send attributes of axis from a group of client to other group of clients/servers 
+    on supposing that these attributes are not distributed among the sending group
+    In the future, if new attributes are added, they should also be processed in this function
+  */
+  void CAxis::sendNonDistributedAttributes()
+  {
+    CContext* context = CContext::getCurrent();
+
+    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
+
+      CEventClient event(getType(), EVENT_ID_NON_DISTRIBUTED_ATTRIBUTES);
+      size_t nbIndex = index.numElements();
+      size_t nbDataIndex = 0;
+
+      for (int idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        int ind = data_index(idx);
+        if (ind >= 0 && ind < nbIndex) ++nbDataIndex;
+      }
+
+      CArray<int,1> dataIndex(nbDataIndex);
+      nbDataIndex = 0;
+      for (int idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        int ind = data_index(idx);
+        if (ind >= 0 && ind < nbIndex)
+        {
+          dataIndex(nbDataIndex) = ind;
+          ++nbDataIndex;
+        }
+      }
+
+      if (client->isServerLeader())
+      {
+        std::list<CMessage> msgs;
+
+        const std::list<int>& ranks = client->getRanksServerLeader();
+        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        {
+          msgs.push_back(CMessage());
+          CMessage& msg = msgs.back();
+          msg << this->getId();
+          msg << index.getValue() << dataIndex << mask.getValue();
+          msg << hasValue;
+          if (hasValue) msg << value.getValue();
+          msg << hasBounds;
+          if (hasBounds) msg << bounds.getValue();
+          msg << hasLabel;
+          if (hasLabel) msg << label.getValue();
+
+          event.push(*itRank, 1, msg);
+        }
+        client->sendEvent(event);
+      }
+      else client->sendEvent(event);
+    }
+  }
+
+  /*
+    Receive the non-distributed attributes from another group of clients
+    \param [in] event event containing data of these attributes
+  */
+  void CAxis::recvNonDistributedAttributes(CEventServer& event)
+  {
+    list<CEventServer::SSubEvent>::iterator it;
+    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+      CBufferIn* buffer = it->buffer;
+      string axisId;
+      *buffer >> axisId;
+      get(axisId)->recvNonDistributedAttributes(it->rank, *buffer);
+    }
+  }
+
+  /*
+    Receive the non-distributed attributes from another group of clients
+    \param [in] rank rank of the sender
+    \param [in] buffer buffer containing data sent from the sender
+  */
+  void CAxis::recvNonDistributedAttributes(int rank, CBufferIn& buffer)
+  { 
+    CArray<int,1> tmp_index, tmp_data_index, tmp_zoom_index;
+    CArray<bool,1> tmp_mask;
+    CArray<double,1> tmp_val;
+    CArray<double,2> tmp_bnds;
+    CArray<string,1> tmp_label;
+
+    buffer >> tmp_index;
+    index.reference(tmp_index);
+    buffer >> tmp_data_index;
+    data_index.reference(tmp_data_index);
+    buffer >> tmp_mask;
+    mask.reference(tmp_mask);
+
+    buffer >> hasValue;
+    if (hasValue)
+    {
+      buffer >> tmp_val;
+      value.reference(tmp_val);
+    }
+
+    buffer >> hasBounds;
+    if (hasBounds)
+    {
+      buffer >> tmp_bnds;
+      bounds.reference(tmp_bnds);
+    }
+
+    buffer >> hasLabel;
+    if (hasLabel)
+    {
+      buffer >> tmp_label;
+      label.reference(tmp_label);
+    }
+
+    // Some value should be reset here
+    data_begin.setValue(0);
+    globalLocalIndexMap_.rehash(std::ceil(index.numElements()/globalLocalIndexMap_.max_load_factor()));
+    for (int idx = 0; idx < index.numElements(); ++idx) globalLocalIndexMap_[idx] = index(idx);
+  }
+
+  /*
+    Send attributes of axis from a group of client to other group of clients/servers 
+    on supposing that these attributes are distributed among the clients of the sending group
+    In the future, if new attributes are added, they should also be processed in this function
+  */
+  void CAxis::sendDistributedAttributes(void)
+  {
+    int ns, n, i, j, ind, nv, idx;
+    CContext* context = CContext::getCurrent();
+    
+    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
+
+      CEventClient eventData(getType(), EVENT_ID_DISTRIBUTED_ATTRIBUTES);
+
+      list<CMessage> listData;
+      list<CArray<int,1> > list_indi, list_dataInd, list_zoomInd;
+      list<CArray<bool,1> > list_mask;
+      list<CArray<double,1> > list_val;
+      list<CArray<double,2> > list_bounds;
+      list<CArray<string,1> > list_label;
+
+      int nbIndex = index.numElements();
+      CArray<int,1> dataIndex(nbIndex);
+      dataIndex = -1;
+      for (idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        if (0 <= data_index(idx) && data_index(idx) < nbIndex)
+          dataIndex(idx) = 1;
+      }
+
+      boost::unordered_map<int, std::vector<size_t> >::const_iterator it, iteMap;
+      iteMap = indSrv_.end();
+      for (int k = 0; k < connectedServerRank_.size(); ++k)
+      {
+        int nbData = 0;
+        int rank = connectedServerRank_[k];        
+        it = indSrv_.find(rank);
+        if (iteMap != it)
+          nbData = it->second.size();
+
+        list_indi.push_back(CArray<int,1>(nbData));
+        list_dataInd.push_back(CArray<int,1>(nbData));        
+        list_mask.push_back(CArray<bool,1>(nbData));
+
+        if (hasValue)
+          list_val.push_back(CArray<double,1>(nbData));
+
+        if (hasBounds)        
+          list_bounds.push_back(CArray<double,2>(2,nbData));
+
+        if (hasLabel)
+          list_label.push_back(CArray<string,1>(nbData));
+
+        CArray<int,1>& indi = list_indi.back();
+        CArray<int,1>& dataIndi = list_dataInd.back();        
+        CArray<bool,1>& maskIndi = list_mask.back();
+
+        for (n = 0; n < nbData; ++n)
+        {
+          idx = static_cast<int>(it->second[n]);
+          indi(n) = idx;
+
+          ind = globalLocalIndexMap_[idx];
+          dataIndi(n) = dataIndex(ind);
+          maskIndi(n) = mask(ind);
+
+          if (hasValue)
+          {
+            CArray<double,1>& val = list_val.back();
+            val(n) = value(ind);
+          }
+
+          if (hasBounds)
+          {
+            CArray<double,2>& boundsVal = list_bounds.back();
+            boundsVal(0, n) = bounds(0,n);
+            boundsVal(1, n) = bounds(1,n);
+          }
+
+          if (hasLabel)
+          {
+            CArray<string,1>& labelVal = list_label.back();
+            labelVal(n) = label(ind); 
+          }
+        }
+
+        listData.push_back(CMessage());
+        listData.back() << this->getId()
+                        << list_indi.back() << list_dataInd.back() << list_mask.back();
+
+        listData.back() << hasValue;
+        if (hasValue)
+          listData.back() << list_val.back();
+
+        listData.back() << hasBounds;
+        if (hasBounds)
+          listData.back() << list_bounds.back();
+
+        listData.back() << hasLabel;
+        if (hasLabel)
+          listData.back() << list_label.back();
+
+        eventData.push(rank, nbConnectedClients_[rank], listData.back());
+      }
+
+      client->sendEvent(eventData);
+    }
+  }
+
+  /*
+    Receive the distributed attributes from another group of clients
+    \param [in] event event containing data of these attributes
+  */
+  void CAxis::recvDistributedAttributes(CEventServer& event)
+  {
+    string axisId;
+    vector<int> ranks;
+    vector<CBufferIn*> buffers;
+
+    list<CEventServer::SSubEvent>::iterator it;
+    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+      ranks.push_back(it->rank);
+      CBufferIn* buffer = it->buffer;
+      *buffer >> axisId;
+      buffers.push_back(buffer);
+    }
+    get(axisId)->recvDistributedAttributes(ranks, buffers);
+  }
+
+  /*
+    Receive the non-distributed attributes from another group of clients
+    \param [in] ranks rank of the sender
+    \param [in] buffers buffer containing data sent from the sender
+  */
+  void CAxis::recvDistributedAttributes(vector<int>& ranks, vector<CBufferIn*> buffers)
+  {
+    int nbReceived = ranks.size(), idx, ind, gloInd, locInd;
+    vector<CArray<int,1> > vec_indi(nbReceived), vec_dataInd(nbReceived), vec_zoomInd(nbReceived);    
+    vector<CArray<bool,1> > vec_mask(nbReceived);
+    vector<CArray<double,1> > vec_val(nbReceived);
+    vector<CArray<double,2> > vec_bounds(nbReceived);
+    vector<CArray<string,1> > vec_label(nbReceived);
+    
+    for (idx = 0; idx < nbReceived; ++idx)
+    {      
+      CBufferIn& buffer = *buffers[idx];
+      buffer >> vec_indi[idx];
+      buffer >> vec_dataInd[idx];      
+      buffer >> vec_mask[idx];
+
+      buffer >> hasValue;
+      if (hasValue)
+        buffer >> vec_val[idx];
+
+      buffer >> hasBounds;
+      if (hasBounds)
+        buffer >> vec_bounds[idx];
+
+      buffer >> hasLabel;
+      if (hasLabel)
+        buffer >> vec_label[idx]; 
+    }
+
+    // Estimate size of index array
+    int nbIndexGlob = 0;
+    for (idx = 0; idx < nbReceived; ++idx)
+    {
+      nbIndexGlob += vec_indi[idx].numElements();
+    }
+
+    // Recompute global index
+    // Take account of the overlapped index 
+    index.resize(nbIndexGlob);
+    globalLocalIndexMap_.rehash(std::ceil(index.numElements()/globalLocalIndexMap_.max_load_factor()));
+    nbIndexGlob = 0;
+    for (idx = 0; idx < nbReceived; ++idx)
+    {
+      CArray<int,1>& tmp = vec_indi[idx];
+      for (ind = 0; ind < tmp.numElements(); ++ind)
+      {
+         gloInd = tmp(ind);
+         if (0 == globalLocalIndexMap_.count(gloInd))
+         {
+           index(nbIndexGlob) = gloInd % n_glo;           
+           globalLocalIndexMap_[gloInd] = nbIndexGlob;  
+           ++nbIndexGlob;
+         } 
+      } 
+    }
+
+    // Resize index to its real size
+    index.resizeAndPreserve(nbIndexGlob);
+
+    int nbData = nbIndexGlob;
+    CArray<int,1> nonCompressedData(nbData);
+    nonCompressedData = -1;   
+    mask.resize(nbData);
+    if (hasValue)
+      value.resize(nbData);
+    if (hasBounds)
+      bounds.resize(2,nbData);
+    if (hasLabel)
+      label.resize(nbData);
+
+    nbData = 0;
+    for (idx = 0; idx < nbReceived; ++idx)
+    {
+      CArray<int,1>& indi = vec_indi[idx];
+      CArray<int,1>& dataIndi = vec_dataInd[idx];
+      CArray<bool,1>& maskIndi = vec_mask[idx];
+      int nb = indi.numElements();
+      for (int n = 0; n < nb; ++n)
+      { 
+        locInd = globalLocalIndexMap_[size_t(indi(n))];
+
+        nonCompressedData(locInd) = (-1 == nonCompressedData(locInd)) ? dataIndi(n) : nonCompressedData(locInd);
+
+        if (!mask(locInd)) // Only rewrite mask if it's not true
+          mask(locInd) = maskIndi(n);
+        
+        if (hasValue)
+          value(locInd) = vec_val[idx](n);
+
+        if (hasBounds)
+        {
+          bounds(0,locInd) = vec_bounds[idx](0,n);
+          bounds(1,locInd) = vec_bounds[idx](1,n);
+        }
+
+        if (hasLabel)
+          label(locInd) = vec_label[idx](n);
+      }
+    }
+    
+    int nbCompressedData = 0; 
+    for (idx = 0; idx < nonCompressedData.numElements(); ++idx)
+    {
+      if (0 <= nonCompressedData(idx))
+        ++nbCompressedData;        
+    }
+
+    data_index.resize(nbCompressedData);
+    nbCompressedData = 0;
+    for (idx = 0; idx < nonCompressedData.numElements(); ++idx)
+    {
+      if (0 <= nonCompressedData(idx))
+      {
+        data_index(nbCompressedData) = idx % n;
+        ++nbCompressedData;        
+      }
+    }
+
+    data_begin.setValue(0);
+  }
+
+
   /*!
     Compare two axis objects. 
@@ -1148 +1277 @@
   }
 
+  /*
+    Add transformation into axis. This function only servers for Fortran interface
+    \param [in] transType transformation type
+    \param [in] id identifier of the transformation object
+  */
   CTransformation<CAxis>* CAxis::addTransformation(ETranformationType transType, const StdString& id)
   {
@@ -1154 +1288 @@
   }
 
+  /*
+    Check whether an axis has (spatial) transformation
+  */
   bool CAxis::hasTransformation()
   {
@@ -1159 +1296 @@
   }
 
+  /*
+    Set transformation
+    \param [in] axisTrans transformation to set
+  */
   void CAxis::setTransformations(const TransMapTypes& axisTrans)
   {
@@ -1164 +1305 @@
   }
 
+  /*
+    Return all transformation held by the axis
+    \return transformation the axis has
+  */
   CAxis::TransMapTypes CAxis::getAllTransformations(void)
   {
@@ -1169 +1314 @@
   }
 
+  /*
+    Duplicate transformation of another axis
+    \param [in] src axis whose transformations are copied
+  */
   void CAxis::duplicateTransformation(CAxis* src)
   {

XIOS/dev/XIOS_DEV_CMIP6/src/node/axis.hpp

@@ -42 +42 @@
          typedef CAxisAttributes SuperClassAttribute;
          
-      public :
+      public:
          enum EEventId
          {
@@ -52 +52 @@
          } ;
 
-
-
-      public :
-
+      public:
          typedef CAxisAttributes RelAttributes;
          typedef CAxisGroup      RelGroup;
@@ -101 +98 @@
          static ENodeType GetType(void);
 
-         static bool dispatchEvent(CEventServer& event);
-         static void recvDistributionAttribute(CEventServer& event);
-         void recvDistributionAttribute(CBufferIn& buffer) ;
+         static bool dispatchEvent(CEventServer& event);         
+         
          void checkAttributesOnClient();
          void checkAttributesOnClientAfterTransformation(const std::vector<int>& globalDim, int orderPositionInGrid,
@@ -122 +118 @@
          bool zoomByIndex();
 
-      public:                
-        CArray<StdString,1> label_srv;
-        bool hasValue;
-        CArray<int,1> globalDimGrid;
-        int orderPosInGrid;
+      public: 
+        bool hasValue;        
         CArray<size_t,1> localIndexToWriteOnServer;
         CArray<int, 1> compressedIndexToWriteOnServer;
@@ -140 +133 @@
          void sendDistributionAttribute(const std::vector<int>& globalDim, int orderPositionInGrid,
                                         CServerDistributionDescription::ServerDistributionType distType);
-         void computeConnectedServer(const std::vector<int>& globalDim, int orderPositionInGrid,
+         void computeConnectedClients(const std::vector<int>& globalDim, int orderPositionInGrid,
                                      CServerDistributionDescription::ServerDistributionType distType);
 
@@ -148 +141 @@
          static void recvNonDistributedAttributes(CEventServer& event);
          static void recvDistributedAttributes(CEventServer& event);
+         static void recvDistributionAttribute(CEventServer& event);
          void recvNonDistributedAttributes(int rank, CBufferIn& buffer);
          void recvDistributedAttributes(vector<int>& rank, vector<CBufferIn*> buffers);
+         void recvDistributionAttribute(CBufferIn& buffer);
 
          void setTransformations(const TransMapTypes&);
@@ -167 +162 @@
          std::vector<int> indexesToWrite;
          int numberWrittenIndexes_, totalNumberWrittenIndexes_, offsetWrittenIndexes_;
-         std::vector<int> connectedServerRank_;
-         std::map<int, CArray<int,1> > indiSrv_;
-         bool hasBounds_;
+         std::vector<int> connectedServerRank_;         
+         bool hasBounds;
          bool hasLabel;         
-         bool computedWrittenIndex_;
+         bool computedWrittenIndex_;                  
 
        private:

XIOS/dev/XIOS_DEV_CMIP6/src/node/field.cpp

@@ -139 +139 @@
     list<CArray<double,1> > list_data;
 
-    if (!grid->doGridHaveDataDistributed())
+    if (!grid->doGridHaveDataDistributed(client))
     {
        if (client->isServerLeader())
@@ -358 +358 @@
 
     map<int, CArray<double,1> >::iterator it;
-    if (!grid->doGridHaveDataDistributed())
+    if (!grid->doGridHaveDataDistributed(client))
     {
        if (client->isServerLeader())

XIOS/dev/XIOS_DEV_CMIP6/src/node/grid.cpp

@@ -659 +659 @@
 
             
-            if (nbIndex != localIndex.numElements())
+            if (doGridHaveDataDistributed(client) && (nbIndex != localIndex.numElements()))
                  ERROR("void CGrid::computeClientIndex()",
                     << "Number of local index on client is different from number of received global index" 
@@ -698 +698 @@
        connectedServerRank_[p].clear();
 
-       if (!doGridHaveDataDistributed())
+       if (!doGridHaveDataDistributed(client))
        {
           if (client->isServerLeader())
@@ -769 +769 @@
        if (connectedServerRank_[p].empty())
         connectedServerRank_[p].push_back(client->clientRank % client->serverSize);
-      
+
        nbSenders[p] = clientServerMap_->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[p]);
      }
@@ -1373 +1373 @@
       itIndex = itbIndex;                                                              
 
-      if (!doGridHaveDataDistributed())
+      if (!doGridHaveDataDistributed(client))
       {
         if (client->isServerLeader())
@@ -1593 +1593 @@
           for (int i = 0; i < nZoomSize.size(); ++i)
             dataSize *= nZoomSize[i];
-          // serverDistribution_ = new CDistributionServer(server->intraCommRank, nZoomBegin, nZoomSize,
-          //                                               nZoomBeginGlobal, nGlob);
           serverDistribution_ = new CDistributionServer(server->intraCommRank, 
                                                         globalZoomIndex, axis_domain_order,
@@ -1604 +1602 @@
         outGlobalIndexFromClient.insert(std::make_pair(rank, outIndex));        
 
-        if (isDataDistributed_)
+        if (doGridHaveDataDistributed(client))
         {}
         else
@@ -1827 +1825 @@
   }
 
-  bool CGrid::doGridHaveDataDistributed()
+  bool CGrid::doGridHaveDataDistributed(CContextClient* client)
   {
     if (isScalarGrid()) return false;
+    else if (0 != client)
+    {
+      return  (isDataDistributed_ ||  (1 != client->clientSize) || (1 != client->serverSize));
+    }
     else
-      return isDataDistributed_;
+      return isDataDistributed_;    
   }
 

XIOS/dev/XIOS_DEV_CMIP6/src/node/grid.hpp

@@ -179 +179 @@
 
          bool doGridHaveDataToWrite();
-         bool doGridHaveDataDistributed();
+         bool doGridHaveDataDistributed(CContextClient* client = 0);
          size_t getWrittenDataSize() const;
          int getNumberWrittenIndexes() const;

XIOS/dev/XIOS_DEV_CMIP6/src/server_distribution_description.cpp

@@ -29 +29 @@
 { /* Nothing to do */ }
 
+int CServerDistributionDescription::defaultDistributedDimension(int gridDimension,                                   
+                                                                ServerDistributionType serType)
+{  
+  switch (serType) 
+  {
+    case BAND_DISTRIBUTION:       
+       return ((1 == gridDimension) ? 0 : 1);
+      break;
+    default:
+      break;
+  } 
+}
+
 /*!
   Compute pre-defined global index distribution of server(s).

XIOS/dev/XIOS_DEV_CMIP6/src/server_distribution_description.hpp

@@ -51 +51 @@
     int getDimensionDistributed();
 
+    static int defaultDistributedDimension(int gridDimension,                                   
+                                           ServerDistributionType serType=BAND_DISTRIBUTION);
+
   protected:
     int computeBandDistribution(int nServer, int positionDimensionDistributed = 1);