Changeset 1460 for XIOS/dev/branch_openmp/src/node

XIOS/dev/branch_openmp/src/node/axis.cpp

-                      r1334
+                      r1460
 #include "context_server.hpp"
 #include "xios_spl.hpp"
-#include "inverse_axis.hpp"
-#include "zoom_axis.hpp"
-#include "interpolate_axis.hpp"
 #include "server_distribution_description.hpp"
 #include "client_server_mapping_distributed.hpp"
 #include "distribution_client.hpp"
+using namespace ep_lib;
 namespace xios {
    /// ////////////////////// DÃ©finitions ////////////////////// ///
+   /// ////////////////////// Definitions ////////////////////// ///
    CAxis::CAxis(void)
       : CObjectTemplate<CAxis>()
+      , CAxisAttributes(), isChecked(false), relFiles(), areClientAttributesChecked_(false)
+      , CAxisAttributes(), isChecked(false), relFiles()
+      , areClientAttributesChecked_(false)
       , isClientAfterTransformationChecked(false)
+      , isDistributed_(false), hasBounds_(false), isCompressible_(false)
+      , numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
+      , transformationMap_(), hasValue(false), hasLabel(false)
+      , hasBounds(false), isCompressible_(false)
+      , numberWrittenIndexes_(), totalNumberWrittenIndexes_(), offsetWrittenIndexes_()
+      , transformationMap_()
+      , hasValue(false), hasLabel(false)
+      , computedWrittenIndex_(false)
+      , clients()
+   {
+   }
 …
    CAxis::CAxis(const StdString & id)
       : CObjectTemplate<CAxis>(id)
+      , CAxisAttributes(), isChecked(false), relFiles(), areClientAttributesChecked_(false)
+      , CAxisAttributes(), isChecked(false), relFiles()
+      , areClientAttributesChecked_(false)
       , isClientAfterTransformationChecked(false)
+      , isDistributed_(false), hasBounds_(false), isCompressible_(false)
+      , numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
+      , transformationMap_(), hasValue(false), hasLabel(false)
+      , hasBounds(false), isCompressible_(false)
+      , numberWrittenIndexes_(), totalNumberWrittenIndexes_(), offsetWrittenIndexes_()
+      , transformationMap_()
+      , hasValue(false), hasLabel(false)
+      , computedWrittenIndex_(false)
+      , clients()
+   {
+   }
 …
      m["inverse_axis"] = TRANS_INVERSE_AXIS;
      m["reduce_domain"] = TRANS_REDUCE_DOMAIN_TO_AXIS;
+     m["reduce_axis"] = TRANS_REDUCE_AXIS_TO_AXIS;
      m["extract_domain"] = TRANS_EXTRACT_DOMAIN_TO_AXIS;
+     m["temporal_splitting"] = TRANS_TEMPORAL_SPLITTING;
+     m["duplicate_scalar"] = TRANS_DUPLICATE_SCALAR_TO_AXIS;
+   }
 …
      (*CAxis::transformationMapList_ptr)["reduce_domain"]    = TRANS_REDUCE_DOMAIN_TO_AXIS;
      (*CAxis::transformationMapList_ptr)["extract_domain"]   = TRANS_EXTRACT_DOMAIN_TO_AXIS;
+     (*CAxis::transformationMapList_ptr)["reduce_axis"]      = TRANS_REDUCE_AXIS_TO_AXIS;
+     (*CAxis::transformationMapList_ptr)["temporal_splitting"] = TRANS_TEMPORAL_SPLITTING;
+     (*CAxis::transformationMapList_ptr)["duplicate_scalar"] = TRANS_DUPLICATE_SCALAR_TO_AXIS;
+   }
 …
    bool CAxis::isDistributed(void) const
+   {
+      return isDistributed_;
+      bool distributed = (!this->begin.isEmpty() && !this->n.isEmpty() && (this->begin + this->n < this->n_glo)) ||
+             (!this->n.isEmpty() && (this->n != this->n_glo));
+      // A same stupid condition to make sure that if there is only one client, axis
+      // should be considered to be distributed. This should be a temporary solution
+      distributed |= (1 == CContext::getCurrent()->client->clientSize);
+      return distributed;
+   }
 …
    //----------------------------------------------------------------
-   const std::vector<int>& CAxis::getIndexesToWrite(void) const
+   {
-     return indexesToWrite;
+   }
    /*!
      Returns the number of indexes written by each server.
      \return the number of indexes written by each server
    */
+   int CAxis::getNumberWrittenIndexes() const
+   {
+     return numberWrittenIndexes_;
+   int CAxis::getNumberWrittenIndexes(ep_lib::MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     ep_lib::MPI_Comm_size(writtenCom, &writtenSize);
+     return numberWrittenIndexes_[writtenSize];
+   }
 …
      \return the total number of indexes written by the servers
    */
+   int CAxis::getTotalNumberWrittenIndexes() const
+   {
+     return totalNumberWrittenIndexes_;
+   int CAxis::getTotalNumberWrittenIndexes(ep_lib::MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     ep_lib::MPI_Comm_size(writtenCom, &writtenSize);
+     return totalNumberWrittenIndexes_[writtenSize];
+   }
 …
      \return the offset of indexes written by each server
    */
+   int CAxis::getOffsetWrittenIndexes() const
+   {
+     return offsetWrittenIndexes_;
+   int CAxis::getOffsetWrittenIndexes(ep_lib::MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     ep_lib::MPI_Comm_size(writtenCom, &writtenSize);
+     return offsetWrittenIndexes_[writtenSize];
+   }
+   CArray<int, 1>& CAxis::getCompressedIndexToWriteOnServer(ep_lib::MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     ep_lib::MPI_Comm_size(writtenCom, &writtenSize);
+     return compressedIndexToWriteOnServer[writtenSize];
+   }
    //----------------------------------------------------------------
    /*!
+      /*!
     * Compute the minimum buffer size required to send the attributes to the server(s).
+    *
     * \return A map associating the server rank with its minimum buffer size.
     */
+   std::map<int, StdSize> CAxis::getAttributesBufferSize()
+   {
+     CContextClient* client = CContext::getCurrent()->client;
+     std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes();
+     bool isNonDistributed = (n == n_glo);
+   std::map<int, StdSize> CAxis::getAttributesBufferSize(CContextClient* client, const std::vector<int>& globalDim, int orderPositionInGrid,
+                                                         CServerDistributionDescription::ServerDistributionType distType)
+   {
+     std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes(client);
+//     bool isNonDistributed = (n_glo == n);
+     bool isDistributed = (orderPositionInGrid == CServerDistributionDescription::defaultDistributedDimension(globalDim.size(), distType))
+                         || (index.numElements() != n_glo);
      if (client->isServerLeader())
 …
        size_t size = 6 * sizeof(size_t);
        // size estimation for sendNonDistributedValue
+       if (isNonDistributed)
+         size = std::max(size, CArray<double,1>::size(n_glo) + (isCompressible_ ? CArray<int,1>::size(n_glo) : 0));
+       if (!isDistributed)
+       {
+//         size = std::max(size, CArray<double,1>::size(n_glo) + (isCompressible_ ? CArray<int,1>::size(n_glo) : 0));
+         size += CArray<int,1>::size(n_glo);
+         size += CArray<int,1>::size(n_glo);
+         size += CArray<bool,1>::size(n_glo);
+         size += CArray<double,1>::size(n_glo);
+         if (hasBounds)
+           size += CArray<double,2>::size(2*n_glo);
+         if (hasLabel)
+          size += CArray<StdString,1>::size(n_glo);
+       }
        size += CEventClient::headerSize + getId().size() + sizeof(size_t);
 …
            attributesSizes[*itRank] = size;
+       }
+       const std::list<int>& ranksNonLeaders = client->getRanksServerNotLeader();
+       for (std::list<int>::const_iterator itRank = ranksNonLeaders.begin(), itRankEnd = ranksNonLeaders.end(); itRank != itRankEnd; ++itRank)
+       {
+         if (size > attributesSizes[*itRank])
+           attributesSizes[*itRank] = size;
+       }
+     }
      if (!isNonDistributed)
+     if (isDistributed)
+     {
        // size estimation for sendDistributedValue
        std::map<int, std::vector<size_t> >::const_iterator it, ite = indSrv_.end();
        for (it = indSrv_.begin(); it != ite; ++it)
+       boost::unordered_map<int, vector<size_t> >::const_iterator it, ite = indSrv_[client->serverSize].end();
+       for (it = indSrv_[client->serverSize].begin(); it != ite; ++it)
+       {
+         size_t sizeIndexEvent = CArray<int,1>::size(it->second.size());
+         if (isCompressible_)
+           sizeIndexEvent += CArray<int,1>::size(indWrittenSrv_[it->first].size());
+         size_t sizeValEvent = CArray<double,1>::size(it->second.size());
+         if (hasBounds_)
+           sizeValEvent += CArray<double,2>::size(2 * it->second.size());
+         size_t size = 6 * sizeof(size_t);
+         size += CArray<int,1>::size(it->second.size());
+         size += CArray<int,1>::size(it->second.size());
+         size += CArray<bool,1>::size(it->second.size());
+         size += CArray<double,1>::size(it->second.size());
+         if (hasBounds)
+           size += CArray<double,2>::size(2 * it->second.size());
          if (hasLabel)
            sizeValEvent += CArray<StdString,1>::size(it->second.size());
          size_t size = CEventClient::headerSize + getId().size() + sizeof(size_t) + std::max(sizeIndexEvent, sizeValEvent);
+           size += CArray<StdString,1>::size(it->second.size());
+         size += CEventClient::headerSize + getId().size() + sizeof(size_t);
          if (size > attributesSizes[it->first])
            attributesSizes[it->first] = size;
 …
+   }
+   void CAxis::fillInValues(const CArray<double,1>& values)
+   {
+     this->value = values;
+   }
+   /*!
+     Check common attributes of an axis.
+     This check should be done in the very beginning of work flow
+   */
    void CAxis::checkAttributes(void)
+   {
 …
+      }
+      // Remove this check because it doen't make sense in case of a hole or overlapping axes
       if (!this->value.isEmpty())
+      {
         StdSize true_size = value.numElements();
         if (this->n.getValue() != true_size)
           ERROR("CAxis::checkAttributes(void)",
                 << "[ id = '" << getId() << "' , context = '" << CObjectFactory::GetCurrentContextId() << "' ] "
                 << "The axis is wrongly defined, attribute 'value' has a different size (" << true_size << ") than the one defined by the \'size\' attribute (" << n.getValue() << ").");
+//        StdSize true_size = value.numElements();
+//        if (this->n.getValue() != true_size)
+//          ERROR("CAxis::checkAttributes(void)",
+//                << "[ id = '" << getId() << "' , context = '" << CObjectFactory::GetCurrentContextId() << "' ] "
+//                << "The axis is wrongly defined, attribute 'value' has a different size (" << true_size << ") than the one defined by the \'size\' attribute (" << n.getValue() << ").");
         this->hasValue = true;
+      }
-      this->checkData();
-      this->checkZoom();
-      this->checkMask();
       this->checkBounds();
+      this->checkLabel();
+      isDistributed_ = (!this->begin.isEmpty() && !this->n.isEmpty() && (this->begin + this->n < this->n_glo)) ||
+                       (!this->n.isEmpty() && (this->n != this->n_glo));
+      // A same stupid condition to make sure that if there is only one client, axis
+      // should be considered to be distributed. This should be a temporary solution
+      isDistributed_ |= (1 == CContext::getCurrent()->client->clientSize);
+   }
+      CContext* context=CContext::getCurrent();
+      if (context->hasClient)
+      {
+        this->checkData();
+        this->checkZoom();
+        this->checkMask();
+        this->checkLabel();
+      }
+   }
+   /*!
+      Check the validity of data and fill in values if any.
+   */
    void CAxis::checkData()
+   {
 …
+   }
+   /*!
+     Check validity of zoom info and fill in values if any.
+   */
    void CAxis::checkZoom(void)
+   {
      if (global_zoom_begin.isEmpty()) global_zoom_begin.setValue(0);
      if (global_zoom_n.isEmpty()) global_zoom_n.setValue(n_glo.getValue());
+   }
+     if (zoom_index.isEmpty())
+     {
+       zoom_index.setValue(index.getValue());
+     }
+     if (zoom_n.isEmpty()) zoom_n.setValue(n);
+     if (zoom_begin.isEmpty()) zoom_begin.setValue(begin);
+   }
+    size_t CAxis::getGlobalWrittenSize(void)
+    {
+      if (zoomByIndex()) return  global_zoom_index.numElements();
+      else return global_zoom_n ;
+    }
+   /*!
+     Check validity of mask info and fill in values if any.
+   */
    void CAxis::checkMask()
+   {
 …
+   }
+  void CAxis::checkBounds()
+  {
+    if (!bounds.isEmpty())
+    {
+      if (bounds.extent(0) != 2 || bounds.extent(1) != n)
+        ERROR("CAxis::checkAttributes(void)",
+              << "The bounds array of the axis [ id = '" << getId() << "' , context = '" << CObjectFactory::GetCurrentContextId() << "' ] must be of dimension 2 x axis size." << std::endl
+              << "Axis size is " << n.getValue() << "." << std::endl
+              << "Bounds size is "<< bounds.extent(0) << " x " << bounds.extent(1) << ".");
+      hasBounds_ = true;
+    }
+    else hasBounds_ = false;
+  }
+   /*!
+     Check validity of bounds info and fill in values if any.
+   */
+   void CAxis::checkBounds()
+   {
+     if (!bounds.isEmpty())
+     {
+       if (bounds.extent(0) != 2 || bounds.extent(1) != n)
+         ERROR("CAxis::checkAttributes(void)",
+               << "The bounds array of the axis [ id = '" << getId() << "' , context = '" << CObjectFactory::GetCurrentContextId() << "' ] must be of dimension 2 x axis size." << std::endl
+               << "Axis size is " << n.getValue() << "." << std::endl
+               << "Bounds size is "<< bounds.extent(0) << " x " << bounds.extent(1) << ".");
+       hasBounds = true;
+     }
+     else hasBounds = false;
+   }
   void CAxis::checkLabel()
 …
     else hasLabel = false;
+  }
+  /*!
+    Check whether we can do compressed output
+  */
   void CAxis::checkEligibilityForCompressedOutput()
+  {
 …
+  }
+  /*
+    Check whether we do zooming by indexing
+    return true if do zooming by index
+  */
   bool CAxis::zoomByIndex()
+  {
 …
+  }
+  /*!
+    Dispatch event from the lower communication layer then process event according to its type
+  */
   bool CAxis::dispatchEvent(CEventServer& event)
+   {
+      if (SuperClass::dispatchEvent(event)) return true;
+      else
+      {
+        switch(event.type)
+        {
+           case EVENT_ID_SERVER_ATTRIBUT :
+             recvServerAttribut(event);
+             return true;
+             break;
+           case EVENT_ID_INDEX:
+            recvIndex(event);
+  {
+     if (SuperClass::dispatchEvent(event)) return true;
+     else
+     {
+       switch(event.type)
+       {
+          case EVENT_ID_DISTRIBUTION_ATTRIBUTE :
+            recvDistributionAttribute(event);
             return true;
             break;
+          case EVENT_ID_DISTRIBUTED_VALUE:
+            recvDistributedValue(event);
+            return true;
+            break;
+          case EVENT_ID_NON_DISTRIBUTED_VALUE:
+            recvNonDistributedValue(event);
+            return true;
+            break;
+           default :
+             ERROR("bool CAxis::dispatchEvent(CEventServer& event)",
+                    << "Unknown Event");
+           return false;
+         }
+      }
+   }
+         case EVENT_ID_NON_DISTRIBUTED_ATTRIBUTES:
+           recvNonDistributedAttributes(event);
+           return true;
+           break;
+         case EVENT_ID_DISTRIBUTED_ATTRIBUTES:
+           recvDistributedAttributes(event);
+           return true;
+           break;
+          default :
+            ERROR("bool CAxis::dispatchEvent(CEventServer& event)",
+                   << "Unknown Event");
+          return false;
+        }
+     }
+  }
+  /*!
+     Check attributes on client side (This name is still adequate???)
+   */
    void CAxis::checkAttributesOnClient()
+   {
      if (this->areClientAttributesChecked_) return;
+     this->checkAttributes();
+     CContext* context=CContext::getCurrent();
+     if (context->hasClient && !context->hasServer) this->checkAttributes();
      this->areClientAttributesChecked_ = true;
+   }
+   /*
+     The (spatial) transformation sometimes can change attributes of an axis (e.g zoom can change mask or generate can change whole attributes)
+     Therefore, we should recheck them.
+   */
    void CAxis::checkAttributesOnClientAfterTransformation(const std::vector<int>& globalDim, int orderPositionInGrid,
                                                           CServerDistributionDescription::ServerDistributionType distType)
 …
      if (this->isClientAfterTransformationChecked) return;
      if (context->hasClient)
+     {
+       if (n.getValue() != n_glo.getValue()) computeConnectedServer(globalDim, orderPositionInGrid, distType);
+     {
+       if (orderPositionInGrid == CServerDistributionDescription::defaultDistributedDimension(globalDim.size(), distType))
+         computeConnectedClients(globalDim, orderPositionInGrid, distType);
+       else if (index.numElements() != n_glo) computeConnectedClients(globalDim, orderPositionInGrid,  CServerDistributionDescription::ROOT_DISTRIBUTION);
+     }
 …
+   }
+   // Send all checked attributes to server
+   /*
+     Send all checked attributes to server? (We dont have notion of server any more so client==server)
+     \param [in] globalDim global dimension of grid containing this axis
+     \param [in] orderPositionInGrid the relative order of this axis in the grid (e.g grid composed of domain+axis -> orderPositionInGrid is 2)
+     \param [in] distType distribution type of the server. For now, we only have band distribution.
+   */
    void CAxis::sendCheckedAttributes(const std::vector<int>& globalDim, int orderPositionInGrid,
                                      CServerDistributionDescription::ServerDistributionType distType)
 …
      if (this->isChecked) return;
+     if (context->hasClient)
+     if (context->hasClient) sendAttributes(globalDim, orderPositionInGrid, distType);
+     this->isChecked = true;
+   }
+   /*!
+    Send attributes from one client to other clients
+    \param[in] globalDim global dimension of grid which contains this axis
+    \param[in] order
+  */
+  void CAxis::sendAttributes(const std::vector<int>& globalDim, int orderPositionInGrid,
+                             CServerDistributionDescription::ServerDistributionType distType)
+  {
+     sendDistributionAttribute(globalDim, orderPositionInGrid, distType);
+     // if (index.numElements() == n_glo.getValue())
+     if ((orderPositionInGrid == CServerDistributionDescription::defaultDistributedDimension(globalDim.size(), distType))
+         || (index.numElements() != n_glo))
+     {
+       sendServerAttribut(globalDim, orderPositionInGrid, distType);
+       if (hasValue) sendValue();
+       sendDistributedAttributes();
+     }
-     this->isChecked = true;
+   }
-  void CAxis::sendValue()
+  {
-     if (n.getValue() == n_glo.getValue())
-       sendNonDistributedValue();
      else
+       sendDistributedValue();
+  }
+  void CAxis::computeConnectedServer(const std::vector<int>& globalDim, int orderPositionInGrid,
+     {
+       sendNonDistributedAttributes();
+     }
+  }
+  /*
+    Compute the connection between group of clients (or clients/servers).
+    (E.g: Suppose we have 2 group of clients in two model: A (client role) connect to B (server role),
+    this function calculate number of clients B connect to one client of A)
+     \param [in] globalDim global dimension of grid containing this axis
+     \param [in] orderPositionInGrid the relative order of this axis in the grid (e.g grid composed of domain+axis -> orderPositionInGrid is 2)
+     \param [in] distType distribution type of the server. For now, we only have band distribution.
+  */
+  void CAxis::computeConnectedClients(const std::vector<int>& globalDim, int orderPositionInGrid,
                                      CServerDistributionDescription::ServerDistributionType distType)
+  {
     CContext* context = CContext::getCurrent();
+    CContextClient* client = context->client;
+    int nbServer = client->serverSize;
+    int range, clientSize = client->clientSize;
+    int rank = client->clientRank;
+    size_t ni = this->n.getValue();
+    size_t ibegin = this->begin.getValue();
+    size_t zoom_end = global_zoom_begin+global_zoom_n-1;
+    size_t nZoomCount = 0;
+    size_t nbIndex = index.numElements();
+    int end = (0 == n) ? begin : begin + n - 1;
+    int zoom_size = zoomByIndex() ? global_zoom_index.numElements() : global_zoom_n;
+    int minInd = min(index);
+    int maxInd = max(index);
+    for (size_t idx = 0; idx < zoom_size; ++idx)
+    {
+      size_t globalZoomIndex = zoomByIndex() ? global_zoom_index(idx) : global_zoom_begin + idx;
+      if (globalZoomIndex >= minInd && globalZoomIndex <= maxInd) ++nZoomCount;
+    }
+/*    for (size_t idx = 0; idx < nbIndex; ++idx)
+    {
+      size_t globalIndex = index(idx);
+      if (globalIndex >= global_zoom_begin && globalIndex <= zoom_end) ++nZoomCount;
+    }*/
+    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
+    connectedServerRank_.clear();
+    nbSenders.clear();
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
+      int nbServer = client->serverSize;
+      int range, clientSize = client->clientSize;
+      int rank = client->clientRank;
+      if (connectedServerRank_.find(nbServer) == connectedServerRank_.end())
+      {
+        size_t ni = this->n.getValue();
+        size_t ibegin = this->begin.getValue();
+        size_t global_zoom_end = global_zoom_begin+global_zoom_n-1;
+        size_t nZoomCount = 0;
+        size_t nbIndex = index.numElements();
+        // First of all, we should compute the mapping of the global index and local index of the current client
+        if (globalLocalIndexMap_.empty())
+        {
+          for (size_t idx = 0; idx < nbIndex; ++idx)
+          {
+            globalLocalIndexMap_[index(idx)] = idx;
+          }
+        }
+        // Calculate the compressed index if any
+        std::set<int> writtenInd;
+        if (isCompressible_)
+        {
+          for (int idx = 0; idx < data_index.numElements(); ++idx)
+          {
+            int ind = CDistributionClient::getAxisIndex(data_index(idx), data_begin, ni);
+            if (ind >= 0 && ind < ni && mask(ind))
+            {
+              ind += ibegin;
+              if (ind >= global_zoom_begin && ind <= global_zoom_end)
+                writtenInd.insert(ind);
+            }
+          }
+        }
+        // Compute the global index of the current client (process) hold
+        std::vector<int> nGlobAxis(1);
+        nGlobAxis[0] = n_glo.getValue();
+        size_t globalSizeIndex = 1, indexBegin, indexEnd;
+        for (int i = 0; i < nGlobAxis.size(); ++i) globalSizeIndex *= nGlobAxis[i];
+        indexBegin = 0;
+        if (globalSizeIndex <= clientSize)
+        {
+          indexBegin = rank%globalSizeIndex;
+          indexEnd = indexBegin;
+        }
+        else
+        {
+          for (int i = 0; i < clientSize; ++i)
+          {
+            range = globalSizeIndex / clientSize;
+            if (i < (globalSizeIndex%clientSize)) ++range;
+            if (i == client->clientRank) break;
+            indexBegin += range;
+          }
+          indexEnd = indexBegin + range - 1;
+        }
+        CArray<size_t,1> globalIndex(index.numElements());
+        for (size_t idx = 0; idx < globalIndex.numElements(); ++idx)
+          globalIndex(idx) = index(idx);
+        // Describe the distribution of server side
+        CServerDistributionDescription serverDescription(nGlobAxis, nbServer, distType);
+        std::vector<int> serverZeroIndex;
+        serverZeroIndex = serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 0);
+        std::list<int> serverZeroIndexLeader;
+        std::list<int> serverZeroIndexNotLeader;
+        CContextClient::computeLeader(client->clientRank, client->clientSize, serverZeroIndex.size(), serverZeroIndexLeader, serverZeroIndexNotLeader);
+        for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+          *it = serverZeroIndex[*it];
+        // Find out the connection between client and server side
+        CClientServerMapping* clientServerMap = new CClientServerMappingDistributed(serverDescription.getGlobalIndexRange(), client->intraComm);
+        clientServerMap->computeServerIndexMapping(globalIndex, nbServer);
+        CClientServerMapping::GlobalIndexMap& globalIndexAxisOnServer = clientServerMap->getGlobalIndexOnServer();
+        indSrv_[nbServer].swap(globalIndexAxisOnServer);
+        if (distType==CServerDistributionDescription::ROOT_DISTRIBUTION)
+        {
+          for(int i=1; i<nbServer; ++i)
+          {
+            indSrv_[nbServer].insert(pair<int, vector<size_t> >(i,indSrv_[nbServer][0]) ) ;
+          }
+          serverZeroIndexLeader.clear() ;
+        }
+        CClientServerMapping::GlobalIndexMap::const_iterator it  = indSrv_[nbServer].begin(),
+                                                             ite = indSrv_[nbServer].end();
+        for (it = indSrv_[nbServer].begin(); it != ite; ++it) connectedServerRank_[nbServer].push_back(it->first);
+        for (std::list<int>::const_iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+          connectedServerRank_[nbServer].push_back(*it);
+         // Even if a client has no index, it must connect to at least one server and
+         // send an "empty" data to this server
+         if (connectedServerRank_[nbServer].empty())
+          connectedServerRank_[nbServer].push_back(client->clientRank % client->serverSize);
+         nbSenders[nbServer] = CClientServerMapping::computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[nbServer]);
+        delete clientServerMap;
+      }
+    }
+  }
+  /*
+    Compute the index of data to write into file
+    (Different from the previous version, this version of XIOS allows data be written into file (classical role),
+    or transfered to another clients)
+  */
+  void CAxis::computeWrittenIndex()
+  {
+    if (computedWrittenIndex_) return;
+    computedWrittenIndex_ = true;
+    CContext* context=CContext::getCurrent();
+    CContextServer* server = context->server;
+    // We describe the distribution of client (server) on which data are written
+    std::vector<int> nBegin(1), nSize(1), nBeginGlobal(1), nGlob(1);
+    nBegin[0]       = zoom_begin;
+    nSize[0]        = zoom_n;
+    nBeginGlobal[0] = 0;
+    nGlob[0]        = n_glo;
+    CDistributionServer srvDist(server->intraCommSize, nBegin, nSize, nBeginGlobal, nGlob);
+    const CArray<size_t,1>& writtenGlobalIndex  = srvDist.getGlobalIndex();
+    // Because all written data are local on a client,
+    // we need to compute the local index on the server from its corresponding global index
+    size_t nbWritten = 0, indGlo;
+    boost::unordered_map<size_t,size_t>::const_iterator itb = globalLocalIndexMap_.begin(),
+                                                        ite = globalLocalIndexMap_.end(), it;
+    CArray<size_t,1>::const_iterator itSrvb = writtenGlobalIndex.begin(),
+                                     itSrve = writtenGlobalIndex.end(), itSrv;
+    if (!zoomByIndex())
+    {
+      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+      {
+        indGlo = *itSrv;
+        if (ite != globalLocalIndexMap_.find(indGlo))
+        {
+          ++nbWritten;
+        }
+      }
+      localIndexToWriteOnServer.resize(writtenGlobalIndex.numElements());
+//      localIndexToWriteOnServer.resize(nbWritten);
+      nbWritten = 0;
+      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+      {
+        indGlo = *itSrv;
+        if (ite != globalLocalIndexMap_.find(indGlo))
+        {
+          localIndexToWriteOnServer(nbWritten) = globalLocalIndexMap_[indGlo];
+          ++nbWritten;
+        }
+      }
+    }
+    else
+    {
+      nbWritten = 0;
+      boost::unordered_map<size_t,size_t>::const_iterator itb = globalLocalIndexMap_.begin(),
+                                                          ite = globalLocalIndexMap_.end(), it;
+      for (int i = 0; i < zoom_index.numElements(); ++i)
+      {
+         if (ite != globalLocalIndexMap_.find(zoom_index(i)))
+          ++nbWritten;
+      }
+      localIndexToWriteOnServer.resize(nbWritten);
+      nbWritten = 0;
+      for (int i = 0; i < zoom_index.numElements(); ++i)
+      {
+         if (ite != globalLocalIndexMap_.find(zoom_index(i)))
+         {
+           localIndexToWriteOnServer(nbWritten) = globalLocalIndexMap_[zoom_index(i)];
+           ++nbWritten;
+         }
+      }
+    }
+  }
+  void CAxis::computeWrittenCompressedIndex(ep_lib::MPI_Comm writtenComm)
+  {
+    int writtenCommSize;
+    ep_lib::MPI_Comm_size(writtenComm, &writtenCommSize);
+    if (compressedIndexToWriteOnServer.find(writtenCommSize) != compressedIndexToWriteOnServer.end())
+      return;
+    if (isCompressible())
+    {
+      size_t nbWritten = 0, indGlo;
+      CContext* context=CContext::getCurrent();
+      CContextServer* server = context->server;
+      // We describe the distribution of client (server) on which data are written
+      std::vector<int> nBegin(1), nSize(1), nBeginGlobal(1), nGlob(1);
+      nBegin[0]       = zoom_begin;
+      nSize[0]        = zoom_n;
+      nBeginGlobal[0] = 0;
+      nGlob[0]        = n_glo;
+      CDistributionServer srvDist(server->intraCommSize, nBegin, nSize, nBeginGlobal, nGlob);
+      const CArray<size_t,1>& writtenGlobalIndex  = srvDist.getGlobalIndex();
+      boost::unordered_map<size_t,size_t>::const_iterator itb = globalLocalIndexMap_.begin(),
+                                                          ite = globalLocalIndexMap_.end(), it;
+      CArray<size_t,1>::const_iterator itSrvb = writtenGlobalIndex.begin(),
+                                       itSrve = writtenGlobalIndex.end(), itSrv;
+      boost::unordered_map<size_t,size_t> localGlobalIndexMap;
+      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+      {
+        indGlo = *itSrv;
+        if (ite != globalLocalIndexMap_.find(indGlo))
+        {
+          localGlobalIndexMap[localIndexToWriteOnServer(nbWritten)] = indGlo;
+          ++nbWritten;
+        }
+      }
+      nbWritten = 0;
+      for (int idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
+        {
+          ++nbWritten;
+        }
+      }
+      compressedIndexToWriteOnServer[writtenCommSize].resize(nbWritten);
+      nbWritten = 0;
+      for (int idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
+        {
+          compressedIndexToWriteOnServer[writtenCommSize](nbWritten) = localGlobalIndexMap[data_index(idx)];
+          ++nbWritten;
+        }
+      }
+      numberWrittenIndexes_[writtenCommSize] = nbWritten;
+      if (isDistributed())
+      {
+        ep_lib::MPI_Allreduce(&numberWrittenIndexes_[writtenCommSize], &totalNumberWrittenIndexes_[writtenCommSize], 1, MPI_INT, MPI_SUM, writtenComm);
+        ep_lib::MPI_Scan(&numberWrittenIndexes_[writtenCommSize], &offsetWrittenIndexes_[writtenCommSize], 1, MPI_INT, MPI_SUM, writtenComm);
+        offsetWrittenIndexes_[writtenCommSize] -= numberWrittenIndexes_[writtenCommSize];
+      }
+      else
+        totalNumberWrittenIndexes_[writtenCommSize] = numberWrittenIndexes_[writtenCommSize];
+    }
+  }
+  /*!
+    Send distribution information from a group of client (client role) to another group of client (server role)
+    The distribution of a group of client (server role) is imposed by the group of client (client role)
+    \param [in] globalDim global dimension of grid containing this axis
+    \param [in] orderPositionInGrid the relative order of this axis in the grid (e.g grid composed of domain+axis -> orderPositionInGrid is 2)
+    \param [in] distType distribution type of the server. For now, we only have band distribution.
+  */
+  void CAxis::sendDistributionAttribute(const std::vector<int>& globalDim, int orderPositionInGrid,
+                                        CServerDistributionDescription::ServerDistributionType distType)
+  {
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      CContextClient* client = *it;
+      int nbServer = client->serverSize;
+      CServerDistributionDescription serverDescription(globalDim, nbServer);
+      serverDescription.computeServerDistribution();
+      std::vector<std::vector<int> > serverIndexBegin = serverDescription.getServerIndexBegin();
+      std::vector<std::vector<int> > serverDimensionSizes = serverDescription.getServerDimensionSizes();
+      CEventClient event(getType(),EVENT_ID_DISTRIBUTION_ATTRIBUTE);
+      if (client->isServerLeader())
+      {
+        std::list<CMessage> msgs;
+        const std::list<int>& ranks = client->getRanksServerLeader();
+        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        {
+          // Use const int to ensure CMessage holds a copy of the value instead of just a reference
+          const int begin = serverIndexBegin[*itRank][orderPositionInGrid];
+          const int ni    = serverDimensionSizes[*itRank][orderPositionInGrid];
+          const int end   = begin + ni - 1;
+          msgs.push_back(CMessage());
+          CMessage& msg = msgs.back();
+          msg << this->getId();
+          msg << ni << begin << end;
+          msg << isCompressible_;
+          event.push(*itRank,1,msg);
+        }
+        client->sendEvent(event);
+      }
+      else client->sendEvent(event);
+    }
+  }
+  /*
+    Receive distribution attribute from another client
+    \param [in] event event containing data of these attributes
+  */
+  void CAxis::recvDistributionAttribute(CEventServer& event)
+  {
+    CBufferIn* buffer = event.subEvents.begin()->buffer;
+    string axisId;
+    *buffer >> axisId;
+    get(axisId)->recvDistributionAttribute(*buffer);
+  }
+  /*
+    Receive distribution attribute from another client
+    \param [in] buffer buffer containing data of these attributes
+  */
+  void CAxis::recvDistributionAttribute(CBufferIn& buffer)
+  {
+    int ni_srv, begin_srv, end_srv;
+    int global_zoom_end, zoom_end;
+    bool zoomIndex = zoomByIndex();
+    CArray<size_t,1> globalIndexAxis(nbIndex);
+    for (size_t idx = 0; idx < nbIndex; ++idx)
+    {
+      globalIndexAxis(idx) = (size_t)index(idx);
+    }
+    std::vector<size_t> globalAxisZoom(nZoomCount);
+    nZoomCount = 0;
+    for (size_t idx = 0; idx < zoom_size; ++idx)
+    {
+      size_t globalZoomIndex = zoomByIndex() ? global_zoom_index(idx) : global_zoom_begin + idx;
+      if (globalZoomIndex >= minInd && globalZoomIndex <= maxInd)
+      {
+        globalAxisZoom[nZoomCount] = globalZoomIndex;
+        ++nZoomCount;
+      }
+    }
+    std::set<int> writtenInd;
+    if (isCompressible_)
+    {
+    std::vector<int> zoom_index_tmp;
+    std::vector<int>::iterator itZoomBegin, itZoomEnd, itZoom;
+    buffer >> ni_srv >> begin_srv >> end_srv;
+    buffer >> isCompressible_;
+    // Set up new local size of axis on the receiving clients
+    n.setValue(ni_srv);
+    begin.setValue(begin_srv);
+    // If we have zoom by index then process it
+    if (zoomIndex)
+    {
+      zoom_index_tmp.resize(global_zoom_index.numElements());
+      std::copy(global_zoom_index.begin(), global_zoom_index.end(), zoom_index_tmp.begin());
+      std::sort(zoom_index_tmp.begin(), zoom_index_tmp.end());
+      itZoomBegin = std::lower_bound(zoom_index_tmp.begin(), zoom_index_tmp.end(), begin_srv);
+      itZoomEnd   = std::upper_bound(zoom_index_tmp.begin(), zoom_index_tmp.end(), end_srv);
+      int sz = std::distance(itZoomBegin, itZoomEnd);
+      zoom_index.resize(sz);
+      itZoom = itZoomBegin;
+      for (int i = 0; i < sz; ++i, ++itZoom)
+      {
+        zoom_index(i) = *(itZoom);
+      }
+    }
+    global_zoom_begin = zoomIndex ? 0 : global_zoom_begin ;
+    global_zoom_n     = zoomIndex ? zoom_index_tmp.size() : global_zoom_n;
+    global_zoom_end   = global_zoom_begin + global_zoom_n - 1;
+    zoom_begin = zoomIndex ? std::distance(zoom_index_tmp.begin(), itZoomBegin)
+                           : global_zoom_begin > begin_srv ? global_zoom_begin : begin_srv ;
+    zoom_end   = zoomIndex ? std::distance(zoom_index_tmp.begin(), itZoomEnd) - 1
+                           : global_zoom_end < end_srv ? global_zoom_end : end_srv ;
+    zoom_n     = zoom_end - zoom_begin + 1;
+    if (zoom_n<=0)
+    {
+      zoom_n = 0; zoom_begin=global_zoom_begin; //0; zoom_begin = 0;
+    }
+    if (n_glo == n)
+    {
+      zoom_begin = zoomIndex ? std::distance(itZoomBegin, zoom_index_tmp.begin())
+                             : global_zoom_begin;
+      zoom_n     = zoomIndex ? zoom_index_tmp.size() : global_zoom_n;
+    }
+  }
+  /*
+    Send attributes of axis from a group of client to other group of clients/servers
+    on supposing that these attributes are not distributed among the sending group
+    In the future, if new attributes are added, they should also be processed in this function
+  */
+  void CAxis::sendNonDistributedAttributes()
+  {
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+  {
+    CContextClient* client = *it;
+      CEventClient event(getType(), EVENT_ID_NON_DISTRIBUTED_ATTRIBUTES);
+      size_t nbIndex = index.numElements();
+      size_t nbDataIndex = 0;
       for (int idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        int ind = CDistributionClient::getAxisIndex(data_index(idx), data_begin, ni);
+        if (ind >= 0 && ind < ni && mask(ind))
+        {
+          ind += ibegin;
+          if (ind >= global_zoom_begin && ind <= zoom_end)
+            writtenInd.insert(ind);
+        }
+      }
+    }
+    CServerDistributionDescription serverDescriptionGlobal(globalDim, nbServer);
+    int distributedDimensionOnServer = serverDescriptionGlobal.getDimensionDistributed();
+    CClientServerMapping::GlobalIndexMap globalIndexAxisOnServer;
+    if (distributedDimensionOnServer == orderPositionInGrid) // So we have distributed axis on client side and also on server side*
+    {
+      std::vector<int> nGlobAxis(1);
+      nGlobAxis[0] = n_glo.getValue();
+      size_t globalSizeIndex = 1, indexBegin, indexEnd;
+      for (int i = 0; i < nGlobAxis.size(); ++i) globalSizeIndex *= nGlobAxis[i];
+      indexBegin = 0;
+      if (globalSizeIndex <= clientSize)
+      {
+        indexBegin = rank%globalSizeIndex;
+        indexEnd = indexBegin;
+      }
+      else
+      {
+        for (int i = 0; i < clientSize; ++i)
+        {
+          range = globalSizeIndex / clientSize;
+          if (i < (globalSizeIndex%clientSize)) ++range;
+          if (i == client->clientRank) break;
+          indexBegin += range;
+        }
+        indexEnd = indexBegin + range - 1;
+      }
+      CServerDistributionDescription serverDescription(nGlobAxis, nbServer);
+      serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd));
+      CClientServerMapping* clientServerMap = new CClientServerMappingDistributed(serverDescription.getGlobalIndexRange(), client->intraComm);
+      clientServerMap->computeServerIndexMapping(globalIndexAxis);
+      globalIndexAxisOnServer = clientServerMap->getGlobalIndexOnServer();
+      delete clientServerMap;
+    }
+    else
+    {
+      std::vector<size_t> globalIndexServer(n_glo.getValue());
+      for (size_t idx = 0; idx < n_glo.getValue(); ++idx)
+      {
+        globalIndexServer[idx] = idx;
+      }
+      for (int idx = 0; idx < nbServer; ++idx)
+      {
+        globalIndexAxisOnServer[idx] = globalIndexServer;
+      }
+    }
+    CClientServerMapping::GlobalIndexMap::const_iterator it = globalIndexAxisOnServer.begin(),
+                                                         ite = globalIndexAxisOnServer.end();
+    std::vector<size_t>::const_iterator itbVec = (globalAxisZoom).begin(),
+                                        iteVec = (globalAxisZoom).end();
+    indSrv_.clear();
+    indWrittenSrv_.clear();
+    for (; it != ite; ++it)
+    {
+      int rank = it->first;
+      const std::vector<size_t>& globalIndexTmp = it->second;
+      int nb = globalIndexTmp.size();
+      for (int i = 0; i < nb; ++i)
+      {
+        if (std::binary_search(itbVec, iteVec, globalIndexTmp[i]))
+        {
+          indSrv_[rank].push_back(globalIndexTmp[i]);
+        }
+        if (writtenInd.count(globalIndexTmp[i]))
+        {
+          indWrittenSrv_[rank].push_back(globalIndexTmp[i]);
+        }
+      }
+    }
+    connectedServerRank_.clear();
+    for (it = globalIndexAxisOnServer.begin(); it != ite; ++it) {
+      connectedServerRank_.push_back(it->first);
+    }
+    if (!indSrv_.empty())
+    {
+      std::map<int, vector<size_t> >::const_iterator itIndSrv  = indSrv_.begin(),
+                                                     iteIndSrv = indSrv_.end();
+      connectedServerRank_.clear();
+      for (; itIndSrv != iteIndSrv; ++itIndSrv)
+        connectedServerRank_.push_back(itIndSrv->first);
+    }
+    nbConnectedClients_ = CClientServerMapping::computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_);
+  }
+  void CAxis::sendNonDistributedValue()
+  {
+    CContext* context = CContext::getCurrent();
+    CContextClient* client = context->client;
+    CEventClient event(getType(), EVENT_ID_NON_DISTRIBUTED_VALUE);
+    int zoom_end = global_zoom_begin + global_zoom_n - 1;
+    int nb = 0;
+/*    for (size_t idx = 0; idx < n; ++idx)
+    {
+      size_t globalIndex = begin + idx;
+      if (globalIndex >= global_zoom_begin && globalIndex <= zoom_end) ++nb;
+    }*/
+    int end = (0 == n) ? begin : begin + n - 1;
+    int zoom_size = zoomByIndex() ? global_zoom_index.numElements() : global_zoom_n;
+    for (size_t idx = 0; idx < zoom_size; ++idx)
+    {
+      size_t globalZoomIndex = zoomByIndex() ? global_zoom_index(idx) : global_zoom_begin + idx;
+      if (globalZoomIndex >= begin && globalZoomIndex <= end) ++nb;
+    }
+    int nbWritten = 0;
+    if (isCompressible_)
+    {
+        int ind = data_index(idx);
+        if (ind >= 0 && ind < nbIndex) ++nbDataIndex;
+      }
+      CArray<int,1> dataIndex(nbDataIndex);
+      nbDataIndex = 0;
       for (int idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        int ind = CDistributionClient::getAxisIndex(data_index(idx), data_begin, n);
+        if (ind >= 0 && ind < n && mask(ind))
+        {
+          ind += begin;
+          if (ind >= global_zoom_begin && ind <= zoom_end)
+            ++nbWritten;
+        }
+      }
+    }
+    CArray<double,1> val(nb);
+    nb = 0;
+/*    for (size_t idx = 0; idx < n; ++idx)
+    {
+      size_t globalIndex = begin + idx;
+      if (globalIndex >= global_zoom_begin && globalIndex <= zoom_end)
+      {
+        val(nb) = value(idx);
+        ++nb;
+      }
+    }*/
+    for (size_t idx = 0; idx < zoom_size; ++idx)
+    {
+      size_t globalZoomIndex = zoomByIndex() ? global_zoom_index(idx) : global_zoom_begin + idx;
+      if (globalZoomIndex >= begin && globalZoomIndex <= end)
+      {
+        val(nb) = value(globalZoomIndex-begin);
+        ++nb;
+      }
+    }
+    CArray<int, 1> writtenInd(nbWritten);
+    nbWritten = 0;
+    if (isCompressible_)
+    {
+      for (int idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        int ind = CDistributionClient::getAxisIndex(data_index(idx), data_begin, n);
+        if (ind >= 0 && ind < n && mask(ind))
+        {
+          ind += begin;
+          if (ind >= global_zoom_begin && ind <= zoom_end)
+          {
+            writtenInd(nbWritten) = ind;
+            ++nbWritten;
+          }
+        }
+      }
+    }
+    if (client->isServerLeader())
+    {
+      std::list<CMessage> msgs;
+      const std::list<int>& ranks = client->getRanksServerLeader();
+      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+      {
+        msgs.push_back(CMessage());
+        CMessage& msg = msgs.back();
+        msg << this->getId();
+        msg << val;
+        if (isCompressible_)
+          msg << writtenInd;
+        event.push(*itRank, 1, msg);
+      }
+      client->sendEvent(event);
+    }
+    else client->sendEvent(event);
+  }
+  void CAxis::sendDistributedValue(void)
+  {
+    int ns, n, i, j, ind, nv, idx;
+    CContext* context = CContext::getCurrent();
+    CContextClient* client=context->client;
+    // send value for each connected server
+    CEventClient eventIndex(getType(), EVENT_ID_INDEX);
+    CEventClient eventVal(getType(), EVENT_ID_DISTRIBUTED_VALUE);
+    list<CMessage> list_msgsIndex, list_msgsVal;
+    list<CArray<int,1> > list_indi;
+    list<CArray<int,1> > list_writtenInd;
+    list<CArray<double,1> > list_val;
+    list<CArray<double,2> > list_bounds;
+    list<CArray<StdString,1> > list_label;
+    std::map<int, std::vector<size_t> >::const_iterator it, iteMap;
+    iteMap = indSrv_.end();
+    for (int k = 0; k < connectedServerRank_.size(); ++k)
+    {
+      int nbData = 0;
+      int rank = connectedServerRank_[k];
+      it = indSrv_.find(rank);
+      if (iteMap != it)
+        nbData = it->second.size();
+      list_indi.push_back(CArray<int,1>(nbData));
+      list_val.push_back(CArray<double,1>(nbData));
+      if (hasBounds_)
+      {
+        list_bounds.push_back(CArray<double,2>(2,nbData));
+      }
+      if (hasLabel)
+      {
+        list_label.push_back(CArray<StdString,1>(nbData));
+      }
+      CArray<int,1>& indi = list_indi.back();
+      CArray<double,1>& val = list_val.back();
+      for (n = 0; n < nbData; ++n)
+      {
+        idx = static_cast<int>(it->second[n]);
+        ind = idx - begin;
+        val(n) = value(ind);
+        indi(n) = idx;
+        if (hasBounds_)
+        {
+          CArray<double,2>& boundsVal = list_bounds.back();
+          boundsVal(0, n) = bounds(0,n);
+          boundsVal(1, n) = bounds(1,n);
+        }
+        if (hasLabel)
+        {
+          CArray<StdString,1>& labelVal = list_label.back();
+          labelVal(n) = label(n);
+        }
+      }
+      list_msgsIndex.push_back(CMessage());
+      list_msgsIndex.back() << this->getId() << list_indi.back();
+      if (isCompressible_)
+      {
+        std::vector<int>& writtenIndSrc = indWrittenSrv_[rank];
+        list_writtenInd.push_back(CArray<int,1>(writtenIndSrc.size()));
+        CArray<int,1>& writtenInd = list_writtenInd.back();
+        for (n = 0; n < writtenInd.numElements(); ++n)
+          writtenInd(n) = writtenIndSrc[n];
+        list_msgsIndex.back() << writtenInd;
+      }
+      list_msgsVal.push_back(CMessage());
+      list_msgsVal.back() << this->getId() << list_val.back();
+      if (hasBounds_)
+      {
+        list_msgsVal.back() << list_bounds.back();
+      }
+      if (hasLabel)
+      {
+        list_msgsVal.back() << list_label.back();
+      }
+      eventIndex.push(rank, nbConnectedClients_[rank], list_msgsIndex.back());
+      eventVal.push(rank, nbConnectedClients_[rank], list_msgsVal.back());
+    }
+    client->sendEvent(eventIndex);
+    client->sendEvent(eventVal);
+  }
+  void CAxis::recvIndex(CEventServer& event)
+  {
+    CAxis* axis;
+        int ind = data_index(idx);
+        if (ind >= 0 && ind < nbIndex)
+        {
+          dataIndex(nbDataIndex) = ind;
+          ++nbDataIndex;
+        }
+      }
+      if (client->isServerLeader())
+      {
+        std::list<CMessage> msgs;
+        const std::list<int>& ranks = client->getRanksServerLeader();
+        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        {
+          msgs.push_back(CMessage());
+          CMessage& msg = msgs.back();
+          msg << this->getId();
+          msg << index.getValue() << dataIndex << mask.getValue();
+          msg << hasValue;
+          if (hasValue) msg << value.getValue();
+          msg << hasBounds;
+          if (hasBounds) msg << bounds.getValue();
+          msg << hasLabel;
+          if (hasLabel) msg << label.getValue();
+          event.push(*itRank, 1, msg);
+        }
+        client->sendEvent(event);
+      }
+      else client->sendEvent(event);
+    }
+  }
+  /*
+    Receive the non-distributed attributes from another group of clients
+    \param [in] event event containing data of these attributes
+  */
+  void CAxis::recvNonDistributedAttributes(CEventServer& event)
+  {
     list<CEventServer::SSubEvent>::iterator it;
     for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
 …
       string axisId;
       *buffer >> axisId;
+      axis = get(axisId);
+      axis->recvIndex(it->rank, *buffer);
+    }
+    if (axis->isCompressible_)
+    {
+      std::sort(axis->indexesToWrite.begin(), axis->indexesToWrite.end());
+      CContextServer* server = CContext::getCurrent()->server;
+      axis->numberWrittenIndexes_ = axis->indexesToWrite.size();
+      MPI_Allreduce(&axis->numberWrittenIndexes_, &axis->totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      MPI_Scan(&axis->numberWrittenIndexes_, &axis->offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      axis->offsetWrittenIndexes_ -= axis->numberWrittenIndexes_;
+    }
+  }
+  void CAxis::recvIndex(int rank, CBufferIn& buffer)
+  {
+    buffer >> indiSrv_[rank];
+    if (isCompressible_)
+    {
+      CArray<int, 1> writtenIndexes;
+      buffer >> writtenIndexes;
+      indexesToWrite.reserve(indexesToWrite.size() + writtenIndexes.numElements());
+      for (int i = 0; i < writtenIndexes.numElements(); ++i)
+        indexesToWrite.push_back(writtenIndexes(i));
+    }
+  }
+  void CAxis::recvDistributedValue(CEventServer& event)
+  {
+      get(axisId)->recvNonDistributedAttributes(it->rank, *buffer);
+    }
+  }
+  /*
+    Receive the non-distributed attributes from another group of clients
+    \param [in] rank rank of the sender
+    \param [in] buffer buffer containing data sent from the sender
+  */
+  void CAxis::recvNonDistributedAttributes(int rank, CBufferIn& buffer)
+  {
+    CArray<int,1> tmp_index, tmp_data_index, tmp_zoom_index;
+    CArray<bool,1> tmp_mask;
+    CArray<double,1> tmp_val;
+    CArray<double,2> tmp_bnds;
+    CArray<string,1> tmp_label;
+    buffer >> tmp_index;
+    index.reference(tmp_index);
+    buffer >> tmp_data_index;
+    data_index.reference(tmp_data_index);
+    buffer >> tmp_mask;
+    mask.reference(tmp_mask);
+    buffer >> hasValue;
+    if (hasValue)
+    {
+      buffer >> tmp_val;
+      value.reference(tmp_val);
+    }
+    buffer >> hasBounds;
+    if (hasBounds)
+    {
+      buffer >> tmp_bnds;
+      bounds.reference(tmp_bnds);
+    }
+    buffer >> hasLabel;
+    if (hasLabel)
+    {
+      buffer >> tmp_label;
+      label.reference(tmp_label);
+    }
+    // Some value should be reset here
+    data_begin.setValue(0);
+    globalLocalIndexMap_.rehash(std::ceil(index.numElements()/globalLocalIndexMap_.max_load_factor()));
+//    for (int idx = 0; idx < index.numElements(); ++idx) globalLocalIndexMap_[idx] = index(idx);
+    for (int idx = 0; idx < index.numElements(); ++idx) globalLocalIndexMap_[index(idx)] = idx;
+  }
+/*
+    Send attributes of axis from a group of client to other group of clients/servers
+    on supposing that these attributes are distributed among the clients of the sending group
+    In the future, if new attributes are added, they should also be processed in this function
+  */
+  void CAxis::sendDistributedAttributes(void)
+  {
+    int ns, n, i, j, ind, nv, idx;
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      CContextClient* client = *it;
+      int nbServer = client->serverSize;
+      CEventClient eventData(getType(), EVENT_ID_DISTRIBUTED_ATTRIBUTES);
+      list<CMessage> listData;
+      list<CArray<int,1> > list_indi, list_dataInd, list_zoomInd;
+      list<CArray<bool,1> > list_mask;
+      list<CArray<double,1> > list_val;
+      list<CArray<double,2> > list_bounds;
+      list<CArray<string,1> > list_label;
+      int nbIndex = index.numElements();
+      CArray<int,1> dataIndex(nbIndex);
+      dataIndex = -1;
+      for (idx = 0; idx < data_index.numElements(); ++idx)
+      {
+        if (0 <= data_index(idx) && data_index(idx) < nbIndex)
+          dataIndex(idx) = 1;
+      }
+      boost::unordered_map<int, std::vector<size_t> >::const_iterator it, iteMap;
+      iteMap = indSrv_[nbServer].end();
+      for (int k = 0; k < connectedServerRank_[nbServer].size(); ++k)
+      {
+        int nbData = 0;
+        int rank = connectedServerRank_[nbServer][k];
+        it = indSrv_[nbServer].find(rank);
+        if (iteMap != it)
+          nbData = it->second.size();
+        list_indi.push_back(CArray<int,1>(nbData));
+        list_dataInd.push_back(CArray<int,1>(nbData));
+        list_mask.push_back(CArray<bool,1>(nbData));
+        if (hasValue)
+          list_val.push_back(CArray<double,1>(nbData));
+        if (hasBounds)
+          list_bounds.push_back(CArray<double,2>(2,nbData));
+        if (hasLabel)
+          list_label.push_back(CArray<string,1>(nbData));
+        CArray<int,1>& indi = list_indi.back();
+        CArray<int,1>& dataIndi = list_dataInd.back();
+        CArray<bool,1>& maskIndi = list_mask.back();
+        for (n = 0; n < nbData; ++n)
+        {
+          idx = static_cast<int>(it->second[n]);
+          indi(n) = idx;
+          ind = globalLocalIndexMap_[idx];
+          dataIndi(n) = dataIndex(ind);
+          maskIndi(n) = mask(ind);
+          if (hasValue)
+          {
+            CArray<double,1>& val = list_val.back();
+            val(n) = value(ind);
+          }
+          if (hasBounds)
+          {
+            CArray<double,2>& boundsVal = list_bounds.back();
+            boundsVal(0, n) = bounds(0,ind);
+            boundsVal(1, n) = bounds(1,ind);
+          }
+          if (hasLabel)
+          {
+            CArray<string,1>& labelVal = list_label.back();
+            labelVal(n) = label(ind);
+          }
+        }
+        listData.push_back(CMessage());
+        listData.back() << this->getId()
+                        << list_indi.back() << list_dataInd.back() << list_mask.back();
+        listData.back() << hasValue;
+        if (hasValue)
+          listData.back() << list_val.back();
+        listData.back() << hasBounds;
+        if (hasBounds)
+          listData.back() << list_bounds.back();
+        listData.back() << hasLabel;
+        if (hasLabel)
+          listData.back() << list_label.back();
+        eventData.push(rank, nbSenders[nbServer][rank], listData.back());
+      }
+      client->sendEvent(eventData);
+    }
+  }
+  /*
+    Receive the distributed attributes from another group of clients
+    \param [in] event event containing data of these attributes
+  */
+  void CAxis::recvDistributedAttributes(CEventServer& event)
+  {
+    string axisId;
+    vector<int> ranks;
+    vector<CBufferIn*> buffers;
     list<CEventServer::SSubEvent>::iterator it;
     for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+      ranks.push_back(it->rank);
       CBufferIn* buffer = it->buffer;
-      string axisId;
       *buffer >> axisId;
+      get(axisId)->recvDistributedValue(it->rank, *buffer);
+    }
+  }
+  void CAxis::recvDistributedValue(int rank, CBufferIn& buffer)
+  {
+    CArray<int,1> &indi = indiSrv_[rank];
+    CArray<double,1> val;
+    CArray<double,2> boundsVal;
+    CArray<StdString,1> labelVal;
+    buffer >> val;
+    if (hasBounds_) buffer >> boundsVal;
+    if (hasLabel) buffer >> labelVal;
+    int i, j, ind_srv;
+    for (int ind = 0; ind < indi.numElements(); ++ind)
+    {
+      i = indi(ind);
+      ind_srv = i - zoom_begin_srv;
+      value_srv(ind_srv) = val(ind);
+      if (hasBounds_)
+      {
+        bound_srv(0,ind_srv) = boundsVal(0, ind);
+        bound_srv(1,ind_srv) = boundsVal(1, ind);
+      }
+      buffers.push_back(buffer);
+    }
+    get(axisId)->recvDistributedAttributes(ranks, buffers);
+  }
+  /*
+    Receive the non-distributed attributes from another group of clients
+    \param [in] ranks rank of the sender
+    \param [in] buffers buffer containing data sent from the sender
+  */
+  void CAxis::recvDistributedAttributes(vector<int>& ranks, vector<CBufferIn*> buffers)
+  {
+    int nbReceived = ranks.size(), idx, ind, gloInd, locInd;
+    vector<CArray<int,1> > vec_indi(nbReceived), vec_dataInd(nbReceived), vec_zoomInd(nbReceived);
+    vector<CArray<bool,1> > vec_mask(nbReceived);
+    vector<CArray<double,1> > vec_val(nbReceived);
+    vector<CArray<double,2> > vec_bounds(nbReceived);
+    vector<CArray<string,1> > vec_label(nbReceived);
+    for (idx = 0; idx < nbReceived; ++idx)
+    {
+      CBufferIn& buffer = *buffers[idx];
+      buffer >> vec_indi[idx];
+      buffer >> vec_dataInd[idx];
+      buffer >> vec_mask[idx];
+      buffer >> hasValue;
+      if (hasValue)
+        buffer >> vec_val[idx];
+      buffer >> hasBounds;
+      if (hasBounds)
+        buffer >> vec_bounds[idx];
+      buffer >> hasLabel;
       if (hasLabel)
+      {
+        label_srv(ind_srv) = labelVal( ind);
+      }
+    }
+  }
+   void CAxis::recvNonDistributedValue(CEventServer& event)
+  {
+    CAxis* axis;
+    list<CEventServer::SSubEvent>::iterator it;
+    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+      CBufferIn* buffer = it->buffer;
+      string axisId;
+      *buffer >> axisId;
+      axis = get(axisId);
+      axis->recvNonDistributedValue(it->rank, *buffer);
+    }
+    if (axis->isCompressible_)
+    {
+      std::sort(axis->indexesToWrite.begin(), axis->indexesToWrite.end());
+      axis->numberWrittenIndexes_ = axis->totalNumberWrittenIndexes_ = axis->indexesToWrite.size();
+      axis->offsetWrittenIndexes_ = 0;
+    }
+  }
+  void CAxis::recvNonDistributedValue(int rank, CBufferIn& buffer)
+  {
+    CArray<double,1> val;
+    buffer >> val;
+    for (int ind = 0; ind < val.numElements(); ++ind)
+    {
+      value_srv(ind) = val(ind);
+      if (hasBounds_)
+      {
+        bound_srv(0,ind) = bounds(0,ind);
+        bound_srv(1,ind) = bounds(1,ind);
+      }
+      if (hasLabel)
+      {
+        label_srv(ind) = label(ind);
+      }
+    }
+    if (isCompressible_)
+    {
+      CArray<int, 1> writtenIndexes;
+      buffer >> writtenIndexes;
+      indexesToWrite.reserve(indexesToWrite.size() + writtenIndexes.numElements());
+      for (int i = 0; i < writtenIndexes.numElements(); ++i)
+        indexesToWrite.push_back(writtenIndexes(i));
+    }
+  }
+  void CAxis::sendServerAttribut(const std::vector<int>& globalDim, int orderPositionInGrid,
+                                 CServerDistributionDescription::ServerDistributionType distType)
+  {
+    CContext* context = CContext::getCurrent();
+    CContextClient* client = context->client;
+    int nbServer = client->serverSize;
+    CServerDistributionDescription serverDescription(globalDim, nbServer);
+    serverDescription.computeServerDistribution();
+    std::vector<std::vector<int> > serverIndexBegin = serverDescription.getServerIndexBegin();
+    std::vector<std::vector<int> > serverDimensionSizes = serverDescription.getServerDimensionSizes();
+    CEventClient event(getType(),EVENT_ID_SERVER_ATTRIBUT);
+    if (client->isServerLeader())
+    {
+      std::list<CMessage> msgs;
+      const std::list<int>& ranks = client->getRanksServerLeader();
+      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+      {
+        // Use const int to ensure CMessage holds a copy of the value instead of just a reference
+        const int begin = serverIndexBegin[*itRank][orderPositionInGrid];
+        const int ni    = serverDimensionSizes[*itRank][orderPositionInGrid];
+        const int end   = begin + ni - 1;
+        const bool zoomIndex = zoomByIndex();
+        msgs.push_back(CMessage());
+        CMessage& msg = msgs.back();
+        msg << this->getId();
+        msg << ni << begin << end;
+        msg << global_zoom_begin.getValue() << global_zoom_n.getValue();
+        msg << isCompressible_;
+        msg << zoomIndex;
+        if (zoomIndex)
+          msg << global_zoom_index.getValue();
+        event.push(*itRank,1,msg);
+      }
+      client->sendEvent(event);
+    }
+    else client->sendEvent(event);
+  }
+  void CAxis::recvServerAttribut(CEventServer& event)
+  {
+    CBufferIn* buffer = event.subEvents.begin()->buffer;
+    string axisId;
+    *buffer >> axisId;
+    get(axisId)->recvServerAttribut(*buffer);
+  }
+  void CAxis::recvServerAttribut(CBufferIn& buffer)
+  {
+    int ni_srv, begin_srv, end_srv, global_zoom_begin_tmp, global_zoom_n_tmp;
+    bool zoomIndex;
+    CArray<int,1> zoom_index_recv;
+    std::vector<int> zoom_index_tmp;
+    std::vector<int>::iterator itZoomBeginSrv, itZoomEndSrv, itZoomSrv;
+    buffer >> ni_srv >> begin_srv >> end_srv;
+    buffer >> global_zoom_begin_tmp >> global_zoom_n_tmp;
+    buffer >> isCompressible_;
+    buffer >> zoomIndex;
+    if (zoomIndex)
+    {
+      buffer >> zoom_index_recv;
+      global_zoom_index.reference(zoom_index_recv);
+      zoom_index_tmp.resize(global_zoom_index.numElements());
+      std::copy(global_zoom_index.begin(), global_zoom_index.end(), zoom_index_tmp.begin());
+      std::sort(zoom_index_tmp.begin(), zoom_index_tmp.end());
+      itZoomBeginSrv = std::lower_bound(zoom_index_tmp.begin(), zoom_index_tmp.end(), begin_srv);
+      itZoomEndSrv   = std::upper_bound(zoom_index_tmp.begin(), zoom_index_tmp.end(), end_srv);
+      int sz = std::distance(itZoomBeginSrv, itZoomEndSrv);
+      zoom_index_srv.resize(sz);
+      itZoomSrv = itZoomBeginSrv;
+      for (int i = 0; i < sz; ++i, ++itZoomSrv)
+      {
+        zoom_index_srv(i) = *(itZoomSrv);
+      }
+    }
+    global_zoom_begin = global_zoom_begin_tmp;
+    global_zoom_n  = global_zoom_n_tmp;
+    int global_zoom_end = global_zoom_begin + global_zoom_n - 1;
+    zoom_begin_srv = zoomIndex ? std::distance(itZoomBeginSrv, zoom_index_tmp.begin())
+                                 : global_zoom_begin > begin_srv ? global_zoom_begin : begin_srv ;
+    zoom_end_srv   = zoomIndex ? std::distance(zoom_index_tmp.begin(), itZoomEndSrv) - 1
+                                 : global_zoom_end < end_srv ? global_zoom_end : end_srv ;
+    zoom_size_srv  = zoom_end_srv - zoom_begin_srv + 1;
+    global_zoom_begin_srv = zoomIndex ? 0 : global_zoom_begin ;
+    global_zoom_size_srv  = zoomIndex ? zoom_index_tmp.size() : global_zoom_n;
+    if (zoom_size_srv<=0)
+    {
+      zoom_begin_srv = 0; zoom_end_srv = 0; zoom_size_srv = 0;
+    }
+    if (n_glo == n)
+    {
+      zoom_begin_srv = zoomIndex ? std::distance(itZoomBeginSrv, zoom_index_tmp.begin())
+                                   : global_zoom_begin;
+      zoom_size_srv  = zoomIndex ? zoom_index_tmp.size()
+                                   : global_zoom_n;
+    }
+        buffer >> vec_label[idx];
+    }
+    // Estimate size of index array
+    int nbIndexGlob = 0;
+    for (idx = 0; idx < nbReceived; ++idx)
+    {
+      nbIndexGlob += vec_indi[idx].numElements();
+    }
+    // Recompute global index
+    // Take account of the overlapped index
+    index.resize(nbIndexGlob);
+    globalLocalIndexMap_.rehash(std::ceil(index.numElements()/globalLocalIndexMap_.max_load_factor()));
+    nbIndexGlob = 0;
+    for (idx = 0; idx < nbReceived; ++idx)
+    {
+      CArray<int,1>& tmp = vec_indi[idx];
+      for (ind = 0; ind < tmp.numElements(); ++ind)
+      {
+         gloInd = tmp(ind);
+         if (0 == globalLocalIndexMap_.count(gloInd))
+         {
+           index(nbIndexGlob) = gloInd % n_glo;
+           globalLocalIndexMap_[gloInd] = nbIndexGlob;
+           ++nbIndexGlob;
+         }
+      }
+    }
+    // Resize index to its real size
+    if (nbIndexGlob==0) index.resize(nbIndexGlob) ;
+    else index.resizeAndPreserve(nbIndexGlob);
+    int nbData = nbIndexGlob;
+    CArray<int,1> nonCompressedData(nbData);
+    nonCompressedData = -1;
+    mask.resize(nbData);
     if (hasValue)
+    {
+      value_srv.resize(zoom_size_srv);
+      if (hasBounds_)  bound_srv.resize(2,zoom_size_srv);
+      if (hasLabel)  label_srv.resize(zoom_size_srv);
+    }
+      value.resize(nbData);
+    if (hasBounds)
+      bounds.resize(2,nbData);
+    if (hasLabel)
+      label.resize(nbData);
+    nbData = 0;
+    for (idx = 0; idx < nbReceived; ++idx)
+    {
+      CArray<int,1>& indi = vec_indi[idx];
+      CArray<int,1>& dataIndi = vec_dataInd[idx];
+      CArray<bool,1>& maskIndi = vec_mask[idx];
+      int nb = indi.numElements();
+      for (int n = 0; n < nb; ++n)
+      {
+        locInd = globalLocalIndexMap_[size_t(indi(n))];
+        nonCompressedData(locInd) = (-1 == nonCompressedData(locInd)) ? dataIndi(n) : nonCompressedData(locInd);
+        if (!mask(locInd)) // Only rewrite mask if it's not true
+          mask(locInd) = maskIndi(n);
+        if (hasValue)
+          value(locInd) = vec_val[idx](n);
+        if (hasBounds)
+        {
+          bounds(0,locInd) = vec_bounds[idx](0,n);
+          bounds(1,locInd) = vec_bounds[idx](1,n);
+        }
+        if (hasLabel)
+          label(locInd) = vec_label[idx](n);
+      }
+    }
+    int nbCompressedData = 0;
+    for (idx = 0; idx < nonCompressedData.numElements(); ++idx)
+    {
+      if (0 <= nonCompressedData(idx))
+        ++nbCompressedData;
+    }
+    data_index.resize(nbCompressedData);
+    nbCompressedData = 0;
+    for (idx = 0; idx < nonCompressedData.numElements(); ++idx)
+    {
+      if (0 <= nonCompressedData(idx))
+      {
+        data_index(nbCompressedData) = idx % n;
+        ++nbCompressedData;
+      }
+    }
+    data_begin.setValue(0);
+  }
 …
+  }
+  /*
+    Add transformation into axis. This function only servers for Fortran interface
+    \param [in] transType transformation type
+    \param [in] id identifier of the transformation object
+  */
   CTransformation<CAxis>* CAxis::addTransformation(ETranformationType transType, const StdString& id)
+  {
 …
+  }
+  /*
+    Check whether an axis has (spatial) transformation
+  */
   bool CAxis::hasTransformation()
+  {
 …
+  }
+  /*
+    Set transformation
+    \param [in] axisTrans transformation to set
+  */
   void CAxis::setTransformations(const TransMapTypes& axisTrans)
+  {
 …
+  }
+  /*
+    Return all transformation held by the axis
+    \return transformation the axis has
+  */
   CAxis::TransMapTypes CAxis::getAllTransformations(void)
+  {
 …
+  }
+  /*
+    Duplicate transformation of another axis
+    \param [in] src axis whose transformations are copied
+  */
   void CAxis::duplicateTransformation(CAxis* src)
+  {
 …
+  }
+  void CAxis::setContextClient(CContextClient* contextClient)
+  {
+    if (clientsSet.find(contextClient)==clientsSet.end())
+    {
+      clients.push_back(contextClient) ;
+      clientsSet.insert(contextClient);
+    }
+}
   void CAxis::parse(xml::CXMLNode & node)
+  {

XIOS/dev/branch_openmp/src/node/axis.hpp

-                      r1331
+                      r1460
 #include "transformation.hpp"
 #include "transformation_enum.hpp"
+#include "mpi_std.hpp"
 namespace xios {
 …
       , public CAxisAttributes
+   {
+               /// typedef ///
+         typedef CObjectTemplate<CAxis>   SuperClass;
+         typedef CAxisAttributes SuperClassAttribute;
+      public:
          enum EEventId
+         {
+           EVENT_ID_SERVER_ATTRIBUT,
+           EVENT_ID_INDEX,
+           EVENT_ID_DISTRIBUTION_ATTRIBUTE,
            EVENT_ID_DISTRIBUTED_VALUE,
+           EVENT_ID_NON_DISTRIBUTED_VALUE
+           EVENT_ID_NON_DISTRIBUTED_VALUE,
+           EVENT_ID_NON_DISTRIBUTED_ATTRIBUTES,
+           EVENT_ID_DISTRIBUTED_ATTRIBUTES
          } ;
+         /// typedef ///
+         typedef CObjectTemplate<CAxis>   SuperClass;
+         typedef CAxisAttributes SuperClassAttribute;
+      public :
+      public:
          typedef CAxisAttributes RelAttributes;
          typedef CAxisGroup      RelGroup;
 …
          const std::set<StdString> & getRelFiles(void) const;
          const std::vector<int>& getIndexesToWrite(void) const;
          int getNumberWrittenIndexes() const;
          int getTotalNumberWrittenIndexes() const;
          int getOffsetWrittenIndexes() const;
+         int getNumberWrittenIndexes(ep_lib::MPI_Comm writtenCom);
+         int getTotalNumberWrittenIndexes(ep_lib::MPI_Comm writtenCom);
+         int getOffsetWrittenIndexes(ep_lib::MPI_Comm writtenCom);
+         CArray<int, 1>& getCompressedIndexToWriteOnServer(ep_lib::MPI_Comm writtenCom);
+         std::map<int, StdSize> getAttributesBufferSize();
+         std::map<int, StdSize> getAttributesBufferSize(CContextClient* client, const std::vector<int>& globalDim, int orderPositionInGrid,
+                                                        CServerDistributionDescription::ServerDistributionType disType = CServerDistributionDescription::BAND_DISTRIBUTION);
          /// Test ///
 …
          virtual void parse(xml::CXMLNode & node);
+         void setContextClient(CContextClient* contextClient);
          /// Accesseurs statiques ///
          static StdString GetName(void);
 …
          static ENodeType GetType(void);
+         void sendServerAttribut(const std::vector<int>& globalDim, int orderPositionInGrid,
+                                 CServerDistributionDescription::ServerDistributionType distType);
+         static bool dispatchEvent(CEventServer& event);
+         static void recvServerAttribut(CEventServer& event);
+         void recvServerAttribut(CBufferIn& buffer) ;
+         static bool dispatchEvent(CEventServer& event);
          void checkAttributesOnClient();
          void checkAttributesOnClientAfterTransformation(const std::vector<int>& globalDim, int orderPositionInGrid,
 …
          void checkEligibilityForCompressedOutput();
+         size_t getGlobalWrittenSize(void) ;
+         void computeWrittenIndex();
+         void computeWrittenCompressedIndex(ep_lib::MPI_Comm);
          bool hasTransformation();
          void solveInheritanceTransformation();
+         TransMapTypes getAllTransformations();
+         void fillInValues(const CArray<double,1>& values);
+         TransMapTypes getAllTransformations();
          void duplicateTransformation(CAxis*);
          CTransformation<CAxis>* addTransformation(ETranformationType transType, const StdString& id="");
          bool isEqual(CAxis* axis);
+         bool zoomByIndex();
+      public:
+        int zoom_begin_srv, zoom_end_srv, zoom_size_srv;
+        int ni_srv, begin_srv, end_srv;
+        int global_zoom_begin_srv, global_zoom_end_srv, global_zoom_size_srv;
+        CArray<double,1> value_srv;
+        CArray<double,2> bound_srv;
+        CArray<StdString,1> label_srv;
+        CArray<int,1> zoom_index_srv;
+        bool hasValue;
+      public:
+        bool hasValue;
+        CArray<size_t,1> localIndexToWriteOnServer;
       private:
 …
          void checkZoom();
          void checkBounds();
+         void checkLabel();
+         void sendValue();
+         void computeConnectedServer(const std::vector<int>& globalDim, int orderPositionInGrid,
+         void checkLabel();
+         void sendAttributes(const std::vector<int>& globalDim, int orderPositionInGrid,
+                             CServerDistributionDescription::ServerDistributionType distType);
+         void sendDistributionAttribute(const std::vector<int>& globalDim, int orderPositionInGrid,
+                                        CServerDistributionDescription::ServerDistributionType distType);
+         void computeConnectedClients(const std::vector<int>& globalDim, int orderPositionInGrid,
                                      CServerDistributionDescription::ServerDistributionType distType);
-         void sendDistributedValue();
-         void sendNonDistributedValue();
-         bool zoomByIndex();
+         static void recvIndex(CEventServer& event);
+         static void recvDistributedValue(CEventServer& event);
+         static void recvNonDistributedValue(CEventServer& event);
+         void recvIndex(int rank, CBufferIn& buffer);
+         void recvDistributedValue(int rank, CBufferIn& buffer);
+         void recvNonDistributedValue(int rank, CBufferIn& buffer);
+         void sendNonDistributedAttributes(void);
+         void sendDistributedAttributes(void);
+         static void recvNonDistributedAttributes(CEventServer& event);
+         static void recvDistributedAttributes(CEventServer& event);
+         static void recvDistributionAttribute(CEventServer& event);
+         void recvNonDistributedAttributes(int rank, CBufferIn& buffer);
+         void recvDistributedAttributes(vector<int>& rank, vector<CBufferIn*> buffers);
+         void recvDistributionAttribute(CBufferIn& buffer);
          void setTransformations(const TransMapTypes&);
       private:
+/** Clients that have to send a domain. There can be multiple clients in case of secondary server, otherwise only one client. */
+         std::list<CContextClient*> clients;
+         std::set<CContextClient*> clientsSet;
          bool isChecked;
          bool areClientAttributesChecked_;
          bool isClientAfterTransformationChecked;
          std::set<StdString> relFiles, relFilesCompressed;
+         TransMapTypes transformationMap_;
+         bool isDistributed_;
+         TransMapTypes transformationMap_;
          //! True if and only if the data defined on the axis can be outputted in a compressed way
          bool isCompressible_;
+         std::map<int,int> nbConnectedClients_; // Mapping of number of communicating client to a server
+         std::map<int, vector<size_t> > indSrv_; // Global index of each client sent to server
+         std::map<int, vector<int> > indWrittenSrv_; // Global written index of each client sent to server
+         std::map<int, map<int,int> > nbSenders; // Mapping of number of communicating client to a server
+         std::map<int, boost::unordered_map<int, vector<size_t> > > indSrv_; // Global index of each client sent to server
+         // std::map<int, vector<int> > indWrittenSrv_; // Global written index of each client sent to server
+         boost::unordered_map<size_t,size_t> globalLocalIndexMap_;
          std::vector<int> indexesToWrite;
+         int numberWrittenIndexes_, totalNumberWrittenIndexes_, offsetWrittenIndexes_;
+         std::vector<int> connectedServerRank_;
+         std::map<int, CArray<int,1> > indiSrv_;
+         bool hasBounds_;
+         bool hasLabel;
+         std::map<int,int> numberWrittenIndexes_, totalNumberWrittenIndexes_, offsetWrittenIndexes_;
+         std::map<int, CArray<int, 1> > compressedIndexToWriteOnServer;
+         std::map<int, std::vector<int> > connectedServerRank_;
+         bool hasBounds;
+         bool hasLabel;
+         bool computedWrittenIndex_;
        private:

XIOS/dev/branch_openmp/src/node/context.cpp

-                      r1342
+                      r1460
 #include "context.hpp"
 #include "attribute_template.hpp"
 …
 #include "timer.hpp"
 #include "memtrack.hpp"
+#include <limits>
+#include <fstream>
+#include "server.hpp"
+#include "distribute_file_server2.hpp"
 using namespace ep_lib;
 namespace xios {
   boost::shared_ptr<CContextGroup> * CContext::root_ptr = 0;
    /// ////////////////////// Dfinitions ////////////////////// ///
+   /// ////////////////////// DÃ©finitions ////////////////////// ///
    CContext::CContext(void)
       : CObjectTemplate<CContext>(), CContextAttributes()
+      , calendar(), hasClient(false), hasServer(false), isPostProcessed(false), finalized(false)
+      , calendar(), hasClient(false), hasServer(false)
+      , isPostProcessed(false), finalized(false)
       , idServer_(), client(0), server(0)
+      , allProcessed(false), countChildCtx_(0)
    { /* Ne rien faire de plus */ }
    CContext::CContext(const StdString & id)
       : CObjectTemplate<CContext>(id), CContextAttributes()
+      , calendar(), hasClient(false), hasServer(false), isPostProcessed(false), finalized(false)
+      , calendar(), hasClient(false), hasServer(false)
+      , isPostProcessed(false), finalized(false)
       , idServer_(), client(0), server(0)
+      , allProcessed(false), countChildCtx_(0)
    { /* Ne rien faire de plus */ }
 …
      delete client;
      delete server;
+     for (std::vector<CContextClient*>::iterator it = clientPrimServer.begin(); it != clientPrimServer.end(); it++)  delete *it;
+     for (std::vector<CContextServer*>::iterator it = serverPrimServer.begin(); it != serverPrimServer.end(); it++)  delete *it;
+   }
 …
       if (!this->hasChild())
+      {
          //oss << "<!-- No definition -->" << std::endl; // fait planter l'incrmentation
+         //oss << "<!-- No definition -->" << std::endl; // fait planter l'incrÃ©mentation
+      }
       else
 …
    void CContext::initClient(ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* cxtServer /*= 0*/)
+   {
+     hasClient=true;
+     client = new CContextClient(this,intraComm, interComm, cxtServer);
+     registryIn=new CRegistry(intraComm);
+     registryIn->setPath(getId()) ;
+     if (client->clientRank==0) registryIn->fromFile("xios_registry.bin") ;
+     registryIn->bcastRegistry() ;
+     registryOut=new CRegistry(intraComm) ;
+     registryOut->setPath(getId()) ;
+     hasClient = true;
      ep_lib::MPI_Comm intraCommServer, interCommServer;
+     if (cxtServer) // Attached mode
+     {
+       intraCommServer = intraComm;
+       interCommServer = interComm;
+     if (CServer::serverLevel != 1)
+      // initClient is called by client
+     {
+       client = new CContextClient(this, intraComm, interComm, cxtServer);
+       if (cxtServer) // Attached mode
+       {
+         intraCommServer = intraComm;
+         interCommServer = interComm;
+       }
+       else
+       {
+         ep_lib::MPI_Comm_dup(intraComm, &intraCommServer);
+         comms.push_back(intraCommServer);
+         ep_lib::MPI_Comm_dup(interComm, &interCommServer);
+         comms.push_back(interCommServer);
+       }
+/* for registry take the id of client context */
+/* for servers, supress the _server_ from id  */
+       string contextRegistryId=getId() ;
+       size_t pos=contextRegistryId.find("_server_") ;
+       if (pos!=std::string::npos)  contextRegistryId=contextRegistryId.substr(0,pos) ;
+       registryIn=new CRegistry(intraComm);
+       registryIn->setPath(contextRegistryId) ;
+       if (client->clientRank==0) registryIn->fromFile("xios_registry.bin") ;
+       registryIn->bcastRegistry() ;
+       registryOut=new CRegistry(intraComm) ;
+       registryOut->setPath(contextRegistryId) ;
+       server = new CContextServer(this, intraCommServer, interCommServer);
+     }
      else
+     {
+     // initClient is called by primary server
+     {
+       clientPrimServer.push_back(new CContextClient(this, intraComm, interComm));
        ep_lib::MPI_Comm_dup(intraComm, &intraCommServer);
        comms.push_back(intraCommServer);
        ep_lib::MPI_Comm_dup(interComm, &interCommServer);
        comms.push_back(interCommServer);
+     }
+     server = new CContextServer(this,intraCommServer,interCommServer);
+   }
+   void CContext::setClientServerBuffer()
+   {
+     // Estimated minimum event size for small events (10 is an arbitrary constant just for safety)
+       serverPrimServer.push_back(new CContextServer(this, intraCommServer, interCommServer));
+     }
+   }
+   /*!
+    Sets client buffers.
+    \param [in] contextClient
+    \param [in] bufferForWriting True if buffers are used for sending data for writing
+    This flag is only true for client and server-1 for communication with server-2
+  */
+   void CContext::setClientServerBuffer(CContextClient* contextClient, bool bufferForWriting)
+   {
+      // Estimated minimum event size for small events (10 is an arbitrary constant just for safety)
      const size_t minEventSize = CEventClient::headerSize + getIdServer().size() + 10 * sizeof(int);
+     // Ensure there is at least some room for 20 of such events in the buffers
+     size_t minBufferSize = std::max(CXios::minBufferSize, 20 * minEventSize);
+      // Ensure there is at least some room for 20 of such events in the buffers
+      size_t minBufferSize = std::max(CXios::minBufferSize, 20 * minEventSize);
 #define DECLARE_NODE(Name_, name_)    \
      if (minBufferSize < sizeof(C##Name_##Definition)) minBufferSize = sizeof(C##Name_##Definition);
 …
      // Compute the buffer sizes needed to send the attributes and data corresponding to fields
      std::map<int, StdSize> maxEventSize;
      std::map<int, StdSize> bufferSize = getAttributesBufferSize(maxEventSize);
      std::map<int, StdSize> dataBufferSize = getDataBufferSize(maxEventSize);
+     std::map<int, StdSize> bufferSize = getAttributesBufferSize(maxEventSize, contextClient, bufferForWriting);
+     std::map<int, StdSize> dataBufferSize = getDataBufferSize(maxEventSize, contextClient, bufferForWriting);
      std::map<int, StdSize>::iterator it, ite = dataBufferSize.end();
 …
        if (it->second > bufferSize[it->first]) bufferSize[it->first] = it->second;
      // Apply the buffer size factor and check that we are above the minimum buffer size
+     // Apply the buffer size factor, check that we are above the minimum buffer size and below the maximum size
      ite = bufferSize.end();
      for (it = bufferSize.begin(); it != ite; ++it)
 …
        it->second *= CXios::bufferSizeFactor;
        if (it->second < minBufferSize) it->second = minBufferSize;
+       if (it->second > CXios::maxBufferSize) it->second = CXios::maxBufferSize;
+     }
      // Leaders will have to send some control events so ensure there is some room for those in the buffers
      if (client->isServerLeader())
+     {
        const std::list<int>& ranks = client->getRanksServerLeader();
+     if (contextClient->isServerLeader())
+     {
+       const std::list<int>& ranks = contextClient->getRanksServerLeader();
        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+       {
 …
+       }
+     }
+     client->setBufferSize(bufferSize, maxEventSize);
+     contextClient->setBufferSize(bufferSize, maxEventSize);
+   }
 …
+   }
-   //! Initialize server
    void CContext::initServer(ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* cxtClient /*= 0*/)
+   {
 …
      server = new CContextServer(this,intraComm,interComm);
+/* for registry take the id of client context */
+/* for servers, supress the _server_ from id  */
+     string contextRegistryId=getId() ;
+     size_t pos=contextRegistryId.find("_server_") ;
+     if (pos!=std::string::npos)  contextRegistryId=contextRegistryId.substr(0,pos) ;
      registryIn=new CRegistry(intraComm);
      registryIn->setPath(getId()) ;
+     registryIn->setPath(contextRegistryId) ;
      if (server->intraCommRank==0) registryIn->fromFile("xios_registry.bin") ;
      registryIn->bcastRegistry() ;
      registryOut=new CRegistry(intraComm) ;
      registryOut->setPath(getId()) ;
+     registryOut->setPath(contextRegistryId) ;
      ep_lib::MPI_Comm intraCommClient, interCommClient;
 …
    //! Try to send the buffers and receive possible answers
+   bool CContext::checkBuffersAndListen(void)
+   {
+     client->checkBuffers();
+     bool hasTmpBufferedEvent = client->hasTemporarilyBufferedEvent();
+     if (hasTmpBufferedEvent)
+       hasTmpBufferedEvent = !client->sendTemporarilyBufferedEvent();
+     // Don't process events if there is a temporarily buffered event
+     return server->eventLoop(!hasTmpBufferedEvent);
+   }
+  bool CContext::checkBuffersAndListen(bool enableEventsProcessing /*= true*/)
+  {
+    bool clientReady, serverFinished;
+    // Only classical servers are non-blocking
+    if (CServer::serverLevel == 0)
+    {
+      client->checkBuffers();
+      bool hasTmpBufferedEvent = client->hasTemporarilyBufferedEvent();
+      if (hasTmpBufferedEvent)
+        hasTmpBufferedEvent = !client->sendTemporarilyBufferedEvent();
+      // Don't process events if there is a temporarily buffered event
+      return server->eventLoop(!hasTmpBufferedEvent || !enableEventsProcessing);
+    }
+    else if (CServer::serverLevel == 1)
+    {
+      if (!finalized)
+        client->checkBuffers();
+      bool serverFinished = true;
+      if (!finalized)
+        serverFinished = server->eventLoop(enableEventsProcessing);
+      bool serverPrimFinished = true;
+      for (int i = 0; i < clientPrimServer.size(); ++i)
+      {
+        if (!finalized)
+          clientPrimServer[i]->checkBuffers();
+        if (!finalized)
+          serverPrimFinished *= serverPrimServer[i]->eventLoop(enableEventsProcessing);
+      }
+      return ( serverFinished && serverPrimFinished);
+    }
+    else if (CServer::serverLevel == 2)
+    {
+      client->checkBuffers();
+      return server->eventLoop(enableEventsProcessing);
+    }
+  }
    //! Terminate a context
    void CContext::finalize(void)
+   {
       if (!finalized)
+      if (hasClient && !hasServer) // For now we only use server level 1 to read data
+      {
+        finalized = true;
+        if (hasClient) sendRegistry() ;
+        client->finalize();
+        while (!server->hasFinished())
+        {
+          server->eventLoop();
+        }
+        if (hasServer)
+        {
+          closeAllFile();
+          registryOut->hierarchicalGatherRegistry() ;
+          if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
+        }
+        for (std::list<ep_lib::MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
+          ep_lib::MPI_Comm_free(&(*it));
+        comms.clear();
+        doPreTimestepOperationsForEnabledReadModeFiles();
+      }
+     // Send registry upon calling the function the first time
+     if (countChildCtx_ == 0)
+       if (hasClient) sendRegistry() ;
+     // Client:
+     // (1) blocking send context finalize to its server
+     // (2) blocking receive context finalize from its server
+     // (3) some memory deallocations
+     if (CXios::isClient)
+     {
+       // Make sure that client (model) enters the loop only once
+       if (countChildCtx_ < 1)
+       {
+         ++countChildCtx_;
+         client->finalize();
+         while (client->havePendingRequests())
+            client->checkBuffers();
+         while (!server->hasFinished())
+           server->eventLoop();
+         if (hasServer) // Mode attache
+         {
+           closeAllFile();
+           registryOut->hierarchicalGatherRegistry() ;
+           if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
+         }
+         //! Deallocate client buffers
+         client->releaseBuffers();
+         //! Free internally allocated communicators
+         for (std::list<ep_lib::MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
+           ep_lib::MPI_Comm_free(&(*it));
+         comms.clear();
+         #pragma omp critical (_output)
+         info(20)<<"CContext: Context <"<<getId()<<"> is finalized."<<endl;
+       }
+     }
+     else if (CXios::isServer)
+     {
+       // First context finalize message received from a model
+       // Send context finalize to its child contexts (if any)
+       if (countChildCtx_ == 0)
+         for (int i = 0; i < clientPrimServer.size(); ++i)
+           clientPrimServer[i]->finalize();
+       // (Last) context finalized message received
+       if (countChildCtx_ == clientPrimServer.size())
+       {
+         // Blocking send of context finalize message to its client (e.g. primary server or model)
+         #pragma omp critical (_output)
+         info(100)<<"DEBUG: context "<<getId()<<" Send client finalize<<"<<endl ;
+         client->finalize();
+         bool bufferReleased;
+         do
+         {
+           client->checkBuffers();
+           bufferReleased = !client->havePendingRequests();
+         } while (!bufferReleased);
+         finalized = true;
+         closeAllFile(); // Just move to here to make sure that server-level 1 can close files
+         if (hasServer && !hasClient)
+         {
+           registryOut->hierarchicalGatherRegistry() ;
+           if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
+         }
+         //! Deallocate client buffers
+         client->releaseBuffers();
+         for (int i = 0; i < clientPrimServer.size(); ++i)
+           clientPrimServer[i]->releaseBuffers();
+         //! Free internally allocated communicators
+         for (std::list<ep_lib::MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
+           ep_lib::MPI_Comm_free(&(*it));
+         comms.clear();
+         #pragma omp critical (_output)
+         info(20)<<"CContext: Context <"<<getId()<<"> is finalized."<<endl;
+       }
+       ++countChildCtx_;
+     }
+   }
+   //! Free internally allocated communicators
+   void CContext::freeComms(void)
+   {
+     for (std::list<ep_lib::MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
+       ep_lib::MPI_Comm_free(&(*it));
+     comms.clear();
+   }
+   //! Deallocate buffers allocated by clientContexts
+   void CContext::releaseClientBuffers(void)
+   {
+     client->releaseBuffers();
+     for (int i = 0; i < clientPrimServer.size(); ++i)
+       clientPrimServer[i]->releaseBuffers();
+   }
+   void CContext::postProcessingGlobalAttributes()
+   {
+     if (allProcessed) return;
+     // After xml is parsed, there are some more works with post processing
+     postProcessing();
+     // Check grid and calculate its distribution
+     checkGridEnabledFields();
+     // Distribute files between secondary servers according to the data size
+     distributeFiles();
+     setClientServerBuffer(client, (hasClient && !hasServer));
+     for (int i = 0; i < clientPrimServer.size(); ++i)
+         setClientServerBuffer(clientPrimServer[i], true);
+     if (hasClient)
+     {
+      // Send all attributes of current context to server
+      this->sendAllAttributesToServer();
+      // Send all attributes of current calendar
+      CCalendarWrapper::get(CCalendarWrapper::GetDefName())->sendAllAttributesToServer();
+      // We have enough information to send to server
+      // First of all, send all enabled files
+      sendEnabledFiles(this->enabledWriteModeFiles);
+      // We only use server-level 1 (for now) to read data
+      if (!hasServer)
+        sendEnabledFiles(this->enabledReadModeFiles);
+      // Then, send all enabled fields
+      sendEnabledFieldsInFiles(this->enabledWriteModeFiles);
+      if (!hasServer)
+        sendEnabledFieldsInFiles(this->enabledReadModeFiles);
+      // Then, check whether we have domain_ref, axis_ref or scalar_ref attached to the enabled fields
+      // If any, so send them to server
+       sendRefDomainsAxisScalars(this->enabledWriteModeFiles);
+      if (!hasServer)
+        sendRefDomainsAxisScalars(this->enabledReadModeFiles);
+       // Check whether enabled fields have grid_ref, if any, send this info to server
+      sendRefGrid(this->enabledFiles);
+      // This code may be useful in the future when we want to seperate completely read and write
+      // sendRefGrid(this->enabledWriteModeFiles);
+      // if (!hasServer)
+      //   sendRefGrid(this->enabledReadModeFiles);
+      // A grid of enabled fields composed of several components which must be checked then their
+      // checked attributes should be sent to server
+      sendGridComponentEnabledFieldsInFiles(this->enabledFiles); // This code can be seperated in two (one for reading, another for writing)
+       // We have a xml tree on the server side and now, it should be also processed
+      sendPostProcessing();
+      // Finally, we send information of grid itself to server
+      sendGridEnabledFieldsInFiles(this->enabledWriteModeFiles);
+      if (!hasServer)
+        sendGridEnabledFieldsInFiles(this->enabledReadModeFiles);
+     }
+     allProcessed = true;
+   }
+   void CContext::sendPostProcessingGlobalAttributes()
+   {
+      // Use correct context client to send message
+     // int nbSrvPools = (hasServer) ? clientPrimServer.size() : 1;
+    int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
+     for (int i = 0; i < nbSrvPools; ++i)
+     {
+       CContextClient* contextClientTmp = (0 != clientPrimServer.size()) ? clientPrimServer[i] : client;
+       CEventClient event(getType(),EVENT_ID_POST_PROCESS_GLOBAL_ATTRIBUTES);
+       if (contextClientTmp->isServerLeader())
+       {
+         CMessage msg;
+         if (hasServer)
+           msg<<this->getIdServer(i);
+         else
+           msg<<this->getIdServer();
+         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         contextClientTmp->sendEvent(event);
+       }
+       else contextClientTmp->sendEvent(event);
+     }
+   }
+   void CContext::recvPostProcessingGlobalAttributes(CEventServer& event)
+   {
+      CBufferIn* buffer=event.subEvents.begin()->buffer;
+      string id;
+      *buffer>>id;
+      get(id)->recvPostProcessingGlobalAttributes(*buffer);
+   }
+   void CContext::recvPostProcessingGlobalAttributes(CBufferIn& buffer)
+   {
+      postProcessingGlobalAttributes();
+   }
 …
    void CContext::closeDefinition(void)
+   {
+     CTimer::get("Context : close definition").resume() ;
+     // There is nothing client need to send to server
+     if (hasClient)
+     {
+       // After xml is parsed, there are some more works with post processing
+       postProcessing();
+     }
+     setClientServerBuffer();
+    CTimer::get("Context : close definition").resume() ;
+    postProcessingGlobalAttributes();
+    if (hasClient) sendPostProcessingGlobalAttributes();
+    // There are some processings that should be done after all of above. For example: check mask or index
+    this->buildFilterGraphOfEnabledFields();
      if (hasClient && !hasServer)
+     {
+      // Send all attributes of current context to server
+      this->sendAllAttributesToServer();
+      // Send all attributes of current calendar
+      CCalendarWrapper::get(CCalendarWrapper::GetDefName())->sendAllAttributesToServer();
+      // We have enough information to send to server
+      // First of all, send all enabled files
+       sendEnabledFiles();
+      // Then, send all enabled fields
+       sendEnabledFields();
+      // At last, we have all info of domain and axis, then send them
+       sendRefDomainsAxis();
+      // After that, send all grid (if any)
+       sendRefGrid();
+    {
+      buildFilterGraphOfFieldsWithReadAccess();
+      postProcessFilterGraph();
+    }
+    // We have a xml tree on the server side and now, it should be also processed
+    if (hasClient && !hasServer) sendPostProcessing();
+    // There are some processings that should be done after all of above. For example: check mask or index
+    if (hasClient)
+    {
+      this->buildFilterGraphOfEnabledFields();
+      buildFilterGraphOfFieldsWithReadAccess();
+      this->solveAllRefOfEnabledFields(true);
+    }
+    // Now tell server that it can process all messages from client
+    if (hasClient && !hasServer) this->sendCloseDefinition();
+    checkGridEnabledFields();
+    if (hasClient) this->sendProcessingGridOfEnabledFields();
+    if (hasClient) this->sendCloseDefinition();
     // Nettoyage de l'arborescence
     if (hasClient && !hasServer) CleanTree(); // Only on client side??
+    if (hasClient) CleanTree(); // Only on client side??
     if (hasClient)
+    {
       sendCreateFileHeader();
+      startPrefetchingOfEnabledReadModeFiles();
+      if (!hasServer) startPrefetchingOfEnabledReadModeFiles();
+    }
     CTimer::get("Context : close definition").suspend() ;
+   }
+   void CContext::findAllEnabledFields(void)
+   {
+     for (unsigned int i = 0; i < this->enabledFiles.size(); i++)
+     (void)this->enabledFiles[i]->getEnabledFields();
+   }
+   void CContext::findAllEnabledFieldsInReadModeFiles(void)
+   {
+     for (unsigned int i = 0; i < this->enabledReadModeFiles.size(); ++i)
+     (void)this->enabledReadModeFiles[i]->getEnabledFields();
+   void CContext::findAllEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     for (unsigned int i = 0; i < activeFiles.size(); i++)
+     (void)activeFiles[i]->getEnabledFields();
+   }
 …
+   }
+   void CContext::sendGridComponentEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       activeFiles[i]->sendGridComponentOfEnabledFields();
+     }
+   }
+   /*!
+      Send active (enabled) fields in file from a client to others
+      \param [in] activeFiles files contains enabled fields to send
+   */
+   void CContext::sendGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       activeFiles[i]->sendGridOfEnabledFields();
+     }
+   }
+   void CContext::checkGridEnabledFields()
+   {
+     int size = enabledFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       enabledFiles[i]->checkGridOfEnabledFields();
+     }
+   }
+   /*!
+      Check grid of active (enabled) fields in file
+      \param [in] activeFiles files contains enabled fields whose grid needs checking
+   */
+   void CContext::checkGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       activeFiles[i]->checkGridOfEnabledFields();
+     }
+   }
+    /*!
+      Go up the hierachical tree via field_ref and do check of attributes of fields
+      This can be done in a client then all computed information will be sent from this client to others
+      \param [in] sendToServer Flag to indicate whether calculated information will be sent
+   */
    void CContext::solveOnlyRefOfEnabledFields(bool sendToServer)
+   {
 …
+   }
+   void CContext::solveAllRefOfEnabledFields(bool sendToServer)
+    /*!
+      Go up the hierachical tree via field_ref and do check of attributes of fields.
+      The transformation can be done in this step.
+      All computed information will be sent from this client to others.
+      \param [in] sendToServer Flag to indicate whether calculated information will be sent
+   */
+   void CContext::solveAllRefOfEnabledFieldsAndTransform(bool sendToServer)
+   {
      int size = this->enabledFiles.size();
      for (int i = 0; i < size; ++i)
+     {
        this->enabledFiles[i]->solveAllRefOfEnabledFields(sendToServer);
+       this->enabledFiles[i]->solveAllRefOfEnabledFieldsAndTransform(sendToServer);
+     }
+   }
 …
+   }
+   void CContext::postProcessFilterGraph()
+   {
+     int size = enabledFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+        enabledFiles[i]->postProcessFilterGraph();
+     }
+   }
    void CContext::startPrefetchingOfEnabledReadModeFiles()
+   {
 …
+     {
         enabledReadModeFiles[i]->prefetchEnabledReadModeFields();
+     }
+   }
+   void CContext::doPreTimestepOperationsForEnabledReadModeFiles()
+   {
+     int size = enabledReadModeFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+        enabledReadModeFiles[i]->doPreTimestepOperationsForEnabledReadModeFields();
+     }
+   }
 …
+  }
    void CContext::solveAllInheritance(bool apply) // default : apply = true
+   {
      // Rsolution des hritages descendants (cd des hritages de groupes)
+   void CContext::solveAllInheritance(bool apply)
+   {
+     // RÃ©solution des hÃ©ritages descendants (cÃ d des hÃ©ritages de groupes)
      // pour chacun des contextes.
       solveDescInheritance(apply);
      // Rsolution des hritages par rfrence au niveau des fichiers.
+     // RÃ©solution des hÃ©ritages par rÃ©fÃ©rence au niveau des fichiers.
       const vector<CFile*> allFiles=CFile::getAll();
       const vector<CGrid*> allGrids= CGrid::getAll();
      //if (hasClient && !hasServer)
       if (hasClient)
+      if (hasClient && !hasServer)
+      //if (hasClient)
+      {
         for (unsigned int i = 0; i < allFiles.size(); i++)
 …
       for (unsigned int i = 0; i < allFiles.size(); i++)
          if (!allFiles[i]->enabled.isEmpty()) // Si l'attribut 'enabled' est dfini.
+         if (!allFiles[i]->enabled.isEmpty()) // Si l'attribut 'enabled' est dÃ©fini.
+         {
             if (allFiles[i]->enabled.getValue()) // Si l'attribut 'enabled' est fix  vrai.
+            if (allFiles[i]->enabled.getValue()) // Si l'attribut 'enabled' est fixÃ© Ã  vrai.
+            {
               if ((initDate + allFiles[i]->output_freq.getValue()) < (initDate + this->getCalendar()->getTimeStep()))
 …
       if (enabledFiles.size() == 0)
          DEBUG(<<"Aucun fichier ne va tre sorti dans le contexte nomm \""
+         DEBUG(<<"Aucun fichier ne va Ãªtre sorti dans le contexte nommÃ© \""
                << getId() << "\" !");
+   }
+   void CContext::findEnabledReadModeFiles(void)
+   }
+   void CContext::distributeFiles(void)
+   {
+     bool distFileMemory=false ;
+     distFileMemory=CXios::getin<bool>("server2_dist_file_memory", distFileMemory);
+     if (distFileMemory) distributeFileOverMemoryBandwith() ;
+     else distributeFileOverBandwith() ;
+   }
+   void CContext::distributeFileOverBandwith(void)
+   {
+     double eps=std::numeric_limits<double>::epsilon()*10 ;
+     // If primary server
+     if (hasServer && hasClient)
+     {
+       std::ofstream ofs(("distribute_file_"+getId()+".dat").c_str(), std::ofstream::out);
+       int nbPools = clientPrimServer.size();
+       // (1) Find all enabled files in write mode
+       // for (int i = 0; i < this->enabledFiles.size(); ++i)
+       // {
+       //   if (enabledFiles[i]->mode.isEmpty() || (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
+       //    enabledWriteModeFiles.push_back(enabledFiles[i]);
+       // }
+       // (2) Estimate the data volume for each file
+       int size = this->enabledWriteModeFiles.size();
+       std::vector<std::pair<double, CFile*> > dataSizeMap;
+       double dataPerPool = 0;
+       int nfield=0 ;
+       ofs<<size<<endl ;
+       for (size_t i = 0; i < size; ++i)
+       {
+         CFile* file = this->enabledWriteModeFiles[i];
+         ofs<<file->getId()<<endl ;
+         StdSize dataSize=0;
+         std::vector<CField*> enabledFields = file->getEnabledFields();
+         size_t numEnabledFields = enabledFields.size();
+         ofs<<numEnabledFields<<endl ;
+         for (size_t j = 0; j < numEnabledFields; ++j)
+         {
+           dataSize += enabledFields[j]->getGlobalWrittenSize() ;
+           ofs<<enabledFields[j]->grid->getId()<<endl ;
+           ofs<<enabledFields[j]->getGlobalWrittenSize()<<endl ;
+         }
+         double outFreqSec = (Time)(calendar->getCurrentDate()+file->output_freq)-(Time)(calendar->getCurrentDate()) ;
+         double dataSizeSec= dataSize/ outFreqSec;
+         ofs<<dataSizeSec<<endl ;
+         nfield++ ;
+// add epsilon*nField to dataSizeSec in order to  preserve reproductive ordering when sorting
+         dataSizeMap.push_back(make_pair(dataSizeSec + dataSizeSec * eps * nfield , file));
+         dataPerPool += dataSizeSec;
+       }
+       dataPerPool /= nbPools;
+       std::sort(dataSizeMap.begin(), dataSizeMap.end());
+       // (3) Assign contextClient to each enabled file
+       std::multimap<double,int> poolDataSize ;
+// multimap is not garanty to preserve stable sorting in c++98 but it seems it does for c++11
+       int j;
+       double dataSize ;
+       for (j = 0 ; j < nbPools ; ++j) poolDataSize.insert(std::pair<double,int>(0.,j)) ;
+       for (int i = dataSizeMap.size()-1; i >= 0; --i)
+       {
+         dataSize=(*poolDataSize.begin()).first ;
+         j=(*poolDataSize.begin()).second ;
+         dataSizeMap[i].second->setContextClient(clientPrimServer[j]);
+         dataSize+=dataSizeMap[i].first;
+         poolDataSize.erase(poolDataSize.begin()) ;
+         poolDataSize.insert(std::pair<double,int>(dataSize,j)) ;
+       }
+       for (std::multimap<double,int>:: iterator it=poolDataSize.begin() ; it!=poolDataSize.end(); ++it)
+       {
+         #pragma omp critical (_output)
+         info(30)<<"Load Balancing for servers (perfect=1) : "<<it->second<<" :  ratio "<<it->first*1./dataPerPool<<endl ;
+       }
+       for (int i = 0; i < this->enabledReadModeFiles.size(); ++i)
+       {
+         enabledReadModeFiles[i]->setContextClient(client);
+       }
+     }
+     else
+     {
+       for (int i = 0; i < this->enabledFiles.size(); ++i)
+         enabledFiles[i]->setContextClient(client);
+     }
+   }
+   void CContext::distributeFileOverMemoryBandwith(void)
+   {
+     // If primary server
+     if (hasServer && hasClient)
+     {
+       int nbPools = clientPrimServer.size();
+       double ratio=0.5 ;
+       ratio=CXios::getin<double>("server2_dist_file_memory_ratio", ratio);
+       int nFiles = this->enabledWriteModeFiles.size();
+       vector<SDistFile> files(nFiles);
+       vector<SDistGrid> grids;
+       map<string,int> gridMap ;
+       string gridId;
+       int gridIndex=0 ;
+       for (size_t i = 0; i < nFiles; ++i)
+       {
+         StdSize dataSize=0;
+         CFile* file = this->enabledWriteModeFiles[i];
+         std::vector<CField*> enabledFields = file->getEnabledFields();
+         size_t numEnabledFields = enabledFields.size();
+         files[i].id_=file->getId() ;
+         files[i].nbGrids_=numEnabledFields;
+         files[i].assignedGrid_ = new int[files[i].nbGrids_] ;
+         for (size_t j = 0; j < numEnabledFields; ++j)
+         {
+           gridId=enabledFields[j]->grid->getId() ;
+           if (gridMap.find(gridId)==gridMap.end())
+           {
+              gridMap[gridId]=gridIndex  ;
+              SDistGrid newGrid;
+              grids.push_back(newGrid) ;
+              gridIndex++ ;
+           }
+           files[i].assignedGrid_[j]=gridMap[gridId] ;
+           grids[files[i].assignedGrid_[j]].size_=enabledFields[j]->getGlobalWrittenSize() ;
+           dataSize += enabledFields[j]->getGlobalWrittenSize() ; // usefull
+         }
+         double outFreqSec = (Time)(calendar->getCurrentDate()+file->output_freq)-(Time)(calendar->getCurrentDate()) ;
+         files[i].bandwith_= dataSize/ outFreqSec ;
+       }
+       double bandwith=0 ;
+       double memory=0 ;
+       for(int i=0; i<nFiles; i++)  bandwith+=files[i].bandwith_ ;
+       for(int i=0; i<nFiles; i++)  files[i].bandwith_ = files[i].bandwith_/bandwith * ratio ;
+       for(int i=0; i<grids.size(); i++)  memory+=grids[i].size_ ;
+       for(int i=0; i<grids.size(); i++)  grids[i].size_ = grids[i].size_ / memory * (1.0-ratio) ;
+       distributeFileOverServer2(nbPools, grids.size(), &grids[0], nFiles, &files[0]) ;
+       vector<double> memorySize(nbPools,0.) ;
+       vector< set<int> > serverGrids(nbPools) ;
+       vector<double> bandwithSize(nbPools,0.) ;
+       for (size_t i = 0; i < nFiles; ++i)
+       {
+         bandwithSize[files[i].assignedServer_] += files[i].bandwith_* bandwith /ratio ;
+         for(int j=0 ; j<files[i].nbGrids_;j++)
+         {
+           if (serverGrids[files[i].assignedServer_].find(files[i].assignedGrid_[j]) == serverGrids[files[i].assignedServer_].end())
+           {
+             memorySize[files[i].assignedServer_]+= grids[files[i].assignedGrid_[j]].size_ * memory / (1.0-ratio);
+             serverGrids[files[i].assignedServer_].insert(files[i].assignedGrid_[j]) ;
+           }
+         }
+         enabledWriteModeFiles[i]->setContextClient(clientPrimServer[files[i].assignedServer_]) ;
+         delete [] files[i].assignedGrid_ ;
+       }
+       for (int i = 0; i < nbPools; ++i)
+       {
+         #pragma omp critical (_output)
+         info(100)<<"Pool server level2 "<<i<<"   assigned file bandwith "<<bandwithSize[i]*86400.*4./1024/1024.<<" Mb / days"<<endl ;
+       }
+       for (int i = 0; i < nbPools; ++i)
+       {
+         #pragma omp critical (_output)
+         info(100)<<"Pool server level2 "<<i<<"   assigned grid memory "<<memorySize[i]*100/1024./1024.<<" Mb"<<endl ;
+       }
+       for (int i = 0; i < this->enabledReadModeFiles.size(); ++i)
+       {
+         enabledReadModeFiles[i]->setContextClient(client);
+       }
+   }
+   else
+   {
+     for (int i = 0; i < this->enabledFiles.size(); ++i)
+        enabledFiles[i]->setContextClient(client);
+   }
+}
+   /*!
+      Find all files in write mode
+   */
+   void CContext::findEnabledWriteModeFiles(void)
+   {
      int size = this->enabledFiles.size();
      for (int i = 0; i < size; ++i)
+     {
+       if (enabledFiles[i]->mode.isEmpty() ||
+          (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
+        enabledWriteModeFiles.push_back(enabledFiles[i]);
+     }
+   }
+   /*!
+      Find all files in read mode
+   */
+   void CContext::findEnabledReadModeFiles(void)
+   {
+     int size = this->enabledFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
        if (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::read)
         enabledReadModeFiles.push_back(enabledFiles[i]);
 …
      for (; it != end; it++)
+     {
+       #pragma omp critical (_output)
        info(30)<<"Closing File : "<<(*it)->getId()<<endl;
        (*it)->close();
 …
              recvRegistry(event);
              return true;
+            break;
+             break;
+            case EVENT_ID_POST_PROCESS_GLOBAL_ATTRIBUTES:
+             recvPostProcessingGlobalAttributes(event);
+             return true;
+             break;
+            case EVENT_ID_PROCESS_GRID_ENABLED_FIELDS:
+             recvProcessingGridOfEnabledFields(event);
+             return true;
+             break;
            default :
              ERROR("bool CContext::dispatchEvent(CEventServer& event)",
 …
    void CContext::sendCloseDefinition(void)
+   {
+     CEventClient event(getType(),EVENT_ID_CLOSE_DEFINITION);
+     if (client->isServerLeader())
+     {
+       CMessage msg;
+       msg<<this->getIdServer();
+       const std::list<int>& ranks = client->getRanksServerLeader();
+       for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+         event.push(*itRank,1,msg);
+       client->sendEvent(event);
+     }
+     else client->sendEvent(event);
+     // Use correct context client to send message
+     int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
+     for (int i = 0; i < nbSrvPools; ++i)
+     {
+       CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
+       CEventClient event(getType(),EVENT_ID_CLOSE_DEFINITION);
+       if (contextClientTmp->isServerLeader())
+       {
+         CMessage msg;
+         if (hasServer)
+           msg<<this->getIdServer(i);
+         else
+           msg<<this->getIdServer();
+         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         contextClientTmp->sendEvent(event);
+       }
+       else contextClientTmp->sendEvent(event);
+     }
+   }
 …
    void CContext::recvCloseDefinition(CEventServer& event)
+   {
       CBufferIn* buffer=event.subEvents.begin()->buffer;
       string id;
 …
    void CContext::sendUpdateCalendar(int step)
+   {
+     if (!hasServer)
+     {
+     // Use correct context client to send message
+    int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
+     for (int i = 0; i < nbSrvPools; ++i)
+     {
+       CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
        CEventClient event(getType(),EVENT_ID_UPDATE_CALENDAR);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg<<this->getIdServer()<<step;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+         if (contextClientTmp->isServerLeader())
+         {
+           CMessage msg;
+           if (hasServer)
+             msg<<this->getIdServer(i)<<step;
+           else
+             msg<<this->getIdServer()<<step;
+           const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
+           for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+             event.push(*itRank,1,msg);
+           contextClientTmp->sendEvent(event);
+         }
+         else contextClientTmp->sendEvent(event);
+     }
+   }
 …
       buffer>>step;
       updateCalendar(step);
+      if (hasClient && hasServer)
+      {
+        sendUpdateCalendar(step);
+      }
+   }
 …
    void CContext::sendCreateFileHeader(void)
+   {
+     CEventClient event(getType(),EVENT_ID_CREATE_FILE_HEADER);
+     if (client->isServerLeader())
+     {
+       CMessage msg;
+       msg<<this->getIdServer();
+       const std::list<int>& ranks = client->getRanksServerLeader();
+       for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+         event.push(*itRank,1,msg) ;
+       client->sendEvent(event);
+     }
+     else client->sendEvent(event);
+     // Use correct context client to send message
+     // int nbSrvPools = (hasServer) ? clientPrimServer.size() : 1;
+     int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
+     for (int i = 0; i < nbSrvPools; ++i)
+     {
+       CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
+       CEventClient event(getType(),EVENT_ID_CREATE_FILE_HEADER);
+       if (contextClientTmp->isServerLeader())
+       {
+         CMessage msg;
+         if (hasServer)
+           msg<<this->getIdServer(i);
+         else
+           msg<<this->getIdServer();
+         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg) ;
+         contextClientTmp->sendEvent(event);
+       }
+       else contextClientTmp->sendEvent(event);
+     }
+   }
 …
    void CContext::recvCreateFileHeader(CBufferIn& buffer)
+   {
+      createFileHeader();
+      if (!hasClient && hasServer)
+        createFileHeader();
+   }
+   //! Client side: Send a message to do some post processing on server
+   void CContext::sendProcessingGridOfEnabledFields()
+   {
+      // Use correct context client to send message
+     int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
+     for (int i = 0; i < nbSrvPools; ++i)
+     {
+       CContextClient* contextClientTmp = (0 != clientPrimServer.size()) ? clientPrimServer[i] : client;
+       CEventClient event(getType(),EVENT_ID_PROCESS_GRID_ENABLED_FIELDS);
+       if (contextClientTmp->isServerLeader())
+       {
+         CMessage msg;
+         if (hasServer)
+           msg<<this->getIdServer(i);
+         else
+           msg<<this->getIdServer();
+         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         contextClientTmp->sendEvent(event);
+       }
+       else contextClientTmp->sendEvent(event);
+     }
+   }
+   //! Server side: Receive a message to do some post processing
+   void CContext::recvProcessingGridOfEnabledFields(CEventServer& event)
+   {
+      CBufferIn* buffer=event.subEvents.begin()->buffer;
+      string id;
+      *buffer>>id;
+   }
 …
    void CContext::sendPostProcessing()
+   {
+     if (!hasServer)
+     {
+      // Use correct context client to send message
+     // int nbSrvPools = (hasServer) ? clientPrimServer.size() : 1;
+     int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
+     for (int i = 0; i < nbSrvPools; ++i)
+     {
+       CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
        CEventClient event(getType(),EVENT_ID_POST_PROCESS);
        if (client->isServerLeader())
+       if (contextClientTmp->isServerLeader())
+       {
          CMessage msg;
+         msg<<this->getIdServer();
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         if (hasServer)
+           msg<<this->getIdServer(i);
+         else
+           msg<<this->getIdServer();
+         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
            event.push(*itRank,1,msg);
          client->sendEvent(event);
+       }
        else client->sendEvent(event);
+         event.push(*itRank,1,msg);
+         contextClientTmp->sendEvent(event);
+       }
+       else contextClientTmp->sendEvent(event);
+     }
+   }
 …
+   }
+   const StdString& CContext::getIdServer(const int i)
+   {
+     idServer_ = this->getId();
+     idServer_ += "_server_";
+     idServer_ += boost::lexical_cast<string>(i);
+     return idServer_;
+   }
    /*!
    \brief Do some simple post processings after parsing xml file
 …
       this->solveAllInheritance();
+//      ShowTree(info(10));
       // Check if some axis, domains or grids are eligible to for compressed indexed output.
       // Warning: This must be done after solving the inheritance and before the rest of post-processing
       checkAxisDomainsGridsEligibilityForCompressedOutput();
+      checkAxisDomainsGridsEligibilityForCompressedOutput();
       // Check if some automatic time series should be generated
+      // Warning: This must be done after solving the inheritance and before the rest of post-processing
+      prepareTimeseries();
+      //Initialisation du vecteur 'enabledFiles' contenant la liste des fichiers  sortir.
+      this->findEnabledFiles();
+      this->findEnabledReadModeFiles();
+      // Find all enabled fields of each file
+      this->findAllEnabledFields();
+      this->findAllEnabledFieldsInReadModeFiles();
+     if (hasClient && !hasServer)
+     {
+      // Try to read attributes of fields in file then fill in corresponding grid (or domain, axis)
+      this->readAttributesOfEnabledFieldsInReadModeFiles();
+     }
+      // Warning: This must be done after solving the inheritance and before the rest of post-processing
+      // The timeseries should only be prepared in client
+      if (hasClient && !hasServer) prepareTimeseries();
+      //Initialisation du vecteur 'enabledFiles' contenant la liste des fichiers Ã  sortir.
+      findEnabledFiles();
+      findEnabledWriteModeFiles();
+      findEnabledReadModeFiles();
+      // For now, only read files with client and only one level server
+      // if (hasClient && !hasServer) findEnabledReadModeFiles();
+      // Find all enabled fields of each file
+      findAllEnabledFieldsInFiles(this->enabledWriteModeFiles);
+      findAllEnabledFieldsInFiles(this->enabledReadModeFiles);
+      // For now, only read files with client and only one level server
+      // if (hasClient && !hasServer)
+      //   findAllEnabledFieldsInFiles(this->enabledReadModeFiles);
+      if (hasClient && !hasServer)
+      {
+        initReadFiles();
+        // Try to read attributes of fields in file then fill in corresponding grid (or domain, axis)
+        this->readAttributesOfEnabledFieldsInReadModeFiles();
+      }
       // Only search and rebuild all reference objects of enable fields, don't transform
       this->solveOnlyRefOfEnabledFields(false);
       // Search and rebuild all reference object of enabled fields
       this->solveAllRefOfEnabledFields(false);
+      // Search and rebuild all reference object of enabled fields, and transform
+      this->solveAllRefOfEnabledFieldsAndTransform(false);
       // Find all fields with read access from the public API
       findFieldsWithReadAccess();
+      if (hasClient && !hasServer) findFieldsWithReadAccess();
       // and solve the all reference for them
       solveAllRefOfFieldsWithReadAccess();
+      if (hasClient && !hasServer) solveAllRefOfFieldsWithReadAccess();
       isPostProcessed = true;
 …
    /*!
     * Compute the required buffer size to send the attributes (mostly those grid related).
+    *
     * \param maxEventSize [in/out] the size of the bigger event for each connected server
+    * \param [in] contextClient
+    * \param [in] bufferForWriting True if buffers are used for sending data for writing
+      This flag is only true for client and server-1 for communication with server-2
     */
+   std::map<int, StdSize> CContext::getAttributesBufferSize(std::map<int, StdSize>& maxEventSize)
+   {
+     std::map<int, StdSize> attributesSize;
+     if (hasClient)
+     {
+       size_t numEnabledFiles = this->enabledFiles.size();
+       for (size_t i = 0; i < numEnabledFiles; ++i)
+       {
+         CFile* file = this->enabledFiles[i];
+   std::map<int, StdSize> CContext::getAttributesBufferSize(std::map<int, StdSize>& maxEventSize,
+                                                           CContextClient* contextClient, bool bufferForWriting /*= "false"*/)
+   {
+         // As calendar attributes are sent even if there are no active files or fields, maps are initialized according the size of calendar attributes
+     std::map<int, StdSize> attributesSize = CCalendarWrapper::get(CCalendarWrapper::GetDefName())->getMinimumBufferSizeForAttributes(contextClient);
+     maxEventSize = CCalendarWrapper::get(CCalendarWrapper::GetDefName())->getMinimumBufferSizeForAttributes(contextClient);
+     std::vector<CFile*>& fileList = this->enabledFiles;
+     size_t numEnabledFiles = fileList.size();
+     for (size_t i = 0; i < numEnabledFiles; ++i)
+     {
+//         CFile* file = this->enabledWriteModeFiles[i];
+        CFile* file = fileList[i];
+        std::vector<CField*> enabledFields = file->getEnabledFields();
+        size_t numEnabledFields = enabledFields.size();
+        for (size_t j = 0; j < numEnabledFields; ++j)
+        {
+          const std::map<int, StdSize> mapSize = enabledFields[j]->getGridAttributesBufferSize(contextClient, bufferForWriting);
+          std::map<int, StdSize>::const_iterator it = mapSize.begin(), itE = mapSize.end();
+          for (; it != itE; ++it)
+          {
+                     // If attributesSize[it->first] does not exist, it will be zero-initialized
+                     // so we can use it safely without checking for its existence
+             if (attributesSize[it->first] < it->second)
+                           attributesSize[it->first] = it->second;
+                     if (maxEventSize[it->first] < it->second)
+                           maxEventSize[it->first] = it->second;
+          }
+        }
+     }
+     return attributesSize;
+   }
+   /*!
+    * Compute the required buffer size to send the fields data.
+    * \param maxEventSize [in/out] the size of the bigger event for each connected server
+    * \param [in] contextClient
+    * \param [in] bufferForWriting True if buffers are used for sending data for writing
+      This flag is only true for client and server-1 for communication with server-2
+    */
+   std::map<int, StdSize> CContext::getDataBufferSize(std::map<int, StdSize>& maxEventSize,
+                                                      CContextClient* contextClient, bool bufferForWriting /*= "false"*/)
+   {
+     std::map<int, StdSize> dataSize;
+     // Find all reference domain and axis of all active fields
+     std::vector<CFile*>& fileList = bufferForWriting ? this->enabledWriteModeFiles : this->enabledReadModeFiles;
+     size_t numEnabledFiles = fileList.size();
+     for (size_t i = 0; i < numEnabledFiles; ++i)
+     {
+//       CFile* file = this->enabledFiles[i];
+       CFile* file = fileList[i];
+       if (file->getContextClient() == contextClient)
+       {
          std::vector<CField*> enabledFields = file->getEnabledFields();
          size_t numEnabledFields = enabledFields.size();
          for (size_t j = 0; j < numEnabledFields; ++j)
+         {
+           const std::map<int, StdSize> mapSize = enabledFields[j]->getGridAttributesBufferSize();
+           std::map<int, StdSize>::const_iterator it = mapSize.begin(), itE = mapSize.end();
+           for (; it != itE; ++it)
+           {
+             // If attributesSize[it->first] does not exist, it will be zero-initialized
+             // so we can use it safely without checking for its existance
+             if (attributesSize[it->first] < it->second)
+               attributesSize[it->first] = it->second;
+             if (maxEventSize[it->first] < it->second)
+               maxEventSize[it->first] = it->second;
+           }
+         }
+       }
+     }
+     return attributesSize;
+   }
+   /*!
+    * Compute the required buffer size to send the fields data.
+    *
+    * \param maxEventSize [in/out] the size of the bigger event for each connected server
+    */
+   std::map<int, StdSize> CContext::getDataBufferSize(std::map<int, StdSize>& maxEventSize)
+   {
+     CFile::mode_attr::t_enum mode = hasClient ? CFile::mode_attr::write : CFile::mode_attr::read;
+     std::map<int, StdSize> dataSize;
+     // Find all reference domain and axis of all active fields
+     size_t numEnabledFiles = this->enabledFiles.size();
+     for (size_t i = 0; i < numEnabledFiles; ++i)
+     {
+       CFile* file = this->enabledFiles[i];
+       CFile::mode_attr::t_enum fileMode = file->mode.isEmpty() ? CFile::mode_attr::write : file->mode.getValue();
+       if (fileMode == mode)
+       {
+         std::vector<CField*> enabledFields = file->getEnabledFields();
+         size_t numEnabledFields = enabledFields.size();
+         for (size_t j = 0; j < numEnabledFields; ++j)
+         {
+           const std::map<int, StdSize> mapSize = enabledFields[j]->getGridDataBufferSize();
+           // const std::vector<std::map<int, StdSize> > mapSize = enabledFields[j]->getGridDataBufferSize(contextClient);
+           const std::map<int, StdSize> mapSize = enabledFields[j]->getGridDataBufferSize(contextClient,bufferForWriting);
            std::map<int, StdSize>::const_iterator it = mapSize.begin(), itE = mapSize.end();
            for (; it != itE; ++it)
 …
              // If dataSize[it->first] does not exist, it will be zero-initialized
              // so we can use it safely without checking for its existance
              if (CXios::isOptPerformance)
+                 if (CXios::isOptPerformance)
                dataSize[it->first] += it->second;
              else if (dataSize[it->first] < it->second)
                dataSize[it->first] = it->second;
              if (maxEventSize[it->first] < it->second)
+                 if (maxEventSize[it->first] < it->second)
                maxEventSize[it->first] = it->second;
+           }
 …
+       }
+     }
      return dataSize;
+   }
    //! Client side: Send infomation of active files (files are enabled to write out)
    void CContext::sendEnabledFiles()
+   {
      int size = this->enabledFiles.size();
+   void CContext::sendEnabledFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
      // In a context, each type has a root definition, e.g: axis, domain, field.
 …
      for (int i = 0; i < size; ++i)
+     {
+       cfgrpPtr->sendCreateChild(this->enabledFiles[i]->getId());
+       this->enabledFiles[i]->sendAllAttributesToServer();
+       this->enabledFiles[i]->sendAddAllVariables();
+       CFile* f = activeFiles[i];
+       cfgrpPtr->sendCreateChild(f->getId(),f->getContextClient());
+       f->sendAllAttributesToServer(f->getContextClient());
+       f->sendAddAllVariables(f->getContextClient());
+     }
+   }
    //! Client side: Send information of active fields (ones are written onto files)
    void CContext::sendEnabledFields()
+   {
      int size = this->enabledFiles.size();
+   void CContext::sendEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
      for (int i = 0; i < size; ++i)
+     {
        this->enabledFiles[i]->sendEnabledFields();
+       activeFiles[i]->sendEnabledFields(activeFiles[i]->getContextClient());
+     }
+   }
 …
    //! Client side: Send information of reference grid of active fields
    void CContext::sendRefGrid()
+   void CContext::sendRefGrid(const std::vector<CFile*>& activeFiles)
+   {
      std::set<StdString> gridIds;
      int sizeFile = this->enabledFiles.size();
+     int sizeFile = activeFiles.size();
      CFile* filePtr(NULL);
 …
      for (int i = 0; i < sizeFile; ++i)
+     {
        filePtr = this->enabledFiles[i];
+       filePtr = activeFiles[i];
        std::vector<CField*> enabledFields = filePtr->getEnabledFields();
        int sizeField = enabledFields.size();
 …
+   }
+   //! Client side: Send information of reference domain and axis of active fields
+   void CContext::sendRefDomainsAxis()
+   //! Client side: Send information of reference domain, axis and scalar of active fields
+   void CContext::sendRefDomainsAxisScalars(const std::vector<CFile*>& activeFiles)
+   {
      std::set<StdString> domainIds, axisIds, scalarIds;
      // Find all reference domain and axis of all active fields
      int numEnabledFiles = this->enabledFiles.size();
+     int numEnabledFiles = activeFiles.size();
      for (int i = 0; i < numEnabledFiles; ++i)
+     {
        std::vector<CField*> enabledFields = this->enabledFiles[i]->getEnabledFields();
+       std::vector<CField*> enabledFields = activeFiles[i]->getEnabledFields();
        int numEnabledFields = enabledFields.size();
        for (int j = 0; j < numEnabledFields; ++j)
 …
    void CContext::updateCalendar(int step)
+   {
+      #pragma omp critical (_output)
+      info(50) << "updateCalendar : before : " << calendar->getCurrentDate() << endl;
+      calendar->update(step);
+      #pragma omp critical (_output)
+      info(50) << "updateCalendar : after : " << calendar->getCurrentDate() << endl;
+#ifdef XIOS_MEMTRACK_LIGHT
+      #pragma omp critical (_output)
+      info(50) << " Current memory used by XIOS : "<<  MemTrack::getCurrentMemorySize()*1.0/(1024*1024)<<" Mbyte, at timestep "<<step<<" of context "<<this->getId()<<endl ;
+#endif
+      if (hasClient)
+      int prevStep = calendar->getStep();
+      if (prevStep < step)
+      {
+        doPostTimestepOperationsForEnabledReadModeFiles();
+        garbageCollector.invalidate(calendar->getCurrentDate());
+        if (hasClient && !hasServer) // For now we only use server level 1 to read data
+        {
+          doPreTimestepOperationsForEnabledReadModeFiles();
+        }
+        #pragma omp critical (_output)
+        info(50) << "updateCalendar : before : " << calendar->getCurrentDate() << endl;
+        calendar->update(step);
+        #pragma omp critical (_output)
+        info(50) << "updateCalendar : after : " << calendar->getCurrentDate() << endl;
+  #ifdef XIOS_MEMTRACK_LIGHT
+        #pragma omp critical (_output)
+        info(50) << " Current memory used by XIOS : "<<  MemTrack::getCurrentMemorySize()*1.0/(1024*1024)<<" Mbyte, at timestep "<<step<<" of context "<<this->getId()<<endl ;
+  #endif
+        if (hasClient && !hasServer) // For now we only use server level 1 to read data
+        {
+          doPostTimestepOperationsForEnabledReadModeFiles();
+          garbageCollector.invalidate(calendar->getCurrentDate());
+        }
+      }
+      else if (prevStep == step)
+      {
+        #pragma omp critical (_output)
+        info(50) << "updateCalendar: already at step " << step << ", no operation done." << endl;
+      }
+      else // if (prevStep > step)
+        ERROR("void CContext::updateCalendar(int step)",
+              << "Illegal calendar update: previous step was " << prevStep << ", new step " << step << "is in the past!")
+   }
+   void CContext::initReadFiles(void)
+   {
+      vector<CFile*>::const_iterator it;
+      for (it=enabledReadModeFiles.begin(); it != enabledReadModeFiles.end(); it++)
+      {
+         (*it)->initRead();
+      }
+   }
    //! Server side: Create header of netcdf file
    void CContext::createFileHeader(void )
+   void CContext::createFileHeader(void)
+   {
       vector<CFile*>::const_iterator it;
       for (it=enabledFiles.begin(); it != enabledFiles.end(); it++)
+      // for (it=enabledWriteModeFiles.begin(); it != enabledWriteModeFiles.end(); it++)
+      {
          (*it)->initFile();
+         (*it)->initWrite();
+      }
+   }
 …
     return (context);
+  }
 …
     registryOut->hierarchicalGatherRegistry() ;
+    CEventClient event(CContext::GetType(), CContext::EVENT_ID_SEND_REGISTRY);
+    if (client->isServerLeader())
+    // Use correct context client to send message
+    int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
+    for (int i = 0; i < nbSrvPools; ++i)
+    {
+       CMessage msg ;
+       msg<<this->getIdServer();
+       if (client->clientRank==0) msg<<*registryOut ;
+       const std::list<int>& ranks = client->getRanksServerLeader();
+       for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+         event.push(*itRank,1,msg);
+       client->sendEvent(event);
+     }
+     else client->sendEvent(event);
+      CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
+      CEventClient event(CContext::GetType(), CContext::EVENT_ID_SEND_REGISTRY);
+        if (contextClientTmp->isServerLeader())
+        {
+           CMessage msg ;
+           if (hasServer)
+             msg<<this->getIdServer(i);
+           else
+             msg<<this->getIdServer();
+           if (contextClientTmp->clientRank==0) msg<<*registryOut ;
+           const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
+           for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+             event.push(*itRank,1,msg);
+           contextClientTmp->sendEvent(event);
+         }
+         else contextClientTmp->sendEvent(event);
+    }
+  }
+  /*!
+  * \fn bool CContext::isFinalized(void)
+  * Context is finalized if it received context post finalize event.
+  */
+  bool CContext::isFinalized(void)
+  {
+    return finalized;
+  }
 } // namespace xios

XIOS/dev/branch_openmp/src/node/context.hpp

-                      r1334
+                      r1460
 #include "garbage_collector.hpp"
 #include "registry.hpp"
+//#include "mpi.hpp"
 …
            EVENT_ID_CLOSE_DEFINITION,EVENT_ID_UPDATE_CALENDAR,
            EVENT_ID_CREATE_FILE_HEADER,EVENT_ID_CONTEXT_FINALIZE,
+           EVENT_ID_POST_PROCESS, EVENT_ID_SEND_REGISTRY
+           EVENT_ID_POST_PROCESS, EVENT_ID_SEND_REGISTRY,
+           EVENT_ID_POST_PROCESS_GLOBAL_ATTRIBUTES,
+           EVENT_ID_PROCESS_GRID_ENABLED_FIELDS
          };
 …
       public :
          // Initialize server or client
+         void initClient(ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* cxtServer = 0);
          void initServer(ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* cxtClient = 0);
-         void initClient(ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* cxtServer = 0);
          bool isInitialized(void);
          // Put sever or client into loop state
          bool checkBuffersAndListen(void);
+         bool checkBuffersAndListen(bool enableEventsProcessing=true);
          // Finalize a context
          void finalize(void);
+         bool isFinalized(void);
          void closeDefinition(void);
          // Some functions to process context
+         void findAllEnabledFields(void);
+         void findAllEnabledFieldsInReadModeFiles(void);
+         void findAllEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles);
+         // void findAllEnabledFields(void);
+         // void findAllEnabledFieldsInReadModeFiles(void);
          void readAttributesOfEnabledFieldsInReadModeFiles();
          void solveAllInheritance(bool apply=true);
          void findEnabledFiles(void);
+         void findEnabledWriteModeFiles(void);
          void findEnabledReadModeFiles(void);
          void closeAllFile(void);
          void updateCalendar(int step);
+         void createFileHeader(void );
+         void createFileHeader(void);
+         void initReadFiles(void);
          void checkAxisDomainsGridsEligibilityForCompressedOutput();
          void prepareTimeseries(void);
+         void solveOnlyRefOfEnabledFields(bool sendToServer);
+         void solveAllRefOfEnabledFields(bool sendToServer);
+         void solveOnlyRefOfEnabledFields(bool sendToServer);
          void buildFilterGraphOfEnabledFields();
+         void postProcessFilterGraph();
          void startPrefetchingOfEnabledReadModeFiles();
+         void doPreTimestepOperationsForEnabledReadModeFiles();
          void doPostTimestepOperationsForEnabledReadModeFiles();
          void findFieldsWithReadAccess(void);
 …
          void buildFilterGraphOfFieldsWithReadAccess();
          void postProcessing();
+         std::map<int, StdSize> getAttributesBufferSize(std::map<int, StdSize>& maxEventSize);
+         std::map<int, StdSize> getDataBufferSize(std::map<int, StdSize>& maxEventSize);
+         void setClientServerBuffer();
+         void postProcessingGlobalAttributes();
+         void solveAllRefOfEnabledFieldsAndTransform(bool sendToServer);
+         void checkGridEnabledFields();
+         void checkGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles);
+         void sendGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles);
+         void sendGridComponentEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)       ;
+         std::map<int, StdSize> getAttributesBufferSize(std::map<int, StdSize>& maxEventSize, CContextClient* contextClient, bool bufferForWriting = false);
+         std::map<int, StdSize> getDataBufferSize(std::map<int, StdSize>& maxEventSize, CContextClient* contextClient, bool bufferForWriting = false);
+         void setClientServerBuffer(CContextClient* contextClient, bool bufferForWriting = false);
+         // Distribute files (in write mode) among secondary-server pools according to the estimated data flux
+         void distributeFiles(void);
+         void distributeFileOverBandwith() ;
+         void distributeFileOverMemoryBandwith() ;
          // Send context close definition
 …
          void sendUpdateCalendar(int step);
          void sendCreateFileHeader(void);
          void sendEnabledFiles();
          void sendEnabledFields();
          void sendRefDomainsAxis();
          void sendRefGrid();
+         void sendEnabledFiles(const std::vector<CFile*>& activeFiles);
+         void sendEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles);
+         void sendRefDomainsAxisScalars(const std::vector<CFile*>& activeFiles);
+         void sendRefGrid(const std::vector<CFile*>& activeFiles);
          void sendPostProcessing();
+         void sendRegistry(void) ; //!< after be gathered to the root process of the context, merged registry is sent to the root process of the servers
+         void sendPostProcessingGlobalAttributes();
+         void sendProcessingGridOfEnabledFields();
+         //!< after be gathered to the root process of the context, merged registry is sent to the root process of the servers
+         void sendRegistry(void) ;
          const StdString& getIdServer();
+         const StdString& getIdServer(const int srvPoolNb);
          // Client side: Receive and process messages
 …
          static void recvPostProcessing(CEventServer& event);
          void recvPostProcessing(CBufferIn& buffer);
+         static void recvProcessingGridOfEnabledFields(CEventServer& event);
+         static void recvPostProcessingGlobalAttributes(CEventServer& event);
+         void recvPostProcessingGlobalAttributes(CBufferIn& buffer);
          static void recvRegistry(CEventServer& event) ;
+         void recvRegistry(CBufferIn& buffer) ; //!< registry is received by the root process of the servers
+         void recvRegistry(CBufferIn& buffer) ; //!< registry is received by the servers
+         void freeComms(void);                  //!< Free internally allcoated communicators
+         void releaseClientBuffers(void);       //! Deallocate buffers allocated by clientContexts
          // dispatch event
 …
          virtual bool hasChild(void) const;
       public :
          // Calendar of context
 …
          // List of all enabled files in read mode (files on which fields are read)
          std::vector<CFile*> enabledReadModeFiles;
+         // List of all enabled files in write mode
+         std::vector<CFile*> enabledWriteModeFiles;
          // List of all enabled fields whose instant data is accessible from the public API
 …
          bool hasServer;
          // Concrete context server
          CContextServer* server;
          // Concrete contex client
+         CContextClient* client;
          CRegistry* registryIn ;  //!< input registry which is read from file
          CRegistry* registryOut ; //!< output registry which will be wrote on file at the finalize
+         CContextServer* server;    //!< Concrete context server
+         CContextClient* client;    //!< Concrete contex client
+         std::vector<CContextServer*> serverPrimServer;
+         std::vector<CContextClient*> clientPrimServer;
+         CRegistry* registryIn ;    //!< input registry which is read from file
+         CRegistry* registryOut ;   //!< output registry which will be written into file at the finalize
       private:
          bool isPostProcessed;
+         bool allProcessed;
          bool finalized;
+         int countChildCtx_;        //!< Counter of child contexts (for now it is the number of secondary server pools)
          StdString idServer_;
          CGarbageCollector garbageCollector;

XIOS/dev/branch_openmp/src/node/domain.cpp

-                      r1334
+                      r1460
 #include "server_distribution_description.hpp"
 #include "client_server_mapping_distributed.hpp"
+#include "zoom_domain.hpp"
+#include "interpolate_domain.hpp"
+#include "generate_rectilinear_domain.hpp"
+using namespace ep_lib;
 #include <algorithm>
 …
    CDomain::CDomain(void)
       : CObjectTemplate<CDomain>(), CDomainAttributes()
       , isChecked(false), relFiles(), isClientChecked(false), nbConnectedClients_(), indSrv_(), connectedServerRank_()
       , hasBounds(false), hasArea(false), isDistributed_(false), isCompressible_(false), isUnstructed_(false)
+      , isChecked(false), relFiles(), isClientChecked(false), nbSenders(), indSrv_(), connectedServerRank_()
+      , hasBounds(false), hasArea(false), isCompressible_(false), isUnstructed_(false)
       , isClientAfterTransformationChecked(false), hasLonLat(false)
+      , lonvalue_client(), latvalue_client(), bounds_lon_client(), bounds_lat_client()
+      , isRedistributed_(false), hasPole(false)
+      , isRedistributed_(false), hasPole(false), doZoomByIndex_(false)
+      , lonvalue(), latvalue(), bounds_lonvalue(), bounds_latvalue()
+      , globalLocalIndexMap_(), computedWrittenIndex_(false)
+          , clients()
+   {
+   }
 …
    CDomain::CDomain(const StdString & id)
       : CObjectTemplate<CDomain>(id), CDomainAttributes()
       , isChecked(false), relFiles(), isClientChecked(false), nbConnectedClients_(), indSrv_(), connectedServerRank_()
       , hasBounds(false), hasArea(false), isDistributed_(false), isCompressible_(false), isUnstructed_(false)
+      , isChecked(false), relFiles(), isClientChecked(false), nbSenders(), indSrv_(), connectedServerRank_()
+      , hasBounds(false), hasArea(false), isCompressible_(false), isUnstructed_(false)
       , isClientAfterTransformationChecked(false), hasLonLat(false)
+      , lonvalue_client(), latvalue_client(), bounds_lon_client(), bounds_lat_client()
+      , isRedistributed_(false), hasPole(false)
+   {
+         }
+      , isRedistributed_(false), hasPole(false), doZoomByIndex_(false)
+      , lonvalue(), latvalue(), bounds_lonvalue(), bounds_latvalue()
+      , globalLocalIndexMap_(), computedWrittenIndex_(false)
+          , clients()
+   {
+    }
    CDomain::~CDomain(void)
 …
      m["compute_connectivity_domain"] = TRANS_COMPUTE_CONNECTIVITY_DOMAIN;
      m["expand_domain"] = TRANS_EXPAND_DOMAIN;
+     m["reorder_domain"] = TRANS_REORDER_DOMAIN;
+   }
 …
      (*CDomain::transformationMapList_ptr)["compute_connectivity_domain"] = TRANS_COMPUTE_CONNECTIVITY_DOMAIN;
      (*CDomain::transformationMapList_ptr)["expand_domain"] = TRANS_EXPAND_DOMAIN;
+     (*CDomain::transformationMapList_ptr)["reorder_domain"] = TRANS_REORDER_DOMAIN;
+   }
 …
+   }
-   const std::vector<int>& CDomain::getIndexesToWrite(void) const
+   {
-     return indexesToWrite;
+   }
    /*!
 …
      \return the number of indexes written by each server
    */
+   int CDomain::getNumberWrittenIndexes() const
+   {
+     return numberWrittenIndexes_;
+   int CDomain::getNumberWrittenIndexes(ep_lib::MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     ep_lib::MPI_Comm_size(writtenCom, &writtenSize);
+     return numberWrittenIndexes_[writtenSize];
+   }
 …
      \return the total number of indexes written by the servers
    */
+   int CDomain::getTotalNumberWrittenIndexes() const
+   {
+     return totalNumberWrittenIndexes_;
+   int CDomain::getTotalNumberWrittenIndexes(ep_lib::MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     ep_lib::MPI_Comm_size(writtenCom, &writtenSize);
+     return totalNumberWrittenIndexes_[writtenSize];
+   }
 …
      \return the offset of indexes written by each server
    */
+   int CDomain::getOffsetWrittenIndexes() const
+   {
+     return offsetWrittenIndexes_;
+   int CDomain::getOffsetWrittenIndexes(ep_lib::MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     ep_lib::MPI_Comm_size(writtenCom, &writtenSize);
+     return offsetWrittenIndexes_[writtenSize];
+   }
+   CArray<int, 1>& CDomain::getCompressedIndexToWriteOnServer(ep_lib::MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     ep_lib::MPI_Comm_size(writtenCom, &writtenSize);
+     return compressedIndexToWriteOnServer[writtenSize];
+   }
 …
     * \return A map associating the server rank with its minimum buffer size.
     */
+   std::map<int, StdSize> CDomain::getAttributesBufferSize()
+   {
+     CContextClient* client = CContext::getCurrent()->client;
+     std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes();
+   std::map<int, StdSize> CDomain::getAttributesBufferSize(CContextClient* client, bool bufferForWriting /*= false*/)
+   {
+     std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes(client);
      if (client->isServerLeader())
+     {
        // size estimation for sendServerAttribut
+       // size estimation for sendDistributionAttribut
        size_t size = 11 * sizeof(size_t);
 …
+     }
      std::map<int, std::vector<size_t> >::const_iterator itIndexEnd = indSrv_.end();
      std::map<int, std::vector<int> >::const_iterator itWrittenIndexEnd = indWrittenSrv_.end();
      for (size_t k = 0; k < connectedServerRank_.size(); ++k)
+     {
        int rank = connectedServerRank_[k];
        std::map<int, std::vector<size_t> >::const_iterator it = indSrv_.find(rank);
+     boost::unordered_map<int, vector<size_t> >::const_iterator itIndexEnd = indSrv_[client->serverSize].end();
+     // std::map<int, std::vector<int> >::const_iterator itWrittenIndexEnd = indWrittenSrv_.end();
+     for (size_t k = 0; k < connectedServerRank_[client->serverSize].size(); ++k)
+     {
+       int rank = connectedServerRank_[client->serverSize][k];
+       boost::unordered_map<int, std::vector<size_t> >::const_iterator it = indSrv_[client->serverSize].find(rank);
        size_t idxCount = (it != itIndexEnd) ? it->second.size() : 0;
        // size estimation for sendIndex (and sendArea which is always smaller or equal)
        size_t sizeIndexEvent = 2 * sizeof(size_t) + 2 * CArray<int,1>::size(idxCount);
        if (isCompressible_)
+       {
          std::map<int, std::vector<int> >::const_iterator itWritten = indWrittenSrv_.find(rank);
          size_t writtenIdxCount = (itWritten != itWrittenIndexEnd) ? itWritten->second.size() : 0;
          sizeIndexEvent += CArray<int,1>::size(writtenIdxCount);
+       }
+       // if (isCompressible_)
+       // {
+       //   std::map<int, std::vector<int> >::const_iterator itWritten = indWrittenSrv_.find(rank);
+       //   size_t writtenIdxCount = (itWritten != itWrittenIndexEnd) ? itWritten->second.size() : 0;
+       //   sizeIndexEvent += CArray<int,1>::size(writtenIdxCount);
+       // }
        // size estimation for sendLonLat
 …
    bool CDomain::isEmpty(void) const
+   {
       return ((this->zoom_ni_srv == 0) ||
+              (this->zoom_nj_srv == 0));
+      return ((this->zoom_i_index.isEmpty()) || (0 == this->zoom_i_index.numElements()));
+   }
 …
    bool CDomain::isDistributed(void) const
+   {
+      return isDistributed_;
+      bool distributed =  !((!ni.isEmpty() && (ni == ni_glo) && !nj.isEmpty() && (nj == nj_glo)) ||
+              (!i_index.isEmpty() && i_index.numElements() == ni_glo*nj_glo));
+      distributed |= (1 == CContext::getCurrent()->client->clientSize);
+      return distributed;
+   }
 …
    /*!
      Redistribute RECTILINEAR domain with a number of local domains.
+     Redistribute RECTILINEAR or CURVILINEAR domain with a number of local domains.
    All attributes ni,nj,ibegin,jbegin (if defined) will be rewritten
    The optional attributes lonvalue, latvalue will be added. Because this function only serves (for now)
 …
      this->isRedistributed_ = true;
      CContext* context = CContext::getCurrent();
+     CContextClient* client = context->client;
+     // For now the assumption is that secondary server pools consist of the same number of procs.
+     // CHANGE the line below if the assumption changes.
+     CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client;
      int rankClient = client->clientRank;
      int rankOnDomain = rankClient%nbLocalDomain;
 …
        for (int jdx = 0; jdx < nj; ++jdx)
         for (int idx = 0; idx < ni; ++idx)
          lonvalue_2d(idx,jdx) = lonvalue_curvilinear_read_from_file(idx+ibegin, jdx+jbegin);
+         lonvalue_2d(idx,jdx) = lonvalue_curvilinear_read_from_file(idx, jdx);
        lonvalue_curvilinear_read_from_file.free();
 …
        for (int jdx = 0; jdx < nj; ++jdx)
         for (int idx = 0; idx < ni; ++idx)
          latvalue_2d(idx,jdx) = latvalue_curvilinear_read_from_file(idx+ibegin, jdx+jbegin);
+           latvalue_2d(idx,jdx) = latvalue_curvilinear_read_from_file(idx, jdx);
        latvalue_curvilinear_read_from_file.free();
 …
         for (int idx = 0; idx < ni; ++idx)
           for (int ndx = 0; ndx < nvertex; ++ndx)
          bounds_lon_2d(ndx,idx,jdx) = bounds_lonvalue_curvilinear_read_from_file(ndx,idx+ibegin, jdx+jbegin);
+            bounds_lon_2d(ndx,idx,jdx) = bounds_lonvalue_curvilinear_read_from_file(ndx,idx, jdx);
        bounds_lonvalue_curvilinear_read_from_file.free();
 …
         for (int idx = 0; idx < ni; ++idx)
           for (int ndx = 0; ndx < nvertex; ++ndx)
             bounds_lat_2d(ndx,idx,jdx) = bounds_latvalue_curvilinear_read_from_file(ndx,idx+ibegin, jdx+jbegin);
+            bounds_lat_2d(ndx,idx,jdx) = bounds_latvalue_curvilinear_read_from_file(ndx,idx, jdx);
        bounds_latvalue_curvilinear_read_from_file.free();
 …
         lonvalue_1d.resize(ni);
         for (int idx = 0; idx < ni; ++idx)
           lonvalue_1d(idx) = lonvalue_unstructured_read_from_file(i_index(idx));
+          lonvalue_1d(idx) = lonvalue_unstructured_read_from_file(idx);
         // We dont need these values anymore, so just delete them
 …
         latvalue_1d.resize(ni);
         for (int idx = 0; idx < ni; ++idx)
           latvalue_1d(idx) =  latvalue_unstructured_read_from_file(i_index(idx));
+          latvalue_1d(idx) =  latvalue_unstructured_read_from_file(idx);
         // We dont need these values anymore, so just delete them
 …
         for (int idx = 0; idx < ni; ++idx)
           for (int jdx = 0; jdx < nbVertex; ++jdx)
             bounds_lon_1d(jdx,idx) = bounds_lonvalue_unstructured_read_from_file(jdx, i_index(idx));
+            bounds_lon_1d(jdx,idx) = bounds_lonvalue_unstructured_read_from_file(jdx, idx);
         // We dont need these values anymore, so just delete them
         lonvalue_unstructured_read_from_file.free();
+        bounds_lonvalue_unstructured_read_from_file.free();
+     }
 …
         for (int idx = 0; idx < ni; ++idx)
           for (int jdx = 0; jdx < nbVertex; ++jdx)
             bounds_lat_1d(jdx,idx) = bounds_latvalue_unstructured_read_from_file(jdx, i_index(idx));
+            bounds_lat_1d(jdx,idx) = bounds_latvalue_unstructured_read_from_file(jdx, idx);
         // We dont need these values anymore, so just delete them
         lonvalue_unstructured_read_from_file.free();
+        bounds_latvalue_unstructured_read_from_file.free();
+     }
+   }
 …
    void CDomain::AllgatherRectilinearLonLat(CArray<double,1>& lon, CArray<double,1>& lat, CArray<double,1>& lon_g, CArray<double,1>& lat_g)
+   {
+          CContext* context = CContext::getCurrent();
+      CContextClient* client = context->client;
+          lon_g.resize(ni_glo) ;
+          lat_g.resize(nj_glo) ;
+          int* ibegin_g = new int[client->clientSize] ;
+          int* jbegin_g = new int[client->clientSize] ;
+          int* ni_g = new int[client->clientSize] ;
+          int* nj_g = new int[client->clientSize] ;
+          int v ;
+          v=ibegin ;
+          MPI_Allgather(&v,1,MPI_INT,ibegin_g,1,MPI_INT,client->intraComm) ;
+          v=jbegin ;
+          MPI_Allgather(&v,1,MPI_INT,jbegin_g,1,MPI_INT,client->intraComm) ;
+          v=ni ;
+          MPI_Allgather(&v,1,MPI_INT,ni_g,1,MPI_INT,client->intraComm) ;
+          v=nj ;
+          MPI_Allgather(&v,1,MPI_INT,nj_g,1,MPI_INT,client->intraComm) ;
+          MPI_Allgatherv(lon.dataFirst(),ni,MPI_DOUBLE,lon_g.dataFirst(),ni_g, ibegin_g,MPI_DOUBLE,client->intraComm) ;
+          MPI_Allgatherv(lat.dataFirst(),nj,MPI_DOUBLE,lat_g.dataFirst(),nj_g, jbegin_g,MPI_DOUBLE,client->intraComm) ;
+     CContext* context = CContext::getCurrent();
+    // For now the assumption is that secondary server pools consist of the same number of procs.
+    // CHANGE the line below if the assumption changes.
+    CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client;
+     lon_g.resize(ni_glo) ;
+     lat_g.resize(nj_glo) ;
+     int* ibegin_g = new int[client->clientSize] ;
+     int* jbegin_g = new int[client->clientSize] ;
+     int* ni_g = new int[client->clientSize] ;
+     int* nj_g = new int[client->clientSize] ;
+     int v ;
+     v=ibegin ;
+     ep_lib::MPI_Allgather(&v,1,MPI_INT,ibegin_g,1,MPI_INT,client->intraComm) ;
+     v=jbegin ;
+     ep_lib::MPI_Allgather(&v,1,MPI_INT,jbegin_g,1,MPI_INT,client->intraComm) ;
+     v=ni ;
+     ep_lib::MPI_Allgather(&v,1,MPI_INT,ni_g,1,MPI_INT,client->intraComm) ;
+     v=nj ;
+     ep_lib::MPI_Allgather(&v,1,MPI_INT,nj_g,1,MPI_INT,client->intraComm) ;
+     ep_lib::MPI_Allgatherv(lon.dataFirst(),ni,MPI_DOUBLE,lon_g.dataFirst(),ni_g, ibegin_g,MPI_DOUBLE,client->intraComm) ;
+     ep_lib::MPI_Allgatherv(lat.dataFirst(),nj,MPI_DOUBLE,lat_g.dataFirst(),nj_g, jbegin_g,MPI_DOUBLE,client->intraComm) ;
       delete[] ibegin_g ;
 …
+   }
+   /*
+     General check of the domain to verify its mandatory attributes
+   */
    void CDomain::checkDomain(void)
+   {
 …
      if (type == type_attr::gaussian)
+     {
            hasPole=true ;
              type.setValue(type_attr::unstructured) ;
+           }
            else if (type == type_attr::rectilinear) hasPole=true ;
+        hasPole=true ;
+        type.setValue(type_attr::unstructured) ;
+      }
+      else if (type == type_attr::rectilinear) hasPole=true ;
      if (type == type_attr::unstructured)
+     {
 …
      checkZoom();
-     isDistributed_ = !((!ni.isEmpty() && (ni == ni_glo) && !nj.isEmpty() && (nj == nj_glo)) ||
-                        (!i_index.isEmpty() && i_index.numElements() == ni_glo*nj_glo));
-     // A stupid condition to make sure that if there is only one client, domain
-     // should be considered to be distributed. This should be a temporary solution
-     isDistributed_ |= (1 == CContext::getCurrent()->client->clientSize);
+   }
 …
      if (global_zoom_nj.isEmpty())
       global_zoom_nj.setValue(nj_glo);
+   }
+    if (zoom_i_index.isEmpty()) zoom_i_index.setValue(i_index.getValue());
+    if (zoom_j_index.isEmpty()) zoom_j_index.setValue(j_index.getValue());
+    if (zoom_ibegin.isEmpty()) zoom_ibegin.setValue(ibegin);
+    if (zoom_ni.isEmpty()) zoom_ni.setValue(ni);
+    if (zoom_jbegin.isEmpty()) zoom_jbegin.setValue(jbegin);
+    if (zoom_nj.isEmpty()) zoom_nj.setValue(nj);
+   }
+   size_t CDomain::getGlobalWrittenSize(void)
+   {
+      return global_zoom_ni*global_zoom_nj ;
+   }
    //----------------------------------------------------------------
 …
       if (!mask_2d.isEmpty())
+      {
         mask_1d.resize(mask_2d.extent(0) * mask_2d.extent(1));
+        domainMask.resize(mask_2d.extent(0) * mask_2d.extent(1));
         for (int j = 0; j < nj; ++j)
           for (int i = 0; i < ni; ++i) mask_1d(i+j*ni) = mask_2d(i,j);
         mask_2d.reset();
+          for (int i = 0; i < ni; ++i) domainMask(i+j*ni) = mask_2d(i,j);
+//        mask_2d.reset();
+      }
       else if (mask_1d.isEmpty())
+      {
+        mask_1d.resize(i_index.numElements());
+        for (int i = 0; i < i_index.numElements(); ++i) mask_1d(i) = true;
+      }
+        domainMask.resize(i_index.numElements());
+        for (int i = 0; i < i_index.numElements(); ++i) domainMask(i) = true;
+      }
+      else
+      {
+      domainMask.resize(mask_1d.numElements());
+      domainMask=mask_1d ;
+     }
+   }
 …
    void CDomain::computeLocalMask(void)
+   {
      localMask.resize(ni*nj) ;
+     localMask.resize(i_index.numElements()) ;
      localMask=false ;
-     size_t zoom_ibegin=global_zoom_ibegin ;
-     size_t zoom_iend=global_zoom_ibegin+global_zoom_ni-1 ;
-     size_t zoom_jbegin=global_zoom_jbegin ;
-     size_t zoom_jend=global_zoom_jbegin+global_zoom_nj-1 ;
      size_t dn=data_i_index.numElements() ;
 …
           i=data_i_index(k)+data_ibegin ;
           j=data_j_index(k)+data_jbegin ;
+          if (i>=0 && i<ni && j>=0 && j<nj)
+          {
+            ind=j*ni+i ;
+            localMask(ind)=domainMask(ind) ;
+          }
+       }
        else
+       {
+          i=(data_i_index(k)+data_ibegin)%ni ;
+          j=(data_i_index(k)+data_ibegin)/ni ;
+          i=data_i_index(k)+data_ibegin ;
+          if (i>=0 && i<i_index.numElements())
+          {
+            ind=i ;
+            localMask(ind)=domainMask(ind) ;
+          }
+       }
+       if (i>=0 && i<ni && j>=0 && j<nj)
+         if (i+ibegin>=zoom_ibegin && i+ibegin<=zoom_iend && j+jbegin>=zoom_jbegin && j+jbegin<=zoom_jend)
+         {
+           ind=i+ni*j ;
+           localMask(ind)=mask_1d(ind) ;
+         }
+     }
+   }
+     }
+   }
    void CDomain::checkEligibilityForCompressedOutput(void)
 …
    //----------------------------------------------------------------
+   /*
+     Fill in longitude and latitude value from clients (or models) into internal values lonvalue, latvalue which
+     will be used by XIOS.
+   */
    void CDomain::completeLonLatClient(void)
+   {
+     if (!lonvalue_2d.isEmpty())
+     {
+       lonvalue_client.resize(ni * nj);
+       latvalue_client.resize(ni * nj);
+     bool lonlatValueExisted = (0 != lonvalue.numElements()) || (0 != latvalue.numElements());
+     checkBounds() ;
+     if (!lonvalue_2d.isEmpty() && !lonlatValueExisted)
+     {
+       lonvalue.resize(ni * nj);
+       latvalue.resize(ni * nj);
        if (hasBounds)
+       {
          bounds_lon_client.resize(nvertex, ni * nj);
          bounds_lat_client.resize(nvertex, ni * nj);
+         bounds_lonvalue.resize(nvertex, ni * nj);
+         bounds_latvalue.resize(nvertex, ni * nj);
+       }
 …
            int k = j * ni + i;
            lonvalue_client(k) = lonvalue_2d(i,j);
            latvalue_client(k) = latvalue_2d(i,j);
+           lonvalue(k) = lonvalue_2d(i,j);
+           latvalue(k) = latvalue_2d(i,j);
            if (hasBounds)
 …
              for (int n = 0; n < nvertex; ++n)
+             {
                bounds_lon_client(n,k) = bounds_lon_2d(n,i,j);
                bounds_lat_client(n,k) = bounds_lat_2d(n,i,j);
+               bounds_lonvalue(n,k) = bounds_lon_2d(n,i,j);
+               bounds_latvalue(n,k) = bounds_lat_2d(n,i,j);
+             }
+           }
 …
+       }
+     }
      else if (!lonvalue_1d.isEmpty())
+     else if (!lonvalue_1d.isEmpty()  && !lonlatValueExisted)
+     {
        if (type_attr::rectilinear == type)
 …
          if (ni == lonvalue_1d.numElements() && nj == latvalue_1d.numElements())
+         {
            lonvalue_client.resize(ni * nj);
            latvalue_client.resize(ni * nj);
+           lonvalue.resize(ni * nj);
+           latvalue.resize(ni * nj);
            if (hasBounds)
+           {
              bounds_lon_client.resize(nvertex, ni * nj);
              bounds_lat_client.resize(nvertex, ni * nj);
+             bounds_lonvalue.resize(nvertex, ni * nj);
+             bounds_latvalue.resize(nvertex, ni * nj);
+           }
 …
                int k = j * ni + i;
                lonvalue_client(k) = lonvalue_1d(i);
                latvalue_client(k) = latvalue_1d(j);
+               lonvalue(k) = lonvalue_1d(i);
+               latvalue(k) = latvalue_1d(j);
                if (hasBounds)
 …
                  for (int n = 0; n < nvertex; ++n)
+                 {
                    bounds_lon_client(n,k) = bounds_lon_1d(n,i);
                    bounds_lat_client(n,k) = bounds_lat_1d(n,j);
+                   bounds_lonvalue(n,k) = bounds_lon_1d(n,i);
+                   bounds_latvalue(n,k) = bounds_lat_1d(n,j);
+                 }
+               }
 …
+           }
+         }
          else if (i_index.numElements() == lonvalue_1d.numElements() && j_index.numElements() == latvalue_1d.numElements())
+         else if (i_index.numElements() == lonvalue_1d.numElements() && j_index.numElements() == latvalue_1d.numElements()  && !lonlatValueExisted)
+         {
            lonvalue_client.reference(lonvalue_1d);
            latvalue_client.reference(latvalue_1d);
+           lonvalue.reference(lonvalue_1d);
+           latvalue.reference(latvalue_1d);
             if (hasBounds)
+           {
              bounds_lon_client.reference(bounds_lon_1d);
              bounds_lat_client.reference(bounds_lat_1d);
+             bounds_lonvalue.reference(bounds_lon_1d);
+             bounds_latvalue.reference(bounds_lat_1d);
+           }
+         }
 …
                  << i_index.numElements() << " and "  << j_index.numElements() << ".");
+       }
        else if (type == type_attr::curvilinear || type == type_attr::unstructured)
+       else if (type == type_attr::curvilinear || type == type_attr::unstructured  && !lonlatValueExisted)
+       {
          lonvalue_client.reference(lonvalue_1d);
          latvalue_client.reference(latvalue_1d);
+         lonvalue.reference(lonvalue_1d);
+         latvalue.reference(latvalue_1d);
          if (hasBounds)
+         {
            bounds_lon_client.reference(bounds_lon_1d);
            bounds_lat_client.reference(bounds_lat_1d);
+           bounds_lonvalue.reference(bounds_lon_1d);
+           bounds_latvalue.reference(bounds_lat_1d);
+         }
+       }
 …
+   }
+   /*
+     Convert internal longitude latitude value used by XIOS to "lonvalue_*" which can be retrieved with Fortran interface
+   */
+   void CDomain::convertLonLatValue(void)
+   {
+     bool lonlatValueExisted = (0 != lonvalue.numElements()) || (0 != latvalue.numElements());
+     if (!lonvalue_2d.isEmpty() && lonlatValueExisted)
+     {
+       lonvalue_2d.resize(ni,nj);
+       latvalue_2d.resize(ni,nj);
+       if (hasBounds)
+       {
+         bounds_lon_2d.resize(nvertex, ni, nj);
+         bounds_lat_2d.resize(nvertex, ni, nj);
+       }
+       for (int j = 0; j < nj; ++j)
+       {
+         for (int i = 0; i < ni; ++i)
+         {
+           int k = j * ni + i;
+           lonvalue_2d(i,j) = lonvalue(k);
+           latvalue_2d(i,j) = latvalue(k);
+           if (hasBounds)
+           {
+             for (int n = 0; n < nvertex; ++n)
+             {
+               bounds_lon_2d(n,i,j) = bounds_lonvalue(n,k);
+               bounds_lat_2d(n,i,j) = bounds_latvalue(n,k);
+             }
+           }
+         }
+       }
+     }
+     else if (!lonvalue_1d.isEmpty()  && lonlatValueExisted)
+     {
+       if (type_attr::rectilinear == type)
+       {
+         if (ni == lonvalue_1d.numElements() && nj == latvalue_1d.numElements())
+         {
+           lonvalue.resize(ni * nj);
+           latvalue.resize(ni * nj);
+           if (hasBounds)
+           {
+             bounds_lonvalue.resize(nvertex, ni * nj);
+             bounds_latvalue.resize(nvertex, ni * nj);
+           }
+           for (int j = 0; j < nj; ++j)
+           {
+             for (int i = 0; i < ni; ++i)
+             {
+               int k = j * ni + i;
+               lonvalue(k) = lonvalue_1d(i);
+               latvalue(k) = latvalue_1d(j);
+               if (hasBounds)
+               {
+                 for (int n = 0; n < nvertex; ++n)
+                 {
+                   bounds_lonvalue(n,k) = bounds_lon_1d(n,i);
+                   bounds_latvalue(n,k) = bounds_lat_1d(n,j);
+                 }
+               }
+             }
+           }
+         }
+         else if (i_index.numElements() == lonvalue_1d.numElements() && j_index.numElements() == latvalue_1d.numElements()  && !lonlatValueExisted)
+         {
+           lonvalue.reference(lonvalue_1d);
+           latvalue.reference(latvalue_1d);
+            if (hasBounds)
+           {
+             bounds_lonvalue.reference(bounds_lon_1d);
+             bounds_latvalue.reference(bounds_lat_1d);
+           }
+         }
+         else
+           ERROR("CDomain::completeLonClient(void)",
+                 << "[ id = " << this->getId() << " , context = '" << CObjectFactory::GetCurrentContextId() << " ] "
+                 << "'lonvalue_1d' and 'latvalue_1d' does not have the same size as the local domain." << std::endl
+                 << "'lonvalue_1d' size is " << lonvalue_1d.numElements()
+                 << " and 'latvalue_1d' size is " << latvalue_1d.numElements() << std::endl
+                 << " They should be correspondingly " << ni.getValue() << " and "  << nj.getValue() << " or " << std::endl
+                 << i_index.numElements() << " and "  << j_index.numElements() << ".");
+       }
+       else if (type == type_attr::curvilinear || type == type_attr::unstructured  && !lonlatValueExisted)
+       {
+         lonvalue.reference(lonvalue_1d);
+         latvalue.reference(latvalue_1d);
+         if (hasBounds)
+         {
+           bounds_lonvalue.reference(bounds_lon_1d);
+           bounds_latvalue.reference(bounds_lat_1d);
+         }
+       }
+     }
+   }
    void CDomain::checkBounds(void)
+   {
+     if (!nvertex.isEmpty() && nvertex > 0)
+     bool hasBoundValues = (0 != bounds_lonvalue.numElements()) || (0 != bounds_latvalue.numElements());
+     if (!nvertex.isEmpty() && nvertex > 0 && !hasBoundValues)
+     {
        if (!bounds_lon_1d.isEmpty() && !bounds_lon_2d.isEmpty())
 …
        hasBounds = true;
+     }
+     else if (hasBoundValues)
+     {
+       hasBounds = true;
+     }
      else
+     {
        hasBounds = false;
-       nvertex = 0;
+     }
+   }
 …
    void CDomain::checkArea(void)
+   {
+     bool hasAreaValue = (!areavalue.isEmpty() && 0 != areavalue.numElements());
      hasArea = !area.isEmpty();
      if (hasArea)
+     if (hasArea && !hasAreaValue)
+     {
        if (area.extent(0) != ni || area.extent(1) != nj)
 …
                << "Area size is " << area.extent(0) << " x " << area.extent(1) << ".");
+       }
+       if (areavalue.isEmpty())
+       {
+          areavalue.resize(ni*nj);
+         for (int j = 0; j < nj; ++j)
+         {
+           for (int i = 0; i < ni; ++i)
+           {
+             int k = j * ni + i;
+             areavalue(k) = area(i,j);
+           }
+         }
+       }
+     }
+   }
 …
    void CDomain::checkLonLat()
+   {
+     hasLonLat = (!latvalue_1d.isEmpty() && !lonvalue_1d.isEmpty()) ||
+                 (!latvalue_2d.isEmpty() && !lonvalue_2d.isEmpty());
+     if (hasLonLat)
+     if (!hasLonLat) hasLonLat = (!latvalue_1d.isEmpty() && !lonvalue_1d.isEmpty()) ||
+                                 (!latvalue_2d.isEmpty() && !lonvalue_2d.isEmpty());
+     bool hasLonLatValue = (0 != lonvalue.numElements()) || (0 != latvalue.numElements());
+     if (hasLonLat && !hasLonLatValue)
+     {
        if (!lonvalue_1d.isEmpty() && !lonvalue_2d.isEmpty())
 …
      if (context->hasClient)
+     {
+       this->checkMask();
+       if (hasLonLat || hasArea || isCompressible_) this->computeConnectedServer();
+       if (hasLonLat) this->completeLonLatClient();
+      this->computeConnectedClients();
+       if (hasLonLat)
+         if (!context->hasServer)
+           this->completeLonLatClient();
+     }
 …
      CContext* context=CContext::getCurrent();
+      this->checkDomain();
+      this->checkBounds();
+      this->checkArea();
+      this->checkLonLat();
+      if (context->hasClient)
+      { // CÃŽtÃ© client uniquement
+      if (context->hasClient && !context->hasServer)
+      {
+        this->checkDomain();
+        this->checkBounds();
+        this->checkArea();
+        this->checkLonLat();
+      }
+      if (context->hasClient && !context->hasServer)
+      { // Ct client uniquement
          this->checkMask();
          this->checkDomainData();
 …
+      }
       else
       { // CÃŽtÃ© serveur uniquement
+      { // Ct serveur uniquement
+      }
 …
      if (context->hasClient)
+     {
+       sendServerAttribut();
+       if (hasLonLat || hasArea || isCompressible_) sendLonLatArea();
+       sendAttributes();
+     }
      this->isChecked = true;
 …
       if (context->hasClient)
       { // CÃŽtÃ© client uniquement
+      { // Ct client uniquement
          this->checkMask();
          this->checkDomainData();
 …
+      }
       else
       { // CÃŽtÃ© serveur uniquement
+      { // Ct serveur uniquement
+      }
       if (context->hasClient)
+      {
         this->computeConnectedServer();
+        this->computeConnectedClients();
         this->completeLonLatClient();
-        this->sendServerAttribut();
-        this->sendLonLatArea();
+      }
 …
+   }
+  void CDomain::sendServerAttribut(void)
+  {
+    CContext* context = CContext::getCurrent();
+    CContextClient* client = context->client;
+    int nbServer = client->serverSize;
+    CServerDistributionDescription serverDescription(getNbGlob(), nbServer);
+    if (isUnstructed_) serverDescription.computeServerDistribution(false, 0);
+    else serverDescription.computeServerDistribution(false, 1);
+    std::vector<std::vector<int> > serverIndexBegin = serverDescription.getServerIndexBegin();
+    std::vector<std::vector<int> > serverDimensionSizes = serverDescription.getServerDimensionSizes();
+    CEventClient event(getType(),EVENT_ID_SERVER_ATTRIBUT);
+    if (client->isServerLeader())
+  /*!
+     Compute the connection of a client to other clients to determine which clients to send attributes to.
+     The sending clients are supposed to already know the distribution of receiving clients (In simple cases, it's band)
+     The connection among clients is calculated by using global index.
+     A client connects to other clients which holds the same global index as it.
+  */
+  void CDomain::computeConnectedClients()
+  {
+    CContext* context=CContext::getCurrent() ;
+    // This line should be changed soon.
+    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
+    nbSenders.clear();
+    connectedServerRank_.clear();
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      std::list<CMessage> msgs;
+      const std::list<int>& ranks = client->getRanksServerLeader();
+      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+      {
+        // Use const int to ensure CMessage holds a copy of the value instead of just a reference
+        const int ibegin_srv = serverIndexBegin[*itRank][0];
+        const int jbegin_srv = serverIndexBegin[*itRank][1];
+        const int ni_srv = serverDimensionSizes[*itRank][0];
+        const int nj_srv = serverDimensionSizes[*itRank][1];
+        const int iend_srv = ibegin_srv + ni_srv - 1;
+        const int jend_srv = jbegin_srv + nj_srv - 1;
+        msgs.push_back(CMessage());
+        CMessage& msg = msgs.back();
+        msg << this->getId() ;
+        msg << ni_srv << ibegin_srv << iend_srv << nj_srv << jbegin_srv << jend_srv;
+        msg << global_zoom_ni.getValue() << global_zoom_ibegin.getValue() << global_zoom_nj.getValue() << global_zoom_jbegin.getValue();
+        msg << isCompressible_;
+        event.push(*itRank,1,msg);
+      }
+      client->sendEvent(event);
+    }
+    else client->sendEvent(event);
+  }
+  std::vector<int> CDomain::getNbGlob()
+  {
+     std::vector<int> nbGlob(2);
+     nbGlob[0] = ni_glo.getValue();
+     nbGlob[1] = nj_glo.getValue();
+     return nbGlob;
+  }
+  void CDomain::computeConnectedServer(void)
+  {
+    CContext* context=CContext::getCurrent() ;
+    CContextClient* client=context->client ;
+    int nbServer=client->serverSize;
+    int rank = client->clientRank;
+    bool doComputeGlobalIndexServer = true;
+    int i,j,i_ind,j_ind, nbIndex;
+    int global_zoom_iend=global_zoom_ibegin+global_zoom_ni-1 ;
+    int global_zoom_jend=global_zoom_jbegin+global_zoom_nj-1 ;
+    // Precompute number of index
+    int globalIndexCountZoom = 0;
+    nbIndex = i_index.numElements();
+    for (i = 0; i < nbIndex; ++i)
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
+      int nbServer = client->serverSize;
+      int nbClient = client->clientSize;
+      int rank     = client->clientRank;
+      bool doComputeGlobalIndexServer = true;
+      if (connectedServerRank_.find(nbServer) == connectedServerRank_.end())
+      {
+        if (indSrv_.find(nbServer) == indSrv_.end())
+        {
+          int i,j,i_ind,j_ind, nbIndex, nbIndexZoom;
+          int global_zoom_iend=global_zoom_ibegin+global_zoom_ni-1;
+          int global_zoom_jend=global_zoom_jbegin+global_zoom_nj-1;
+           // Precompute number of index
+           int globalIndexCountZoom = 0;
+           nbIndex = i_index.numElements();
+           if (doZoomByIndex_)
+           {
+             globalIndexCountZoom = zoom_i_index.numElements();
+           }
+           else
+           {
+             for (i = 0; i < nbIndex; ++i)
+             {
+               i_ind=i_index(i);
+               j_ind=j_index(i);
+               if (i_ind >= global_zoom_ibegin && i_ind <= global_zoom_iend && j_ind >= global_zoom_jbegin && j_ind <= global_zoom_jend)
+               {
+                  ++globalIndexCountZoom;
+               }
+             }
+           }
+           // Fill in index
+           CArray<size_t,1> globalIndexDomainZoom(globalIndexCountZoom);
+           CArray<size_t,1> localIndexDomainZoom(globalIndexCountZoom);
+           CArray<size_t,1> globalIndexDomain(nbIndex);
+           size_t globalIndex;
+           int globalIndexCount = 0;
+           for (i = 0; i < nbIndex; ++i)
+           {
+             i_ind=i_index(i);
+             j_ind=j_index(i);
+             globalIndex = i_ind + j_ind * ni_glo;
+             globalIndexDomain(i) = globalIndex;
+           }
+           if (globalLocalIndexMap_.empty())
+           {
+             for (i = 0; i < nbIndex; ++i)
+               globalLocalIndexMap_[globalIndexDomain(i)] = i;
+           }
+           globalIndexCountZoom = 0;
+           if (doZoomByIndex_)
+           {
+             int nbIndexZoom = zoom_i_index.numElements();
+             for (i = 0; i < nbIndexZoom; ++i)
+             {
+               i_ind=zoom_i_index(i);
+               j_ind=zoom_j_index(i);
+               globalIndex = i_ind + j_ind * ni_glo;
+               globalIndexDomainZoom(globalIndexCountZoom) = globalIndex;
+               ++globalIndexCountZoom;
+             }
+           }
+           else
+           {
+             int global_zoom_iend=global_zoom_ibegin+global_zoom_ni-1;
+             int global_zoom_jend=global_zoom_jbegin+global_zoom_nj-1;
+             for (i = 0; i < nbIndex; ++i)
+             {
+               i_ind=i_index(i);
+               j_ind=j_index(i);
+               globalIndex = i_ind + j_ind * ni_glo;
+               if (i_ind >= global_zoom_ibegin && i_ind <= global_zoom_iend && j_ind >= global_zoom_jbegin && j_ind <= global_zoom_jend)
+               {
+                  globalIndexDomainZoom(globalIndexCountZoom) = globalIndex;
+                  ++globalIndexCountZoom;
+               }
+             }
+             int iend = ibegin + ni -1;
+             int jend = jbegin + nj -1;
+             zoom_ibegin = global_zoom_ibegin > ibegin ? global_zoom_ibegin : ibegin;
+             int zoom_iend  = global_zoom_iend < iend ? zoom_iend : iend ;
+             zoom_ni     = zoom_iend-zoom_ibegin+1 ;
+             zoom_jbegin = global_zoom_jbegin > jbegin ? global_zoom_jbegin : jbegin ;
+             int zoom_jend   = global_zoom_jend < jend ? zoom_jend : jend;
+             zoom_nj     = zoom_jend-zoom_jbegin+1;
+           }
+           size_t globalSizeIndex = 1, indexBegin, indexEnd;
+           int range, clientSize = client->clientSize;
+           std::vector<int> nGlobDomain(2);
+           nGlobDomain[0] = this->ni_glo;
+           nGlobDomain[1] = this->nj_glo;
+           for (int i = 0; i < nGlobDomain.size(); ++i) globalSizeIndex *= nGlobDomain[i];
+           indexBegin = 0;
+           if (globalSizeIndex <= clientSize)
+           {
+             indexBegin = rank%globalSizeIndex;
+             indexEnd = indexBegin;
+           }
+           else
+           {
+             for (int i = 0; i < clientSize; ++i)
+             {
+               range = globalSizeIndex / clientSize;
+               if (i < (globalSizeIndex%clientSize)) ++range;
+               if (i == client->clientRank) break;
+               indexBegin += range;
+             }
+             indexEnd = indexBegin + range - 1;
+           }
+           // Even if servers have no index, they must received something from client
+           // We only use several client to send "empty" message to these servers
+           CServerDistributionDescription serverDescription(nGlobDomain, nbServer);
+           std::vector<int> serverZeroIndex;
+           if (isUnstructed_) serverZeroIndex = serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 0);
+           else serverZeroIndex = serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 1);
+           std::list<int> serverZeroIndexLeader;
+           std::list<int> serverZeroIndexNotLeader;
+           CContextClient::computeLeader(client->clientRank, client->clientSize, serverZeroIndex.size(), serverZeroIndexLeader, serverZeroIndexNotLeader);
+           for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+              *it = serverZeroIndex[*it];
+           CClientServerMapping* clientServerMap = new CClientServerMappingDistributed(serverDescription.getGlobalIndexRange(),
+                 client->intraComm);
+           clientServerMap->computeServerIndexMapping(globalIndexDomain, nbServer);
+           CClientServerMapping::GlobalIndexMap& globalIndexDomainOnServer = clientServerMap->getGlobalIndexOnServer();
+           CClientServerMapping::GlobalIndexMap::const_iterator it  = globalIndexDomainOnServer.begin(),
+                 ite = globalIndexDomainOnServer.end();
+           indSrv_[nbServer].swap(globalIndexDomainOnServer);
+           connectedServerRank_[nbServer].clear();
+           for (it = indSrv_[nbServer].begin(); it != ite; ++it)
+             connectedServerRank_[nbServer].push_back(it->first);
+           for (std::list<int>::const_iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+              connectedServerRank_[nbServer].push_back(*it);
+           // Even if a client has no index, it must connect to at least one server and
+           // send an "empty" data to this server
+           if (connectedServerRank_[nbServer].empty())
+              connectedServerRank_[nbServer].push_back(client->clientRank % client->serverSize);
+           nbSenders[nbServer] = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[nbServer]);
+           delete clientServerMap;
+        }
+      }
+    }
+  }
+   /*!
+     Compute index to write data. We only write data on the zoomed region, therefore, there should
+     be a map between the complete grid and the reduced grid where we write data.
+     By using global index we can easily create this kind of mapping.
+   */
+   void CDomain::computeWrittenIndex()
+   {
+      if (computedWrittenIndex_) return;
+      computedWrittenIndex_ = true;
+      CContext* context=CContext::getCurrent();
+      CContextServer* server = context->server;
+      std::vector<int> nBegin(2), nSize(2), nBeginGlobal(2), nGlob(2);
+      nBegin[0]       = zoom_ibegin;  nBegin[1] = zoom_jbegin;
+      nSize[0]        = zoom_ni;      nSize[1]  = zoom_nj;
+      nBeginGlobal[0] = 0; nBeginGlobal[1] = 0;
+      nGlob[0]        = ni_glo;   nGlob[1] = nj_glo;
+      CDistributionServer srvDist(server->intraCommSize, nBegin, nSize, nBeginGlobal, nGlob);
+      const CArray<size_t,1>& writtenGlobalIndex  = srvDist.getGlobalIndex();
+      size_t nbWritten = 0, indGlo;
+      boost::unordered_map<size_t,size_t>::const_iterator itb = globalLocalIndexMap_.begin(),
+                                                          ite = globalLocalIndexMap_.end(), it;
+      CArray<size_t,1>::const_iterator itSrvb = writtenGlobalIndex.begin(),
+                                       itSrve = writtenGlobalIndex.end(), itSrv;
+//      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+//      {
+//        indGlo = *itSrv;
+//        if (ite != globalLocalIndexMap_.find(indGlo))
+//        {
+//          ++nbWritten;
+//        }
+//      }
+//      localIndexToWriteOnServer.resize(nbWritten);
+      localIndexToWriteOnServer.resize(writtenGlobalIndex.numElements());
+      nbWritten = 0;
+      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+      {
+        indGlo = *itSrv;
+        if (ite != globalLocalIndexMap_.find(indGlo))
+        {
+          localIndexToWriteOnServer(nbWritten) = globalLocalIndexMap_[indGlo];
+          ++nbWritten;
+        }
+        else
+        {
+          localIndexToWriteOnServer(nbWritten) = 0;
+          ++nbWritten;
+        }
+      }
+      // if (isCompressible())
+      // {
+      //   nbWritten = 0;
+      //   boost::unordered_map<size_t,size_t> localGlobalIndexMap;
+      //   for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+      //   {
+      //     indGlo = *itSrv;
+      //     if (ite != globalLocalIndexMap_.find(indGlo))
+      //     {
+      //       localGlobalIndexMap[localIndexToWriteOnServer(nbWritten)] = indGlo;
+      //       ++nbWritten;
+      //     }
+      //   }
+      //   nbWritten = 0;
+      //   for (int idx = 0; idx < data_i_index.numElements(); ++idx)
+      //   {
+      //     if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_i_index(idx)))
+      //     {
+      //       ++nbWritten;
+      //     }
+      //   }
+      //   compressedIndexToWriteOnServer.resize(nbWritten);
+      //   nbWritten = 0;
+      //   for (int idx = 0; idx < data_i_index.numElements(); ++idx)
+      //   {
+      //     if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_i_index(idx)))
+      //     {
+      //       compressedIndexToWriteOnServer(nbWritten) = localGlobalIndexMap[data_i_index(idx)];
+      //       ++nbWritten;
+      //     }
+      //   }
+      //   numberWrittenIndexes_ = nbWritten;
+      //   if (isDistributed())
+      //   {
+      //     ep_lib::MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      //     ep_lib::MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      //     offsetWrittenIndexes_ -= numberWrittenIndexes_;
+      //   }
+      //   else
+      //     totalNumberWrittenIndexes_ = numberWrittenIndexes_;
+      // }
+   }
+  void CDomain::computeWrittenCompressedIndex(ep_lib::MPI_Comm writtenComm)
+  {
+    int writtenCommSize;
+    ep_lib::MPI_Comm_size(writtenComm, &writtenCommSize);
+    if (compressedIndexToWriteOnServer.find(writtenCommSize) != compressedIndexToWriteOnServer.end())
+      return;
+    if (isCompressible())
+    {
+      i_ind=i_index(i);
+      j_ind=j_index(i);
+      if (i_ind >= global_zoom_ibegin && i_ind <= global_zoom_iend && j_ind >= global_zoom_jbegin && j_ind <= global_zoom_jend)
+      {
+        ++globalIndexCountZoom;
+      }
+    }
+    int globalIndexWrittenCount = 0;
+    if (isCompressible_)
+    {
+      for (i = 0; i < data_i_index.numElements(); ++i)
+      {
+        i_ind = CDistributionClient::getDomainIndex(data_i_index(i), data_j_index(i),
+                                                    data_ibegin, data_jbegin, data_dim, ni,
+                                                    j_ind);
+        if (i_ind >= 0 && i_ind < ni && j_ind >= 0 && j_ind < nj && mask_1d(i_ind + j_ind * ni))
+        {
+          i_ind += ibegin;
+          j_ind += jbegin;
+          if (i_ind >= global_zoom_ibegin && i_ind <= global_zoom_iend && j_ind >= global_zoom_jbegin && j_ind <= global_zoom_jend)
+            ++globalIndexWrittenCount;
+        }
+      }
+    }
+    // Fill in index
+    CArray<size_t,1> globalIndexDomainZoom(globalIndexCountZoom);
+    CArray<size_t,1> localIndexDomainZoom(globalIndexCountZoom);
+    CArray<size_t,1> globalIndexDomain(nbIndex);
+    size_t globalIndex;
+    int globalIndexCount = 0;
+    globalIndexCountZoom = 0;
+    for (i = 0; i < nbIndex; ++i)
+    {
+      i_ind=i_index(i);
+      j_ind=j_index(i);
+      globalIndex = i_ind + j_ind * ni_glo;
+      globalIndexDomain(globalIndexCount) = globalIndex;
+      ++globalIndexCount;
+      if (i_ind >= global_zoom_ibegin && i_ind <= global_zoom_iend && j_ind >= global_zoom_jbegin && j_ind <= global_zoom_jend)
+      {
+        globalIndexDomainZoom(globalIndexCountZoom) = globalIndex;
+        localIndexDomainZoom(globalIndexCountZoom) = i;
+        ++globalIndexCountZoom;
+      }
+    }
+    CArray<int,1> globalIndexWrittenDomain(globalIndexWrittenCount);
+    if (isCompressible_)
+    {
+      globalIndexWrittenCount = 0;
+      for (i = 0; i < data_i_index.numElements(); ++i)
+      {
+        i_ind = CDistributionClient::getDomainIndex(data_i_index(i), data_j_index(i),
+                                                    data_ibegin, data_jbegin, data_dim, ni,
+                                                    j_ind);
+        if (i_ind >= 0 && i_ind < ni && j_ind >= 0 && j_ind < nj && mask_1d(i_ind + j_ind * ni))
+        {
+          i_ind += ibegin;
+          j_ind += jbegin;
+          if (i_ind >= global_zoom_ibegin && i_ind <= global_zoom_iend && j_ind >= global_zoom_jbegin && j_ind <= global_zoom_jend)
+          {
+            globalIndexWrittenDomain(globalIndexWrittenCount) = i_ind + j_ind * ni_glo;
+            ++globalIndexWrittenCount;
+          }
+        }
+      }
+    }
+    size_t globalSizeIndex = 1, indexBegin, indexEnd;
+    int range, clientSize = client->clientSize;
+    for (int i = 0; i < getNbGlob().size(); ++i) globalSizeIndex *= getNbGlob()[i];
+    indexBegin = 0;
+    if (globalSizeIndex <= clientSize)
+    {
+      indexBegin = rank%globalSizeIndex;
+      indexEnd = indexBegin;
+    }
+    else
+    {
+      for (int i = 0; i < clientSize; ++i)
+      {
+        range = globalSizeIndex / clientSize;
+        if (i < (globalSizeIndex%clientSize)) ++range;
+        if (i == client->clientRank) break;
+        indexBegin += range;
+      }
+      indexEnd = indexBegin + range - 1;
+    }
+    CServerDistributionDescription serverDescription(getNbGlob(), nbServer);
+    if (isUnstructed_) serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 0);
+    else serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 1);
+    CClientServerMapping* clientServerMap = new CClientServerMappingDistributed(serverDescription.getGlobalIndexRange(),
+                                                                                client->intraComm);
+    clientServerMap->computeServerIndexMapping(globalIndexDomain);
+    const CClientServerMapping::GlobalIndexMap& globalIndexDomainOnServer = clientServerMap->getGlobalIndexOnServer();
+    CClientServerMapping::GlobalIndexMap::const_iterator it  = globalIndexDomainOnServer.begin(),
+                                                         ite = globalIndexDomainOnServer.end();
+    typedef XIOSBinarySearchWithIndex<size_t> BinarySearch;
+    std::vector<int>::iterator itVec;
+    indSrv_.clear();
+    indWrittenSrv_.clear();
+    for (; it != ite; ++it)
+    {
+      int rank = it->first;
+      int indexSize = it->second.size();
+      std::vector<int> permutIndex(indexSize);
+      XIOSAlgorithms::fillInIndex(indexSize, permutIndex);
+      XIOSAlgorithms::sortWithIndex<size_t, CVectorStorage>(it->second, permutIndex);
+      BinarySearch binSearch(it->second);
+      int nb = globalIndexDomainZoom.numElements();
+      for (int i = 0; i < nb; ++i)
+      {
+        if (binSearch.search(permutIndex.begin(), permutIndex.end(), globalIndexDomainZoom(i), itVec))
+        {
+          indSrv_[rank].push_back(localIndexDomainZoom(i));
+        }
+      }
+      for (int i = 0; i < globalIndexWrittenDomain.numElements(); ++i)
+      {
+        if (binSearch.search(permutIndex.begin(), permutIndex.end(), globalIndexWrittenDomain(i), itVec))
+        {
+          indWrittenSrv_[rank].push_back(globalIndexWrittenDomain(i));
+        }
+      }
+    }
+    connectedServerRank_.clear();
+    for (it = globalIndexDomainOnServer.begin(); it != ite; ++it) {
+      connectedServerRank_.push_back(it->first);
+    }
+    nbConnectedClients_ = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_);
+    delete clientServerMap;
+  }
+  const std::map<int, vector<size_t> >& CDomain::getIndexServer() const
+  {
+    return indSrv_;
+      size_t nbWritten = 0, indGlo;
+      CContext* context=CContext::getCurrent();
+      CContextServer* server = context->server;
+      std::vector<int> nBegin(2), nSize(2), nBeginGlobal(2), nGlob(2);
+      nBegin[0]       = zoom_ibegin;  nBegin[1] = zoom_jbegin;
+      nSize[0]        = zoom_ni;      nSize[1]  = zoom_nj;
+      nBeginGlobal[0] = 0; nBeginGlobal[1] = 0;
+      nGlob[0]        = ni_glo;   nGlob[1] = nj_glo;
+      CDistributionServer srvDist(server->intraCommSize, nBegin, nSize, nBeginGlobal, nGlob);
+      const CArray<size_t,1>& writtenGlobalIndex  = srvDist.getGlobalIndex();
+      boost::unordered_map<size_t,size_t>::const_iterator itb = globalLocalIndexMap_.begin(),
+                                                          ite = globalLocalIndexMap_.end(), it;
+      CArray<size_t,1>::const_iterator itSrvb = writtenGlobalIndex.begin(),
+                                       itSrve = writtenGlobalIndex.end(), itSrv;
+      boost::unordered_map<size_t,size_t> localGlobalIndexMap;
+      for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+      {
+        indGlo = *itSrv;
+        if (ite != globalLocalIndexMap_.find(indGlo))
+        {
+          localGlobalIndexMap[localIndexToWriteOnServer(nbWritten)] = indGlo;
+          ++nbWritten;
+        }
+      }
+      nbWritten = 0;
+      for (int idx = 0; idx < data_i_index.numElements(); ++idx)
+      {
+        if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_i_index(idx)))
+        {
+          ++nbWritten;
+        }
+      }
+      compressedIndexToWriteOnServer[writtenCommSize].resize(nbWritten);
+      nbWritten = 0;
+      for (int idx = 0; idx < data_i_index.numElements(); ++idx)
+      {
+        if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_i_index(idx)))
+        {
+          compressedIndexToWriteOnServer[writtenCommSize](nbWritten) = localGlobalIndexMap[data_i_index(idx)];
+          ++nbWritten;
+        }
+      }
+      numberWrittenIndexes_[writtenCommSize] = nbWritten;
+      if (isDistributed())
+      {
+        ep_lib::MPI_Allreduce(&numberWrittenIndexes_[writtenCommSize], &totalNumberWrittenIndexes_[writtenCommSize], 1, MPI_INT, MPI_SUM, writtenComm);
+        ep_lib::MPI_Scan(&numberWrittenIndexes_[writtenCommSize], &offsetWrittenIndexes_[writtenCommSize], 1, MPI_INT, MPI_SUM, writtenComm);
+        offsetWrittenIndexes_[writtenCommSize] -= numberWrittenIndexes_[writtenCommSize];
+      }
+      else
+        totalNumberWrittenIndexes_[writtenCommSize] = numberWrittenIndexes_[writtenCommSize];
+      }
+  }
   /*!
+    Send index from client to server(s)
+    Send all attributes from client to connected clients
+    The attributes will be rebuilt on receiving side
+  */
+  void CDomain::sendAttributes()
+  {
+    sendDistributionAttributes();
+    sendIndex();
+    sendMask();
+    sendLonLat();
+    sendArea();
+    sendDataIndex();
+  }
+  /*!
+    Send global index and zoom index from client to connected client(s)
+    zoom index can be smaller than global index
   */
   void CDomain::sendIndex()
+  {
     int ns, n, i, j, ind, nv, idx;
+    CContext* context = CContext::getCurrent();
+    CContextClient* client=context->client;
+    CEventClient eventIndex(getType(), EVENT_ID_INDEX);
+    list<CMessage> list_msgsIndex;
+    list<CArray<int,1> > list_indi, list_indj, list_writtenInd;
+    std::map<int, std::vector<size_t> >::const_iterator it, iteMap;
+    iteMap = indSrv_.end();
+    for (int k = 0; k < connectedServerRank_.size(); ++k)
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      int nbData = 0;
+      int rank = connectedServerRank_[k];
+      it = indSrv_.find(rank);
+      if (iteMap != it)
+        nbData = it->second.size();
+      list_indi.push_back(CArray<int,1>(nbData));
+      list_indj.push_back(CArray<int,1>(nbData));
+      CArray<int,1>& indi = list_indi.back();
+      CArray<int,1>& indj = list_indj.back();
+      const std::vector<size_t>& temp = it->second;
+      for (n = 0; n < nbData; ++n)
+      {
+        idx = static_cast<int>(it->second[n]);
+        indi(n) = i_index(idx);
+        indj(n) = j_index(idx);
+      }
+      list_msgsIndex.push_back(CMessage());
+      list_msgsIndex.back() << this->getId() << (int)type; // enum ne fonctionne pour les message => ToFix
+      list_msgsIndex.back() << isCurvilinear;
+      list_msgsIndex.back() << list_indi.back() << list_indj.back();
+      if (isCompressible_)
+      {
+        std::vector<int>& writtenIndSrc = indWrittenSrv_[rank];
+        list_writtenInd.push_back(CArray<int,1>(writtenIndSrc.size()));
+        CArray<int,1>& writtenInd = list_writtenInd.back();
+        for (n = 0; n < writtenInd.numElements(); ++n)
+          writtenInd(n) = writtenIndSrc[n];
+        list_msgsIndex.back() << writtenInd;
+      }
+      eventIndex.push(rank, nbConnectedClients_[rank], list_msgsIndex.back());
+    }
+    client->sendEvent(eventIndex);
+      CContextClient* client = *it;
+      int serverSize = client->serverSize;
+      CEventClient eventIndex(getType(), EVENT_ID_INDEX);
+      list<CMessage> list_msgsIndex;
+      list<CArray<int,1> > list_indZoom, list_writtenInd, list_indGlob;
+      boost::unordered_map<int, vector<size_t> >::const_iterator itIndex, iteIndex;
+      iteIndex = indSrv_[serverSize].end();
+      for (int k = 0; k < connectedServerRank_[serverSize].size(); ++k)
+      {
+        int nbIndGlob = 0;
+        int rank = connectedServerRank_[serverSize][k];
+        itIndex = indSrv_[serverSize].find(rank);
+        if (iteIndex != itIndex)
+          nbIndGlob = itIndex->second.size();
+        list_indGlob.push_back(CArray<int,1>(nbIndGlob));
+        CArray<int,1>& indGlob = list_indGlob.back();
+        for (n = 0; n < nbIndGlob; ++n)
+        {
+          indGlob(n) = static_cast<int>(itIndex->second[n]);
+        }
+        list_msgsIndex.push_back(CMessage());
+        list_msgsIndex.back() << this->getId() << (int)type; // enum ne fonctionne pour les message => ToFix
+        list_msgsIndex.back() << isCurvilinear;
+        list_msgsIndex.back() << list_indGlob.back(); //list_indi.back() << list_indj.back();
+        eventIndex.push(rank, nbSenders[serverSize][rank], list_msgsIndex.back());
+      }
+      client->sendEvent(eventIndex);
+    }
+  }
   /*!
+    Send area from client to server(s)
+    Send distribution from client to other clients
+    Because a client in a level knows correctly the grid distribution of client on the next level
+    it calculates this distribution then sends it to the corresponding clients on the next level
+  */
+  void CDomain::sendDistributionAttributes(void)
+  {
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      CContextClient* client = *it;
+      int nbServer = client->serverSize;
+      std::vector<int> nGlobDomain(2);
+      nGlobDomain[0] = this->ni_glo;
+      nGlobDomain[1] = this->nj_glo;
+      CServerDistributionDescription serverDescription(nGlobDomain, nbServer);
+      if (isUnstructed_) serverDescription.computeServerDistribution(false, 0);
+      else serverDescription.computeServerDistribution(false, 1);
+      std::vector<std::vector<int> > serverIndexBegin = serverDescription.getServerIndexBegin();
+      std::vector<std::vector<int> > serverDimensionSizes = serverDescription.getServerDimensionSizes();
+      CEventClient event(getType(),EVENT_ID_SERVER_ATTRIBUT);
+      if (client->isServerLeader())
+      {
+        std::list<CMessage> msgs;
+        const std::list<int>& ranks = client->getRanksServerLeader();
+        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        {
+          // Use const int to ensure CMessage holds a copy of the value instead of just a reference
+          const int ibegin_srv = serverIndexBegin[*itRank][0];
+          const int jbegin_srv = serverIndexBegin[*itRank][1];
+          const int ni_srv = serverDimensionSizes[*itRank][0];
+          const int nj_srv = serverDimensionSizes[*itRank][1];
+          msgs.push_back(CMessage());
+          CMessage& msg = msgs.back();
+          msg << this->getId() ;
+          msg << isUnstructed_;
+          msg << ni_srv << ibegin_srv << nj_srv << jbegin_srv;
+          msg << global_zoom_ni.getValue() << global_zoom_ibegin.getValue() << global_zoom_nj.getValue() << global_zoom_jbegin.getValue();
+          msg << isCompressible_;
+          event.push(*itRank,1,msg);
+        }
+        client->sendEvent(event);
+      }
+      else client->sendEvent(event);
+    }
+  }
+  /*!
+    Send mask index from client to connected(s) clients
+  */
+  void CDomain::sendMask()
+  {
+    int ns, n, i, j, ind, nv, idx;
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      CContextClient* client = *it;
+      int serverSize = client->serverSize;
+      // send area for each connected server
+      CEventClient eventMask(getType(), EVENT_ID_MASK);
+      list<CMessage> list_msgsMask;
+      list<CArray<bool,1> > list_mask;
+      boost::unordered_map<int, vector<size_t> >::const_iterator it, iteMap;
+      iteMap = indSrv_[serverSize].end();
+      for (int k = 0; k < connectedServerRank_[serverSize].size(); ++k)
+      {
+        int nbData = 0;
+        int rank = connectedServerRank_[serverSize][k];
+        it = indSrv_[serverSize].find(rank);
+        if (iteMap != it)
+          nbData = it->second.size();
+        list_mask.push_back(CArray<bool,1>(nbData));
+        const std::vector<size_t>& temp = it->second;
+        for (n = 0; n < nbData; ++n)
+        {
+          idx = static_cast<int>(it->second[n]);
+          list_mask.back()(n) = domainMask(globalLocalIndexMap_[idx]);
+        }
+        list_msgsMask.push_back(CMessage());
+        list_msgsMask.back() << this->getId() << list_mask.back();
+        eventMask.push(rank, nbSenders[serverSize][rank], list_msgsMask.back());
+      }
+      client->sendEvent(eventMask);
+    }
+  }
+  /*!
+    Send area from client to connected client(s)
   */
   void CDomain::sendArea()
 …
     int ns, n, i, j, ind, nv, idx;
+    CContext* context = CContext::getCurrent();
+    CContextClient* client=context->client;
+    // send area for each connected server
+    CEventClient eventArea(getType(), EVENT_ID_AREA);
+    list<CMessage> list_msgsArea;
+    list<CArray<double,1> > list_area;
+    std::map<int, std::vector<size_t> >::const_iterator it, iteMap;
+    iteMap = indSrv_.end();
+    for (int k = 0; k < connectedServerRank_.size(); ++k)
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      int nbData = 0;
+      int rank = connectedServerRank_[k];
+      it = indSrv_.find(rank);
+      if (iteMap != it)
+        nbData = it->second.size();
+      list_area.push_back(CArray<double,1>(nbData));
+      const std::vector<size_t>& temp = it->second;
+      for (n = 0; n < nbData; ++n)
+      {
+        idx = static_cast<int>(it->second[n]);
+        i = i_index(idx);
+        j = j_index(idx);
+        if (hasArea)
+          list_area.back()(n) = area(i - ibegin, j - jbegin);
+      }
+      list_msgsArea.push_back(CMessage());
+      list_msgsArea.back() << this->getId() << list_area.back();
+      eventArea.push(rank, nbConnectedClients_[rank], list_msgsArea.back());
+    }
+    client->sendEvent(eventArea);
+      CContextClient* client = *it;
+      int serverSize = client->serverSize;
+      // send area for each connected server
+      CEventClient eventArea(getType(), EVENT_ID_AREA);
+      list<CMessage> list_msgsArea;
+      list<CArray<double,1> > list_area;
+      boost::unordered_map<int, vector<size_t> >::const_iterator it, iteMap;
+      iteMap = indSrv_[serverSize].end();
+      for (int k = 0; k < connectedServerRank_[serverSize].size(); ++k)
+      {
+        int nbData = 0;
+        int rank = connectedServerRank_[serverSize][k];
+        it = indSrv_[serverSize].find(rank);
+        if (iteMap != it)
+          nbData = it->second.size();
+        list_area.push_back(CArray<double,1>(nbData));
+        const std::vector<size_t>& temp = it->second;
+        for (n = 0; n < nbData; ++n)
+        {
+          idx = static_cast<int>(it->second[n]);
+          list_area.back()(n) = areavalue(globalLocalIndexMap_[idx]);
+        }
+        list_msgsArea.push_back(CMessage());
+        list_msgsArea.back() << this->getId() << hasArea;
+        list_msgsArea.back() << list_area.back();
+        eventArea.push(rank, nbSenders[serverSize][rank], list_msgsArea.back());
+      }
+      client->sendEvent(eventArea);
+    }
+  }
   /*!
     Send longitude and latitude from client to servers
     Each client send long and lat information to corresponding connected server(s).
+    Each client send long and lat information to corresponding connected clients(s).
     Because longitude and latitude are optional, this function only called if latitude and longitude exist
   */
 …
     int ns, n, i, j, ind, nv, idx;
+    CContext* context = CContext::getCurrent();
+    CContextClient* client=context->client;
+    // send lon lat for each connected server
+    CEventClient eventLon(getType(), EVENT_ID_LON);
+    CEventClient eventLat(getType(), EVENT_ID_LAT);
+    list<CMessage> list_msgsLon, list_msgsLat;
+    list<CArray<double,1> > list_lon, list_lat;
+    list<CArray<double,2> > list_boundslon, list_boundslat;
+    std::map<int, std::vector<size_t> >::const_iterator it, iteMap;
+    iteMap = indSrv_.end();
+    for (int k = 0; k < connectedServerRank_.size(); ++k)
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
       int nbData = 0;
       int rank = connectedServerRank_[k];
+      it = indSrv_.find(rank);
       if (iteMap != it)
         nbData = it->second.size();
+      list_lon.push_back(CArray<double,1>(nbData));
       list_lat.push_back(CArray<double,1>(nbData));
       if (hasBounds)
+      {
         list_boundslon.push_back(CArray<double,2>(nvertex, nbData));
         list_boundslat.push_back(CArray<double,2>(nvertex, nbData));
+      }
       CArray<double,1>& lon = list_lon.back();
       CArray<double,1>& lat = list_lat.back();
       const std::vector<size_t>& temp = it->second;
       for (n = 0; n < nbData; ++n)
+      {
+        idx = static_cast<int>(it->second[n]);
         lon(n) = lonvalue_client(idx);
         lat(n) = latvalue_client(idx);
+      CContextClient* client = *it;
+      int serverSize = client->serverSize;
+      // send lon lat for each connected server
+      CEventClient eventLon(getType(), EVENT_ID_LON);
+      CEventClient eventLat(getType(), EVENT_ID_LAT);
+      list<CMessage> list_msgsLon, list_msgsLat;
+      list<CArray<double,1> > list_lon, list_lat;
+      list<CArray<double,2> > list_boundslon, list_boundslat;
+      boost::unordered_map<int, vector<size_t> >::const_iterator it, iteMap;
+      iteMap = indSrv_[serverSize].end();
+      for (int k = 0; k < connectedServerRank_[serverSize].size(); ++k)
+      {
+        int nbData = 0;
+        int rank = connectedServerRank_[serverSize][k];
+        it = indSrv_[serverSize].find(rank);
+        if (iteMap != it)
+          nbData = it->second.size();
+        list_lon.push_back(CArray<double,1>(nbData));
+        list_lat.push_back(CArray<double,1>(nbData));
         if (hasBounds)
+        {
+          CArray<double,2>& boundslon = list_boundslon.back();
+          CArray<double,2>& boundslat = list_boundslat.back();
+          for (nv = 0; nv < nvertex; ++nv)
+          list_boundslon.push_back(CArray<double,2>(nvertex, nbData));
+          list_boundslat.push_back(CArray<double,2>(nvertex, nbData));
+        }
+        CArray<double,1>& lon = list_lon.back();
+        CArray<double,1>& lat = list_lat.back();
+        const std::vector<size_t>& temp = it->second;
+        for (n = 0; n < nbData; ++n)
+        {
+          idx = static_cast<int>(it->second[n]);
+          int localInd = globalLocalIndexMap_[idx];
+          lon(n) = lonvalue(localInd);
+          lat(n) = latvalue(localInd);
+          if (hasBounds)
+          {
+            boundslon(nv, n) = bounds_lon_client(nv, idx);
+            boundslat(nv, n) = bounds_lat_client(nv, idx);
+            CArray<double,2>& boundslon = list_boundslon.back();
+            CArray<double,2>& boundslat = list_boundslat.back();
+            for (nv = 0; nv < nvertex; ++nv)
+            {
+              boundslon(nv, n) = bounds_lonvalue(nv, localInd);
+              boundslat(nv, n) = bounds_latvalue(nv, localInd);
+            }
+          }
+        }
+      }
+      list_msgsLon.push_back(CMessage());
+      list_msgsLat.push_back(CMessage());
+      list_msgsLon.back() << this->getId() << list_lon.back();
+      list_msgsLat.back() << this->getId() << list_lat.back();
+      if (hasBounds)
+      {
+        list_msgsLon.back() << list_boundslon.back();
+        list_msgsLat.back() << list_boundslat.back();
+      }
+      eventLon.push(rank, nbConnectedClients_[rank], list_msgsLon.back());
+      eventLat.push(rank, nbConnectedClients_[rank], list_msgsLat.back());
+    }
+    client->sendEvent(eventLon);
+    client->sendEvent(eventLat);
+        list_msgsLon.push_back(CMessage());
+        list_msgsLat.push_back(CMessage());
+        list_msgsLon.back() << this->getId() << hasLonLat;
+        if (hasLonLat)
+          list_msgsLon.back() << list_lon.back();
+        list_msgsLon.back()  << hasBounds;
+        if (hasBounds)
+        {
+          list_msgsLon.back() << list_boundslon.back();
+        }
+        list_msgsLat.back() << this->getId() << hasLonLat;
+        if (hasLonLat)
+          list_msgsLat.back() << list_lat.back();
+        list_msgsLat.back() << hasBounds;
+        if (hasBounds)
+        {
+          list_msgsLat.back() << list_boundslat.back();
+        }
+        eventLon.push(rank, nbSenders[serverSize][rank], list_msgsLon.back());
+        eventLat.push(rank, nbSenders[serverSize][rank], list_msgsLat.back());
+      }
+      client->sendEvent(eventLon);
+      client->sendEvent(eventLat);
+    }
+  }
   /*!
+    Send some optional information to server(s)
+    In the future, this function can be extended with more optional information to send
+    Send data index to corresponding connected clients.
+    Data index can be compressed however, we always send decompressed data index
+    and they will be compressed on receiving.
+    The compressed index are represented with 1 and others are represented with -1
   */
+  void CDomain::sendLonLatArea(void)
+  {
+    sendIndex();
+    sendLonLat();
+    sendArea();
+  }
+  void CDomain::sendDataIndex()
+  {
+    int ns, n, i, j, ind, nv, idx;
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      CContextClient* client = *it;
+      int serverSize = client->serverSize;
+      // send area for each connected server
+      CEventClient eventDataIndex(getType(), EVENT_ID_DATA_INDEX);
+      list<CMessage> list_msgsDataIndex;
+      list<CArray<int,1> > list_data_i_index, list_data_j_index;
+      int nbIndex = i_index.numElements();
+      int niByIndex = max(i_index) - min(i_index) + 1;
+      int njByIndex = max(j_index) - min(j_index) + 1;
+      int dataIindexBound = (1 == data_dim) ? (niByIndex * njByIndex) : niByIndex;
+      int dataJindexBound = (1 == data_dim) ? (niByIndex * njByIndex) : njByIndex;
+      CArray<int,1> dataIIndex(nbIndex), dataJIndex(nbIndex);
+      dataIIndex = -1;
+      dataJIndex = -1;
+      ind = 0;
+      for (idx = 0; idx < data_i_index.numElements(); ++idx)
+      {
+        int dataIidx = data_i_index(idx) + data_ibegin;
+        int dataJidx = data_j_index(idx) + data_jbegin;
+        if ((0 <= dataIidx) && (dataIidx < dataIindexBound) &&
+            (0 <= dataJidx) && (dataJidx < dataJindexBound))
+        {
+          dataIIndex((1 == data_dim) ? dataIidx : dataJidx * ni + dataIidx) = 1; //i_index(dataIidx);//dataIidx;
+          dataJIndex((1 == data_dim) ? dataIidx : dataJidx * ni + dataIidx) = 1; //j_index(dataJidx);//
+        }
+      }
+      boost::unordered_map<int, vector<size_t> >::const_iterator it, iteMap;
+      iteMap = indSrv_[serverSize].end();
+      for (int k = 0; k < connectedServerRank_[serverSize].size(); ++k)
+      {
+        int nbData = 0;
+        int rank = connectedServerRank_[serverSize][k];
+        it = indSrv_[serverSize].find(rank);
+        if (iteMap != it)
+          nbData = it->second.size();
+        list_data_i_index.push_back(CArray<int,1>(nbData));
+        list_data_j_index.push_back(CArray<int,1>(nbData));
+        const std::vector<size_t>& temp = it->second;
+        for (n = 0; n < nbData; ++n)
+        {
+          idx = static_cast<int>(it->second[n]);
+          i = globalLocalIndexMap_[idx];
+          list_data_i_index.back()(n) = dataIIndex(i);
+          list_data_j_index.back()(n) = dataJIndex(i);
+        }
+        list_msgsDataIndex.push_back(CMessage());
+        list_msgsDataIndex.back() << this->getId();
+        list_msgsDataIndex.back() << list_data_i_index.back() << list_data_j_index.back();
+        eventDataIndex.push(rank, nbSenders[serverSize][rank], list_msgsDataIndex.back());
+      }
+      client->sendEvent(eventDataIndex);
+    }
+  }
   bool CDomain::dispatchEvent(CEventServer& event)
+  {
 …
+      {
         case EVENT_ID_SERVER_ATTRIBUT:
           recvServerAttribut(event);
+          recvDistributionAttributes(event);
           return true;
           break;
         case EVENT_ID_INDEX:
           recvIndex(event);
+          return true;
+          break;
+        case EVENT_ID_MASK:
+          recvMask(event);
           return true;
           break;
 …
           recvArea(event);
           return true;
+          break;
+        case EVENT_ID_DATA_INDEX:
+          recvDataIndex(event);
+          return true;
           break;
         default:
 …
   /*!
+    Receive index event from clients(s)
+    \param[in] event event contain info about rank and associated index
+  */
+  void CDomain::recvIndex(CEventServer& event)
+  {
+    string domainId;
+    std::map<int, CBufferIn*> rankBuffers;
+    list<CEventServer::SSubEvent>::iterator it;
+    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+      CBufferIn* buffer = it->buffer;
+      *buffer >> domainId;
+      rankBuffers[it->rank] = buffer;
+    }
+    get(domainId)->recvIndex(rankBuffers);
+  }
+  /*!
+    Receive index information from client(s). We use the global index for mapping index between
+    sending clients and receiving clients.
+    \param[in] rankBuffers rank of sending client and the corresponding receive buffer
+  */
+  void CDomain::recvIndex(std::map<int, CBufferIn*>& rankBuffers)
+  {
+    int nbReceived = rankBuffers.size(), i, ind, index, type_int, iIndex, jIndex;
+    recvClientRanks_.resize(nbReceived);
+    std::map<int, CBufferIn*>::iterator it = rankBuffers.begin(), ite = rankBuffers.end();
+    ind = 0;
+    for (ind = 0; it != ite; ++it, ++ind)
+    {
+       recvClientRanks_[ind] = it->first;
+       CBufferIn& buffer = *(it->second);
+       buffer >> type_int >> isCurvilinear >> indGlob_[it->first];
+       type.setValue((type_attr::t_enum)type_int); // probleme des type enum avec les buffers : ToFix
+    }
+    int nbIndGlob = 0;
+    for (i = 0; i < nbReceived; ++i)
+    {
+      nbIndGlob += indGlob_[recvClientRanks_[i]].numElements();
+    }
+    globalLocalIndexMap_.rehash(std::ceil(nbIndGlob/globalLocalIndexMap_.max_load_factor()));
+    i_index.resize(nbIndGlob);
+    j_index.resize(nbIndGlob);
+    int nbIndexGlobMax = nbIndGlob, nbIndLoc;
+    nbIndGlob = 0;
+    for (i = 0; i < nbReceived; ++i)
+    {
+      CArray<int,1>& tmp = indGlob_[recvClientRanks_[i]];
+      for (ind = 0; ind < tmp.numElements(); ++ind)
+      {
+         index = tmp(ind);
+         if (0 == globalLocalIndexMap_.count(index))
+         {
+           iIndex = (index%ni_glo)-ibegin;
+           iIndex = (iIndex < 0) ? 0 : iIndex;
+           jIndex = (index/ni_glo)-jbegin;
+           jIndex = (jIndex < 0) ? 0 : jIndex;
+           nbIndLoc = iIndex + ni * jIndex;
+           if (nbIndLoc < nbIndexGlobMax)
+           {
+             i_index(nbIndLoc) = index % ni_glo;
+             j_index(nbIndLoc) = index / ni_glo;
+             globalLocalIndexMap_[index] = nbIndLoc;
+             ++nbIndGlob;
+           }
+           // i_index(nbIndGlob) = index % ni_glo;
+           // j_index(nbIndGlob) = index / ni_glo;
+           // globalLocalIndexMap_[index] = nbIndGlob;
+           // ++nbIndGlob;
+         }
+      }
+    }
+    if (nbIndGlob==0)
+    {
+      i_index.resize(nbIndGlob);
+      j_index.resize(nbIndGlob);
+    }
+    else
+    {
+      i_index.resizeAndPreserve(nbIndGlob);
+      j_index.resizeAndPreserve(nbIndGlob);
+    }
+  }
+  /*!
     Receive attributes event from clients(s)
     \param[in] event event contain info about rank and associated attributes
   */
   void CDomain::recvServerAttribut(CEventServer& event)
+  void CDomain::recvDistributionAttributes(CEventServer& event)
+  {
     CBufferIn* buffer=event.subEvents.begin()->buffer;
     string domainId ;
     *buffer>>domainId ;
     get(domainId)->recvServerAttribut(*buffer) ;
+    get(domainId)->recvDistributionAttributes(*buffer);
+  }
 …
     \param[in] buffer message containing attributes info
   */
+  void CDomain::recvServerAttribut(CBufferIn& buffer)
+  {
+  void CDomain::recvDistributionAttributes(CBufferIn& buffer)
+  {
+    int ni_tmp, ibegin_tmp, nj_tmp, jbegin_tmp;
     int global_zoom_ni_tmp, global_zoom_ibegin_tmp, global_zoom_nj_tmp, global_zoom_jbegin_tmp;
     buffer >> ni_srv >> ibegin_srv >> iend_srv >> nj_srv >> jbegin_srv >> jend_srv
            >> global_zoom_ni_tmp >> global_zoom_ibegin_tmp >> global_zoom_nj_tmp >> global_zoom_jbegin_tmp
+    buffer >> isUnstructed_ >> ni_tmp >> ibegin_tmp >> nj_tmp >> jbegin_tmp
+           >> global_zoom_ni_tmp >> global_zoom_ibegin_tmp >> global_zoom_nj_tmp >> global_zoom_jbegin_tmp
            >> isCompressible_;
+    ni.setValue(ni_tmp);
+    ibegin.setValue(ibegin_tmp);
+    nj.setValue(nj_tmp);
+    jbegin.setValue(jbegin_tmp);
     global_zoom_ni.setValue(global_zoom_ni_tmp);
 …
     global_zoom_jbegin.setValue(global_zoom_jbegin_tmp);
+    int zoom_iend = global_zoom_ibegin + global_zoom_ni - 1;
+    int zoom_jend = global_zoom_jbegin + global_zoom_nj - 1;
+    zoom_ibegin_srv = global_zoom_ibegin > ibegin_srv ? global_zoom_ibegin : ibegin_srv ;
+    zoom_iend_srv = zoom_iend < iend_srv ? zoom_iend : iend_srv ;
+    zoom_ni_srv=zoom_iend_srv-zoom_ibegin_srv+1 ;
+    zoom_jbegin_srv = global_zoom_jbegin > jbegin_srv ? global_zoom_jbegin : jbegin_srv ;
+    zoom_jend_srv = zoom_jend < jend_srv ? zoom_jend : jend_srv ;
+    zoom_nj_srv=zoom_jend_srv-zoom_jbegin_srv+1 ;
+    if (zoom_ni_srv<=0 || zoom_nj_srv<=0)
+    int iend = ibegin + ni  - 1;
+    int jend = jbegin + nj  - 1;
+    int zoom_iend_glob = global_zoom_ibegin + global_zoom_ni - 1;
+    int zoom_jend_glob = global_zoom_jbegin + global_zoom_nj - 1;
+    zoom_ibegin.setValue(global_zoom_ibegin > ibegin ? global_zoom_ibegin : ibegin);
+    int zoom_iend = zoom_iend_glob < iend ? zoom_iend_glob : iend ;
+    zoom_ni.setValue(zoom_iend-zoom_ibegin+1);
+    zoom_jbegin.setValue(global_zoom_jbegin > jbegin ? global_zoom_jbegin : jbegin);
+    int zoom_jend = zoom_jend_glob < jend ? zoom_jend_glob : jend ;
+    zoom_nj.setValue(zoom_jend-zoom_jbegin+1);
+    if (zoom_ni<=0 || zoom_nj<=0)
+    {
+      zoom_ibegin_srv=0 ; zoom_iend_srv=0 ; zoom_ni_srv=0 ;
+      zoom_jbegin_srv=0 ; zoom_jend_srv=0 ; zoom_nj_srv=0 ;
+    }
+    lonvalue_srv.resize(zoom_ni_srv*zoom_nj_srv) ;
+    lonvalue_srv = 0. ;
+    latvalue_srv.resize(zoom_ni_srv*zoom_nj_srv) ;
+    latvalue_srv = 0. ;
+    if (hasBounds)
+    {
+      bounds_lon_srv.resize(nvertex,zoom_ni_srv*zoom_nj_srv) ;
+      bounds_lon_srv = 0. ;
+      bounds_lat_srv.resize(nvertex,zoom_ni_srv*zoom_nj_srv) ;
+      bounds_lat_srv = 0. ;
+    }
+    if (hasArea)
+    {
+      area_srv.resize(zoom_ni_srv * zoom_nj_srv);
+      area_srv = 0.;
+      zoom_ni=0 ; zoom_ibegin=global_zoom_ibegin ; //=0; zoom_iend=0 ;
+      zoom_nj=0 ; zoom_jbegin=global_zoom_jbegin ; //=0; zoom_jend=0 ;
+    }
 …
   /*!
     Receive index event from clients(s)
     \param[in] event event contain info about rank and associated index
+    Receive area event from clients(s)
+    \param[in] event event contain info about rank and associated area
   */
+  void CDomain::recvIndex(CEventServer& event)
+  {
+    CDomain* domain;
+  void CDomain::recvMask(CEventServer& event)
+  {
+    string domainId;
+    std::map<int, CBufferIn*> rankBuffers;
     list<CEventServer::SSubEvent>::iterator it;
     for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+      CBufferIn* buffer = it->buffer;
+      *buffer >> domainId;
+      rankBuffers[it->rank] = buffer;
+    }
+    get(domainId)->recvMask(rankBuffers);
+  }
+  /*!
+    Receive mask information from client(s)
+    \param[in] rankBuffers rank of sending client and the corresponding receive buffer
+  */
+  void CDomain::recvMask(std::map<int, CBufferIn*>& rankBuffers)
+  {
+    int nbReceived = rankBuffers.size(), i, ind, index, lInd;
+    if (nbReceived != recvClientRanks_.size())
+      ERROR("void CDomain::recvMask(std::map<int, CBufferIn*>& rankBuffers)",
+           << "The number of sending clients is not correct."
+           << "Expected number: " << recvClientRanks_.size() << " but received " << nbReceived);
+    vector<CArray<bool,1> > recvMaskValue(nbReceived);
+    for (i = 0; i < recvClientRanks_.size(); ++i)
+    {
+      CBufferIn* buffer = it->buffer;
+      string domainId;
+      *buffer >> domainId;
+      domain = get(domainId);
+      domain->recvIndex(it->rank, *buffer);
+    }
+    if (domain->isCompressible_)
+      int rank = recvClientRanks_[i];
+      CBufferIn& buffer = *(rankBuffers[rank]);
+      buffer >> recvMaskValue[i];
+    }
+    int nbMaskInd = 0;
+    for (i = 0; i < nbReceived; ++i)
+    {
+      std::sort(domain->indexesToWrite.begin(), domain->indexesToWrite.end());
+      CContextServer* server = CContext::getCurrent()->server;
+      domain->numberWrittenIndexes_ = domain->indexesToWrite.size();
+      MPI_Allreduce(&domain->numberWrittenIndexes_, &domain->totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      MPI_Scan(&domain->numberWrittenIndexes_, &domain->offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      domain->offsetWrittenIndexes_ -= domain->numberWrittenIndexes_;
+    }
+  }
+  /*!
+    Receive index information from client(s)
+    \param[in] rank rank of client source
+    \param[in] buffer message containing index info
+  */
+  void CDomain::recvIndex(int rank, CBufferIn& buffer)
+  {
+    int type_int;
+    buffer >> type_int >> isCurvilinear >> indiSrv[rank] >> indjSrv[rank];
+    type.setValue((type_attr::t_enum)type_int); // probleme des type enum avec les buffers : ToFix
+    if (isCompressible_)
+      nbMaskInd += recvMaskValue[i].numElements();
+    }
+    if (nbMaskInd != globalLocalIndexMap_.size())
+      info (0) << "If the domain " << this->getDomainOutputName() <<" does not have overlapped region between processes."
+               << "Something must be wrong with mask index "<< std::endl;
+    nbMaskInd = globalLocalIndexMap_.size();
+    mask_1d.resize(nbMaskInd);
+    domainMask.resize(nbMaskInd);
+    mask_1d = false;
+    for (i = 0; i < nbReceived; ++i)
+    {
+      CArray<int, 1> writtenIndexes;
+      buffer >> writtenIndexes;
+      indexesToWrite.reserve(indexesToWrite.size() + writtenIndexes.numElements());
+      for (int i = 0; i < writtenIndexes.numElements(); ++i)
+        indexesToWrite.push_back(writtenIndexes(i));
+    }
+      CArray<int,1>& tmpInd = indGlob_[recvClientRanks_[i]];
+      CArray<bool,1>& tmp = recvMaskValue[i];
+      for (ind = 0; ind < tmp.numElements(); ++ind)
+      {
+        lInd = globalLocalIndexMap_[size_t(tmpInd(ind))];
+        if (!mask_1d(lInd)) // Only rewrite mask_1d if it's not true
+          mask_1d(lInd) = tmp(ind);
+      }
+    }
+    domainMask=mask_1d ;
+  }
 …
   void CDomain::recvLon(CEventServer& event)
+  {
+    string domainId;
+    std::map<int, CBufferIn*> rankBuffers;
     list<CEventServer::SSubEvent>::iterator it;
     for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+    {
       CBufferIn* buffer = it->buffer;
-      string domainId;
       *buffer >> domainId;
+      get(domainId)->recvLon(it->rank, *buffer);
+    }
+      rankBuffers[it->rank] = buffer;
+    }
+    get(domainId)->recvLon(rankBuffers);
+  }
   /*!
     Receive longitude information from client(s)
+    \param[in] rank rank of client source
+    \param[in] buffer message containing longitude info
+    \param[in] rankBuffers rank of sending client and the corresponding receive buffer
   */
+  void CDomain::recvLon(int rank, CBufferIn& buffer)
+  {
+    CArray<int,1> &indi = indiSrv[rank], &indj = indjSrv[rank];
+    CArray<double,1> lon;
+    CArray<double,2> boundslon;
+    buffer >> lon;
+    if (hasBounds) buffer >> boundslon;
+    int i, j, ind_srv;
+    for (int ind = 0; ind < indi.numElements(); ind++)
+  void CDomain::recvLon(std::map<int, CBufferIn*>& rankBuffers)
+  {
+    int nbReceived = rankBuffers.size(), i, ind, index, iindex, jindex, lInd;
+    if (nbReceived != recvClientRanks_.size())
+      ERROR("void CDomain::recvLon(std::map<int, CBufferIn*>& rankBuffers)",
+           << "The number of sending clients is not correct."
+           << "Expected number: " << recvClientRanks_.size() << " but received " << nbReceived);
+    vector<CArray<double,1> > recvLonValue(nbReceived);
+    vector<CArray<double,2> > recvBoundsLonValue(nbReceived);
+    for (i = 0; i < recvClientRanks_.size(); ++i)
+    {
+      i = indi(ind); j = indj(ind);
+      ind_srv = (i - zoom_ibegin_srv) + (j - zoom_jbegin_srv) * zoom_ni_srv;
+      lonvalue_srv(ind_srv) = lon(ind);
+      int rank = recvClientRanks_[i];
+      CBufferIn& buffer = *(rankBuffers[rank]);
+      buffer >> hasLonLat;
+      if (hasLonLat)
+        buffer >> recvLonValue[i];
+      buffer >> hasBounds;
       if (hasBounds)
+      {
+        for (int nv = 0; nv < nvertex; ++nv)
+          bounds_lon_srv(nv, ind_srv) = boundslon(nv, ind);
+      }
+        buffer >> recvBoundsLonValue[i];
+    }
+    if (hasLonLat)
+    {
+      int nbLonInd = 0;
+      for (i = 0; i < nbReceived; ++i)
+      {
+        nbLonInd += recvLonValue[i].numElements();
+      }
+      if (nbLonInd != globalLocalIndexMap_.size())
+        info (0) << "If the domain " << this->getDomainOutputName() <<" does not have overlapped region between processes."
+                 << "Something must be wrong with longitude index "<< std::endl;
+      nbLonInd = globalLocalIndexMap_.size();
+      lonvalue.resize(nbLonInd);
+      if (hasBounds)
+      {
+        bounds_lonvalue.resize(nvertex,nbLonInd);
+        bounds_lonvalue = 0.;
+      }
+      nbLonInd = 0;
+      for (i = 0; i < nbReceived; ++i)
+      {
+        CArray<int,1>& tmpInd = indGlob_[recvClientRanks_[i]];
+        CArray<double,1>& tmp = recvLonValue[i];
+        for (ind = 0; ind < tmp.numElements(); ++ind)
+        {
+          lInd = globalLocalIndexMap_[size_t(tmpInd(ind))];
+          lonvalue(lInd) = tmp(ind);
+           if (hasBounds)
+           {
+            for (int nv = 0; nv < nvertex; ++nv)
+              bounds_lonvalue(nv, lInd) = recvBoundsLonValue[i](nv, ind);
+           }
+        }
+      }
+    }
+  }
 …
   void CDomain::recvLat(CEventServer& event)
+  {
+    string domainId;
+    std::map<int, CBufferIn*> rankBuffers;
     list<CEventServer::SSubEvent>::iterator it;
     for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+    {
       CBufferIn* buffer = it->buffer;
-      string domainId;
       *buffer >> domainId;
+      get(domainId)->recvLat(it->rank, *buffer);
+    }
+      rankBuffers[it->rank] = buffer;
+    }
+    get(domainId)->recvLat(rankBuffers);
+  }
   /*!
     Receive latitude information from client(s)
+    \param[in] rank rank of client source
+    \param[in] buffer message containing latitude info
+    \param[in] rankBuffers rank of sending client and the corresponding receive buffer
   */
   void CDomain::recvLat(int rank, CBufferIn& buffer)
+  {
     CArray<int,1> &indi = indiSrv[rank], &indj = indjSrv[rank];
     CArray<double,1> lat;
     CArray<double,2> boundslat;
     buffer >> lat;
+    if (hasBounds) buffer >> boundslat;
     int i, j, ind_srv;
     for (int ind = 0; ind < indi.numElements(); ind++)
+  void CDomain::recvLat(std::map<int, CBufferIn*>& rankBuffers)
+  {
+    int nbReceived = rankBuffers.size(), i, ind, index, iindex, jindex, lInd;
+    if (nbReceived != recvClientRanks_.size())
+      ERROR("void CDomain::recvLat(std::map<int, CBufferIn*>& rankBuffers)",
+           << "The number of sending clients is not correct."
+           << "Expected number: " << recvClientRanks_.size() << " but received " << nbReceived);
+    vector<CArray<double,1> > recvLatValue(nbReceived);
+    vector<CArray<double,2> > recvBoundsLatValue(nbReceived);
+    for (i = 0; i < recvClientRanks_.size(); ++i)
+    {
+      i = indi(ind); j = indj(ind);
+      ind_srv = (i - zoom_ibegin_srv) + (j - zoom_jbegin_srv) * zoom_ni_srv;
+      latvalue_srv(ind_srv) = lat(ind);
+      int rank = recvClientRanks_[i];
+      CBufferIn& buffer = *(rankBuffers[rank]);
+      buffer >> hasLonLat;
+      if (hasLonLat)
+        buffer >> recvLatValue[i];
+      buffer >> hasBounds;
       if (hasBounds)
+      {
+        for (int nv = 0; nv < nvertex; nv++)
+          bounds_lat_srv(nv, ind_srv) = boundslat(nv, ind);
+      }
+        buffer >> recvBoundsLatValue[i];
+    }
+    if (hasLonLat)
+    {
+      int nbLatInd = 0;
+      for (i = 0; i < nbReceived; ++i)
+      {
+        nbLatInd += recvLatValue[i].numElements();
+      }
+      if (nbLatInd != globalLocalIndexMap_.size())
+        info (0) << "If the domain " << this->getDomainOutputName() <<" does not have overlapped region between processes."
+                << "Something must be wrong with latitude index "<< std::endl;
+      nbLatInd = globalLocalIndexMap_.size();
+      latvalue.resize(nbLatInd);
+      if (hasBounds)
+      {
+        bounds_latvalue.resize(nvertex,nbLatInd);
+        bounds_latvalue = 0. ;
+      }
+      nbLatInd = 0;
+      for (i = 0; i < nbReceived; ++i)
+      {
+        CArray<int,1>& tmpInd = indGlob_[recvClientRanks_[i]];
+        CArray<double,1>& tmp = recvLatValue[i];
+        for (ind = 0; ind < tmp.numElements(); ++ind)
+        {
+          lInd = globalLocalIndexMap_[size_t(tmpInd(ind))];
+          latvalue(lInd) = tmp(ind);
+           if (hasBounds)
+           {
+            CArray<double,2>& boundslat = recvBoundsLatValue[i];
+            for (int nv = 0; nv < nvertex; ++nv)
+              bounds_latvalue(nv, lInd) = boundslat(nv, ind);
+           }
+          ++nbLatInd;
+        }
+      }
+    }
+  }
 …
   void CDomain::recvArea(CEventServer& event)
+  {
+    string domainId;
+    std::map<int, CBufferIn*> rankBuffers;
     list<CEventServer::SSubEvent>::iterator it;
     for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+    {
       CBufferIn* buffer = it->buffer;
-      string domainId;
       *buffer >> domainId;
+      get(domainId)->recvArea(it->rank, *buffer);
+    }
+      rankBuffers[it->rank] = buffer;
+    }
+    get(domainId)->recvArea(rankBuffers);
+  }
   /*!
     Receive area information from client(s)
+    \param[in] rank rank of client source
+    \param[in] buffer message containing area info
+    \param[in] rankBuffers rank of sending client and the corresponding receive buffer
   */
+  void CDomain::recvArea(int rank, CBufferIn& buffer)
+  {
+    CArray<int,1> &indi = indiSrv[rank], &indj = indjSrv[rank];
+    CArray<double,1> clientArea;
+    buffer >> clientArea;
+    int i, j, ind_srv;
+    for (int ind = 0; ind < indi.numElements(); ind++)
+  void CDomain::recvArea(std::map<int, CBufferIn*>& rankBuffers)
+  {
+    int nbReceived = rankBuffers.size(), i, ind, index, lInd;
+    if (nbReceived != recvClientRanks_.size())
+      ERROR("void CDomain::recvArea(std::map<int, CBufferIn*>& rankBuffers)",
+           << "The number of sending clients is not correct."
+           << "Expected number: " << recvClientRanks_.size() << " but received " << nbReceived);
+    vector<CArray<double,1> > recvAreaValue(nbReceived);
+    for (i = 0; i < recvClientRanks_.size(); ++i)
+    {
+      i = indi(ind); j = indj(ind);
+      ind_srv = (i - zoom_ibegin_srv) + (j - zoom_jbegin_srv) * zoom_ni_srv;
+      area_srv(ind_srv) = clientArea(ind);
+      int rank = recvClientRanks_[i];
+      CBufferIn& buffer = *(rankBuffers[rank]);
+      buffer >> hasArea;
+      if (hasArea)
+        buffer >> recvAreaValue[i];
+    }
+    if (hasArea)
+    {
+      int nbAreaInd = 0;
+      for (i = 0; i < nbReceived; ++i)
+      {
+        nbAreaInd += recvAreaValue[i].numElements();
+      }
+      if (nbAreaInd != globalLocalIndexMap_.size())
+        info (0) << "If the domain " << this->getDomainOutputName() <<" does not have overlapped region between processes."
+                 << "Something must be wrong with area index "<< std::endl;
+      nbAreaInd = globalLocalIndexMap_.size();
+      areavalue.resize(nbAreaInd);
+      nbAreaInd = 0;
+      for (i = 0; i < nbReceived; ++i)
+      {
+        CArray<int,1>& tmpInd = indGlob_[recvClientRanks_[i]];
+        CArray<double,1>& tmp = recvAreaValue[i];
+        for (ind = 0; ind < tmp.numElements(); ++ind)
+        {
+          lInd = globalLocalIndexMap_[size_t(tmpInd(ind))];
+          areavalue(lInd) = tmp(ind);
+        }
+      }
+    }
+  }
 …
+  }
+  /*!
+    Receive data index event from clients(s)
+    \param[in] event event contain info about rank and associated index
+  */
+  void CDomain::recvDataIndex(CEventServer& event)
+  {
+    string domainId;
+    std::map<int, CBufferIn*> rankBuffers;
+    list<CEventServer::SSubEvent>::iterator it;
+    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
+    {
+      CBufferIn* buffer = it->buffer;
+      *buffer >> domainId;
+      rankBuffers[it->rank] = buffer;
+    }
+    get(domainId)->recvDataIndex(rankBuffers);
+  }
+  /*!
+    Receive data index information from client(s)
+    A client receives data index from different clients to rebuild its own data index.
+    Because we use global index + mask info to calculate the sending data to client(s),
+    this data index must be updated with mask info (maybe it will change in the future)
+    Because the data index is local, to rebuild data index of received client, we should use global index along with.
+    \param[in] rankBuffers rank of sending client and the corresponding receive buffer
+  */
+  void CDomain::recvDataIndex(std::map<int, CBufferIn*>& rankBuffers)
+  {
+    int nbReceived = rankBuffers.size(), i, ind, index, indexI, indexJ, type_int, lInd;
+    if (nbReceived != recvClientRanks_.size())
+      ERROR("void CDomain::recvDataIndex(std::map<int, CBufferIn*>& rankBuffers)",
+           << "The number of sending clients is not correct."
+           << "Expected number: " << recvClientRanks_.size() << " but received " << nbReceived);
+    vector<CArray<int,1> > recvDataIIndex(nbReceived),recvDataJIndex(nbReceived);
+    for (i = 0; i < recvClientRanks_.size(); ++i)
+    {
+      int rank = recvClientRanks_[i];
+      CBufferIn& buffer = *(rankBuffers[rank]);
+      buffer >> recvDataIIndex[i];
+      buffer >> recvDataJIndex[i];
+    }
+    int nbIndex = i_index.numElements();
+    CArray<int,1> dataIIndex(nbIndex), dataJIndex(nbIndex);
+    dataIIndex = -1; dataJIndex = -1;
+    nbIndex = 0;
+    for (i = 0; i < nbReceived; ++i)
+    {
+      CArray<int,1>& tmpInd = indGlob_[recvClientRanks_[i]];
+      CArray<int,1>& tmpI = recvDataIIndex[i];
+      CArray<int,1>& tmpJ = recvDataJIndex[i];
+      if ((tmpI.numElements() != tmpInd.numElements()) || (tmpJ.numElements() != tmpInd.numElements()))
+          ERROR("void CDomain::recvDataIndex(std::map<int, CBufferIn*>& rankBuffers)",
+             << "The number of global received index is not coherent with the number of received data index."
+             << "Expected number of global index: " << tmpI.numElements() << " but received " << tmpInd.numElements());
+      for (ind = 0; ind < tmpI.numElements(); ++ind)
+      {
+         lInd = globalLocalIndexMap_[size_t(tmpInd(ind))];
+         dataIIndex(lInd) = (-1 == dataIIndex(lInd)) ? tmpI(ind) : dataIIndex(lInd); // Only fill in dataIndex if there is no data
+         dataJIndex(lInd) = (-1 == dataJIndex(lInd)) ? tmpJ(ind) : dataJIndex(lInd);
+         if (!domainMask(lInd))   // Include mask info into data index on the RECEIVE getServerDimensionSizes
+         {
+           dataIIndex(lInd) = dataJIndex(lInd) = -1;
+         }
+      }
+    }
+    int nbCompressedData = 0;
+    for (ind = 0; ind < dataIIndex.numElements(); ++ind)
+    {
+       indexI = dataIIndex(ind); indexJ = dataJIndex(ind);
+       if ((0 <= indexI) && (0 <= indexJ))
+         ++nbCompressedData;
+    }
+    data_i_index.resize(nbCompressedData);
+    data_j_index.resize(nbCompressedData);
+    nbCompressedData = 0;
+    for (ind = 0; ind < dataIIndex.numElements(); ++ind)
+    {
+       indexI = dataIIndex(ind); indexJ = dataJIndex(ind);
+       if ((0 <= indexI) && (0 <= indexJ))
+       {
+          data_i_index(nbCompressedData) = (1 == data_dim) ? ind : ind % ni;
+          data_j_index(nbCompressedData) = (1 == data_dim) ? 0   : ind / ni;
+         ++nbCompressedData;
+       }
+    }
+    // Reset data_ibegin, data_jbegin
+    data_ibegin.setValue(0);
+    data_jbegin.setValue(0);
+  }
   CTransformation<CDomain>* CDomain::addTransformation(ETranformationType transType, const StdString& id)
+  {
 …
+  }
+  void CDomain::setContextClient(CContextClient* contextClient)
+  {
+    if (clientsSet.find(contextClient)==clientsSet.end())
+    {
+      clients.push_back(contextClient) ;
+      clientsSet.insert(contextClient);
+    }
+  }
   /*!
     Parse children nodes of a domain in xml file.

XIOS/dev/branch_openmp/src/node/domain.hpp

-                      r1331
+                      r1460
 #include "transformation.hpp"
 #include "transformation_enum.hpp"
+#include "server_distribution_description.hpp"
+#include "mpi_std.hpp"
 #include "mesh.hpp"
 …
       , public CDomainAttributes
+   {
+     /// typedef ///
+     typedef CObjectTemplate<CDomain>   SuperClass;
+     typedef CDomainAttributes SuperClassAttribute;
+     public:
          enum EEventId
+         {
+           EVENT_ID_SERVER_ATTRIBUT, EVENT_ID_INDEX, EVENT_ID_LON, EVENT_ID_LAT, EVENT_ID_AREA
+           EVENT_ID_INDEX, EVENT_ID_LON, EVENT_ID_LAT,
+           EVENT_ID_AREA, EVENT_ID_MASK,
+           EVENT_ID_DATA_INDEX, EVENT_ID_SERVER_ATTRIBUT
          } ;
-         /// typedef ///
-         typedef CObjectTemplate<CDomain>   SuperClass;
-         typedef CDomainAttributes SuperClassAttribute;
       public:
 …
          virtual void parse(xml::CXMLNode & node);
+         void setContextClient(CContextClient* contextClient);
          /// VÃ©rifications ///
          void checkAttributes(void);
          void checkAttributesOnClient();
          void checkAttributesOnClientAfterTransformation();
          void checkEligibilityForCompressedOutput(void);
 …
          bool IsWritten(const StdString & filename) const;
          bool isWrittenCompressed(const StdString& filename) const;
          const std::vector<int>& getIndexesToWrite(void) const;
          int getNumberWrittenIndexes() const;
          int getTotalNumberWrittenIndexes() const;
          int getOffsetWrittenIndexes() const;
          std::map<int, StdSize> getAttributesBufferSize();
+         int getNumberWrittenIndexes(ep_lib::MPI_Comm writtenCom);
+         int getTotalNumberWrittenIndexes(ep_lib::MPI_Comm writtenCom);
+         int getOffsetWrittenIndexes(ep_lib::MPI_Comm writtenCom);
+         CArray<int,1>& getCompressedIndexToWriteOnServer(ep_lib::MPI_Comm writtenCom);
+         std::map<int, StdSize> getAttributesBufferSize(CContextClient* client, bool bufferForWriting = false);
          bool isEmpty(void) const;
          bool isDistributed(void) const;
+         bool isCompressible(void) const;
+         bool distributionAttributesHaveValue() const;
+         int ni_srv,ibegin_srv,iend_srv ;
+         int zoom_ni_srv,zoom_ibegin_srv,zoom_iend_srv ;
+         int nj_srv,jbegin_srv,jend_srv ;
+         int zoom_nj_srv,zoom_jbegin_srv,zoom_jend_srv ;
+         CArray<double, 1> lonvalue_srv, latvalue_srv ;
+         CArray<double, 2> bounds_lon_srv, bounds_lat_srv ;
+         CArray<double, 1> lonvalue_client, latvalue_client;
+         CArray<double, 2> bounds_lon_client, bounds_lat_client;
+         CArray<double, 1> area_srv;
+        vector<int> connectedServer ; // list of connected server
+        vector<int> nbSenders ; // for each communication with a server, number of communicating client
+        vector<int> nbDataSrv ; // size of data to send to each server
+        vector< vector<int> > i_indSrv ; // for each server, i global index to send
+        vector< vector<int> > j_indSrv ; // for each server, j global index to send
+         bool isCompressible(void) const;
          std::vector<int> getNbGlob();
          bool isEqual(CDomain* domain);
+         static bool dispatchEvent(CEventServer& event);
       public:
          /// Mutateur ///
          void addRelFile(const StdString & filename);
+         void addRelFileCompressed(const StdString& filename);
+         void completeLonLatClient(void);
+         void sendServerAttribut(void) ;
+         void sendLonLatArea(void);
+         void computeConnectedServer(void) ;
+         void computeLocalMask(void) ;
+         void addRelFileCompressed(const StdString& filename);
+         void computeWrittenIndex();
+         void computeWrittenCompressedIndex(ep_lib::MPI_Comm);
          void AllgatherRectilinearLonLat(CArray<double,1>& lon, CArray<double,1>& lat,
                                          CArray<double,1>& lon_g, CArray<double,1>& lat_g);
 …
          void fillInLonLat();
+         static bool dispatchEvent(CEventServer& event);
+         static void recvServerAttribut(CEventServer& event);
+         static void recvIndex(CEventServer& event);
+         static void recvLon(CEventServer& event);
+         static void recvLat(CEventServer& event);
+         static void recvArea(CEventServer& event);
+         void recvServerAttribut(CBufferIn& buffer);
+         void recvIndex(int rank, CBufferIn& buffer);
+         void recvLon(int rank, CBufferIn& buffer);
+         void recvLat(int rank, CBufferIn& buffer);
+         void recvArea(int rank, CBufferIn& buffer);
+         bool distributionAttributesHaveValue() const;
+         size_t getGlobalWrittenSize() ;
          /// Destructeur ///
          virtual ~CDomain(void);
 …
          static StdString GetDefName(void);
+         static ENodeType GetType(void);
+         const std::map<int, vector<size_t> >& getIndexServer() const;
+         CArray<bool, 1> localMask;
+         static ENodeType GetType(void);
+      public:
+         CArray<double, 1> lonvalue, latvalue;
+         CArray<double, 2> bounds_lonvalue, bounds_latvalue;
+         CArray<double, 1> areavalue;
+         CArray<size_t,1> localIndexToWriteOnServer;
+         CArray<bool, 1> domainMask; // mask_1d, mask_2d -> domainMask
+         CArray<bool, 1> localMask; // domainMask + indexing
          bool isCurvilinear ;
          bool hasBounds ;
 …
          bool hasPole ;
+         void computeLocalMask(void) ;
       private:
          void checkDomain(void);
 …
          void checkArea(void);
          void checkLonLat();
          void checkZoom(void);
          void setTransformations(const TransMapTypes&);
+         void checkZoom(void);
+         void setTransformations(const TransMapTypes&);
          void computeNGlobDomain();
+         void sendAttributes();
          void sendIndex();
+         void sendDistributionAttributes();
+         void sendMask();
          void sendArea();
+         void sendLonLat();
+         void sendLonLat();
+         void sendDataIndex();
+         void convertLonLatValue();
          void fillInRectilinearLonLat();
          void fillInCurvilinearLonLat();
          void fillInUnstructuredLonLat();
+         static void recvDistributionAttributes(CEventServer& event);
+         static void recvIndex(CEventServer& event);
+         static void recvIndexZoom(CEventServer& event);
+         static void recvMask(CEventServer& event);
+         static void recvLon(CEventServer& event);
+         static void recvLat(CEventServer& event);
+         static void recvArea(CEventServer& event);
+         static void recvDataIndex(CEventServer& event);
+         void recvDistributionAttributes(CBufferIn& buffer);
+         void recvIndex(std::map<int, CBufferIn*>& rankBuffers);
+         void recvMask(std::map<int, CBufferIn*>& rankBuffers);
+         void recvLon(std::map<int, CBufferIn*>& rankBuffers);
+         void recvLat(std::map<int, CBufferIn*>& rankBuffers);
+         void recvArea(std::map<int, CBufferIn*>& rankBuffers);
+         void recvDataIndex(std::map<int, CBufferIn*>& rankBuffers);
+         void completeLonLatClient(void);
+         void computeConnectedClients();
        private:
+         bool isChecked;
+/** Clients that have to send a domain. There can be multiple clients in case of secondary server, otherwise only one client. */
+         std::list<CContextClient*> clients;
+         std::set<CContextClient*> clientsSet;
+         bool doZoomByIndex_;
+         bool isChecked, computedWrittenIndex_;
          std::set<StdString> relFiles, relFilesCompressed;
          bool isClientChecked; // Verify whether all attributes of domain on the client side are good
          bool isClientAfterTransformationChecked;
+         std::map<int, CArray<int,1> > indiSrv, indjSrv;
+         std::map<int,int> nbConnectedClients_; // Mapping of number of communicating client to a server
+         std::map<int, vector<size_t> > indSrv_; // Global index of each client sent to server
+         std::map<int, vector<int> > indWrittenSrv_; // Global written index of each client sent to server
+         std::map<int, CArray<int,1> > indGlob_;
+         std::map<int, map<int,int> > nbSenders; // Mapping of number of communicating client to a server
+/** Global index of each client sent to server: map<serverSize, map<serverRank, indexes>> */
+         std::map<int, boost::unordered_map<int, vector<size_t> > > indSrv_;
+         // std::map<CContextClient*, std::map<int, vector<int> > > indWrittenSrv_; // Global written index of each client sent to server
          std::vector<int> indexesToWrite;
+         int numberWrittenIndexes_, totalNumberWrittenIndexes_, offsetWrittenIndexes_;
+         std::vector<int> connectedServerRank_;
+         bool isDistributed_;
+         std::vector<int> recvClientRanks_;
+         std::map<int,int> numberWrittenIndexes_, totalNumberWrittenIndexes_, offsetWrittenIndexes_;
+         std::map<int, CArray<int, 1> > compressedIndexToWriteOnServer;
+         std::map<int, std::map<int,size_t> > connectedDataSize_;
+         std::map<int, std::vector<int> > connectedServerRank_;
          //! True if and only if the data defined on the domain can be outputted in a compressed way
          bool isCompressible_;
 …
          TransMapTypes transformationMap_;
          bool isUnstructed_;
+         boost::unordered_map<size_t,size_t> globalLocalIndexMap_;
        private:

XIOS/dev/branch_openmp/src/node/field.cpp

-                      r1342
+                      r1460
 #include "temporal_filter.hpp"
 #include "spatial_transform_filter.hpp"
+#include <stdio.h>
+#include "file_server_writer_filter.hpp"
 namespace xios{
+   /// ////////////////////// Dfinitions ////////////////////// ///
+  CField* CField::my_getDirectFieldReference(void) const
+  {
+    // if (this->field_ref.isEmpty())
+    // ERROR("C" #type "* C" #type "::getDirect" #type "Reference(void)",
+    //       << "The " #name_ " with id = '" << getId() << "'"
+    //       << " has no " #name_ "_ref.");
+    // if (!C##type::has(this->name_##_ref))
+    // ERROR("C" #type "* C" #type "::getDirect" #type "Reference(void)",
+    //       << this->name_##_ref
+    //       << " refers to an unknown " #name_ " id.");
+    return CField::get(this->field_ref);
+  }
+   /// ////////////////////// DÃ©finitions ////////////////////// ///
    CField::CField(void)
 …
       , nstep(0), nstepMax(0)
       , hasOutputFile(false)
+      , domAxisScalarIds_(vector<StdString>(3,"")), areAllReferenceSolved(false), isReferenceSolved(false)
+      , domAxisScalarIds_(vector<StdString>(3,""))
+      , areAllReferenceSolved(false), isReferenceSolved(false), isReferenceSolvedAndTransformed(false)
+      , isGridChecked(false)
       , useCompressedOutput(false)
       , hasTimeInstant(false)
 …
       , wasDataRequestedFromServer(false)
       , wasDataAlreadyReceivedFromServer(false)
+      , isEOF(false)
+      , mustAutoTrigger(false)
+      , isEOF(false), nstepMaxRead(false)
    { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
 …
       , nstep(0), nstepMax(0)
       , hasOutputFile(false)
+      , domAxisScalarIds_(vector<StdString>(3,"")), areAllReferenceSolved(false), isReferenceSolved(false)
+      , domAxisScalarIds_(vector<StdString>(3,""))
+      , areAllReferenceSolved(false), isReferenceSolved(false), isReferenceSolvedAndTransformed(false)
+      , isGridChecked(false)
       , useCompressedOutput(false)
       , hasTimeInstant(false)
 …
       , wasDataRequestedFromServer(false)
       , wasDataAlreadyReceivedFromServer(false)
+      , isEOF(false)
+      , mustAutoTrigger(false)
+      , isEOF(false), nstepMaxRead(false)
    { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
 …
     CContext* context = CContext::getCurrent();
+    CContextClient* client = context->client;
+    CContextClient* client = (!context->hasServer) ? context->client : this->file->getContextClient();
+    int receiverSize = client->serverSize;
     CEventClient event(getType(), EVENT_ID_UPDATE_DATA);
 …
     list<CArray<double,1> > list_data;
     if (!grid->doGridHaveDataDistributed())
+    if (!grid->doGridHaveDataDistributed(client))
+    {
        if (client->isServerLeader())
+       {
           for (it = grid->storeIndex_toSrv.begin(); it != grid->storeIndex_toSrv.end(); it++)
+          for (it = grid->storeIndex_toSrv[client].begin(); it != grid->storeIndex_toSrv[client].end(); it++)
+          {
             int rank = it->first;
 …
+          }
           client->sendEvent(event);
+       }
        else client->sendEvent(event);
+        }
+      else client->sendEvent(event);
+    }
     else
+    {
       for (it = grid->storeIndex_toSrv.begin(); it != grid->storeIndex_toSrv.end(); it++)
+      for (it = grid->storeIndex_toSrv[client].begin(); it != grid->storeIndex_toSrv[client].end(); it++)
+      {
         int rank = it->first;
 …
         list_msg.back() << getId() << data_tmp;
         event.push(rank, grid->nbSenders[rank], list_msg.back());
+        event.push(rank, grid->nbSenders[receiverSize][rank], list_msg.back());
+      }
       client->sendEvent(event);
 …
   void CField::recvUpdateData(CEventServer& event)
+  {
+    vector<int> ranks;
+    vector<CBufferIn*> buffers;
+    std::map<int,CBufferIn*> rankBuffers;
     list<CEventServer::SSubEvent>::iterator it;
 …
       CBufferIn* buffer = it->buffer;
       *buffer >> fieldId;
+      ranks.push_back(rank);
+      buffers.push_back(buffer);
+    }
+    get(fieldId)->recvUpdateData(ranks,buffers);
+      rankBuffers[rank] = buffer;
+    }
+    get(fieldId)->recvUpdateData(rankBuffers);
     CTimer::get("Field : recv data").suspend();
+  }
   void  CField::recvUpdateData(vector<int>& ranks, vector<CBufferIn*>& buffers)
+  void  CField::recvUpdateData(std::map<int,CBufferIn*>& rankBuffers)
+  {
+    if (data_srv.empty())
+    {
+      for (map<int, CArray<size_t, 1> >::iterator it = grid->outIndexFromClient.begin(); it != grid->outIndexFromClient.end(); ++it)
+    CContext* context = CContext::getCurrent();
+    size_t sizeData = 0;
+    if (0 == recvDataSrv.numElements())
+    {
+      CArray<int,1>& storeClient = grid->storeIndex_client;
+      // Gather all data from different clients
+      recvDataSrv.resize(storeClient.numElements());
+      recvFoperationSrv = boost::shared_ptr<func::CFunctor>(new func::CInstant(recvDataSrv));
+    }
+    CArray<double,1> recv_data_tmp(recvDataSrv.numElements());
+    const CDate& currDate = context->getCalendar()->getCurrentDate();
+    const CDate opeDate   = (last_operation_srv + context->getCalendar()->getTimeStep()) +freq_op + freq_operation_srv - freq_op - context->getCalendar()->getTimeStep();
+    if (opeDate <= currDate)
+    {
+      for (map<int, CArray<size_t, 1> >::iterator it = grid->outLocalIndexStoreOnClient.begin(); it != grid->outLocalIndexStoreOnClient.end(); ++it)
+      {
+        int rank = it->first;
+        data_srv.insert(std::make_pair(rank, CArray<double,1>(it->second.numElements())));
+        foperation_srv.insert(pair<int,boost::shared_ptr<func::CFunctor> >(rank,boost::shared_ptr<func::CFunctor>(new func::CInstant(data_srv[rank]))));
+      }
+    }
+        CArray<double,1> tmp;
+        CArray<size_t,1>& indexTmp = it->second;
+        *(rankBuffers[it->first]) >> tmp;
+        for (int idx = 0; idx < indexTmp.numElements(); ++idx)
+        {
+          recv_data_tmp(indexTmp(idx)) = tmp(idx);
+        }
+      }
+    }
+    this->setData(recv_data_tmp);
+    // delete incomming flux for server only
+    recvFoperationSrv.reset() ;
+    recvDataSrv.reset() ;
+  }
+  void CField::writeUpdateData(const CArray<double,1>& data)
+  {
     CContext* context = CContext::getCurrent();
     const CDate& currDate = context->getCalendar()->getCurrentDate();
     const CDate opeDate      = last_operation_srv +freq_op + freq_operation_srv - freq_op;
     const CDate writeDate    = last_Write_srv     + freq_write_srv;
+    const CDate opeDate = (last_operation_srv + context->getCalendar()->getTimeStep()) + freq_op + freq_operation_srv - freq_op - context->getCalendar()->getTimeStep();
+    const CDate writeDate = last_Write_srv + freq_write_srv;
     if (opeDate <= currDate)
+    {
+      for (int n = 0; n < ranks.size(); n++)
+      {
+        CArray<double,1> data_tmp;
+        *buffers[n] >> data_tmp;
+        (*foperation_srv[ranks[n]])(data_tmp);
+      }
+      (*recvFoperationSrv)(data);
       last_operation_srv = currDate;
+    }
 …
     if (writeDate < (currDate + freq_operation_srv))
+    {
+      for (int n = 0; n < ranks.size(); n++)
+      {
+        this->foperation_srv[ranks[n]]->final();
+      }
+      recvFoperationSrv->final();
       last_Write_srv = writeDate;
+      grid->computeWrittenIndex();
       writeField();
       lastlast_Write_srv = last_Write_srv;
 …
   void CField::writeField(void)
+  {
     if (!getRelFile()->allDomainEmpty)
+    if (!getRelFile()->isEmptyZone())
+    {
       if (grid->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)
+      {
         getRelFile()->checkFile();
+        getRelFile()->checkWriteFile();
         this->incrementNStep();
         getRelFile()->getDataOutput()->writeFieldData(CField::get(this));
 …
+  }
+  /*
+    Send a request for reading data.
+    Client sends a request to server for demanding server to read data and send back to it.
+    For now, this function is called only by client
+    In the future, it can be called by level-1 servers
+    \param [in] tsDataRequested timestamp when the call is made
+  */
   bool CField::sendReadDataRequest(const CDate& tsDataRequested)
+  {
     CContext* context = CContext::getCurrent();
+    CContextClient* client = context->client;
+    // CContextClient* client = context->client;
+    // This code is for future: If we want to read file with level-2 servers
+    CContextClient* client = (!context->hasServer) ? context->client : this->file->getContextClient();
     lastDataRequestedFromServer = tsDataRequested;
 …
         info(20) << "lastDataRequestedFromServer + file->output_freq.getValue() : " << lastDataRequestedFromServer + file->output_freq << endl ;
+      }
       dataRequested |= sendReadDataRequest(lastDataRequestedFromServer + file->output_freq);
+    }
 …
+  }
+  /*!
+    Receive data request sent from client and process it
+    Every time server receives this request, it will try to read data and sent read data back to client
+    At the moment, this function is called by server level 1
+    In the future, this should (only) be done by the last level servers.
+  */
   void CField::recvReadDataRequest(void)
+  {
 …
     std::list<CMessage> msgs;
     bool hasData = readField();
+    EReadField hasData = readField();
     map<int, CArray<double,1> >::iterator it;
     if (!grid->doGridHaveDataDistributed())
+    if (!grid->doGridHaveDataDistributed(client))
+    {
        if (client->isServerLeader())
+       {
+          if (!data_srv.empty())
+          {
+            it = data_srv.begin();
+          if (0 != recvDataSrv.numElements())
+          {
             const std::list<int>& ranks = client->getRanksServerLeader();
             for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
 …
               CMessage& msg = msgs.back();
               msg << getId();
+              if (hasData)
+                msg << getNStep() - 1 << it->second;
+              else
+                msg << int(-1);
+              switch (hasData)
+              {
+                case RF_DATA:
+                  msg << getNStep() - 1 << recvDataSrv;
+                  break;
+                case RF_NODATA:
+                  msg << int(-2) << recvDataSrv;
+                  break;
+                case RF_EOF:
+                default:
+                  msg << int(-1);
+                  break;
+              }
               event.push(*itRank, 1, msg);
+            }
+          }
           client->sendEvent(event);
+       }
+       else
+       {
+          // if (!data_srv.empty())
+          // {
+          //   it = data_srv.begin();
+          //   const std::list<int>& ranks = client->getRanksServerNotLeader();
+          //   for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+          //   {
+          //     msgs.push_back(CMessage());
+          //     CMessage& msg = msgs.back();
+          //     msg << getId();
+          //     if (hasData)
+          //       msg << getNStep() - 1 << it->second;
+          //     else
+          //       msg << int(-1);
+          //     event.push(*itRank, 1, msg);
+          //   }
+          // }
+       }
+       else
+       {
           client->sendEvent(event);
+       }
 …
     else
+    {
+      for (it = data_srv.begin(); it != data_srv.end(); it++)
+      for (map<int, CArray<size_t, 1> >::iterator it = grid->outLocalIndexStoreOnClient.begin();
+                                                  it != grid->outLocalIndexStoreOnClient.end(); ++it)
+      {
+        CArray<size_t,1>& indexTmp = it->second;
+        CArray<double,1> tmp(indexTmp.numElements());
+        for (int idx = 0; idx < indexTmp.numElements(); ++idx)
+        {
+          tmp(idx) = recvDataSrv(indexTmp(idx));
+        }
         msgs.push_back(CMessage());
         CMessage& msg = msgs.back();
         msg << getId();
+        if (hasData)
+          msg << getNStep() - 1 << it->second;
+        else
+          msg << int(-1);
+        event.push(it->first, grid->nbSenders[it->first], msg);
+        switch (hasData)
+        {
+          case RF_DATA:
+            msg << getNStep() - 1 << tmp;
+            break;
+          case RF_NODATA:
+            msg << int(-2) << tmp;
+            break;
+          case RF_EOF:
+          default:
+            msg << int(-1);
+            break;
+        }
+        event.push(it->first, grid->nbReadSenders[client][it->first], msg);
+      }
       client->sendEvent(event);
 …
+  }
+  bool CField::readField(void)
+  /*!
+    Read field from a file.
+    A field is read with the distribution of data on the server side
+    \return State of field can be read from a file
+  */
+  CField::EReadField CField::readField(void)
+  {
+    if (!getRelFile()->allDomainEmpty)
+    {
+      if (grid->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)
+    CContext* context = CContext::getCurrent();
+    grid->computeWrittenIndex();
+    getRelFile()->initRead();
+    EReadField readState = RF_DATA;
+    if (!getRelFile()->isEmptyZone())
+    {
+      if (grid->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)
+      {
         if (data_srv.empty())
+        {
           for (map<int, CArray<size_t, 1> >::iterator it = grid->outIndexFromClient.begin(); it != grid->outIndexFromClient.end(); ++it)
             data_srv.insert(std::make_pair(it->first, CArray<double,1>(it->second.numElements())));
+        if (0 == recvDataSrv.numElements())
+        {
+          CArray<int,1>& storeClient = grid->storeIndex_client;
+          recvDataSrv.resize(storeClient.numElements());
+        }
+        getRelFile()->checkFile();
+        getRelFile()->checkReadFile();
         if (!nstepMax)
+        {
 …
         if (getNStep() > nstepMax && (getRelFile()->cyclic.isEmpty() || !getRelFile()->cyclic) )
+          return false;
+        getRelFile()->getDataInput()->readFieldData(CField::get(this));
+      }
+    }
+    return true;
+          readState = RF_EOF;
+        if (RF_EOF != readState)
+          getRelFile()->getDataInput()->readFieldData(CField::get(this));
+      }
+    }
+    else
+    {
+      this->incrementNStep();
+      if (getNStep() > nstepMax && (getRelFile()->cyclic.isEmpty() || !getRelFile()->cyclic) )
+        readState = RF_EOF;
+      else
+        readState = RF_NODATA;
+      if (!nstepMaxRead) // This can be a bug if we try to read field from zero time record
+        readState = RF_NODATA;
+    }
+    if (!nstepMaxRead)
+    {
+       //MPI_Allreduce(MPI_IN_PLACE, &nstepMax, 1, MPI_INT, MPI_MAX, context->server->intraComm);
+       MPI_Allreduce(&nstepMax, &nstepMax, 1, MPI_INT, MPI_MAX, context->server->intraComm);
+       nstepMaxRead = true;
+    }
+    return readState;
+  }
+  /*
+    Receive read data from server.
+    At the moment, this function is called in the client side.
+    In the future, this function can be called hiearachically (server n-1, server n -2, ..., client)
+    \param event event containing read data
+  */
   void CField::recvReadDataReady(CEventServer& event)
+  {
 …
+  }
+  /*!
+    Receive read data from server
+    \param [in] ranks Ranks of sending processes
+    \param [in] buffers buffers containing read data
+  */
   void CField::recvReadDataReady(vector<int> ranks, vector<CBufferIn*> buffers)
+  {
 …
     // Check if data previously requested has been received as expected
     if (wasDataRequestedFromServer && (!isEOF || currentDate <= dateEOF))
+    if (wasDataRequestedFromServer && !isEOF)
+    {
       CTimer timer("CField::checkForLateDataFromServer");
 …
+      {
         const CDate nextDataDue = wasDataAlreadyReceivedFromServer ? (lastDataReceivedFromServer + file->output_freq) : context->getCalendar()->getInitDate();
         isDataLate = nextDataDue < currentDate;
+        isDataLate = (nextDataDue <= currentDate);
         if (isDataLate)
 …
+  }
+  void CField::checkIfMustAutoTrigger(void)
+  {
+    mustAutoTrigger = serverSourceFilter ? serverSourceFilter->mustAutoTrigger() : false;
+  }
+  void CField::autoTriggerIfNeeded(void)
+  {
+    if (mustAutoTrigger)
+      serverSourceFilter->trigger(CContext::getCurrent()->getCalendar()->getCurrentDate());
+  }
    //----------------------------------------------------------------
 …
+   {
       this->nstepMax = 0;
+      nstepMaxRead = false;
+   }
 …
    //----------------------------------------------------------------
-   void CField::solveOnlyReferenceEnabledField(bool doSending2Server)
+   {
-     CContext* context = CContext::getCurrent();
-     if (!isReferenceSolved)
+     {
-        isReferenceSolved = true;
-        if (context->hasClient)
+        {
-          solveRefInheritance(true);
-          if (hasDirectFieldReference()) getDirectFieldReference()->solveOnlyReferenceEnabledField(false);
+        }
-        else if (context->hasServer)
-          solveServerOperation();
-        solveGridReference();
-       if (context->hasClient)
+       {
-         solveGenerateGrid();
-         buildGridTransformationGraph();
+       }
+     }
+   }
    /*!
      Build up graph of grids which plays role of destination and source in grid transformation
 …
+   {
      CContext* context = CContext::getCurrent();
      if (context->hasClient)
+     if (context->hasClient && !context->hasServer)
+     {
        if (grid && !grid->isTransformed() && hasDirectFieldReference() && grid != getDirectFieldReference()->grid)
 …
+   {
      CContext* context = CContext::getCurrent();
      if (context->hasClient)
+     if (context->hasClient && !context->hasServer)
+     {
        std::map<CGrid*,std::pair<bool,StdString> >& gridSrcMap = grid->getTransGridSource();
 …
+     }
+   }
+   /*!
+     Solve reference of all enabled fields even the source fields .
+     In this step, we do transformations.
+   */
+   void CField::solveAllEnabledFieldsAndTransform()
+   {
+     CContext* context = CContext::getCurrent();
+     bool hasClient = context->hasClient;
+     bool hasServer = context->hasServer;
+     if (!isReferenceSolvedAndTransformed)
+     {
+        isReferenceSolvedAndTransformed = true;
+        if (hasClient && !hasServer)
+        {
+          solveRefInheritance(true);
+          if (hasDirectFieldReference()) getDirectFieldReference()->solveAllEnabledFieldsAndTransform();
+        }
+        if (hasServer)
+          solveServerOperation();
+        solveGridReference();
+        if (hasClient && !hasServer)
+       {
+         solveGenerateGrid();
+         buildGridTransformationGraph();
+       }
+       solveGridDomainAxisRef(false);
+       if (hasClient && !hasServer)
+       {
+         solveTransformedGrid();
+       }
+       solveGridDomainAxisRef(false);
+     }
+   }
+   void CField::checkGridOfEnabledFields()
+   {
+     if (!isGridChecked)
+     {
+       isGridChecked = true;
+       solveCheckMaskIndex(false);
+     }
+   }
+   void CField::sendGridComponentOfEnabledFields()
+   {
+      solveGridDomainAxisRef(true);
+      // solveCheckMaskIndex(true);
+   }
+   void CField::sendGridOfEnabledFields()
+   {
+      // solveGridDomainAxisRef(true);
+      solveCheckMaskIndex(true);
+   }
+   void CField::solveOnlyReferenceEnabledField(bool doSending2Server)
+   {
+     CContext* context = CContext::getCurrent();
+     if (!isReferenceSolved)
+     {
+        isReferenceSolved = true;
+        if (context->hasClient && !context->hasServer)
+        {
+          solveRefInheritance(true);
+          if (hasDirectFieldReference()) getDirectFieldReference()->solveOnlyReferenceEnabledField(false);
+        }
+        if (context->hasServer)
+          solveServerOperation();
+        solveGridReference();
+        grid->solveDomainAxisRefInheritance(true); // make it again to solve grid reading from file
+        if (context->hasClient && !context->hasServer)
+       {
+         solveGenerateGrid();
+         buildGridTransformationGraph();
+       }
+     }
+   }
    void CField::solveAllReferenceEnabledField(bool doSending2Server)
+   {
 …
      solveOnlyReferenceEnabledField(doSending2Server);
-     //std::cout<<"Field "<<this->getId()<<" areAllReferenceSolved = "<<areAllReferenceSolved<<std::endl;
      if (!areAllReferenceSolved)
+     {
         areAllReferenceSolved = true;
+        //std::cout<<"Field "<<this->getId()<<" all reference solved"<<std::endl;
         if (context->hasClient)
+        if (context->hasClient && !context->hasServer)
+        {
           solveRefInheritance(true);
 …
      solveGridDomainAxisRef(doSending2Server);
      if (context->hasClient)
+     if (context->hasClient && !context->hasServer)
+     {
        solveTransformedGrid();
 …
+   }
+   std::map<int, StdSize> CField::getGridAttributesBufferSize()
+   {
+     return grid->getAttributesBufferSize();
+   }
+   std::map<int, StdSize> CField::getGridDataBufferSize()
+   {
+     return grid->getDataBufferSize(getId());
+   std::map<int, StdSize> CField::getGridAttributesBufferSize(CContextClient* client, bool bufferForWriting /*= "false"*/)
+   {
+     return grid->getAttributesBufferSize(client, bufferForWriting);
+   }
+   std::map<int, StdSize> CField::getGridDataBufferSize(CContextClient* client, bool bufferForWriting /*= "false"*/)
+   {
+     return grid->getDataBufferSize(client, getId(), bufferForWriting);
+   }
+   size_t CField::getGlobalWrittenSize()
+   {
+     return grid->getGlobalWrittenSize();
+   }
 …
     */
    void CField::buildFilterGraph(CGarbageCollector& gc, bool enableOutput)
+   {
+     if (!areAllReferenceSolved) solveAllReferenceEnabledField(false);
+   {
+    if (!isReferenceSolvedAndTransformed) solveAllEnabledFieldsAndTransform();
+    if (!isGridChecked) checkGridOfEnabledFields();
      const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
      const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
+     // Start by building a filter which can provide the field's instant data
+     if (!instantDataFilter)
+     {
+       // Check if we have an expression to parse
+       if (hasExpression())
+       {
+         boost::scoped_ptr<IFilterExprNode> expr(parseExpr(getExpression() + '\0'));
+         boost::shared_ptr<COutputPin> filter = expr->reduce(gc, *this);
+         // Check if a spatial transformation is needed
+         if (!field_ref.isEmpty())
+     CContext* context = CContext::getCurrent();
+     bool hasWriterServer = context->hasServer && !context->hasClient;
+     bool hasIntermediateServer = context->hasServer && context->hasClient;
+     if (hasWriterServer)
+     {
+        if (!instantDataFilter)
+          instantDataFilter = clientSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid,true));
+       // If the field data is to be read by the client or/and written to a file
+       if (enableOutput && !storeFilter && !fileWriterFilter)
+       {
+         if (file && (file->mode.isEmpty() || file->mode == CFile::mode_attr::write))
+         {
+           CGrid* gridRef = CField::get(field_ref)->grid;
+           if (grid && grid != gridRef && grid->hasTransform())
+           fileServerWriterFilter = boost::shared_ptr<CFileServerWriterFilter>(new CFileServerWriterFilter(gc, this));
+           instantDataFilter->connectOutput(fileServerWriterFilter, 0);
+         }
+       }
+     }
+     else if (hasIntermediateServer)
+     {
+       if (!instantDataFilter)
+         instantDataFilter = clientSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true));
+             // If the field data is to be read by the client or/and written to a file
+       if (enableOutput && !storeFilter && !fileWriterFilter)
+       {
+         if (file && (file->mode.isEmpty() || file->mode == CFile::mode_attr::write))
+         {
+           fileWriterFilter = boost::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this));
+           instantDataFilter->connectOutput(fileWriterFilter, 0);
+         }
+       }
+     }
+     else
+     {
+       // Start by building a filter which can provide the field's instant data
+       if (!instantDataFilter)
+       {
+         // Check if we have an expression to parse
+         if (hasExpression())
+         {
+           boost::scoped_ptr<IFilterExprNode> expr(parseExpr(getExpression() + '\0'));
+           boost::shared_ptr<COutputPin> filter = expr->reduce(gc, *this);
+           // Check if a spatial transformation is needed
+           if (!field_ref.isEmpty())
+           {
+             std::pair<boost::shared_ptr<CFilter>, boost::shared_ptr<CFilter> > filters = CSpatialTransformFilter::buildFilterGraph(gc, gridRef, grid, detectMissingValues, defaultValue);
+             filter->connectOutput(filters.first, 0);
+             filter = filters.second;
+             CGrid* gridRef = CField::get(field_ref)->grid;
+             if (grid && grid != gridRef && grid->hasTransform())
+             {
+                 std::pair<boost::shared_ptr<CFilter>, boost::shared_ptr<CFilter> > filters = CSpatialTransformFilter::buildFilterGraph(gc, gridRef, grid, detectMissingValues, defaultValue);
+               filter->connectOutput(filters.first, 0);
+               filter = filters.second;
+             }
+           }
+           instantDataFilter = filter;
+         }
+         instantDataFilter = filter;
+       }
+       // Check if we have a reference on another field
+       else if (!field_ref.isEmpty())
+         instantDataFilter = getFieldReference(gc);
+       // Check if the data is to be read from a file
+       else if (file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read)
+       {
+         checkAttributes();
+         instantDataFilter = serverSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, freq_offset, true,
+                                                                                                     detectMissingValues, defaultValue));
+       }
+       else // The data might be passed from the model
+       {
+          if (check_if_active.isEmpty()) check_if_active = false;
+          instantDataFilter = clientSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, NoneDu, false,
+                                                                                                      detectMissingValues, defaultValue));
+       }
+     }
+     // If the field data is to be read by the client or/and written to a file
+     if (enableOutput && !storeFilter && !fileWriterFilter)
+     {
+       if (!read_access.isEmpty() && read_access)
+       {
+         storeFilter = boost::shared_ptr<CStoreFilter>(new CStoreFilter(gc, CContext::getCurrent(), grid,
+                                                                        detectMissingValues, defaultValue));
+         instantDataFilter->connectOutput(storeFilter, 0);
+       }
+       if (file && (file->mode.isEmpty() || file->mode == CFile::mode_attr::write))
+       {
+         fileWriterFilter = boost::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this));
+         getTemporalDataFilter(gc, file->output_freq)->connectOutput(fileWriterFilter, 0);
+         // Check if we have a reference on another field
+         else if (!field_ref.isEmpty())
+           instantDataFilter = getFieldReference(gc);
+         // Check if the data is to be read from a file
+         else if (file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read)
+         {
+           checkTimeAttributes();
+           instantDataFilter = serverSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, freq_offset, true,
+                                                                                                       detectMissingValues, defaultValue));
+         }
+         else // The data might be passed from the model
+         {
+            if (check_if_active.isEmpty()) check_if_active = false;
+            instantDataFilter = clientSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, false, NoneDu, false,
+                                                                                                        detectMissingValues, defaultValue));
+         }
+       }
+       // If the field data is to be read by the client or/and written to a file
+       if (enableOutput && !storeFilter && !fileWriterFilter)
+       {
+         if (!read_access.isEmpty() && read_access)
+         {
+           storeFilter = boost::shared_ptr<CStoreFilter>(new CStoreFilter(gc, CContext::getCurrent(), grid,
+                                                                          detectMissingValues, defaultValue));
+           instantDataFilter->connectOutput(storeFilter, 0);
+         }
+         if (file && (file->mode.isEmpty() || file->mode == CFile::mode_attr::write))
+         {
+           fileWriterFilter = boost::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this));
+           getTemporalDataFilter(gc, file->output_freq)->connectOutput(fileWriterFilter, 0);
+         }
+       }
+     }
 …
          if (!serverSourceFilter)
+         {
            checkAttributes();
            serverSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, freq_offset, true,
+           checkTimeAttributes();
+           serverSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, freq_offset, true,
                                                                                    detectMissingValues, defaultValue));
+         }
 …
+       {
          CField* fieldRef = CField::get(field_ref);
          fieldRef->buildFilterGraph(gc, false);
+         fieldRef->buildFilterGraph(gc, false);
          selfReferenceFilter = fieldRef->getInstantDataFilter();
+       }
 …
+         {
            if (check_if_active.isEmpty()) check_if_active = false;
            clientSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, NoneDu, false,
+           clientSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, true, NoneDu, false,
                                                                                    detectMissingValues, defaultValue));
+         }
 …
                << "An operation must be defined for field \"" << getId() << "\".");
        checkAttributes();
        const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
+       checkTimeAttributes(&outFreq);
+       const bool detectMissingValues = (!detect_missing_value.isEmpty()  && detect_missing_value == true);
        boost::shared_ptr<CTemporalFilter> temporalFilter(new CTemporalFilter(gc, operation,
                                                                              CContext::getCurrent()->getCalendar()->getInitDate(),
                                                                              freq_op, freq_offset, outFreq,
+                                                                             detectMissingValues, detectMissingValues ? default_value : 0.0));
+                                                                             freq_op, freq_offset, outFreq, detectMissingValues));
        instantDataFilter->connectOutput(temporalFilter, 0);
 …
                << "An operation must be defined for field \"" << getId() << "\".");
        checkAttributes();
        const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
+       checkTimeAttributes(&outFreq);
+       const bool detectMissingValues = (!detect_missing_value.isEmpty() && detect_missing_value == true);
        boost::shared_ptr<CTemporalFilter> temporalFilter(new CTemporalFilter(gc, operation,
                                                                              CContext::getCurrent()->getCalendar()->getInitDate(),
                                                                              freq_op, freq_offset, outFreq,
+                                                                             detectMissingValues, detectMissingValues ? default_value : 0.0));
+                                                                             freq_op, freq_offset, outFreq, detectMissingValues));
        selfReferenceFilter->connectOutput(temporalFilter, 0);
        return temporalFilter ;
 …
+     {
        // Temporarily deactivate the self-transformation of grid
        //grid->transformGrid(grid);
+       // grid->transformGrid(grid);
+     }
+   }
 …
    void CField::scaleFactorAddOffset(double scaleFactor, double addOffset)
+   {
+     map<int, CArray<double,1> >::iterator it;
+     for (it = data_srv.begin(); it != data_srv.end(); it++) it->second = (it->second - addOffset) / scaleFactor;
+     recvDataSrv = (recvDataSrv - addOffset) / scaleFactor;
+   }
    void CField::invertScaleFactorAddOffset(double scaleFactor, double addOffset)
+   {
      map<int, CArray<double,1> >::iterator it;
      for (it = data_srv.begin(); it != data_srv.end(); it++) it->second = it->second * scaleFactor + addOffset;
+   }
    void CField::outputField(CArray<double,3>& fieldOut)
+   {
       map<int, CArray<double,1> >::iterator it;
       for (it = data_srv.begin(); it != data_srv.end(); it++)
+     recvDataSrv = recvDataSrv * scaleFactor + addOffset;
+   }
+   void CField::outputField(CArray<double,1>& fieldOut)
+   {
+      CArray<size_t,1>& outIndexClient = grid->localIndexToWriteOnClient;
+      CArray<size_t,1>& outIndexServer = grid->localIndexToWriteOnServer;
+      for (size_t idx = 0; idx < outIndexServer.numElements(); ++idx)
+      {
+        grid->outputField(it->first, it->second, fieldOut.dataFirst());
+      }
+   }
+   void CField::outputField(CArray<double,2>& fieldOut)
+   {
+      map<int, CArray<double,1> >::iterator it;
+      for(it=data_srv.begin();it!=data_srv.end();it++)
+        fieldOut(outIndexServer(idx)) = recvDataSrv(outIndexClient(idx));
+      }
+   }
+   void CField::inputField(CArray<double,1>& fieldIn)
+   {
+      CArray<size_t,1>& outIndexClient = grid->localIndexToWriteOnClient;
+      CArray<size_t,1>& outIndexServer = grid->localIndexToWriteOnServer;
+      for (size_t idx = 0; idx < outIndexServer.numElements(); ++idx)
+      {
+         grid->outputField(it->first, it->second, fieldOut.dataFirst());
+      }
+   }
+   void CField::outputField(CArray<double,1>& fieldOut)
+   {
+      map<int, CArray<double,1> >::iterator it;
+      for (it = data_srv.begin(); it != data_srv.end(); it++)
+        recvDataSrv(outIndexClient(idx)) = fieldIn(outIndexServer(idx));
+      }
+   }
+   void CField::outputCompressedField(CArray<double,1>& fieldOut)
+   {
+      CArray<size_t,1>& outIndexClient = grid->localIndexToWriteOnClient;
+      CArray<size_t,1>& outIndexServer = grid->localIndexToWriteOnServer;
+      for (size_t idx = 0; idx < outIndexServer.numElements(); ++idx)
+      {
+         grid->outputField(it->first, it->second, fieldOut.dataFirst());
+      }
+   }
+   void CField::inputField(CArray<double,3>& fieldOut)
+   {
+      map<int, CArray<double,1> >::iterator it;
+      for (it = data_srv.begin(); it != data_srv.end(); it++)
+        fieldOut((idx)) = recvDataSrv(outIndexClient(idx));
+      }
+   }
+   ///-------------------------------------------------------------------
+   void CField::parse(xml::CXMLNode& node)
+   {
+      string newContent ;
+      SuperClass::parse(node);
+      if (node.goToChildElement())
+      {
+        grid->inputField(it->first, fieldOut.dataFirst(), it->second);
+      }
+   }
+   void CField::inputField(CArray<double,2>& fieldOut)
+   {
+      map<int, CArray<double,1> >::iterator it;
+      for(it = data_srv.begin(); it != data_srv.end(); it++)
+      {
+         grid->inputField(it->first, fieldOut.dataFirst(), it->second);
+      }
+   }
+   void CField::inputField(CArray<double,1>& fieldOut)
+   {
+      map<int, CArray<double,1> >::iterator it;
+      for (it = data_srv.begin(); it != data_srv.end(); it++)
+      {
+         grid->inputField(it->first, fieldOut.dataFirst(), it->second);
+      }
+   }
+   void CField::outputCompressedField(CArray<double,1>& fieldOut)
+   {
+      map<int, CArray<double,1> >::iterator it;
+      for (it = data_srv.begin(); it != data_srv.end(); it++)
+      {
+         grid->outputCompressedField(it->first, it->second, fieldOut.dataFirst());
+      }
+   }
+   ///-------------------------------------------------------------------
+   void CField::parse(xml::CXMLNode& node)
+   {
+      SuperClass::parse(node);
+      if (!node.getContent(this->content))
+      {
+        if (node.goToChildElement())
+        do
+        {
+          do
+          {
+            if (node.getElementName() == "variable" || node.getElementName() == "variable_group") this->getVirtualVariableGroup()->parseChild(node);
+          } while (node.goToNextElement());
+          node.goToParentElement();
+        }
+      }
+          if (node.getElementName() == "variable" || node.getElementName() == "variable_group") this->getVirtualVariableGroup()->parseChild(node);
+          else if (node.getElementName() == "expr") if (node.getContent(newContent)) content+=newContent ;
+        } while (node.goToNextElement());
+        node.goToParentElement();
+      }
+      if (node.getContent(newContent)) content=newContent ;
+    }
 …
+   }
+   void CField::sendAddAllVariables()
+   void CField::setContextClient(CContextClient* contextClient)
+   {
+     CContext* context = CContext::getCurrent();
+     client = contextClient;
+     if (context->hasClient)
+     {
+       // A grid is sent by a client (both for read or write) or by primary server (write only)
+       if (context->hasServer)
+       {
+         if (file->mode.isEmpty() || (!file->mode.isEmpty() && file->mode == CFile::mode_attr::write))
+           grid->setContextClient(contextClient);
+       }
+       else
+           grid->setContextClient(contextClient);
+     }
+   }
+   CContextClient* CField::getContextClient()
+   {
+     return client;
+   }
+   void CField::sendAddAllVariables(CContextClient* client)
+   {
      std::vector<CVariable*> allVar = getAllVariables();
 …
      for (; it != itE; ++it)
+     {
+       this->sendAddVariable((*it)->getId());
+       (*it)->sendAllAttributesToServer();
+       (*it)->sendValue();
+     }
+   }
+   void CField::sendAddVariable(const string& id)
+   {
+    CContext* context = CContext::getCurrent();
+    if (!context->hasServer)
+    {
+       CContextClient* client = context->client;
+       CEventClient event(this->getType(),EVENT_ID_ADD_VARIABLE);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg << this->getId();
+         msg << id;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+    }
+   }
+   void CField::sendAddVariableGroup(const string& id)
+   {
+    CContext* context = CContext::getCurrent();
+    if (!context->hasServer)
+    {
+       CContextClient* client = context->client;
+       CEventClient event(this->getType(),EVENT_ID_ADD_VARIABLE_GROUP);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg << this->getId();
+         msg << id;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+    }
+       this->sendAddVariable((*it)->getId(), client);
+       (*it)->sendAllAttributesToServer(client);
+       (*it)->sendValue(client);
+     }
+   }
+   void CField::sendAddVariable(const string& id, CContextClient* client)
+   {
+      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE, client);
+   }
+   void CField::sendAddVariableGroup(const string& id, CContextClient* client)
+   {
+      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE_GROUP, client);
+   }
 …
     * Check on freq_off and freq_op attributes.
     */
+   void CField::checkAttributes(void)
+   {
+     bool isFieldRead = file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read;
+   void CField::checkTimeAttributes(CDuration* freqOp)
+   {
+     bool isFieldRead  = file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read;
+     bool isFieldWrite = file && ( file->mode.isEmpty() ||  file->mode == CFile::mode_attr::write);
      if (isFieldRead && operation.getValue() != "instant")
        ERROR("void CField::checkAttributes(void)",
+       ERROR("void CField::checkTimeAttributes(void)",
              << "Unsupported operation for field '" << getFieldOutputName() << "'." << std::endl
              << "Currently only \"instant\" is supported for fields read from file.")
 …
+     {
        if (operation.getValue() == "instant")
+         freq_op.setValue(file->output_freq.getValue());
+       {
+         if (isFieldRead || isFieldWrite) freq_op.setValue(file->output_freq.getValue());
+         else freq_op=*freqOp ;
+       }
        else
          freq_op.setValue(TimeStep);
 …
+   }
    DEFINE_REF_FUNC(Field,field)
 } // namespace xios

XIOS/dev/branch_openmp/src/node/field.hpp

-                      r1328
+                      r1460
 #include "transformation_enum.hpp"
 #include "variable.hpp"
+#include "context_client.hpp"
 …
    class CStoreFilter;
    class CFileWriterFilter;
+   class CFileServerWriterFilter;
    ///--------------------------------------------------------------
 …
          typedef CFieldAttributes SuperClassAttribute;
+         enum EReadField
+         {
+           RF_NODATA, RF_EOF, RF_DATA
+         };
       public:
 …
          void resetNStepMax();
+         std::map<int, StdSize> getGridAttributesBufferSize();
+         std::map<int, StdSize> getGridDataBufferSize();
+         std::map<int, StdSize> getGridAttributesBufferSize(CContextClient* client, bool bufferForWriting = false);
+         // Grid data buffer size for each connection of contextclient
+         std::map<int, StdSize> getGridDataBufferSize(CContextClient* client, bool bufferForWriting = false);
+         void setContextClient(CContextClient* newContextClient);
+         CContextClient* getContextClient();
        public:
 …
          void solveGridDomainAxisBaseRef();
+         void solveAllEnabledFieldsAndTransform();
+         void checkGridOfEnabledFields();
+         void sendGridOfEnabledFields();
+         void sendGridComponentOfEnabledFields();
+         /// VÃ©rifications ///
+         void checkTimeAttributes(CDuration* freqOp=NULL);
          void buildFilterGraph(CGarbageCollector& gc, bool enableOutput);
+         size_t getGlobalWrittenSize(void) ;
          boost::shared_ptr<COutputPin> getFieldReference(CGarbageCollector& gc);
          boost::shared_ptr<COutputPin> getSelfReference(CGarbageCollector& gc);
 …
         static bool dispatchEvent(CEventServer& event);
         void sendUpdateData(const CArray<double,1>& data);
+        void sendUpdateData(const CArray<double,1>& data, CContextClient* client);
         static void recvUpdateData(CEventServer& event);
         void recvUpdateData(vector<int>& ranks, vector<CBufferIn*>& buffers);
+        void recvUpdateData(std::map<int,CBufferIn*>& rankBuffers);
         void writeField(void);
         bool sendReadDataRequest(const CDate& tsDataRequested);
 …
         static void recvReadDataRequest(CEventServer& event);
         void recvReadDataRequest(void);
         bool readField(void);
+        EReadField readField(void);
         static void recvReadDataReady(CEventServer& event);
         void recvReadDataReady(vector<int> ranks, vector<CBufferIn*> buffers);
         void checkForLateDataFromServer(void);
+        void checkIfMustAutoTrigger(void);
+        void autoTriggerIfNeeded(void);
         void outputField(CArray<double,3>& fieldOut);
         void outputField(CArray<double,2>& fieldOut);
 …
         CVariable* addVariable(const string& id = "");
         CVariableGroup* addVariableGroup(const string& id = "");
         void sendAddVariable(const string& id = "");
         void sendAddVariableGroup(const string& id = "");
+        CVariableGroup* addVariableGroup(const string& id = "");
+        void sendAddVariable(const string& id, CContextClient* client);
+        void sendAddVariableGroup(const string& id, CContextClient* client);
         static void recvAddVariable(CEventServer& event);
         void recvAddVariable(CBufferIn& buffer);
         static void recvAddVariableGroup(CEventServer& event);
+        void recvAddVariableGroup(CBufferIn& buffer);
+        void sendAddAllVariables();
+        /// VÃ©rifications ///
+        void checkAttributes(void);
+        void recvAddVariableGroup(CBufferIn& buffer);
+        void sendAddAllVariables(CContextClient* client);
+        void writeUpdateData(const CArray<double,1>& data);
         const std::vector<StdString>& getRefDomainAxisIds();
 …
         const string& getExpression(void);
         bool hasExpression(void) const;
-        CField* my_getDirectFieldReference(void) const;
       public:
 …
          CDate lastDataRequestedFromServer, lastDataReceivedFromServer, dateEOF;
          bool wasDataRequestedFromServer, wasDataAlreadyReceivedFromServer;
+         bool mustAutoTrigger;
          map<int,boost::shared_ptr<func::CFunctor> > foperation_srv;
+         map<int, CArray<double,1> > data_srv;
+         // map<int, CArray<double,1> > data_srv;
+         CArray<double,1> recvDataSrv;
+         boost::shared_ptr<func::CFunctor> recvFoperationSrv;
          string content;
+         std::vector<StdString> domAxisScalarIds_;
+         bool useCompressedOutput;
+         // Two variables to identify the time_counter meta data written in file, which has no time_counter
+         bool hasTimeInstant;
+         bool hasTimeCentered;
+         DECLARE_REF_FUNC(Field,field)
+      private:
+         CContextClient* client;
          bool areAllReferenceSolved;
          bool isReferenceSolved;
+         std::vector<StdString> domAxisScalarIds_;
+         bool useCompressedOutput;
+         // Two variable to identify the time_counter meta data written in file, which has no time_counter
+         bool hasTimeInstant;
+         bool hasTimeCentered;
+         DECLARE_REF_FUNC(Field,field)
+         bool isReferenceSolvedAndTransformed;
+         bool isGridChecked;
+         bool nstepMaxRead;
       private:
 …
          //! The terminal filter which writes the data to file
          boost::shared_ptr<CFileWriterFilter> fileWriterFilter;
+         //! The terminal filter which writes data to file
+         boost::shared_ptr<CFileServerWriterFilter> fileServerWriterFilter;
    }; // class CField

XIOS/dev/branch_openmp/src/node/field_impl.hpp

r1205	r1460
20	20	if (clientSourceFilter)
21	21	{
22		if (~~!check_if_active \|\| isActive(true~~))
	22	if (check_if_active.isEmpty() \|\| (!check_if_active.isEmpty() && (!check_if_active) \|\| isActive(true)))
23	23	clientSourceFilter->streamData(CContext::getCurrent()->getCalendar()->getCurrentDate(), _data);
24	24	}

XIOS/dev/branch_openmp/src/node/file.cpp

-                      r1338
+                      r1460
 #include "mpi.hpp"
 #include "timer.hpp"
+#include "server.hpp"
 namespace xios {
 …
       : CObjectTemplate<CFile>(), CFileAttributes()
       , vFieldGroup(), data_out(), enabledFields()
       , allDomainEmpty(false), isOpen(false)
+      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
+   {
      setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
 …
       : CObjectTemplate<CFile>(id), CFileAttributes()
       , vFieldGroup(), data_out(), enabledFields()
       , allDomainEmpty(false), isOpen(false)
+      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
+    {
       setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
 …
    //! Initialize a file in order to write into it
    void CFile::initFile(void)
+   void CFile::initWrite(void)
+   {
       CContext* context = CContext::getCurrent();
 …
       if (!split_freq.isEmpty())
+      {
         StdString keySuffix("CContext_"+CContext::getCurrent()->getId()+"::CFile_"+getFileOutputName()+"::") ;
+        StdString keySuffix("CFile::"+getFileOutputName()+"::") ;
         if (context->registryIn->foundKey(keySuffix+"splitStart") && context->registryIn->foundKey(keySuffix+"splitEnd"))
+        {
 …
+        }
+      }
+      isOpen = false;
+      allDomainEmpty = true;
+      isOpen = false;
 //      if (!record_offset.isEmpty() && record_offset < 0)
 …
       const int recordOffset = record_offset.isEmpty() ? 0 : record_offset;
-      // set<CAxis*> setAxis;
-      // set<CDomain*> setDomains;
       set<StdString> setAxis;
       set<StdString> setDomains;
       std::vector<CField*>::iterator it, end = this->enabledFields.end();
       for (it = this->enabledFields.begin(); it != end; it++)
+      {
+         CField* field = *it;
+         allDomainEmpty &= !field->grid->doGridHaveDataToWrite();
+         CField* field = *it;
          std::vector<CAxis*> vecAxis = field->grid->getAxis();
          for (size_t i = 0; i < vecAxis.size(); ++i)
+            setAxis.insert(vecAxis[i]->getAxisOutputName());
+            // setAxis.insert(vecAxis[i]);
+           setAxis.insert(vecAxis[i]->getAxisOutputName());
          std::vector<CDomain*> vecDomains = field->grid->getDomains();
          for (size_t i = 0; i < vecDomains.size(); ++i)
+            setDomains.insert(vecDomains[i]->getDomainOutputName());
+            // setDomains.insert(vecDomains[i]);
+           setDomains.insert(vecDomains[i]->getDomainOutputName());
          field->resetNStep(recordOffset);
 …
       // create sub communicator for file
+      int color = allDomainEmpty ? 0 : 1;
+      ep_lib::MPI_Comm_split(server->intraComm, color, server->intraCommRank, &fileComm);
+      if (allDomainEmpty) ep_lib::MPI_Comm_free(&fileComm);
+      createSubComFile();
       // if (time_counter.isEmpty()) time_counter.setValue(time_counter_attr::centered);
 …
+    }
     //! Verify state of a file
     void CFile::checkFile(void)
+    //! Initialize a file in order to write into it
+    void CFile::initRead(void)
+    {
+      if (mode.isEmpty() || mode.getValue() == mode_attr::write)
+      {
+        CTimer::get("Files : create headers").resume();
+        if (!isOpen) createHeader();
+        CTimer::get("Files : create headers").suspend();
+        checkSync();
+      }
+      else
+      {
+        CTimer::get("Files : open headers").resume();
+        if (!isOpen) openInReadMode();
+        CTimer::get("Files : open headers").suspend();
+      }
+      checkSplit();
+      if (checkRead) return;
+      createSubComFile();
+      checkRead = true;
+    }
+    /*!
+      Create a sub communicator in which processes participate in reading/opening file
+    */
+    void CFile::createSubComFile()
+    {
+      CContext* context = CContext::getCurrent();
+      CContextServer* server = context->server;
+      // create sub communicator for file
+      allZoneEmpty = true;
+      std::vector<CField*>::iterator it, end = this->enabledFields.end();
+      for (it = this->enabledFields.begin(); it != end; it++)
+      {
+         CField* field = *it;
+         bool nullGrid = (0 == field->grid);
+         allZoneEmpty &= nullGrid ? false : !field->grid->doGridHaveDataToWrite();
+      }
+      int color = allZoneEmpty ? 0 : 1;
+      ep_lib::MPI_Comm_split(server->intraComm, color, server->intraCommRank, &fileComm);
+      if (allZoneEmpty) ep_lib::MPI_Comm_free(&fileComm);
+    }
+    /*
+       Check condition to write into a file
+       For now, we only use the level-2 server to write files (if this mode is activated)
+       or classical server to do this job.
+    */
+    void CFile::checkWriteFile(void)
+    {
+      CContext* context = CContext::getCurrent();
+      // Done by classical server or secondary server
+      // This condition should be changed soon
+      if (CServer::serverLevel == 0 || CServer::serverLevel == 2)
+      {
+        if (mode.isEmpty() || mode.getValue() == mode_attr::write)
+        {
+          CTimer::get("Files : create headers").resume();
+          if (!isOpen) createHeader();
+          CTimer::get("Files : create headers").suspend();
+          checkSync();
+        }
+        checkSplit(); // REally need this?
+      }
+    }
+    /*
+       Check condition to read from a file
+       For now, we only use the level-1 server to write files (if this mode is activated)
+       or classical server to do this job.
+       This function can be used by client for reading metadata
+    */
+    void CFile::checkReadFile(void)
+    {
+      CContext* context = CContext::getCurrent();
+      // Done by classical server or secondary server
+      // TODO: This condition should be changed soon. It only works with maximum number of level as 2
+      if (CServer::serverLevel == 0 || CServer::serverLevel == 1)
+      {
+        if (!mode.isEmpty() && mode.getValue() == mode_attr::read)
+        {
+          CTimer::get("Files : open headers").resume();
+          if (!isOpen) openInReadMode();
+          CTimer::get("Files : open headers").suspend();
+        }
+        //checkSplit(); // Really need for reading?
+      }
+    }
+    /*!
+      Verify if a process participates in an opening-file communicator
+      \return true if the process doesn't participate in opening file
+    */
+    bool CFile::isEmptyZone()
+    {
+      return allZoneEmpty;
+    }
 …
       CContextServer* server = context->server;
       if (!allDomainEmpty)
+      if (!allZoneEmpty)
+      {
          StdString filename = getFileOutputName();
 …
          if (pos2!=std::string::npos)
+         {
            middlePart=filename.substr(pos1,pos2-pos1) ;
+           middlePart=filename.substr(pos1,pos2-pos1) ;
            pos2+=strEndDate.size() ;
            lastPart=filename.substr(pos2,filename.size()-pos2) ;
 …
            oss << lastPart ;
            StdString keySuffix("CContext_"+CContext::getCurrent()->getId()+"::CFile_"+getFileOutputName()+"::") ;
+           StdString keySuffix("CFile::"+getFileOutputName()+"::") ;
            context->registryOut->setKey(keySuffix+"splitStart", lastSplit);
            context->registryOut->setKey(keySuffix+"splitEnd",   splitEnd);
 …
   \brief Open an existing NetCDF file in read-only mode
   */
   void CFile::openInReadMode(void)
+  void CFile::openInReadMode()
+  {
     CContext* context = CContext::getCurrent();
     CContextServer* server = context->server;
+    if (!allDomainEmpty)
+    ep_lib::MPI_Comm readComm = this->fileComm;
+    if (!allZoneEmpty)
+    {
       StdString filename = getFileOutputName();
 …
+      {
         int commSize, commRank;
         ep_lib::MPI_Comm_size(fileComm, &commSize);
         ep_lib::MPI_Comm_rank(fileComm, &commRank);
+        ep_lib::MPI_Comm_size(readComm, &commSize);
+        ep_lib::MPI_Comm_rank(readComm, &commRank);
         if (server->intraCommSize > 1)
 …
       bool isCollective = par_access.isEmpty() || par_access == par_access_attr::collective;
       #ifdef _usingEP
-        //printf("multifile was %d\n", multifile);
-        //multifile = true;
         if (isOpen) data_out->closeFile();
         if (time_counter_name.isEmpty()) data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), fileComm, multifile, isCollective));
         else data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), fileComm, multifile, isCollective, time_counter_name));
+        if (time_counter_name.isEmpty()) data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective));
+        else data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, time_counter_name));
         isOpen = true;
       #elif _usingMPI
         if (isOpen) data_out->closeFile();
         if (time_counter_name.isEmpty()) data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), fileComm, multifile, isCollective));
         else data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), fileComm, multifile, isCollective, time_counter_name));
+        if (time_counter_name.isEmpty()) data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective));
+        else data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, time_counter_name));
         isOpen = true;
       #endif
 …
    void CFile::close(void)
+   {
      if (!allDomainEmpty)
+     if (!allZoneEmpty)
        if (isOpen)
+       {
 …
          else
           this->data_in->closeFile();
+        isOpen = false;
+       }
+//      if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
+      //if (fileComm.mpi_comm != ::MPI_COMM_NULL) MPI_Comm_free(&fileComm);
+      //if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
+   }
    //----------------------------------------------------------------
 …
      // Just check file and try to open it
-     CContext* context = CContext::getCurrent();
-     CContextClient* client=context->client;
-     // It would probably be better to call initFile() somehow
-     ep_lib::MPI_Comm_dup(client->intraComm, &fileComm);
      if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
      checkFile();
+     checkReadFile();
      for (int idx = 0; idx < enabledFields.size(); ++idx)
 …
         // Read necessary value from file
         #pragma omp critical (_func)
+        {
+          this->data_in->readFieldAttributesValues(enabledFields[idx]);
+        }
+        this->data_in->readFieldAttributesValues(enabledFields[idx]);
         // Fill attributes for base reference
         enabledFields[idx]->solveGridDomainAxisBaseRef();
 …
+     {
        this->enabledFields[i]->solveOnlyReferenceEnabledField(sendToServer);
+//       this->enabledFields[i]->buildGridTransformationGraph();
+     }
+   }
+   void CFile::checkGridOfEnabledFields()
+   {
+     int size = this->enabledFields.size();
+     for (int i = 0; i < size; ++i)
+     {
+       this->enabledFields[i]->checkGridOfEnabledFields();
+     }
+   }
+   void CFile::sendGridComponentOfEnabledFields()
+   {
+     int size = this->enabledFields.size();
+     for (int i = 0; i < size; ++i)
+     {
+       this->enabledFields[i]->sendGridComponentOfEnabledFields();
+     }
+   }
+   void CFile::sendGridOfEnabledFields()
+   {
+     int size = this->enabledFields.size();
+     for (int i = 0; i < size; ++i)
+     {
+       this->enabledFields[i]->sendGridOfEnabledFields();
+     }
+   }
 …
    \param [in] sendToServer: Send all info to server (true) or only a part of it (false)
    */
    void CFile::solveAllRefOfEnabledFields(bool sendToServer)
+   void CFile::solveAllRefOfEnabledFieldsAndTransform(bool sendToServer)
+   {
      int size = this->enabledFields.size();
      for (int i = 0; i < size; ++i)
+     {
        this->enabledFields[i]->solveAllReferenceEnabledField(sendToServer);
+       this->enabledFields[i]->solveAllEnabledFieldsAndTransform();
+     }
+   }
 …
    /*!
+    * Post-process the filter graph for each active field.
+    */
+   void CFile::postProcessFilterGraph()
+   {
+     int size = this->enabledFields.size();
+     for (int i = 0; i < size; ++i)
+     {
+       this->enabledFields[i]->checkIfMustAutoTrigger();
+     }
+   }
+   /*!
      Prefetching the data for enabled fields read from file.
    */
 …
    /*!
+     Do all pre timestep operations for enabled fields in read mode:
+      - Check that the data excepted from server has been received
+      - Check if some filters must auto-trigger
+   */
+   void CFile::doPreTimestepOperationsForEnabledReadModeFields(void)
+   {
+     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
+       return;
+     int size = this->enabledFields.size();
+     for (int i = 0; i < size; ++i)
+     {
+       this->enabledFields[i]->checkForLateDataFromServer();
+       this->enabledFields[i]->autoTriggerIfNeeded();
+     }
+   }
+   /*!
      Do all post timestep operations for enabled fields in read mode:
       - Prefetch the data read from file when needed
-      - Check that the data excepted from server has been received
    */
    void CFile::doPostTimestepOperationsForEnabledReadModeFields(void)
 …
      for (int i = 0; i < size; ++i)
+     {
-       this->enabledFields[i]->checkForLateDataFromServer();
        this->enabledFields[i]->sendReadDataRequestIfNeeded();
+     }
 …
+   }
+   void CFile::setContextClient(CContextClient* newContextClient)
+   {
+     client = newContextClient;
+     size_t size = this->enabledFields.size();
+     for (size_t i = 0; i < size; ++i)
+     {
+       this->enabledFields[i]->setContextClient(newContextClient);
+     }
+   }
+   CContextClient* CFile::getContextClient()
+   {
+     return client;
+   }
+   void CFile::setReadContextClient(CContextClient* readContextclient)
+   {
+     read_client = readContextclient;
+   }
+   CContextClient* CFile::getReadContextClient()
+   {
+     return read_client;
+   }
    /*!
    \brief Send a message to create a field on server side
    \param[in] id String identity of field that will be created on server
    */
+   void CFile::sendAddField(const string& id)
+   {
+    CContext* context = CContext::getCurrent();
+    if (! context->hasServer )
+    {
+       CContextClient* client = context->client;
+       CEventClient event(this->getType(),EVENT_ID_ADD_FIELD);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg << this->getId();
+         msg << id;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+    }
+   void CFile::sendAddField(const string& id, CContextClient* client)
+   {
+      sendAddItem(id, EVENT_ID_ADD_FIELD, client);
+   }
 …
    \param[in] id String identity of field group that will be created on server
    */
+   void CFile::sendAddFieldGroup(const string& id)
+   {
+    CContext* context = CContext::getCurrent();
+    if (! context->hasServer )
+    {
+       CContextClient* client = context->client;
+       CEventClient event(this->getType(),EVENT_ID_ADD_FIELD_GROUP);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg << this->getId();
+         msg << id;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+    }
+   void CFile::sendAddFieldGroup(const string& id, CContextClient* client)
+   {
+      sendAddItem(id, (int)EVENT_ID_ADD_FIELD_GROUP, client);
+   }
 …
    is to duplicate this value on server, too.
    */
    void CFile::sendAddAllVariables()
+   void CFile::sendAddAllVariables(CContextClient* client)
+   {
      std::vector<CVariable*> allVar = getAllVariables();
 …
      for (; it != itE; ++it)
+     {
+       this->sendAddVariable((*it)->getId());
+       (*it)->sendAllAttributesToServer();
+       (*it)->sendValue();
+     }
+   }
+   /*!
+   \brief Send a message to create a variable on server side
+      A variable always belongs to a variable group
+   \param[in] id String identity of variable that will be created on server
+   */
+   void CFile::sendAddVariable(const string& id)
+   {
+    CContext* context = CContext::getCurrent();
+    if (! context->hasServer )
+    {
+       CContextClient* client = context->client;
+       CEventClient event(this->getType(),EVENT_ID_ADD_VARIABLE);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg << this->getId();
+         msg << id;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+    }
+       this->sendAddVariable((*it)->getId(), client);
+       (*it)->sendAllAttributesToServer(client);
+       (*it)->sendValue(client);
+     }
+   }
 …
    \brief Send a message to create a variable group on server side
    \param[in] id String identity of variable group that will be created on server
+   */
+   void CFile::sendAddVariableGroup(const string& id)
+   {
+    CContext* context = CContext::getCurrent();
+    if (! context->hasServer )
+    {
+       CContextClient* client = context->client;
+       CEventClient event(this->getType(),EVENT_ID_ADD_VARIABLE_GROUP);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg << this->getId();
+         msg << id;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+    }
+   \param [in] client client to which we will send this adding action
+   */
+   void CFile::sendAddVariableGroup(const string& id, CContextClient* client)
+   {
+      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE_GROUP, client);
+   }
+   /*
+     Send message to add a variable into a file within a certain client
+     \param [in] id String identity of a variable
+     \param [in] client client to which we will send this adding action
+   */
+   void CFile::sendAddVariable(const string& id, CContextClient* client)
+   {
+      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE, client);
+   }
 …
    Remark: This function must be called AFTER all active (enabled) files have been created on the server side
    */
    void CFile::sendEnabledFields()
+   void CFile::sendEnabledFields(CContextClient* client)
+   {
      size_t size = this->enabledFields.size();
 …
+     {
        CField* field = this->enabledFields[i];
+       this->sendAddField(field->getId());
+       field->checkAttributes();
+       field->sendAllAttributesToServer();
+       field->sendAddAllVariables();
+     }
+   }
+       this->sendAddField(field->getId(), client);
+       field->checkTimeAttributes();
+       field->sendAllAttributesToServer(client);
+       field->sendAddAllVariables(client);
+     }
+   }
    /*!

XIOS/dev/branch_openmp/src/node/file.hpp

-                      r1328
+                      r1460
 #include "attribute_enum.hpp"
 #include "attribute_enum_impl.hpp"
+#include "context_client.hpp"
+//#include "mpi.hpp"
 namespace xios {
 …
          bool checkSplit(void);
          bool checkSync(void);
+         void checkFile(void);
+         void initFile(void);
+         void checkWriteFile(void);
+         void checkReadFile(void);
+         void initWrite(void);
+         void initRead(void);
+         bool isEmptyZone();
          /// Mutateurs ///
 …
          void solveOnlyRefOfEnabledFields(bool sendToServer);
          void generateNewTransformationGridDest();
          void solveAllRefOfEnabledFields(bool sendToServer);
          void buildFilterGraphOfEnabledFields(CGarbageCollector& gc);
+         void postProcessFilterGraph();
          void prefetchEnabledReadModeFields();
+         void doPreTimestepOperationsForEnabledReadModeFields();
          void doPostTimestepOperationsForEnabledReadModeFields();
+         void solveAllRefOfEnabledFieldsAndTransform(bool sendToServer);
+         void checkGridOfEnabledFields();
+         void sendGridOfEnabledFields();
+         void sendGridComponentOfEnabledFields();
          // Add component into file
 …
          CVariable* addVariable(const string& id = "");
          CVariableGroup* addVariableGroup(const string& id = "");
+         // Send info to serever
+         void sendEnabledFields();
+         void sendAddField(const string& id = "");
+         void sendAddFieldGroup(const string& id = "");
+         void sendAddAllVariables();
+         void sendAddVariable(const string& id = "");
+         void sendAddVariableGroup(const string& id = "");
+         void setContextClient(CContextClient* newContextClient);
+         CContextClient* getContextClient();
+         void setReadContextClient(CContextClient* newContextClient);
+         CContextClient* getReadContextClient();
+         // Send info to server
+         void sendEnabledFields(CContextClient* client);
+         void sendAddField(const string& id, CContextClient* client);
+         void sendAddFieldGroup(const string& id, CContextClient* client);
+         void sendAddVariable(const string& id, CContextClient* client);
+         void sendAddVariableGroup(const string& id, CContextClient* client);
+         void sendAddAllVariables(CContextClient* client);
          // Receive info from client
          static void recvAddField(CEventServer& event);
 …
          CDate lastSplit;
          int nbAxis, nbDomains;
+         bool isOpen;
+         bool allDomainEmpty;
+         bool isOpen;
          ep_lib::MPI_Comm fileComm;
+      private:
+         void createSubComFile();
+         bool checkRead;
+         bool allZoneEmpty;
       private :
          /// PropriÃ©tÃ©s privÃ©es ///
+         CContextClient* client;
+         CContextClient* read_client; // Context client for reading (channel between server 1 and client)
          CFieldGroup* vFieldGroup;
          CVariableGroup* vVariableGroup;
 …
          std::vector<CField*> enabledFields;
       public:
         //         virtual void toBinary  (StdOStream& os) const;

XIOS/dev/branch_openmp/src/node/grid.cpp

-                      r1334
+                      r1460
 #include "grid_transformation.hpp"
 #include "grid_generate.hpp"
+#include "server.hpp"
 namespace xios {
 …
       , clientDistribution_(0), isIndexSent(false) , serverDistribution_(0), clientServerMap_(0)
       , writtenDataSize_(0), numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
+      , connectedDataSize_(), connectedServerRank_(), isDataDistributed_(true), isCompressible_(false)
+      , connectedDataSize_(), connectedServerRank_(), connectedServerRankRead_(), connectedDataSizeRead_()
+          , isDataDistributed_(true), isCompressible_(false)
       , transformations_(0), isTransformed_(false)
       , axisPositionInGrid_(), hasDomainAxisBaseRef_(false)
       , gridSrc_(), hasTransform_(false), isGenerated_(false), order_(), globalIndexOnServer_()
+      , computedWrittenIndex_(false)
+      , clients()
+   {
      setVirtualDomainGroup(CDomainGroup::create(getId() + "_virtual_domain_group"));
 …
       , clientDistribution_(0), isIndexSent(false) , serverDistribution_(0), clientServerMap_(0)
       , writtenDataSize_(0), numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
+      , connectedDataSize_(), connectedServerRank_(), isDataDistributed_(true), isCompressible_(false)
+      , connectedDataSize_(), connectedServerRank_(), connectedServerRankRead_(), connectedDataSizeRead_()
+          , isDataDistributed_(true), isCompressible_(false)
       , transformations_(0), isTransformed_(false)
       , axisPositionInGrid_(), hasDomainAxisBaseRef_(false)
       , gridSrc_(), hasTransform_(false), isGenerated_(false), order_(), globalIndexOnServer_()
+      , computedWrittenIndex_(false)
+      , clients()
+   {
      setVirtualDomainGroup(CDomainGroup::create(getId() + "_virtual_domain_group"));
 …
+     {
        std::vector<int> dataNindex = clientDistribution_->getDataNIndex();
        for (int i = 0; i < dataNindex.size(); ++i) retvalue *= dataNindex[i];
+       for (int i = 0; i < dataNindex.size(); ++i) retvalue *= dataNindex[i];
+     }
      return retvalue;
 …
+    *
     * \return A map associating the server rank with its minimum buffer size.
+    * TODO: Refactor code
     */
    std::map<int, StdSize> CGrid::getAttributesBufferSize()
+   {
      std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes();
+   std::map<int, StdSize> CGrid::getAttributesBufferSize(CContextClient* client, bool bufferForWriting)
+   {
+     std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes(client);
      // The grid indexes require a similar size as the actual data
      std::map<int, StdSize> dataSizes = getDataBufferSize();
+     std::map<int, StdSize> dataSizes = getDataBufferSize(client, "", bufferForWriting);
      std::map<int, StdSize>::iterator it, itE = dataSizes.end();
      for (it = dataSizes.begin(); it != itE; ++it)
 …
          attributesSizes[it->first] = it->second;
+     }
      // Account for the axis attributes
      std::vector<CAxis*> axisList = getAxis();
      for (size_t i = 0; i < axisList.size(); ++i)
+     {
        std::map<int, StdSize> axisAttBuffSize = axisList[i]->getAttributesBufferSize();
+       std::map<int, StdSize> axisAttBuffSize = axisList[i]->getAttributesBufferSize(client, getGlobalDimension(),axisPositionInGrid_[i]);
        for (it = axisAttBuffSize.begin(), itE = axisAttBuffSize.end(); it != itE; ++it)
+       {
+         it->second += 2 * sizeof(bool);
          if (it->second > attributesSizes[it->first])
            attributesSizes[it->first] = it->second;
 …
      for (size_t i = 0; i < domList.size(); ++i)
+     {
        std::map<int, StdSize> domAttBuffSize = domList[i]->getAttributesBufferSize();
+       std::map<int, StdSize> domAttBuffSize = domList[i]->getAttributesBufferSize(client);
        for (it = domAttBuffSize.begin(), itE = domAttBuffSize.end(); it != itE; ++it)
+       {
+         it->second += 2 * sizeof(bool);
          if (it->second > attributesSizes[it->first])
            attributesSizes[it->first] = it->second;
 …
      return attributesSizes;
+   }
+  }
    /*!
     * Compute the minimum buffer size required to send the data to the server(s).
+    *
+    * Compute the minimum buffer size required to send the data.
+    * \param client contextClient used to determine the size of connected receivers
     * \param id the id used to tag the data
+    * \return A map associating the server rank with its minimum buffer size.
+    * \param bufferForWriting flag indicating if a buffer is used to send data for writing
+    * \return A map associating the sender rank with its minimum buffer size.
     */
+   std::map<int, StdSize> CGrid::getDataBufferSize(const std::string& id /*= ""*/)
+   {
+     std::map<int, StdSize> dataSizes;
+   std::map<int, StdSize> CGrid::getDataBufferSize(CContextClient* client, const std::string& id /*= ""*/, bool bufferForWriting /*= "false"*/)
+   {
      // The record index is sometimes sent along with the data but we always
      // include it in the size calculation for the sake of simplicity
+     const size_t extraSize = CEventClient::headerSize + (id.empty() ? getId() : id).size() + 2 * sizeof(size_t);
+     std::map<int, size_t>::const_iterator itEnd = connectedDataSize_.end();
+     for (size_t k = 0; k < connectedServerRank_.size(); ++k)
+     const size_t extraSize = CEventClient::headerSize + (id.empty() ? getId() : id).size()
+                                                       + 2 * sizeof(size_t)
+                                                       + sizeof(size_t);
+     std::map<int, StdSize> dataSizes;
+     int receiverSize = client->serverSize;
+     std::map<int,size_t>& dataSizeMap = bufferForWriting ? connectedDataSize_[receiverSize]: connectedDataSizeRead_;
+     std::vector<int>& connectedServerRanks = bufferForWriting ? connectedServerRank_[receiverSize] : connectedServerRankRead_;
+     std::map<int, size_t>::const_iterator itEnd = dataSizeMap.end();
+     for (size_t k = 0; k < connectedServerRanks.size(); ++k)
+     {
        int rank = connectedServerRank_[k];
        std::map<int, size_t>::const_iterator it = connectedDataSize_.find(rank);
+       int rank = connectedServerRanks[k];
+       std::map<int, size_t>::const_iterator it = dataSizeMap.find(rank);
        size_t count = (it != itEnd) ? it->second : 0;
 …
+   }
+   size_t CGrid::getGlobalWrittenSize(void)
+   {
+         std::vector<CDomain*> domainP = this->getDomains();
+     std::vector<CAxis*> axisP = this->getAxis();
+     size_t globalGridSize=1 ;
+     for (std::vector<CDomain*>::iterator it=domainP.begin(); it!=domainP.end();++it) globalGridSize*=(*it)->getGlobalWrittenSize() ;
+     for (std::vector<CAxis*>::iterator it=axisP.begin(); it!=axisP.end();++it) globalGridSize*=(*it)->getGlobalWrittenSize() ;
+     return globalGridSize ;
+   }
    void CGrid::checkAttributesAfterTransformation()
+   {
 …
    //---------------------------------------------------------------
+   /*
+     Find all reference of grid's components and inherite attributes if necessary
+   */
    void CGrid::solveDomainAxisRef(bool areAttributesChecked)
+   {
 …
+   }
+   /*
+     Go up hierachy reference and fill in the base reference with attributes of the children
+     This function should be only used after reading component's attributes from file
+   */
    void CGrid::solveDomainAxisBaseRef()
+   {
 …
+   {
      CContext* context = CContext::getCurrent();
+     CContextClient* client=context->client;
+     if (isScalarGrid())
+     {
+       if (context->hasClient)
+          if (this->isChecked && doSendingIndex && !isIndexSent) { sendIndexScalarGrid(); this->isIndexSent = true; }
+       if (this->isChecked) return;
+       if (context->hasClient)
+       {
+          this->computeIndexScalarGrid();
+     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
+     nbSrvPools = 1;
+     for (int p = 0; p < nbSrvPools; ++p)
+     {
+       if (context->hasClient && this->isChecked && doSendingIndex && !isIndexSent)
+       {
+         if (isScalarGrid())
+           sendIndexScalarGrid();
+         else
+           sendIndex();
+         this->isIndexSent = true;
+       }
+       if (!(this->hasTransform() && !this->isTransformed()))
+        this->isChecked = true;
+       return;
+       // Not sure about this
+       //if (!(this->hasTransform() && !this->isTransformed()))
+       // this->isChecked = true;
+       //return;
+     }
+     if (context->hasClient)
+      if (this->isChecked && doSendingIndex && !isIndexSent) { sendIndex(); this->isIndexSent = true; }
      if (this->isChecked) return;
+     if (context->hasClient)
+     {
+        this->checkAttributesAfterTransformation();
+        this->checkMask();
+        this->computeIndex();
+     }
+     if (!(this->hasTransform() && !this->isTransformed()))
+     this->checkAttributesAfterTransformation();
+     // TODO: Transfer grid attributes
+     //if (!context->hasClient && context->hasServer) this->createMask();
+     this->computeIndex();
+     if (!(this->hasTransform() && !this->isTransformed()))
       this->isChecked = true;
      if (!(this->hasTransform() && (!this->isGenerated())))
+      this->isChecked = true;
+   }
+      this->isChecked = true;
+   }
+   /*
+     Create mask of grid from mask of its components
+   */
    void CGrid::createMask(void)
+   {
 …
       std::vector<CArray<bool,1>* > domainMasks(domainP.size());
       for (int i = 0; i < domainMasks.size(); ++i) domainMasks[i] = &(domainP[i]->mask_1d);
+      for (int i = 0; i < domainMasks.size(); ++i) domainMasks[i] = &(domainP[i]->domainMask);
       std::vector<CArray<bool,1>* > axisMasks(axisP.size());
       for (int i = 0; i < axisMasks.size(); ++i) axisMasks[i] = &(axisP[i]->mask);
 …
+   }
+   /*
+     Check validity of grid's mask by using the masks of its components
+   */
    void CGrid::checkMask(void)
+   {
 …
       std::vector<CArray<bool,1>* > domainMasks(domainP.size());
       for (int i = 0; i < domainMasks.size(); ++i) domainMasks[i] = &(domainP[i]->mask_1d);
+      for (int i = 0; i < domainMasks.size(); ++i) domainMasks[i] = &(domainP[i]->domainMask);
       std::vector<CArray<bool,1>* > axisMasks(axisP.size());
       for (int i = 0; i < axisMasks.size(); ++i) axisMasks[i] = &(axisP[i]->mask);
 …
+   }
+   void CGrid::modifyMask(const CArray<int,1>& indexToModify)
+/*!
+  A grid can have multiple dimension, so can its mask in the form of multi-dimension array.
+It's not a good idea to store all multi-dimension arrays corresponding to each mask.
+One of the ways is to convert this array into 1-dimension one and every process is taken place on it.
+  \param [in] multi-dimension array grid mask
+*/
+  void CGrid::getLocalMask(CArray<bool,1>& localMask)
+  {
+      std::vector<CDomain*> domainP = this->getDomains();
+      std::vector<CAxis*> axisP = this->getAxis();
+      int dim = domainP.size() * 2 + axisP.size();
+      switch (dim)
+      {
+        case 0:
+          getLocalMask(mask_0d, localMask);
+          break;
+        case 1:
+          getLocalMask(mask_1d, localMask);
+          break;
+        case 2:
+          getLocalMask(mask_2d, localMask);
+          break;
+        case 3:
+          getLocalMask(mask_3d, localMask);
+          break;
+        case 4:
+          getLocalMask(mask_4d, localMask);
+          break;
+        case 5:
+          getLocalMask(mask_5d, localMask);
+          break;
+        case 6:
+          getLocalMask(mask_6d, localMask);
+          break;
+        case 7:
+          getLocalMask(mask_7d, localMask);
+          break;
+        default:
+          break;
+      }
+  }
+   /*
+     Modify value of mask in a certain index
+     This function can be used to correct the mask of grid after being constructed with createMask
+     \param [in] indexToModify
+     \param [in] modifyValue
+   */
+   void CGrid::modifyMask(const CArray<int,1>& indexToModify, bool modifyValue)
+   {
       using namespace std;
 …
       switch (dim) {
+        case 0:
+          modifyGridMask(mask_0d, indexToModify, modifyValue);
+          break;
         case 1:
           modifyGridMask(mask_1d, indexToModify);
+          modifyGridMask(mask_1d, indexToModify, modifyValue);
           break;
         case 2:
           modifyGridMask(mask_2d, indexToModify);
+          modifyGridMask(mask_2d, indexToModify, modifyValue);
           break;
         case 3:
           modifyGridMask(mask_3d, indexToModify);
+          modifyGridMask(mask_3d, indexToModify, modifyValue);
           break;
         case 4:
           modifyGridMask(mask_1d, indexToModify);
+          modifyGridMask(mask_4d, indexToModify, modifyValue);
           break;
         case 5:
           modifyGridMask(mask_2d, indexToModify);
+          modifyGridMask(mask_5d, indexToModify, modifyValue);
           break;
         case 6:
           modifyGridMask(mask_3d, indexToModify);
+          modifyGridMask(mask_6d, indexToModify, modifyValue);
           break;
         case 7:
+          modifyGridMask(mask_3d, indexToModify);
+          modifyGridMask(mask_7d, indexToModify, modifyValue);
+          break;
+        default:
+          break;
+      }
+   }
+   /*
+     Change the mask size. This function is used on reconstructing mask in server side
+     \param [in] newDimensionSize
+     \param [in] newValue
+   */
+   void CGrid::modifyMaskSize(const std::vector<int>& newDimensionSize, bool newValue)
+   {
+      std::vector<CDomain*> domainP = this->getDomains();
+      std::vector<CAxis*> axisP = this->getAxis();
+      int dim = domainP.size() * 2 + axisP.size();
+      switch (dim) {
+        case 0:
+          modifyGridMaskSize(mask_0d, newDimensionSize, newValue);
+          break;
+        case 1:
+          modifyGridMaskSize(mask_1d, newDimensionSize, newValue);
+          break;
+        case 2:
+          modifyGridMaskSize(mask_2d, newDimensionSize, newValue);
+          break;
+        case 3:
+          modifyGridMaskSize(mask_3d, newDimensionSize, newValue);
+          break;
+        case 4:
+          modifyGridMaskSize(mask_4d, newDimensionSize, newValue);
+          break;
+        case 5:
+          modifyGridMaskSize(mask_5d, newDimensionSize, newValue);
+          break;
+        case 6:
+          modifyGridMaskSize(mask_6d, newDimensionSize, newValue);
+          break;
+        case 7:
+          modifyGridMaskSize(mask_7d, newDimensionSize, newValue);
           break;
         default:
 …
+   }
+   std::vector<int> CGrid::getAxisPositionInGrid() const
+   {
+     return axisPositionInGrid_;
+   /*!
+      Compute the index to for write data into a file
+   */
+   void CGrid::computeWrittenIndex()
+   {
+      if (computedWrittenIndex_) return;
+      computedWrittenIndex_ = true;
+      if (isScalarGrid())
+      {
+        size_t nbWritten = 1;
+        int writtenIndex = 0;
+        localIndexToWriteOnClient.resize(nbWritten);
+        localIndexToWriteOnServer.resize(nbWritten);
+        localIndexToWriteOnServer(0) = writtenIndex;
+        localIndexToWriteOnClient(0) = writtenIndex;
+        return;
+      }
+      size_t nbWritten = 0, indGlo;
+      CDistributionClient::GlobalLocalDataMap& globalDataIndex = clientDistribution_->getGlobalDataIndexOnClient();
+      CDistributionClient::GlobalLocalDataMap::const_iterator itb = globalDataIndex.begin(),
+                                                              ite = globalDataIndex.end(), it;
+      const CDistributionServer::GlobalLocalMap& globalLocalIndex = serverDistribution_->getGlobalLocalIndex();
+      CDistributionServer::GlobalLocalMap::const_iterator itSrvb = globalLocalIndex.begin(),
+                                                          itSrve = globalLocalIndex.end(), itSrv;
+      for (it = itb; it != ite; ++it)
+      {
+        indGlo = it->first;
+        if (globalLocalIndex.end() != globalLocalIndex.find(indGlo)) ++nbWritten;
+      }
+      localIndexToWriteOnClient.resize(nbWritten);
+      localIndexToWriteOnServer.resize(nbWritten);
+      {
+        numberWrittenIndexes_ = nbWritten;
+        if (isDataDistributed_)
+        {
+          CContextServer* server = CContext::getCurrent()->server;
+          MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+          MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+          offsetWrittenIndexes_ -= numberWrittenIndexes_;
+        }
+        else
+          totalNumberWrittenIndexes_ = numberWrittenIndexes_;
+      }
+      nbWritten = 0;
+      for (it = itb; it != ite; ++it)
+      {
+        indGlo = it->first;
+        itSrv = globalLocalIndex.find(indGlo);
+        if (itSrve != itSrv)
+        {
+          localIndexToWriteOnServer(nbWritten) = itSrv->second;
+          localIndexToWriteOnClient(nbWritten) = it->second;
+          ++nbWritten;
+        }
+      }
+   }
    //---------------------------------------------------------------
+   /*
+     Compute the global index and its local index taking account mask and data index.
+     These global indexes will be used to compute the connection of this client (sender) to its servers (receivers)
+     (via function computeConnectedClient)
+     These global indexes also correspond to data sent to servers (if any)
+   */
+   void CGrid::computeClientIndex()
+   {
+     CContext* context = CContext::getCurrent();
+     CContextClient* client = context->client;  // Here it's not important which contextClient to recuperate
+     int rank = client->clientRank;
+     clientDistribution_ = new CDistributionClient(rank, this);
+     // Get local data index on client
+     storeIndex_client.resize(clientDistribution_->getLocalDataIndexOnClient().size());
+     int nbStoreIndex = storeIndex_client.numElements();
+     for (int idx = 0; idx < nbStoreIndex; ++idx) storeIndex_client(idx) = (clientDistribution_->getLocalDataIndexOnClient())[idx];
+     if (0 == serverDistribution_) isDataDistributed_= clientDistribution_->isDataDistributed();
+     else
+     {
+        // Mapping global index received from clients to the storeIndex_client
+        CDistributionClient::GlobalLocalDataMap& globalDataIndex = clientDistribution_->getGlobalDataIndexOnClient();
+        CDistributionClient::GlobalLocalDataMap::const_iterator itGloe = globalDataIndex.end();
+        map<int, CArray<size_t, 1> >::iterator itb = outGlobalIndexFromClient.begin(),
+                                               ite = outGlobalIndexFromClient.end(), it;
+        for (it = itb; it != ite; ++it)
+        {
+          int rank = it->first;
+          CArray<size_t,1>& globalIndex = outGlobalIndexFromClient[rank];
+          outLocalIndexStoreOnClient.insert(make_pair(rank, CArray<size_t,1>(globalIndex.numElements())));
+          CArray<size_t,1>& localIndex = outLocalIndexStoreOnClient[rank];
+          size_t nbIndex = 0;
+          // Keep this code for this moment but it should be removed (or moved to DEBUG) to improve performance
+          for (size_t idx = 0; idx < globalIndex.numElements(); ++idx)
+          {
+            if (itGloe != globalDataIndex.find(globalIndex(idx)))
+            {
+              ++nbIndex;
+            }
+          }
+          if (doGridHaveDataDistributed(client) && (nbIndex != localIndex.numElements()))
+               ERROR("void CGrid::computeClientIndex()",
+                  << "Number of local index on client is different from number of received global index"
+                  << "Rank of sent client " << rank <<"."
+                  << "Number of local index " << nbIndex << ". "
+                  << "Number of received global index " << localIndex.numElements() << ".");
+          nbIndex = 0;
+          for (size_t idx = 0; idx < globalIndex.numElements(); ++idx)
+          {
+            if (itGloe != globalDataIndex.find(globalIndex(idx)))
+            {
+              localIndex(idx) = globalDataIndex[globalIndex(idx)];
+            }
+          }
+        }
+      }
+   }
+   /*!
+     Compute connected receivers and indexes to be sent to these receivers.
+   */
+   void CGrid::computeConnectedClients()
+   {
+     CContext* context = CContext::getCurrent();
+     int nbSrvPools = (context->clientPrimServer.size() == 0) ? 1 : context->clientPrimServer.size();
+     connectedServerRank_.clear();
+     connectedDataSize_.clear();
+     globalIndexOnServer_.clear();
+     nbSenders.clear();
+     for (int p = 0; p < nbSrvPools; ++p)
+     {
+       CContextClient* client = (context->clientPrimServer.size() == 0) ? context->client : context->clientPrimServer[p];
+       int receiverSize = client->serverSize;
+//       connectedServerRank_[client].clear();
+       if (connectedServerRank_.find(receiverSize) == connectedServerRank_.end())
+       {
+        if (!doGridHaveDataDistributed(client))
+         {
+            if (client->isServerLeader())
+            {
+              size_t ssize = clientDistribution_->getLocalDataIndexOnClient().size();
+              const std::list<int>& ranks = client->getRanksServerLeader();
+              for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+              {
+                connectedServerRank_[receiverSize].push_back(*itRank);
+                connectedDataSize_[receiverSize][*itRank] = ssize;
+              }
+            }
+            return;
+         }
+         // Compute mapping between client and server
+         std::vector<boost::unordered_map<size_t,std::vector<int> > > indexServerOnElement;
+         CServerDistributionDescription serverDistributionDescription(getGlobalDimension(), client->serverSize);
+         std::vector<int> serverZeroIndex = serverDistributionDescription.computeServerGlobalByElement(indexServerOnElement,
+                                                                                                    client->clientRank,
+                                                                                                    client->clientSize,
+                                                                                                    axis_domain_order,
+                                                                                                    getDistributedDimension());
+         // Even if servers have no index, they must received something from client
+         // We only use several client to send "empty" message to these servers
+         std::list<int> serverZeroIndexLeader;
+         std::list<int> serverZeroIndexNotLeader;
+         CContextClient::computeLeader(client->clientRank, client->clientSize, serverZeroIndex.size(), serverZeroIndexLeader, serverZeroIndexNotLeader);
+         for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+           *it = serverZeroIndex[*it];
+         if (globalIndexOnServer_.find(receiverSize) == globalIndexOnServer_.end())
+           computeIndexByElement(indexServerOnElement, client, globalIndexOnServer_[receiverSize]);
+         const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer();
+         CDistributionClient::GlobalLocalDataMap::const_iterator iteGlobalLocalIndexMap = globalLocalIndexSendToServer.end(), itGlobalLocalIndexMap;
+         CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itbGlobalMap, itGlobalMap;
+         itbGlobalMap = globalIndexOnServer_[receiverSize].begin();
+         iteGlobalMap = globalIndexOnServer_[receiverSize].end();
+         for (itGlobalMap  = itbGlobalMap; itGlobalMap != iteGlobalMap; ++itGlobalMap)
+         {
+           int serverRank = itGlobalMap->first;
+           int indexSize = itGlobalMap->second.size();
+           const std::vector<size_t>& indexVec = itGlobalMap->second;
+           for (int idx = 0; idx < indexSize; ++idx)
+           {
+              itGlobalLocalIndexMap = globalLocalIndexSendToServer.find(indexVec[idx]);
+              if (iteGlobalLocalIndexMap != itGlobalLocalIndexMap)
+              {
+                if (connectedDataSize_[receiverSize].end() == connectedDataSize_[receiverSize].find(serverRank))
+                  connectedDataSize_[receiverSize][serverRank] = 1;
+                else
+                  ++connectedDataSize_[receiverSize][serverRank];
+              }
+           }
+         }
+         // Connected servers which really have index
+         for (itGlobalMap = itbGlobalMap; itGlobalMap != iteGlobalMap; ++itGlobalMap) {
+           connectedServerRank_[receiverSize].push_back(itGlobalMap->first);
+         }
+         // Connected servers which have no index at all
+         for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+           connectedServerRank_[receiverSize].push_back(*it);
+         // Even if a client has no index, it must connect to at least one server and
+         // send an "empty" data to this server
+         if (connectedServerRank_[receiverSize].empty())
+          connectedServerRank_[receiverSize].push_back(client->clientRank % client->serverSize);
+         nbSenders[receiverSize] = clientServerMap_->computeConnectedClients(receiverSize, client->clientSize, client->intraComm, connectedServerRank_[receiverSize]);
+       }
+     }
+   }
    /*!
 …
+   {
      CContext* context = CContext::getCurrent();
+     CContextClient* client = context->client;
+     // First of all, compute distribution on client side
+     clientDistribution_ = new CDistributionClient(client->clientRank, this);
+     // Get local data index on client
+     storeIndex_client.resize(clientDistribution_->getLocalDataIndexOnClient().size());
+     int nbStoreIndex = storeIndex_client.numElements();
+     for (int idx = 0; idx < nbStoreIndex; ++idx) storeIndex_client(idx) = (clientDistribution_->getLocalDataIndexOnClient())[idx];
+     isDataDistributed_= clientDistribution_->isDataDistributed();
+     connectedServerRank_.clear();
+     if (!doGridHaveDataDistributed())
+     if (isScalarGrid())
+     {
+        if (client->isServerLeader())
+        {
+          size_t ssize = clientDistribution_->getLocalDataIndexOnClient().size();
+          const std::list<int>& ranks = client->getRanksServerLeader();
+          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+          {
+            connectedServerRank_.push_back(*itRank);
+            connectedDataSize_[*itRank] = ssize;
+          }
+        }
+        return;
+     }
+     // Compute mapping between client and server
+     std::vector<boost::unordered_map<size_t,std::vector<int> > > indexServerOnElement;
+     CServerDistributionDescription serverDistributionDescription(getGlobalDimension(), client->serverSize);
+     serverDistributionDescription.computeServerGlobalByElement(indexServerOnElement,
+                                                                client->clientRank,
+                                                                client->clientSize,
+                                                                axis_domain_order,
+                                                                getDistributedDimension());
+     computeIndexByElement(indexServerOnElement, globalIndexOnServer_);
+     const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer();
+     CDistributionClient::GlobalLocalDataMap::const_iterator iteGlobalLocalIndexMap = globalLocalIndexSendToServer.end(), itGlobalLocalIndexMap;
+     CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itbGlobalMap, itGlobalMap;
+     itGlobalMap  = itbGlobalMap = globalIndexOnServer_.begin();
+     iteGlobalMap = globalIndexOnServer_.end();
+     for (; itGlobalMap != iteGlobalMap; ++itGlobalMap)
+     {
+       int serverRank = itGlobalMap->first;
+       int indexSize = itGlobalMap->second.size();
+       const std::vector<size_t>& indexVec = itGlobalMap->second;
+       for (int idx = 0; idx < indexSize; ++idx)
+       computeClientIndexScalarGrid();
+       if (context->hasClient)
+       {
+          itGlobalLocalIndexMap = globalLocalIndexSendToServer.find(indexVec[idx]);
+          if (iteGlobalLocalIndexMap != itGlobalLocalIndexMap)
+          {
+             if (connectedDataSize_.end() == connectedDataSize_.find(serverRank))
+               connectedDataSize_[serverRank] = 1;
+             else
+               ++connectedDataSize_[serverRank];
+          }
+         computeConnectedClientsScalarGrid();
+       }
+     }
+     for (itGlobalMap = itbGlobalMap; itGlobalMap != iteGlobalMap; ++itGlobalMap) {
+       connectedServerRank_.push_back(itGlobalMap->first);
+     else
+     {
+       computeClientIndex();
+       if (context->hasClient)
+       {
+         computeConnectedClients();
+       }
+     }
+     nbSenders = clientServerMap_->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_);
+     if (CServer::serverLevel==2)
+     {
+       computeWrittenIndex() ;
+       if (serverDistribution_!=0) serverDistribution_->partialClear() ;
+       if (clientDistribution_!=0) clientDistribution_->partialClear() ;
+       outGlobalIndexFromClient.clear() ;
+     }
+   }
 …
       on each element whose size is much smaller than one of whole grid.
       \param [in] indexServerOnElement global index of each element and the rank of server associated with these index
+      \param [in] client contextClient
       \param [out] globalIndexOnServer global index of grid and its corresponding rank of server.
    */
    void CGrid::computeIndexByElement(const std::vector<boost::unordered_map<size_t,std::vector<int> > >& indexServerOnElement,
+                                     const CContextClient* client,
                                      CClientServerMapping::GlobalIndexMap& globalIndexOnServer)
+   {
-     CContext* context = CContext::getCurrent();
-     CContextClient* client = context->client;
      int serverSize = client->serverSize;
      std::vector<CDomain*> domList = getDomains();
      std::vector<CAxis*> axisList = getAxis();
 …
+       }
+       nbIndexOnServer = 0;
+       for (it = itb; it != ite; ++it)
+     nbIndexOnServer = 0;
+     for (size_t j = 0; j < globalIndexElementOnServerMap.size(); ++j)
+     {
+       it = globalIndexElementOnServerMap.find(globalIndexElementOnClient(j));
+       if (it != ite)
+       {
          const std::vector<int>& tmp = it->second;
 …
+       }
+     }
+   }
     // Determine server which contain global source index
 …
+   }
+/*
    void CGrid::outputField(int rank, const CArray<double, 1>& stored, double* field)
+   {
 …
+     }
+   }
+*/
    //----------------------------------------------------------------
 …
+   }
+  void CGrid::computeIndexScalarGrid()
+   void CGrid::uncompressField_arr(const double* const data, CArray<double, 1>& out) const
+   {
+      const std::vector<int>& localMaskedDataIndex = clientDistribution_->getLocalMaskedDataIndexOnClient();
+      const int size = localMaskedDataIndex.size();
+      for(int i = 0; i < size; ++i) out(localMaskedDataIndex[i]) = data[i];
+   }
+  void CGrid::computeClientIndexScalarGrid()
+  {
+    CContext* context = CContext::getCurrent();
+//    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1; // This should be changed soon
+//    for (int p = 0; p < nbSrvPools; ++p)
+    {
+//      CContextClient* client = (context->hasServer) ? (context->hasClient ? context->clientPrimServer[p] : context->client)
+//                                                    : context->client;
+      CContextClient* client = context->client;
+      int rank = client->clientRank;
+      clientDistribution_ = new CDistributionClient(rank, this);
+      storeIndex_client.resize(1);
+      storeIndex_client(0) = 0;
+      if (0 != serverDistribution_)
+      {
+        map<int, CArray<size_t, 1> >::iterator itb = outGlobalIndexFromClient.begin(),
+                                               ite = outGlobalIndexFromClient.end(), it;
+        for (it = itb; it != ite; ++it)
+        {
+          int rank = it->first;
+          CArray<size_t,1>& globalIndex = outGlobalIndexFromClient[rank];
+          outLocalIndexStoreOnClient.insert(make_pair(rank, CArray<size_t,1>(globalIndex.numElements())));
+          CArray<size_t,1>& localIndex = outLocalIndexStoreOnClient[rank];
+          if (1 != globalIndex.numElements())
+            ERROR("void CGrid::computeClientIndexScalarGrid()",
+              << "Something wrong happened. "
+              << "Number of received global index on scalar grid should equal to 1"
+              << "Number of received global index " << globalIndex.numElements() << ".");
+          localIndex(0) = globalIndex(0);
+        }
+      }
+    }
+  }
+  void CGrid::computeConnectedClientsScalarGrid()
+  {
+    CContext* context = CContext::getCurrent();
+    int nbSrvPools = (context->clientPrimServer.size()==0) ? 1 : context->clientPrimServer.size();
+    connectedServerRank_.clear();
+    connectedDataSize_.clear();
+    nbSenders.clear();
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (context->clientPrimServer.size()==0) ? context->client : context->clientPrimServer[p];
+      int receiverSize = client->serverSize;
+//      connectedServerRank_[client].clear();
+      if (connectedServerRank_.find(receiverSize)==connectedServerRank_.end())
+      {
+        if (client->isServerLeader())
+        {
+          const std::list<int>& ranks = client->getRanksServerLeader();
+          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+          {
+            int rank = *itRank;
+            int nb = 1;
+            connectedServerRank_[receiverSize].push_back(rank);
+            connectedDataSize_[receiverSize][rank] = nb;
+            nbSenders[receiverSize][rank] = nb;
+          }
+        }
+        else
+        {
+          const std::list<int>& ranks = client->getRanksServerNotLeader();
+          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+          {
+            int rank = *itRank;
+            int nb = 1;
+            connectedServerRank_[receiverSize].push_back(rank);
+            connectedDataSize_[receiverSize][rank] = nb;
+            nbSenders[receiverSize][rank] = nb;
+          }
+        }
+      }
+      isDataDistributed_ = false;
+    }
+  }
+  void CGrid::sendIndexScalarGrid()
+  {
     CContext* context = CContext::getCurrent();
+    CContextClient* client=context->client;
+    storeIndex_client.resize(1);
+    storeIndex_client(0) = 0;
+    connectedServerRank_.clear();
+    if (0 == client->clientRank)
+    {
+      for (int rank = 0; rank < client->serverSize; ++rank)
+      {
+        connectedServerRank_.push_back(rank);
+        connectedDataSize_[rank] = 1;
+        nbSenders[rank] = 1;
+      }
+    }
+    isDataDistributed_ = false;
+  }
+  void CGrid::computeCompressedIndex()
+  {
+    std::map<size_t, size_t> indexes;
+    {
+      std::map<int, CArray<size_t,1> >::const_iterator it = outIndexFromClient.begin();
+      std::map<int, CArray<size_t,1> >::const_iterator itEnd = outIndexFromClient.end();
+      for (; it != itEnd; ++it)
+      {
+        for (int i = 0; i < it->second.numElements(); ++i)
+          indexes.insert(std::make_pair(it->second(i), 0));
+        compressedOutIndexFromClient[it->first].resize(it->second.numElements());
+      }
+    }
+    {
+      std::map<size_t, size_t>::iterator it = indexes.begin();
+      std::map<size_t, size_t>::iterator itEnd = indexes.end();
+      for (size_t i = 0; it != itEnd; ++it, ++i)
+        it->second = i;
+    }
+    {
+      std::map<int, CArray<size_t,1> >::iterator it = compressedOutIndexFromClient.begin();
+      std::map<int, CArray<size_t,1> >::iterator itEnd = compressedOutIndexFromClient.end();
+      for (; it != itEnd; ++it)
+      {
+        const CArray<size_t,1>& outIndex = outIndexFromClient[it->first];
+        for (int i = 0; i < it->second.numElements(); ++i)
+          it->second(i) = indexes[outIndex(i)];
+      }
+    }
+  }
+  void CGrid::sendIndexScalarGrid()
+  {
+    CContext* context = CContext::getCurrent();
+    CContextClient* client = context->client;
+    CEventClient event(getType(), EVENT_ID_INDEX);
+    list<CMessage> listMsg;
+    list<CArray<size_t,1> > listOutIndex;
+    if (client->isServerLeader())
+    {
+      const std::list<int>& ranks = client->getRanksServerLeader();
+      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+      {
+        int rank = *itRank;
+        int nb = 1;
+        storeIndex_toSrv.insert(std::make_pair(rank, CArray<int,1>(nb)));
+        listOutIndex.push_back(CArray<size_t,1>(nb));
+        CArray<int, 1>& outLocalIndexToServer = storeIndex_toSrv[rank];
+        CArray<size_t, 1>& outGlobalIndexOnServer = listOutIndex.back();
+        for (int k = 0; k < nb; ++k)
+        {
+          outGlobalIndexOnServer(k) = 0;
+          outLocalIndexToServer(k)  = 0;
+        }
+        storeIndex_fromSrv.insert(std::make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
+        listMsg.push_back(CMessage());
+        listMsg.back() << getId( )<< isDataDistributed_ << isCompressible_ << listOutIndex.back();
+        event.push(rank, 1, listMsg.back());
+      }
+      client->sendEvent(event);
+    }
+    else
+    {
+      const std::list<int>& ranks = client->getRanksServerNotLeader();
+      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+      {
+        int rank = *itRank;
+        int nb = 1;
+        storeIndex_fromSrv.insert(std::make_pair(rank, CArray<int,1>(nb)));
+        CArray<int, 1>& outLocalIndexToServer = storeIndex_fromSrv[rank];
+        for (int k = 0; k < nb; ++k)
+        {
+          outLocalIndexToServer(k)  = 0;
+        }
+      }
+      client->sendEvent(event);
+    }
+  }
+  void CGrid::sendIndex(void)
+  {
+    CContext* context = CContext::getCurrent();
+    CContextClient* client = context->client;
+    CEventClient event(getType(), EVENT_ID_INDEX);
+    int rank;
+    list<CMessage> listMsg;
+    list<CArray<size_t,1> > listOutIndex;
+    const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer();
+    CDistributionClient::GlobalLocalDataMap::const_iterator itIndex = globalLocalIndexSendToServer.begin(),
+                                                           iteIndex = globalLocalIndexSendToServer.end();
+    if (!doGridHaveDataDistributed())
+    {
+    storeIndex_toSrv.clear();
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      CContextClient* client = *it;
+      int receiverSize = client->serverSize;
+      CEventClient event(getType(), EVENT_ID_INDEX);
+      list<CMessage> listMsg;
+      list<CArray<size_t,1> > listOutIndex;
       if (client->isServerLeader())
+      {
-        int indexSize = globalLocalIndexSendToServer.size();
-        CArray<size_t,1> outGlobalIndexOnServer(indexSize);
-        CArray<int,1> outLocalIndexToServer(indexSize);
-        for (int idx = 0; itIndex != iteIndex; ++itIndex, ++idx)
+        {
-          outGlobalIndexOnServer(idx) = itIndex->first;
-          outLocalIndexToServer(idx) = itIndex->second;
+        }
         const std::list<int>& ranks = client->getRanksServerLeader();
         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        {
+          storeIndex_toSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
+          storeIndex_fromSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
+          listOutIndex.push_back(CArray<size_t,1>(outGlobalIndexOnServer));
+          int rank = *itRank;
+          int nb = 1;
+          storeIndex_toSrv[client].insert(std::make_pair(rank, CArray<int,1>(nb)));
+          listOutIndex.push_back(CArray<size_t,1>(nb));
+          CArray<int, 1>& outLocalIndexToServer = storeIndex_toSrv[client][rank];
+          CArray<size_t, 1>& outGlobalIndexOnServer = listOutIndex.back();
+          for (int k = 0; k < nb; ++k)
+          {
+            outGlobalIndexOnServer(k) = 0;
+            outLocalIndexToServer(k)  = 0;
+          }
+          if (context->hasClient && !context->hasServer)
+            storeIndex_fromSrv.insert(std::make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
+          listMsg.push_back(CMessage());
+          listMsg.back() << getId( )<< isDataDistributed_ << isCompressible_ << listOutIndex.back();
+          event.push(rank, 1, listMsg.back());
+        }
+        client->sendEvent(event);
+      }
+      else
+      {
+        const std::list<int>& ranks = client->getRanksServerNotLeader();
+        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        {
+          int rank = *itRank;
+          int nb = 1;
+          CArray<int, 1> outLocalIndexToServer(nb);
+          for (int k = 0; k < nb; ++k)
+          {
+            outLocalIndexToServer(k)  = 0;
+          }
+          if (context->hasClient && !context->hasServer)
+            storeIndex_fromSrv.insert(std::make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
+        }
+        client->sendEvent(event);
+      }
+    }
+  }
+  void CGrid::sendIndex(void)
+  {
+    CContext* context = CContext::getCurrent();
+    storeIndex_toSrv.clear();
+    std::list<CContextClient*>::iterator it;
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      CContextClient* client = *it;
+      int receiverSize = client->serverSize;
+      CEventClient event(getType(), EVENT_ID_INDEX);
+      int rank;
+      list<CMessage> listMsg;
+      list<CArray<size_t,1> > listOutIndex;
+      const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer();
+      CDistributionClient::GlobalLocalDataMap::const_iterator itbIndex = globalLocalIndexSendToServer.begin(), itIndex,
+                                                              iteIndex = globalLocalIndexSendToServer.end();
+      itIndex = itbIndex;
+      if (!doGridHaveDataDistributed(client))
+      {
+        if (client->isServerLeader())
+        {
+          int indexSize = globalLocalIndexSendToServer.size();
+          CArray<size_t,1> outGlobalIndexOnServer(indexSize);
+          CArray<int,1> outLocalIndexToServer(indexSize);
+          for (int idx = 0; itIndex != iteIndex; ++itIndex, ++idx)
+          {
+            outGlobalIndexOnServer(idx) = itIndex->first;
+            outLocalIndexToServer(idx) = itIndex->second;
+          }
+          const std::list<int>& ranks = client->getRanksServerLeader();
+          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+          {
+            storeIndex_toSrv[client].insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
+            if (context->hasClient && !context->hasServer)
+              storeIndex_fromSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
+            listOutIndex.push_back(CArray<size_t,1>(outGlobalIndexOnServer));
+            listMsg.push_back(CMessage());
+            listMsg.back() << getId() << isDataDistributed_ << isCompressible_ << listOutIndex.back();
+            event.push(*itRank, 1, listMsg.back());
+          }
+          client->sendEvent(event);
+        }
+        else
+        {
+           int indexSize = globalLocalIndexSendToServer.size();
+           CArray<int,1> outLocalIndexToServer(indexSize);
+           for (int idx = 0; itIndex != iteIndex; ++itIndex, ++idx)
+           {
+             outLocalIndexToServer(idx) = itIndex->second;
+           }
+           const std::list<int>& ranks = client->getRanksServerNotLeader();
+           for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           {
+             storeIndex_fromSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
+           }
+           client->sendEvent(event);
+         }
+      }
+      else
+      {
+        CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itGlobalMap;
+        itGlobalMap = globalIndexOnServer_[receiverSize].begin();
+        iteGlobalMap = globalIndexOnServer_[receiverSize].end();
+        std::map<int,std::vector<int> >localIndexTmp;
+        std::map<int,std::vector<size_t> > globalIndexTmp;
+        for (; itGlobalMap != iteGlobalMap; ++itGlobalMap)
+        {
+          int serverRank = itGlobalMap->first;
+          int indexSize = itGlobalMap->second.size();
+          const std::vector<size_t>& indexVec = itGlobalMap->second;
+          for (int idx = 0; idx < indexSize; ++idx)
+          {
+            itIndex = globalLocalIndexSendToServer.find(indexVec[idx]);
+            if (iteIndex != itIndex)
+            {
+              globalIndexTmp[serverRank].push_back(itIndex->first);
+              localIndexTmp[serverRank].push_back(itIndex->second);
+            }
+          }
+        }
+        for (int ns = 0; ns < connectedServerRank_[receiverSize].size(); ++ns)
+        {
+          rank = connectedServerRank_[receiverSize][ns];
+          int nb = 0;
+          if (globalIndexTmp.end() != globalIndexTmp.find(rank))
+            nb = globalIndexTmp[rank].size();
+          storeIndex_toSrv[client].insert(make_pair(rank, CArray<int,1>(nb)));
+          listOutIndex.push_back(CArray<size_t,1>(nb));
+          CArray<int, 1>& outLocalIndexToServer = storeIndex_toSrv[client][rank];
+          CArray<size_t, 1>& outGlobalIndexOnServer = listOutIndex.back();
+          for (int k = 0; k < nb; ++k)
+          {
+            outGlobalIndexOnServer(k) = globalIndexTmp[rank].at(k);
+            outLocalIndexToServer(k)  = localIndexTmp[rank].at(k);
+          }
+          storeIndex_fromSrv.insert(make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
           listMsg.push_back(CMessage());
           listMsg.back() << getId() << isDataDistributed_ << isCompressible_ << listOutIndex.back();
+          event.push(*itRank, 1, listMsg.back());
+        }
+          event.push(rank, nbSenders[receiverSize][rank], listMsg.back());
+        }
         client->sendEvent(event);
+      }
-      else
+      {
-        int indexSize = globalLocalIndexSendToServer.size();
-        CArray<int,1> outLocalIndexToServer(indexSize);
-        for (int idx = 0; itIndex != iteIndex; ++itIndex, ++idx)
+        {
-          outLocalIndexToServer(idx) = itIndex->second;
+        }
-        const std::list<int>& ranks = client->getRanksServerNotLeader();
-        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        {
-          storeIndex_fromSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
+        }
-        client->sendEvent(event);
+      }
+    }
-    else
+    {
-      CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itGlobalMap;
-      itGlobalMap = globalIndexOnServer_.begin();
-      iteGlobalMap = globalIndexOnServer_.end();
-      std::map<int,std::vector<int> >localIndexTmp;
-      std::map<int,std::vector<size_t> > globalIndexTmp;
-      for (; itGlobalMap != iteGlobalMap; ++itGlobalMap)
+      {
-        int serverRank = itGlobalMap->first;
-        int indexSize = itGlobalMap->second.size();
-        const std::vector<size_t>& indexVec = itGlobalMap->second;
-        for (int idx = 0; idx < indexSize; ++idx)
+        {
-          itIndex = globalLocalIndexSendToServer.find(indexVec[idx]);
-          if (iteIndex != itIndex)
+          {
-            globalIndexTmp[serverRank].push_back(itIndex->first);
-            localIndexTmp[serverRank].push_back(itIndex->second);
+          }
+        }
+      }
-      for (int ns = 0; ns < connectedServerRank_.size(); ++ns)
+      {
-        rank = connectedServerRank_[ns];
-        int nb = 0;
-        if (globalIndexTmp.end() != globalIndexTmp.find(rank))
-          nb = globalIndexTmp[rank].size();
-        storeIndex_toSrv.insert(make_pair(rank, CArray<int,1>(nb)));
-        listOutIndex.push_back(CArray<size_t,1>(nb));
-        CArray<int, 1>& outLocalIndexToServer = storeIndex_toSrv[rank];
-        CArray<size_t, 1>& outGlobalIndexOnServer = listOutIndex.back();
-        for (int k = 0; k < nb; ++k)
+        {
-          outGlobalIndexOnServer(k) = globalIndexTmp[rank].at(k);
-          outLocalIndexToServer(k)  = localIndexTmp[rank].at(k);
+        }
-        storeIndex_fromSrv.insert(make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
-        listMsg.push_back(CMessage());
-        listMsg.back() << getId() << isDataDistributed_ << isCompressible_ << listOutIndex.back();
-        event.push(rank, nbSenders[rank], listMsg.back());
+      }
-      client->sendEvent(event);
+    }
+  }
 …
+  {
     CContext* context = CContext::getCurrent();
+    CContextServer* server = context->server;
+    numberWrittenIndexes_ = totalNumberWrittenIndexes_ = offsetWrittenIndexes_ = 0;
+    connectedServerRank_ = ranks;
+    for (int n = 0; n < ranks.size(); n++)
+    {
+      int rank = ranks[n];
+      CBufferIn& buffer = *buffers[n];
+      buffer >> isDataDistributed_ >> isCompressible_;
+      size_t dataSize = 0;
+      if (0 == serverDistribution_)
+      {
+        int idx = 0, numElement = axis_domain_order.numElements();
+        int ssize = numElement;
+        std::vector<int> indexMap(numElement);
+        for (int i = 0; i < numElement; ++i)
+        {
+          indexMap[i] = idx;
+          if (2 == axis_domain_order(i))
+    connectedServerRankRead_ = ranks;
+    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
+    nbSrvPools = 1;
+    nbReadSenders.clear();
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextServer* server = (!context->hasClient) ? context->server : context->serverPrimServer[p];
+      CContextClient* client = context->client;   //(!context->hasClient) ? context->client : context->clientPrimServer[p];
+      int idx = 0, numElement = axis_domain_order.numElements();
+      int ssize = numElement;
+      std::vector<int> indexMap(numElement);
+      for (int i = 0; i < numElement; ++i)
+      {
+        indexMap[i] = idx;
+        if (2 == axis_domain_order(i))
+        {
+          ++ssize;
+          idx += 2;
+        }
+        else
+          ++idx;
+      }
+      for (int n = 0; n < ranks.size(); n++)
+      {
+        int rank = ranks[n];
+        CBufferIn& buffer = *buffers[n];
+        buffer >> isDataDistributed_ >> isCompressible_;
+        size_t dataSize = 0;
+        if (0 == serverDistribution_)
+        {
+          int axisId = 0, domainId = 0, scalarId = 0, globalSize = 1;
+          std::vector<CDomain*> domainList = getDomains();
+          std::vector<CAxis*> axisList = getAxis();
+          std::vector<int> nZoomBegin(ssize), nZoomSize(ssize), nGlob(ssize), nZoomBeginGlobal(ssize), nGlobElement(numElement);
+          std::vector<CArray<int,1> > globalZoomIndex(numElement);
+          for (int i = 0; i < numElement; ++i)
+          {
+            ++ssize;
+            idx += 2;
+            nGlobElement[i] = globalSize;
+            if (2 == axis_domain_order(i)) //domain
+            {
+              nZoomBegin[indexMap[i]] = domainList[domainId]->zoom_ibegin;
+              nZoomSize[indexMap[i]]  = domainList[domainId]->zoom_ni;
+              nZoomBeginGlobal[indexMap[i]] = domainList[domainId]->global_zoom_ibegin;
+              nGlob[indexMap[i]] = domainList[domainId]->ni_glo;
+              nZoomBegin[indexMap[i] + 1] = domainList[domainId]->zoom_jbegin;
+              nZoomSize[indexMap[i] + 1] = domainList[domainId]->zoom_nj;
+              nZoomBeginGlobal[indexMap[i] + 1] = domainList[domainId]->global_zoom_jbegin;
+              nGlob[indexMap[i] + 1] = domainList[domainId]->nj_glo;
+              {
+                int count = 0;
+                globalZoomIndex[i].resize(nZoomSize[indexMap[i]]*nZoomSize[indexMap[i]+1]);
+                for (int jdx = 0; jdx < nZoomSize[indexMap[i]+1]; ++jdx)
+                  for (int idx = 0; idx < nZoomSize[indexMap[i]]; ++idx)
+                  {
+                    globalZoomIndex[i](count) = (nZoomBegin[indexMap[i]] + idx) + (nZoomBegin[indexMap[i]+1] + jdx) * nGlob[indexMap[i]];
+                    ++count;
+                  }
+              }
+              ++domainId;
+            }
+            else if (1 == axis_domain_order(i)) // axis
+            {
+              nZoomBegin[indexMap[i]] = axisList[axisId]->zoom_begin;
+              nZoomSize[indexMap[i]]  = axisList[axisId]->zoom_n;
+              nZoomBeginGlobal[indexMap[i]] = axisList[axisId]->global_zoom_begin;
+              nGlob[indexMap[i]] = axisList[axisId]->n_glo;
+              if (axisList[axisId]->zoomByIndex())
+              {
+                globalZoomIndex[i].reference(axisList[axisId]->zoom_index);
+              }
+              else
+              {
+                globalZoomIndex[i].resize(nZoomSize[indexMap[i]]);
+                for (int idx = 0; idx < nZoomSize[indexMap[i]]; ++idx)
+                  globalZoomIndex[i](idx) = nZoomBegin[indexMap[i]] + idx;
+              }
+              ++axisId;
+            }
+            else // scalar
+            {
+              nZoomBegin[indexMap[i]] = 0;
+              nZoomSize[indexMap[i]]  = 1;
+              nZoomBeginGlobal[indexMap[i]] = 0;
+              nGlob[indexMap[i]] = 1;
+              globalZoomIndex[i].resize(1);
+              globalZoomIndex[i](0) = 0;
+              ++scalarId;
+            }
+          }
+          else
+            ++idx;
+        }
+        int axisId = 0, domainId = 0, scalarId = 0;
+          dataSize = 1;
+          for (int i = 0; i < nZoomSize.size(); ++i)
+            dataSize *= nZoomSize[i];
+          serverDistribution_ = new CDistributionServer(server->intraCommRank,
+                                                        globalZoomIndex, axis_domain_order,
+                                                        nZoomBegin, nZoomSize, nZoomBeginGlobal, nGlob);
+        }
+        CArray<size_t,1> outIndex;
+        buffer >> outIndex;
+        outGlobalIndexFromClient.insert(std::make_pair(rank, outIndex));
+        connectedDataSizeRead_[rank] = outIndex.numElements();
+        if (doGridHaveDataDistributed(client))
+        {}
+        else
+        {
+          // THE PROBLEM HERE IS THAT DATA CAN BE NONDISTRIBUTED ON CLIENT AND DISTRIBUTED ON SERVER
+          // BELOW IS THE TEMPORARY FIX only for a single type of element (domain, asix, scalar)
+          dataSize = serverDistribution_->getGridSize();
+        }
+        writtenDataSize_ += dataSize;
+      }
+      // Compute mask of the current grid
+      {
+        int axisId = 0, domainId = 0, scalarId = 0, globalSize = 1;
         std::vector<CDomain*> domainList = getDomains();
         std::vector<CAxis*> axisList = getAxis();
         std::vector<int> nZoomBegin(ssize), nZoomSize(ssize), nGlob(ssize), nZoomBeginGlobal(ssize);
         std::vector<CArray<int,1> > globalZoomIndex(numElement);
+        int dimSize = 2 * domainList.size() + axisList.size();
+        std::vector<int> nBegin(dimSize), nSize(dimSize), nGlob(dimSize), nBeginGlobal(dimSize);
         for (int i = 0; i < numElement; ++i)
+        {
+        {
           if (2 == axis_domain_order(i)) //domain
+          {
             nZoomBegin[indexMap[i]] = domainList[domainId]->zoom_ibegin_srv;
             nZoomSize[indexMap[i]]  = domainList[domainId]->zoom_ni_srv;
             nZoomBeginGlobal[indexMap[i]] = domainList[domainId]->global_zoom_ibegin;
+            nBegin[indexMap[i]] = domainList[domainId]->ibegin;
+            nSize[indexMap[i]]  = domainList[domainId]->ni;
+            nBeginGlobal[indexMap[i]] = 0;
             nGlob[indexMap[i]] = domainList[domainId]->ni_glo;
             nZoomBegin[indexMap[i] + 1] = domainList[domainId]->zoom_jbegin_srv;
             nZoomSize[indexMap[i] + 1] = domainList[domainId]->zoom_nj_srv;
             nZoomBeginGlobal[indexMap[i] + 1] = domainList[domainId]->global_zoom_jbegin;
+            nBegin[indexMap[i] + 1] = domainList[domainId]->jbegin;
+            nSize[indexMap[i] + 1] = domainList[domainId]->nj;
+            nBeginGlobal[indexMap[i] + 1] = 0;
             nGlob[indexMap[i] + 1] = domainList[domainId]->nj_glo;
+            {
-              int count = 0;
-              globalZoomIndex[i].resize(nZoomSize[indexMap[i]]*nZoomSize[indexMap[i]+1]);
-              for (int jdx = 0; jdx < nZoomSize[indexMap[i]+1]; ++jdx)
-                for (int idx = 0; idx < nZoomSize[indexMap[i]]; ++idx)
+                {
-                  globalZoomIndex[i](count) = (nZoomBegin[indexMap[i]] + idx) + (nZoomBegin[indexMap[i]+1] + jdx) * nGlob[indexMap[i]];
-                  ++count;
+                }
+            }
             ++domainId;
+          }
           else if (1 == axis_domain_order(i)) // axis
+          {
+            nZoomBegin[indexMap[i]] = axisList[axisId]->zoom_begin_srv;
+            nZoomSize[indexMap[i]]  = axisList[axisId]->zoom_size_srv;
+            nZoomBeginGlobal[indexMap[i]] = axisList[axisId]->global_zoom_begin_srv;
+            nGlob[indexMap[i]] = axisList[axisId]->n_glo;
+            if (!axisList[axisId]->global_zoom_index.isEmpty())
+            {
+              globalZoomIndex[i].reference(axisList[axisId]->zoom_index_srv);
+            }
+            else
+            {
+              globalZoomIndex[i].resize(nZoomSize[indexMap[i]]);
+              for (int idx = 0; idx < nZoomSize[indexMap[i]]; ++idx)
+                globalZoomIndex[i](idx) = nZoomBegin[indexMap[i]] + idx;
+            }
+            nBegin[indexMap[i]] = axisList[axisId]->begin;
+            nSize[indexMap[i]]  = axisList[axisId]->n;
+            nBeginGlobal[indexMap[i]] = 0;
+            nGlob[indexMap[i]] = axisList[axisId]->n_glo;
             ++axisId;
+          }
           else // scalar
+          {
+          }
+        }
+        if (nSize.empty()) // Scalar grid
+        {
+          nBegin.push_back(0);
+          nSize.push_back(1);
+          nBeginGlobal.push_back(0);
+          nGlob.push_back(1);
+        }
+        modifyMaskSize(nSize, false);
+        // These below codes are reserved for future
+        CDistributionServer srvDist(server->intraCommRank, nBegin, nSize, nBeginGlobal, nGlob);
+        map<int, CArray<size_t, 1> >::iterator itb = outGlobalIndexFromClient.begin(),
+                                               ite = outGlobalIndexFromClient.end(), it;
+        const CDistributionServer::GlobalLocalMap&  globalLocalMask = srvDist.getGlobalLocalIndex();
+        CDistributionServer::GlobalLocalMap::const_iterator itSrv;
+        size_t nb = 0;
+        for (it = itb; it != ite; ++it)
+        {
+          CArray<size_t,1>& globalInd = it->second;
+          for (size_t idx = 0; idx < globalInd.numElements(); ++idx)
+          {
+            nZoomBegin[indexMap[i]] = 0;
+            nZoomSize[indexMap[i]]  = 1;
+            nZoomBeginGlobal[indexMap[i]] = 0;
+            nGlob[indexMap[i]] = 1;
+            globalZoomIndex[i].resize(1);
+            globalZoomIndex[i](0) = 0;
+            ++scalarId;
+            if (globalLocalMask.end() != globalLocalMask.find(globalInd(idx))) ++nb;
+          }
+        }
+        dataSize = 1;
+        for (int i = 0; i < nZoomSize.size(); ++i)
+          dataSize *= nZoomSize[i];
+/*        serverDistribution_ = new CDistributionServer(server->intraCommRank, nZoomBegin, nZoomSize,
+                                                      nZoomBeginGlobal, nGlob);*/
+        serverDistribution_ = new CDistributionServer(server->intraCommRank,
+                                                      globalZoomIndex, axis_domain_order,
+                                                      nZoomBegin, nZoomSize, nZoomBeginGlobal, nGlob);
+      }
+      CArray<size_t,1> outIndex;
+      buffer >> outIndex;
+      if (isDataDistributed_)
+        serverDistribution_->computeLocalIndex(outIndex);
+      else
+      {
+        dataSize = outIndex.numElements();
+        for (int i = 0; i < outIndex.numElements(); ++i) outIndex(i) = i;
+      }
+      writtenDataSize_ += dataSize;
+      outIndexFromClient.insert(std::make_pair(rank, outIndex));
+      connectedDataSize_[rank] = outIndex.numElements();
+      numberWrittenIndexes_ += outIndex.numElements();
+    }
+    // if (isScalarGrid()) return;
+    if (isDataDistributed_)
+    {
+      MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      offsetWrittenIndexes_ -= numberWrittenIndexes_;
+    }
+    else
+      totalNumberWrittenIndexes_ = numberWrittenIndexes_;
+    nbSenders = CClientServerMappingDistributed::computeConnectedClients(context->client->serverSize, context->client->clientSize, context->client->intraComm, ranks);
+        CArray<int,1> indexToModify(nb);
+        nb = 0;
+        for (it = itb; it != ite; ++it)
+        {
+          CArray<size_t,1>& globalInd = it->second;
+          for (size_t idx = 0; idx < globalInd.numElements(); ++idx)
+          {
+            itSrv = globalLocalMask.find(globalInd(idx));
+            if (globalLocalMask.end() != itSrv)
+            {
+              indexToModify(nb) = itSrv->second;
+              ++nb;
+            }
+          }
+        }
+        modifyMask(indexToModify, true);
+      }
+      if (isScalarGrid()) return;
+      nbReadSenders[client] = CClientServerMappingDistributed::computeConnectedClients(context->client->serverSize, context->client->clientSize, context->client->intraComm, ranks);
+    }
+  }
 …
                                         const CArray<int,1>& axisDomainOrder)
+  {
+    globalDim.resize(domains.size()*2+axis.size()+scalars.size());
+ //   globalDim.resize(domains.size()*2+axis.size()+scalars.size());
+    globalDim.resize(domains.size()*2+axis.size());
     int positionDimensionDistributed = 1;
     int idx = 0, idxDomain = 0, idxAxis = 0, idxScalar = 0;
 …
       else
+      {
         globalDim[idx] = 1;
+//        globalDim[idx] = 1;
         ++idxScalar;
         ++idx;
+//        ++idx;
+      }
+    }
 …
+  }
   bool CGrid::doGridHaveDataDistributed()
+  bool CGrid::doGridHaveDataDistributed(CContextClient* client)
+  {
     if (isScalarGrid()) return false;
+    else if (0 != client)
+    {
+      return  (isDataDistributed_ ||  (1 != client->clientSize) || (1 != client->serverSize));
+    }
     else
       return isDataDistributed_;
+      return isDataDistributed_;
+  }
 …
    void CGrid::sendAddDomain(const string& id)
+   {
+    CContext* context=CContext::getCurrent();
+    if (! context->hasServer )
+    {
+       CContextClient* client=context->client;
+       CEventClient event(this->getType(),EVENT_ID_ADD_DOMAIN);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg<<this->getId();
+         msg<<id;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+    }
+      sendAddItem(id, (int)EVENT_ID_ADD_DOMAIN);
+   }
 …
    void CGrid::sendAddAxis(const string& id)
+   {
+    CContext* context=CContext::getCurrent();
+    if (! context->hasServer )
+    {
+       CContextClient* client=context->client;
+       CEventClient event(this->getType(),EVENT_ID_ADD_AXIS);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg<<this->getId();
+         msg<<id;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+    }
+      sendAddItem(id, (int)EVENT_ID_ADD_AXIS);
+   }
 …
    void CGrid::sendAddScalar(const string& id)
+   {
+    CContext* context=CContext::getCurrent();
+    if (! context->hasServer )
+    {
+       CContextClient* client=context->client;
+       CEventClient event(this->getType(),EVENT_ID_ADD_SCALAR);
+       if (client->isServerLeader())
+       {
+         CMessage msg;
+         msg<<this->getId();
+         msg<<id;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event);
+       }
+       else client->sendEvent(event);
+    }
+      sendAddItem(id, (int)EVENT_ID_ADD_SCALAR);
+   }
 …
+    {
       CDomain* pDom = CDomain::get(*it);
       if (context->hasClient)
+      if (context->hasClient && !context->hasServer)
+      {
         pDom->solveRefInheritance(apply);
 …
+    {
       CAxis* pAxis = CAxis::get(*it);
       if (context->hasClient)
+      if (context->hasClient && !context->hasServer)
+      {
         pAxis->solveRefInheritance(apply);
 …
+    {
       CScalar* pScalar = CScalar::get(*it);
       if (context->hasClient)
+      if (context->hasClient && !context->hasServer)
+      {
         pScalar->solveRefInheritance(apply);
 …
+  }
+  void CGrid::setContextClient(CContextClient* contextClient)
+  {
+    if (clientsSet.find(contextClient)==clientsSet.end())
+    {
+      clients.push_back(contextClient) ;
+      clientsSet.insert(contextClient);
+    }
+    for (int i=0; i<this->getDomains().size(); i++)
+        this->getDomains()[i]->setContextClient(contextClient);
+    for (int i=0; i<this->getAxis().size(); i++)
+        this->getAxis()[i]->setContextClient(contextClient);
+  }
   /*!
     Parse a grid, for now, it contains only domain, axis and scalar

XIOS/dev/branch_openmp/src/node/grid.hpp

-                      r1078
+                      r1460
 namespace xios {
    /// ////////////////////// DÃ©clarations ////////////////////// ///
+   /// ////////////////////// Dï¿œï¿œclarations ////////////////////// ///
    class CGridGroup;
 …
          StdSize  getDataSize(void) const;
          /// EntrÃ©es-sorties de champs ///
+         /// Entrï¿œï¿œes-sorties de champs
          template <int n>
          void inputField(const CArray<double,n>& field, CArray<double,1>& stored) const;
          template <int n>
+         void outputField(const CArray<double,1>& stored, CArray<double,n>& field) const;
+         void outputField(int rank, const CArray<double,1>& stored, double* field);
+         void inputField(int rank, const double* const field, CArray<double,1>& stored);
+         void outputCompressedField(int rank, const CArray<double,1>& stored, double* field);
+         void outputField(const CArray<double,1>& stored, CArray<double,n>& field) const;
+         template <int n>
+         void uncompressField(const CArray<double,n>& data, CArray<double,1>& outData) const;
          virtual void parse(xml::CXMLNode& node);
 …
          static CGrid* cloneGrid(const StdString& idNewGrid, CGrid* gridSrc);
+      public:
+         /// EntrÃ©es-sorties de champs (interne) ///
+         void storeField_arr(const double* const data, CArray<double,1>& stored) const;
+         void restoreField_arr(const CArray<double,1>& stored, double* const data) const;
+         /// Traitements protÃ©gÃ©s ///
+      public:
          void computeIndexServer(void);
          void computeIndex(void);
          void computeIndexScalarGrid();
          void computeCompressedIndex();
+         void computeWrittenIndex();
          void solveDomainRef(bool checkAtt);
 …
          void sendIndexScalarGrid();
+         void setContextClient(CContextClient* contextClient);
          void computeDomConServer();
          std::map<int, int> getDomConServerSide();
          std::map<int, StdSize> getAttributesBufferSize();
          std::map<int, StdSize> getDataBufferSize(const std::string& id = "");
+         std::map<int, StdSize> getAttributesBufferSize(CContextClient* client, bool bufferForWriting = false);
+         std::map<int, StdSize> getDataBufferSize(CContextClient* client, const std::string& id = "", bool bufferForWriting = false);
          std::vector<StdString> getDomainList();
          std::vector<StdString> getAxisList();
 …
          std::vector<int> getAxisOrder();
          std::vector<int> getGlobalDimension();
+         bool isScalarGrid() const;
+         std::vector<int> getAxisPositionInGrid() const;
+         bool isScalarGrid() const;
          bool doGridHaveDataToWrite();
          bool doGridHaveDataDistributed();
+         bool doGridHaveDataDistributed(CContextClient* client = 0);
          size_t getWrittenDataSize() const;
          int getNumberWrittenIndexes() const;
 …
          std::map<CGrid*, std::pair<bool,StdString> >& getTransGridSource();
          bool hasTransform();
+         size_t getGlobalWrittenSize(void) ;
+         void getLocalMask(CArray<bool,1>& localMask) ;
+         template<int N>
+         void getLocalMask(const CArray<bool,N>& gridMask, CArray<bool,1>& localMask) ;
       public:
-         /// PropriÃ©tÃ©s privÃ©es ///
-         bool isChecked;
-         bool isDomainAxisChecked;
-         bool isIndexSent;
          CArray<int, 1> storeIndex_client;
+         map<int, CArray<int, 1> > storeIndex_toSrv;
+         map<int, CArray<int, 1> > storeIndex_fromSrv;
+         map<int,int> nbSenders;
+         map<int, CArray<size_t, 1> > outIndexFromClient, compressedOutIndexFromClient;
+/** Map containing indexes that will be sent in sendIndex(). */
+         std::map<CContextClient*, map<int, CArray<int, 1> > > storeIndex_toSrv;
+/** Map storing the number of senders. Key = size of receiver's intracomm */
+         std::map<int, std::map<int,int> > nbSenders;
+         std::map<CContextClient*, std::map<int,int> > nbReadSenders;
+         map<int, CArray<int, 1> > storeIndex_fromSrv; // Support, for now, reading with level-1 server
+         map<int, CArray<size_t, 1> > outIndexFromClient;  // Deprecated
+         map<int, CArray<size_t, 1> > compressedOutIndexFromClient;
+/** Map storing received indexes. Key = sender rank, value = index array. */
+         map<int, CArray<size_t, 1> > outGlobalIndexFromClient;
+// Manh Ha's comment: " A client receives global index from other clients (via recvIndex)
+// then does mapping these index into local index of STORE_CLIENTINDEX
+// In this way, store_clientIndex can be used as an input of a source filter
+// Maybe we need a flag to determine whether a client wants to write. TODO "
+/** Map storing received data. Key = sender rank, value = data array.
+ *  The map is created in CGrid::computeClientIndex and filled upon receiving data in CField::recvUpdateData() */
+         map<int, CArray<size_t, 1> > outLocalIndexStoreOnClient;
+/** Indexes calculated based on server-like distribution.
+ *  They are used for writing/reading data and only calculated for server level that does the writing/reading.
+ *  Along with localIndexToWriteOnClient, these indexes are used to correctly place incoming data. */
+         CArray<size_t,1> localIndexToWriteOnServer;
+/** Indexes calculated based on client-like distribution.
+ *  They are used for writing/reading data and only calculated for server level that does the writing/reading.
+ *  Along with localIndexToWriteOnServer, these indexes are used to correctly place incoming data. */
+         CArray<size_t,1> localIndexToWriteOnClient;
+         CArray<size_t,1> indexFromClients;
          void checkMask(void);
          void createMask(void);
+         void modifyMask(const CArray<int,1>& indexToModify);
+         void modifyMask(const CArray<int,1>& indexToModify, bool valueToModify = false);
+         void modifyMaskSize(const std::vector<int>& newDimensionSize, bool newValue = false);
+         void computeGridGlobalDimension(const std::vector<CDomain*>& domains,
+                                         const std::vector<CAxis*>& axis,
+                                         const std::vector<CScalar*>& scalars,
+                                         const CArray<int,1>& axisDomainOrder);
       private:
 …
                           bool createMask = false);
         template<int N>
+        void modifyGridMask(CArray<bool,N>& gridMask, const CArray<int,1>& indexToModify);
+        void modifyGridMask(CArray<bool,N>& gridMask, const CArray<int,1>& indexToModify, bool valueToModify);
+        template<int N>
+        void modifyGridMaskSize(CArray<bool,N>& gridMask, const std::vector<int>& eachDimSize, bool newValue);
+        void storeField_arr(const double* const data, CArray<double, 1>& stored) const;
+        void restoreField_arr(const CArray<double, 1>& stored, double* const data) const;
+        void uncompressField_arr(const double* const data, CArray<double, 1>& outData) const;
         void setVirtualDomainGroup(CDomainGroup* newVDomainGroup);
 …
         void setTransformationAlgorithms();
         void computeIndexByElement(const std::vector<boost::unordered_map<size_t,std::vector<int> > >& indexServerOnElement,
+                                   const CContextClient* client,
                                    CClientServerMapping::GlobalIndexMap& globalIndexOnServer);
         int computeGridGlobalDimension(std::vector<int>& globalDim,
                                        const std::vector<CDomain*> domains,
 …
                                        const std::vector<CScalar*> scalars,
                                        const CArray<int,1>& axisDomainOrder);
         int getDistributedDimension();
+        void computeClientIndex();
+        void computeConnectedClients();
+        void computeClientIndexScalarGrid();
+        void computeConnectedClientsScalarGrid();
       private:
+/** Clients that have to send a grid. There can be multiple clients in case of secondary server, otherwise only one client. */
+        std::list<CContextClient*> clients;
+        std::set<CContextClient*> clientsSet;
+        bool isChecked;
+        bool isDomainAxisChecked;
+        bool isIndexSent;
         CDomainGroup* vDomainGroup_;
         CAxisGroup* vAxisGroup_;
 …
         bool isAxisListSet, isDomListSet, isScalarListSet;
+/** Client-like distribution calculated based on the knowledge of the entire grid */
         CDistributionClient* clientDistribution_;
+/** Server-like distribution calculated upon receiving indexes */
         CDistributionServer* serverDistribution_;
         CClientServerMapping* clientServerMap_;
         size_t writtenDataSize_;
         int numberWrittenIndexes_, totalNumberWrittenIndexes_, offsetWrittenIndexes_;
+        std::map<int,size_t> connectedDataSize_;
+        std::vector<int> connectedServerRank_;
+        bool isDataDistributed_;
+/** Map storing local ranks of connected receivers. Key = size of receiver's intracomm.
+  * It is calculated in computeConnectedClients(). */
+        std::map<int, std::vector<int> > connectedServerRank_;
+/** Map storing the size of data to be send. Key = size of receiver's intracomm
+  * It is calculated in computeConnectedClients(). */
+        std::map<int, std::map<int,size_t> > connectedDataSize_;
+/** Ranks of connected receivers in case of reading. It is calculated in recvIndex(). */
+        std::vector<int> connectedServerRankRead_;
+/** Size of data to be send in case of reading. It is calculated in recvIndex(). */
+        std::map<int,size_t> connectedDataSizeRead_;
+        bool isDataDistributed_;
          //! True if and only if the data defined on the grid can be outputted in a compressed way
         bool isCompressible_;
         std::set<std::string> relFilesCompressed;
         bool isTransformed_;
         bool isGenerated_;
+        bool isTransformed_, isGenerated_;
+        bool computedWrittenIndex_;
         std::vector<int> axisPositionInGrid_;
         CGridTransformation* transformations_;
 …
         std::map<CGrid*, std::pair<bool,StdString> > gridSrc_;
         bool hasTransform_;
+        CClientServerMapping::GlobalIndexMap globalIndexOnServer_;
+            // List order of axis and domain in a grid, if there is a domain, it will take value 1 (true), axis 0 (false)
+/** Map storing global indexes of server-like (band-wise) distribution for sending to receivers.
+  * Key = size of receiver's intracomm.
+  */
+//        std::map<CContextClient*, CClientServerMapping::GlobalIndexMap> globalIndexOnServer_;
+        std::map<int, CClientServerMapping::GlobalIndexMap> globalIndexOnServer_;
+/** List order of axis and domain in a grid, if there is a domain, it will take value 1 (true), axis 0 (false) */
         std::vector<int> order_;
    }; // class CGrid
 …
    void CGrid::inputField(const CArray<double,n>& field, CArray<double,1>& stored) const
+   {
+//#ifdef __XIOS_DEBUG
       if (this->getDataSize() != field.numElements())
          ERROR("void CGrid::inputField(const  CArray<double,n>& field, CArray<double,1>& stored) const",
 …
                 << "Received data size = "      << field.numElements() << " ] "
                 << "The data array does not have the right size! "
+                << "Grid = " << this->GetName())
+                << "Grid = " << this->getId())
+//#endif
       this->storeField_arr(field.dataFirst(), stored);
+   }
 …
    void CGrid::outputField(const CArray<double,1>& stored, CArray<double,n>& field) const
+   {
+//#ifdef __XIOS_DEBUG
       if (this->getDataSize() != field.numElements())
          ERROR("void CGrid::outputField(const CArray<double,1>& stored, CArray<double,n>& field) const",
 …
                 << "Output data size = "   << field.numElements() << " ] "
                 << "The ouput array does not have the right size! "
+                << "Grid = " << this->GetName())
+                << "Grid = " << this->getId())
+//#endif
       this->restoreField_arr(stored, field.dataFirst());
+   }
+   /*!
+     This function removes the effect of mask on received data on the server.
+     This function only serve for the checking purpose. TODO: Something must be done to seperate mask and data_index from each other in received data
+     \data data received data with masking effect on the server
+     \outData data without masking effect
+   */
+   template <int N>
+   void CGrid::uncompressField(const CArray<double,N>& data, CArray<double,1>& outData) const
+   {
+     uncompressField_arr(data.dataFirst(), outData);
+   }
 …
      int dim = domainMasks.size() * 2 + axisMasks.size();
      std::vector<CDomain*> domainP = this->getDomains();
+     std::vector<CAxis*> axisP = this->getAxis();
      std::vector<int> idxLoop(dim,0), indexMap(numElement), eachDimSize(dim);
 …
+      }
       else if (1 == axisDomainOrder(i)) {
+        eachDimSize[indexMap[i]] = axisMasks[idxAxis]->numElements();
+//        eachDimSize[indexMap[i]] = axisMasks[idxAxis]->numElements();
+        eachDimSize[indexMap[i]] = axisP[idxAxis]->n;
         ++idx; ++idxAxis;
+      }
 …
                 << "Local size of dimension " << i << " is " << eachDimSize[i] << "." << std::endl
                 << "Mask size for dimension " << i << " is " << gridMask.extent(i) << "." << std::endl
                 << "Grid = " << this->GetName())
+                << "Grid = " << this->getId())
+      }
+    }
 …
         if (2 == axisDomainOrder(i))
+        {
+          maskValue = maskValue && (*domainMasks[idxDomain])(idxLoop[indexMap[i]] + idxLoop[indexMap[i]+1] * eachDimSize[indexMap[i]]);
+          int idxTmp = idxLoop[indexMap[i]] + idxLoop[indexMap[i]+1] * eachDimSize[indexMap[i]];
+          if (idxTmp < (*domainMasks[idxDomain]).numElements())
+            maskValue = maskValue && (*domainMasks[idxDomain])(idxTmp);
+          else
+            maskValue = false;
           ++idxDomain;
+        }
         else if (1 == axisDomainOrder(i))
+        {
+          maskValue = maskValue && (*axisMasks[idxAxis])(idxLoop[indexMap[i]]);
+          int idxTmp = idxLoop[indexMap[i]];
+          if (idxTmp < (*axisMasks[idxAxis]).numElements())
+            maskValue = maskValue && (*axisMasks[idxAxis])(idxTmp);
+          else
+            maskValue = false;
           ++idxAxis;
+        }
 …
+   }
+   template<int N>
+   void CGrid::modifyGridMaskSize(CArray<bool,N>& gridMask,
+                                  const std::vector<int>& eachDimSize,
+                                  bool newValue)
+   {
+      if (N != eachDimSize.size())
+      {
+        // ERROR("CGrid::modifyGridMaskSize(CArray<bool,N>& gridMask,
+        //                                  const std::vector<int>& eachDimSize,
+        //                                  bool newValue)",
+        //       << "Dimension size of the mask is different from input dimension size." << std::endl
+        //       << "Mask dimension is " << N << "." << std::endl
+        //       << "Input dimension is " << eachDimSize.size() << "." << std::endl
+        //       << "Grid = " << this->getId())
+      }
+      CArrayBoolTraits<CArray<bool,N> >::resizeArray(gridMask,eachDimSize);
+      gridMask = newValue;
+   }
    /*!
      Modify the current mask of grid, the local index to be modified will take value false
 …
    */
    template<int N>
+   void CGrid::modifyGridMask(CArray<bool,N>& gridMask, const CArray<int,1>& indexToModify)
+   {
+     bool valueToModify = false;
+   void CGrid::modifyGridMask(CArray<bool,N>& gridMask, const CArray<int,1>& indexToModify, bool valueToModify)
+   {
      int num = indexToModify.numElements();
      for (int idx = 0; idx < num; ++idx)
 …
    ///--------------------------------------------------------------
+/*!
+  A grid can have multiple dimension, so can its mask in the form of multi-dimension array.
+It's not a good idea to store all multi-dimension arrays corresponding to each mask.
+One of the ways is to convert this array into 1-dimension one and every process is taken place on it.
+  \param [in] multi-dimension array grid mask
+*/
+template<int N>
+void CGrid::getLocalMask(const CArray<bool,N>& gridMask, CArray<bool,1>& localMask)
+{
+  if (gridMask.isEmpty()) return ;
+  int dim = gridMask.dimensions();
+  std::vector<int> dimensionSizes(dim);
+  for (int i = 0; i < dim; ++i) dimensionSizes[i] = gridMask.extent(i);
+  std::vector<int> idxLoop(dim,0);
+  int ssize = gridMask.numElements(), idx = 0;
+  localMask.resize(ssize);
+  while (idx < ssize)
+  {
+    for (int i = 0; i < dim-1; ++i)
+    {
+      if (idxLoop[i] == dimensionSizes[i])
+      {
+        idxLoop[i] = 0;
+        ++idxLoop[i+1];
+      }
+    }
+    int maskIndex = idxLoop[0];
+    int mulDim = 1;
+    for (int k = 1; k < dim; ++k)
+    {
+      mulDim *= dimensionSizes[k-1];
+      maskIndex += idxLoop[k]*mulDim;
+    }
+    localMask(maskIndex) = *(gridMask.dataFirst()+maskIndex);
+    ++idxLoop[0];
+    ++idx;
+  }
+}
    // Declare/Define CGridGroup and CGridDefinition
    DECLARE_GROUP(CGrid);

XIOS/dev/branch_openmp/src/node/interpolate_domain.cpp

-                      r1014
+                      r1460
+    }
+    bool detect_missing_value=false ;
+    if (!this->detect_missing_value.isEmpty()) detect_missing_value = this->detect_missing_value.getValue();
+    else this->detect_missing_value.setValue(detect_missing_value);
+    bool renormalize=false ;
+    if (!this->renormalize.isEmpty()) renormalize = this->renormalize.getValue();
+    else this->renormalize.setValue(renormalize);
+    bool quantity=false ;
+    if (!this->quantity.isEmpty()) quantity = this->quantity.getValue();
+    else this->quantity.setValue(quantity);
     if (this->mode.isEmpty()) this->mode.setValue(mode_attr::compute);
     if (this->write_weight.isEmpty()) this->write_weight.setValue(false);
+    StdString weightFile;
+    switch (this->mode)
+    {
+      case mode_attr::read:
+        if (this->weight_filename.isEmpty())
+        {
+          if (!this->write_weight)
+            ERROR("void CInterpolateDomain::checkValid(CDomain* domainSrc)",
+                 << "Read mode is activated but there is no file specified." << std::endl
+                 << "Please define a correct file containing interpolation weights with option 'file'. ");
+        }
+        else
+        {
+          weightFile = this->weight_filename;
+          ifstream f(weightFile.c_str());
+          if (!f.good())
+            ERROR("void CInterpolateDomain::checkValid(CDomain* domainSrc)",
+                  << "Read mode is activated but file "  << weightFile << " doesn't exist." << std::endl
+                  << "Please check this file ");
+        }
+        break;
+      case mode_attr::compute:
+        break;
+      case mode_attr::read_or_compute:
+        if (!this->weight_filename.isEmpty() && !this->write_weight)
+        {
+          weightFile = this->weight_filename;
+          ifstream f(weightFile.c_str());
+          if (!f.good())
+            ERROR("void CInterpolateDomain::checkValid(CDomain* domainSrc)",
+                  << "read_or_compute mode is activated but file "  << weightFile << " doesn't exist." << std::endl
+                  << "Please check this file ");
+        }
+        break;
+      default:
+        break;
+    }
+    if (this->read_write_convention.isEmpty()) this->read_write_convention.setValue(read_write_convention_attr::fortran);
+  }

XIOS/dev/branch_openmp/src/node/node_enum.hpp

-                      r976
+                      r1460
          eReduceAxisToScalar,
          eReduceDomainToAxis,
+         eReduceAxisToAxis,
          eExtractDomainToAxis,
          eComputeConnectivityDomain,
          eExpandDomain,
          eExtractAxisToScalar,
+         eReduceDomainToScalar
+      } ENodeType;
+         eReduceDomainToScalar,
+         eTemporalSplitting,
+         eDuplicateScalarToAxis,
+         eReduceScalarToScalar,
+         eReorderDomain
+       } ENodeType;
 } // namespace xios

XIOS/dev/branch_openmp/src/node/node_type.hpp

-                      r976
+                      r1460
 #include "reduce_axis_to_scalar.hpp"
 #include "reduce_domain_to_axis.hpp"
+#include "reduce_axis_to_axis.hpp"
 #include "extract_domain_to_axis.hpp"
 #include "compute_connectivity_domain.hpp"
 …
 #include "extract_axis_to_scalar.hpp"
 #include "reduce_domain_to_scalar.hpp"
+#include "temporal_splitting.hpp"
+#include "duplicate_scalar_to_axis.hpp"
+#include "reduce_scalar_to_scalar.hpp"
+#include "reorder_domain.hpp"

XIOS/dev/branch_openmp/src/node/reduce_domain_to_axis.cpp

-                      r980
+                      r1460
              << "Domain source " <<domainSrc->getId() << std::endl
              << "Axis destination " << axisDst->getId());
+    if (this->local.isEmpty()) local=false ;
-    if (this->direction.isEmpty())
-      ERROR("CReduceDomainToAxis::checkValid(CAxis* axisDst, CDomain* domainSrc)",
-             << "A direction to apply the operation must be defined. It should be: 'iDir' or 'jDir'"
-             << "Domain source " <<domainSrc->getId() << std::endl
-             << "Axis destination " << axisDst->getId());
     switch (direction)
+    {

XIOS/dev/branch_openmp/src/node/reduce_domain_to_scalar.cpp

r976	r1460
39	39	void CReduceDomainToScalar::checkValid(CScalar* scalarDst, CDomain* domainSrc)
40	40	{
	41	if (this->local.isEmpty()) local=false ;
41	42	}
42	43

XIOS/dev/branch_openmp/src/node/scalar.cpp

-                      r1328
+                      r1460
      m["extract_axis"]  = TRANS_EXTRACT_AXIS_TO_SCALAR;
      m["reduce_domain"] = TRANS_REDUCE_DOMAIN_TO_SCALAR;
+     m["reduce_scalar"] = TRANS_REDUCE_SCALAR_TO_SCALAR;
+   }
 …
      (*CScalar::transformationMapList_ptr)["extract_axis"]  = TRANS_EXTRACT_AXIS_TO_SCALAR;
      (*CScalar::transformationMapList_ptr)["reduce_domain"] = TRANS_REDUCE_DOMAIN_TO_SCALAR;
+     (*CScalar::transformationMapList_ptr)["reduce_scalar"] = TRANS_REDUCE_SCALAR_TO_SCALAR;
+   }

XIOS/dev/branch_openmp/src/node/scalar.hpp

-                      r1331
+                      r1460
 #include "attribute_enum.hpp"
 #include "attribute_enum_impl.hpp"
+#include "attribute_array.hpp"
 #include "transformation.hpp"
 #include "transformation_enum.hpp"
 …
            TransMapTypes transformationMap_;
+         private:
+            void setTransformations(const TransMapTypes&);
+           void setTransformations(const TransMapTypes&);
        private:

XIOS/dev/branch_openmp/src/node/transformation.hpp

r1334	r1460
66	66	if (0 == transformationCreationCallBacks_)
67	67	transformationCreationCallBacks_ = new CallBackMap();
	68
68	69	return (*transformationCreationCallBacks_).insert(make_pair(transType, createFn)).second;
69	70	}

XIOS/dev/branch_openmp/src/node/transformation_enum.hpp

-                      r976
+                      r1460
         TRANS_EXTRACT_AXIS_TO_SCALAR = 11,
         TRANS_REDUCE_DOMAIN_TO_SCALAR = 12,
+        TRANS_TEMPORAL_SPLITTING = 13,
+        TRANS_REDUCE_AXIS_TO_AXIS = 14,
+        TRANS_DUPLICATE_SCALAR_TO_AXIS = 15,
+        TRANS_REDUCE_SCALAR_TO_SCALAR = 16,
+        TRANS_REORDER_DOMAIN = 17
       } ETranformationType;

XIOS/dev/branch_openmp/src/node/variable.cpp

-                      r1075
+                      r1460
+   {
      CContext* context=CContext::getCurrent() ;
+     if (!context->hasServer)
+     if (context->hasClient)
+     {
        CContextClient* client=context->client ;
        CEventClient event(this->getType(),EVENT_ID_VARIABLE_VALUE) ;
        if (client->isServerLeader())
+       // Use correct context client to send message
+       // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
+      int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
+       for (int i = 0; i < nbSrvPools; ++i)
+       {
+         CMessage msg ;
+         msg<<this->getId() ;
+         msg<<content ;
+         const std::list<int>& ranks = client->getRanksServerLeader();
+         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+         client->sendEvent(event) ;
+       }
+       else client->sendEvent(event) ;
+    }
+//       CContextClient* contextClientTmp = (0 != context->clientPrimServer) ? context->clientPrimServer
+         CContextClient* contextClientTmp = (context->hasServer) ? context->clientPrimServer[i]
+                                                                             : context->client;
+         CEventClient event(this->getType(),EVENT_ID_VARIABLE_VALUE) ;
+         if (contextClientTmp->isServerLeader())
+         {
+           CMessage msg ;
+           msg<<this->getId() ;
+           msg<<content ;
+           const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
+           for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+             event.push(*itRank,1,msg);
+           contextClientTmp->sendEvent(event) ;
+         }
+         else contextClientTmp->sendEvent(event) ;
+      }
+     }
+   }
+   void CVariable::sendValue(CContextClient* client, bool clientPrim /*= false*/)
+   {
+     CEventClient event(this->getType(),EVENT_ID_VARIABLE_VALUE) ;
+     if (client->isServerLeader())
+     {
+       CMessage msg ;
+       msg<<this->getId() ;
+       msg<<content ;
+       const std::list<int>& ranks = client->getRanksServerLeader();
+       for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+         event.push(*itRank,1,msg);
+       client->sendEvent(event) ;
+     }
+     else client->sendEvent(event) ;
+   }

XIOS/dev/branch_openmp/src/node/variable.hpp

-                      r1041
+                      r1460
          , public CVariableAttributes
+      {
+                     /// typedef ///
+            typedef CObjectTemplate<CVariable>   SuperClass;
+            typedef CVariableAttributes SuperClassAttribute;
+         public :
             enum EEventId
+            {
              EVENT_ID_VARIABLE_VALUE
             };
-            /// typedef ///
-            typedef CObjectTemplate<CVariable>   SuperClass;
-            typedef CVariableAttributes SuperClassAttribute;
             friend class CVariableGroup;
 …
             //! Sending a request to set up variable data
             void sendValue();
+            void sendValue(CContextClient* client, bool clientPrim = false);
             static void recvValue(CEventServer& event) ;

Context Navigation

Legend:

XIOS/dev/branch_openmp/src/node/axis.cpp

XIOS/dev/branch_openmp/src/node/axis.hpp

XIOS/dev/branch_openmp/src/node/context.cpp

XIOS/dev/branch_openmp/src/node/context.hpp

XIOS/dev/branch_openmp/src/node/domain.cpp

XIOS/dev/branch_openmp/src/node/domain.hpp

XIOS/dev/branch_openmp/src/node/field.cpp

XIOS/dev/branch_openmp/src/node/field.hpp

XIOS/dev/branch_openmp/src/node/field_impl.hpp

XIOS/dev/branch_openmp/src/node/file.cpp

XIOS/dev/branch_openmp/src/node/file.hpp

XIOS/dev/branch_openmp/src/node/grid.cpp

XIOS/dev/branch_openmp/src/node/grid.hpp

XIOS/dev/branch_openmp/src/node/interpolate_domain.cpp

XIOS/dev/branch_openmp/src/node/node_enum.hpp

XIOS/dev/branch_openmp/src/node/node_type.hpp

XIOS/dev/branch_openmp/src/node/reduce_domain_to_axis.cpp

XIOS/dev/branch_openmp/src/node/reduce_domain_to_scalar.cpp

XIOS/dev/branch_openmp/src/node/scalar.cpp

XIOS/dev/branch_openmp/src/node/scalar.hpp

XIOS/dev/branch_openmp/src/node/transformation.hpp

XIOS/dev/branch_openmp/src/node/transformation_enum.hpp

XIOS/dev/branch_openmp/src/node/variable.cpp

XIOS/dev/branch_openmp/src/node/variable.hpp

Download in other formats: