Changeset 1460 for XIOS/dev/branch_openmp/src/node/grid.cpp

Timestamp:

03/22/18 10:43:20 (6 years ago)

Author:

yushan

Message:

branch_openmp merged with XIOS_DEV_CMIP6@1459

File:

• XIOS/dev/branch_openmp/src/node/grid.cpp (modified) (36 diffs)

XIOS/dev/branch_openmp/src/node/grid.cpp

@@ -18 +18 @@
 #include "grid_transformation.hpp"
 #include "grid_generate.hpp"
+#include "server.hpp"
 
 namespace xios {
@@ -31 +32 @@
       , clientDistribution_(0), isIndexSent(false) , serverDistribution_(0), clientServerMap_(0)
       , writtenDataSize_(0), numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
-      , connectedDataSize_(), connectedServerRank_(), isDataDistributed_(true), isCompressible_(false)
+      , connectedDataSize_(), connectedServerRank_(), connectedServerRankRead_(), connectedDataSizeRead_()
+          , isDataDistributed_(true), isCompressible_(false)
       , transformations_(0), isTransformed_(false)
       , axisPositionInGrid_(), hasDomainAxisBaseRef_(false)
       , gridSrc_(), hasTransform_(false), isGenerated_(false), order_(), globalIndexOnServer_()
+      , computedWrittenIndex_(false)
+      , clients()
    {
      setVirtualDomainGroup(CDomainGroup::create(getId() + "_virtual_domain_group"));
@@ -49 +53 @@
       , clientDistribution_(0), isIndexSent(false) , serverDistribution_(0), clientServerMap_(0)
       , writtenDataSize_(0), numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
-      , connectedDataSize_(), connectedServerRank_(), isDataDistributed_(true), isCompressible_(false)
+      , connectedDataSize_(), connectedServerRank_(), connectedServerRankRead_(), connectedDataSizeRead_()
+          , isDataDistributed_(true), isCompressible_(false)
       , transformations_(0), isTransformed_(false)
       , axisPositionInGrid_(), hasDomainAxisBaseRef_(false)
       , gridSrc_(), hasTransform_(false), isGenerated_(false), order_(), globalIndexOnServer_()
+      , computedWrittenIndex_(false)
+      , clients()
    {
      setVirtualDomainGroup(CDomainGroup::create(getId() + "_virtual_domain_group"));
@@ -87 +94 @@
      {
        std::vector<int> dataNindex = clientDistribution_->getDataNIndex();
-       for (int i = 0; i < dataNindex.size(); ++i) retvalue *= dataNindex[i];
+       for (int i = 0; i < dataNindex.size(); ++i) retvalue *= dataNindex[i];       
      }
      return retvalue;
@@ -96 +103 @@
     *
     * \return A map associating the server rank with its minimum buffer size.
+    * TODO: Refactor code
     */
-   std::map<int, StdSize> CGrid::getAttributesBufferSize()
-   {
-     std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes();
+   std::map<int, StdSize> CGrid::getAttributesBufferSize(CContextClient* client, bool bufferForWriting)
+   {
+     std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes(client);
 
      // The grid indexes require a similar size as the actual data
-     std::map<int, StdSize> dataSizes = getDataBufferSize();
+     std::map<int, StdSize> dataSizes = getDataBufferSize(client, "", bufferForWriting);
      std::map<int, StdSize>::iterator it, itE = dataSizes.end();
      for (it = dataSizes.begin(); it != itE; ++it)
@@ -110 +118 @@
          attributesSizes[it->first] = it->second;
      }
-
+     
      // Account for the axis attributes
      std::vector<CAxis*> axisList = getAxis();
      for (size_t i = 0; i < axisList.size(); ++i)
      {
-       std::map<int, StdSize> axisAttBuffSize = axisList[i]->getAttributesBufferSize();
+       std::map<int, StdSize> axisAttBuffSize = axisList[i]->getAttributesBufferSize(client, getGlobalDimension(),axisPositionInGrid_[i]);
        for (it = axisAttBuffSize.begin(), itE = axisAttBuffSize.end(); it != itE; ++it)
        {
+         it->second += 2 * sizeof(bool);
          if (it->second > attributesSizes[it->first])
            attributesSizes[it->first] = it->second;
@@ -127 +136 @@
      for (size_t i = 0; i < domList.size(); ++i)
      {
-       std::map<int, StdSize> domAttBuffSize = domList[i]->getAttributesBufferSize();
+       std::map<int, StdSize> domAttBuffSize = domList[i]->getAttributesBufferSize(client);
        for (it = domAttBuffSize.begin(), itE = domAttBuffSize.end(); it != itE; ++it)
        {
+         it->second += 2 * sizeof(bool);
          if (it->second > attributesSizes[it->first])
            attributesSizes[it->first] = it->second;
@@ -136 +146 @@
 
      return attributesSizes;
-   }
+  }
 
    /*!
-    * Compute the minimum buffer size required to send the data to the server(s).
-    *
+    * Compute the minimum buffer size required to send the data.
+    * \param client contextClient used to determine the size of connected receivers
     * \param id the id used to tag the data
-    * \return A map associating the server rank with its minimum buffer size.
+    * \param bufferForWriting flag indicating if a buffer is used to send data for writing
+    * \return A map associating the sender rank with its minimum buffer size.
     */
-   std::map<int, StdSize> CGrid::getDataBufferSize(const std::string& id /*= ""*/)
-   {
-     std::map<int, StdSize> dataSizes;
+   std::map<int, StdSize> CGrid::getDataBufferSize(CContextClient* client, const std::string& id /*= ""*/, bool bufferForWriting /*= "false"*/)
+   {     
      // The record index is sometimes sent along with the data but we always
      // include it in the size calculation for the sake of simplicity
-     const size_t extraSize = CEventClient::headerSize + (id.empty() ? getId() : id).size() + 2 * sizeof(size_t);
-
-     std::map<int, size_t>::const_iterator itEnd = connectedDataSize_.end();
-     for (size_t k = 0; k < connectedServerRank_.size(); ++k)
+     const size_t extraSize = CEventClient::headerSize + (id.empty() ? getId() : id).size() 
+                                                       + 2 * sizeof(size_t) 
+                                                       + sizeof(size_t);
+
+     std::map<int, StdSize> dataSizes;
+     int receiverSize = client->serverSize;
+     std::map<int,size_t>& dataSizeMap = bufferForWriting ? connectedDataSize_[receiverSize]: connectedDataSizeRead_;
+     std::vector<int>& connectedServerRanks = bufferForWriting ? connectedServerRank_[receiverSize] : connectedServerRankRead_;
+
+     std::map<int, size_t>::const_iterator itEnd = dataSizeMap.end();
+     for (size_t k = 0; k < connectedServerRanks.size(); ++k)
      {
-       int rank = connectedServerRank_[k];
-       std::map<int, size_t>::const_iterator it = connectedDataSize_.find(rank);
+       int rank = connectedServerRanks[k];
+       std::map<int, size_t>::const_iterator it = dataSizeMap.find(rank);
        size_t count = (it != itEnd) ? it->second : 0;
 
@@ -164 +181 @@
    }
 
+   size_t CGrid::getGlobalWrittenSize(void)
+   {
+         std::vector<CDomain*> domainP = this->getDomains();
+     std::vector<CAxis*> axisP = this->getAxis();
+     
+     size_t globalGridSize=1 ;
+     for (std::vector<CDomain*>::iterator it=domainP.begin(); it!=domainP.end();++it) globalGridSize*=(*it)->getGlobalWrittenSize() ;
+     for (std::vector<CAxis*>::iterator it=axisP.begin(); it!=axisP.end();++it) globalGridSize*=(*it)->getGlobalWrittenSize() ;
+     return globalGridSize ;
+   }
+   
+   
    void CGrid::checkAttributesAfterTransformation()
    {
@@ -225 +254 @@
 
    //---------------------------------------------------------------
-
+   /*
+     Find all reference of grid's components and inherite attributes if necessary
+   */
    void CGrid::solveDomainAxisRef(bool areAttributesChecked)
    {
@@ -236 +267 @@
    }
 
+   /*
+     Go up hierachy reference and fill in the base reference with attributes of the children
+     This function should be only used after reading component's attributes from file
+   */
    void CGrid::solveDomainAxisBaseRef()
    {
@@ -272 +307 @@
    {
      CContext* context = CContext::getCurrent();
-     CContextClient* client=context->client;
-
-     if (isScalarGrid())
-     {
-       if (context->hasClient)
-          if (this->isChecked && doSendingIndex && !isIndexSent) { sendIndexScalarGrid(); this->isIndexSent = true; }
-
-       if (this->isChecked) return;
-
-       if (context->hasClient)
-       {
-          this->computeIndexScalarGrid();
+     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;   
+     nbSrvPools = 1;  
+     for (int p = 0; p < nbSrvPools; ++p)
+     {    
+       if (context->hasClient && this->isChecked && doSendingIndex && !isIndexSent) 
+       { 
+         if (isScalarGrid())
+           sendIndexScalarGrid();
+         else
+           sendIndex();
+         this->isIndexSent = true; 
        }
 
-       if (!(this->hasTransform() && !this->isTransformed()))
-        this->isChecked = true;
-       return;
+       // Not sure about this
+       //if (!(this->hasTransform() && !this->isTransformed()))
+       // this->isChecked = true;
+       //return;
      }
-
-     if (context->hasClient)
-      if (this->isChecked && doSendingIndex && !isIndexSent) { sendIndex(); this->isIndexSent = true; }
-
+    
      if (this->isChecked) return;
-
-     if (context->hasClient)
-     {
-        this->checkAttributesAfterTransformation();
-        this->checkMask();
-        this->computeIndex();
-     }
-
-     if (!(this->hasTransform() && !this->isTransformed())) 
+     this->checkAttributesAfterTransformation();
+
+     // TODO: Transfer grid attributes
+     //if (!context->hasClient && context->hasServer) this->createMask();
+     this->computeIndex();
+
+     if (!(this->hasTransform() && !this->isTransformed()))
       this->isChecked = true;
 
      if (!(this->hasTransform() && (!this->isGenerated())))
-      this->isChecked = true; 
-   }
-
+      this->isChecked = true;
+   }
+
+   /*
+     Create mask of grid from mask of its components
+   */
    void CGrid::createMask(void)
    {
@@ -318 +351 @@
 
       std::vector<CArray<bool,1>* > domainMasks(domainP.size());
-      for (int i = 0; i < domainMasks.size(); ++i) domainMasks[i] = &(domainP[i]->mask_1d);
+      for (int i = 0; i < domainMasks.size(); ++i) domainMasks[i] = &(domainP[i]->domainMask);
       std::vector<CArray<bool,1>* > axisMasks(axisP.size());
       for (int i = 0; i < axisMasks.size(); ++i) axisMasks[i] = &(axisP[i]->mask);
@@ -349 +382 @@
    }
 
+   /*
+     Check validity of grid's mask by using the masks of its components
+   */
    void CGrid::checkMask(void)
    {
@@ -357 +393 @@
 
       std::vector<CArray<bool,1>* > domainMasks(domainP.size());
-      for (int i = 0; i < domainMasks.size(); ++i) domainMasks[i] = &(domainP[i]->mask_1d);
+      for (int i = 0; i < domainMasks.size(); ++i) domainMasks[i] = &(domainP[i]->domainMask);
       std::vector<CArray<bool,1>* > axisMasks(axisP.size());
       for (int i = 0; i < axisMasks.size(); ++i) axisMasks[i] = &(axisP[i]->mask);
@@ -388 +424 @@
    }
 
-   void CGrid::modifyMask(const CArray<int,1>& indexToModify)
+
+/*!
+  A grid can have multiple dimension, so can its mask in the form of multi-dimension array.
+It's not a good idea to store all multi-dimension arrays corresponding to each mask.
+One of the ways is to convert this array into 1-dimension one and every process is taken place on it.
+  \param [in] multi-dimension array grid mask
+*/
+
+  void CGrid::getLocalMask(CArray<bool,1>& localMask)
+  {
+      std::vector<CDomain*> domainP = this->getDomains();
+      std::vector<CAxis*> axisP = this->getAxis();
+      int dim = domainP.size() * 2 + axisP.size();
+
+      switch (dim)
+      {
+        case 0:
+          getLocalMask(mask_0d, localMask);
+          break;
+        case 1:
+          getLocalMask(mask_1d, localMask);
+          break;
+        case 2:
+          getLocalMask(mask_2d, localMask);
+          break;
+        case 3:
+          getLocalMask(mask_3d, localMask);
+          break;
+        case 4:
+          getLocalMask(mask_4d, localMask);
+          break;
+        case 5:
+          getLocalMask(mask_5d, localMask);
+          break;
+        case 6:
+          getLocalMask(mask_6d, localMask);
+          break;
+        case 7:
+          getLocalMask(mask_7d, localMask);
+          break;
+        default:
+          break;
+      }
+  }
+      
+   /*
+     Modify value of mask in a certain index
+     This function can be used to correct the mask of grid after being constructed with createMask
+     \param [in] indexToModify
+     \param [in] modifyValue
+   */
+   void CGrid::modifyMask(const CArray<int,1>& indexToModify, bool modifyValue)
    {
       using namespace std;
@@ -396 +483 @@
 
       switch (dim) {
+        case 0:
+          modifyGridMask(mask_0d, indexToModify, modifyValue);
+          break;
         case 1:
-          modifyGridMask(mask_1d, indexToModify);
+          modifyGridMask(mask_1d, indexToModify, modifyValue);
           break;
         case 2:
-          modifyGridMask(mask_2d, indexToModify);
+          modifyGridMask(mask_2d, indexToModify, modifyValue);
           break;
         case 3:
-          modifyGridMask(mask_3d, indexToModify);
+          modifyGridMask(mask_3d, indexToModify, modifyValue);
           break;
         case 4:
-          modifyGridMask(mask_1d, indexToModify);
+          modifyGridMask(mask_4d, indexToModify, modifyValue);
           break;
         case 5:
-          modifyGridMask(mask_2d, indexToModify);
+          modifyGridMask(mask_5d, indexToModify, modifyValue);
           break;
         case 6:
-          modifyGridMask(mask_3d, indexToModify);
+          modifyGridMask(mask_6d, indexToModify, modifyValue);
           break;
         case 7:
-          modifyGridMask(mask_3d, indexToModify);
+          modifyGridMask(mask_7d, indexToModify, modifyValue);
+          break;
+        default:
+          break;
+      }
+   }
+
+   /*
+     Change the mask size. This function is used on reconstructing mask in server side
+     \param [in] newDimensionSize
+     \param [in] newValue 
+   */
+   void CGrid::modifyMaskSize(const std::vector<int>& newDimensionSize, bool newValue)
+   {      
+      std::vector<CDomain*> domainP = this->getDomains();
+      std::vector<CAxis*> axisP = this->getAxis();            
+      int dim = domainP.size() * 2 + axisP.size();
+
+      switch (dim) {
+        case 0:
+          modifyGridMaskSize(mask_0d, newDimensionSize, newValue);
+          break;
+        case 1:
+          modifyGridMaskSize(mask_1d, newDimensionSize, newValue);
+          break;
+        case 2:
+          modifyGridMaskSize(mask_2d, newDimensionSize, newValue);
+          break;
+        case 3:
+          modifyGridMaskSize(mask_3d, newDimensionSize, newValue);
+          break;
+        case 4:
+          modifyGridMaskSize(mask_4d, newDimensionSize, newValue);
+          break;
+        case 5:
+          modifyGridMaskSize(mask_5d, newDimensionSize, newValue);
+          break;
+        case 6:
+          modifyGridMaskSize(mask_6d, newDimensionSize, newValue);
+          break;
+        case 7:
+          modifyGridMaskSize(mask_7d, newDimensionSize, newValue);
           break;
         default:
@@ -486 +617 @@
    }
 
-   std::vector<int> CGrid::getAxisPositionInGrid() const
-   {
-     return axisPositionInGrid_;
+   /*!
+      Compute the index to for write data into a file
+   */
+   void CGrid::computeWrittenIndex()
+   {      
+      if (computedWrittenIndex_) return;
+      computedWrittenIndex_ = true;
+
+      if (isScalarGrid())
+      {
+        size_t nbWritten = 1;
+        int writtenIndex = 0;
+
+        localIndexToWriteOnClient.resize(nbWritten);  
+        localIndexToWriteOnServer.resize(nbWritten);
+        localIndexToWriteOnServer(0) = writtenIndex;
+        localIndexToWriteOnClient(0) = writtenIndex;
+        
+        return;
+      }
+
+      size_t nbWritten = 0, indGlo;
+      CDistributionClient::GlobalLocalDataMap& globalDataIndex = clientDistribution_->getGlobalDataIndexOnClient();
+      CDistributionClient::GlobalLocalDataMap::const_iterator itb = globalDataIndex.begin(),
+                                                              ite = globalDataIndex.end(), it;    
+      const CDistributionServer::GlobalLocalMap& globalLocalIndex = serverDistribution_->getGlobalLocalIndex();                                                              
+      CDistributionServer::GlobalLocalMap::const_iterator itSrvb = globalLocalIndex.begin(),
+                                                          itSrve = globalLocalIndex.end(), itSrv;
+      for (it = itb; it != ite; ++it)
+      {
+        indGlo = it->first;
+        if (globalLocalIndex.end() != globalLocalIndex.find(indGlo)) ++nbWritten;                
+      }
+
+      localIndexToWriteOnClient.resize(nbWritten);  
+      localIndexToWriteOnServer.resize(nbWritten);
+      
+      {
+        numberWrittenIndexes_ = nbWritten;
+        if (isDataDistributed_)
+        {
+          CContextServer* server = CContext::getCurrent()->server;      
+          MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+          MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+          offsetWrittenIndexes_ -= numberWrittenIndexes_;
+        }
+        else
+          totalNumberWrittenIndexes_ = numberWrittenIndexes_;
+      }
+
+      nbWritten = 0; 
+      for (it = itb; it != ite; ++it)
+      {
+        indGlo = it->first;
+        itSrv = globalLocalIndex.find(indGlo);
+        if (itSrve != itSrv)
+        {
+          localIndexToWriteOnServer(nbWritten) = itSrv->second;
+          localIndexToWriteOnClient(nbWritten) = it->second;
+          ++nbWritten;                
+        } 
+      }
    }
 
    //---------------------------------------------------------------
+
+   /*
+     Compute the global index and its local index taking account mask and data index.
+     These global indexes will be used to compute the connection of this client (sender) to its servers (receivers)
+     (via function computeConnectedClient)
+     These global indexes also correspond to data sent to servers (if any)
+   */
+   void CGrid::computeClientIndex()
+   {
+     CContext* context = CContext::getCurrent();
+
+     CContextClient* client = context->client;  // Here it's not important which contextClient to recuperate
+     int rank = client->clientRank;
+
+     clientDistribution_ = new CDistributionClient(rank, this);
+     // Get local data index on client
+     storeIndex_client.resize(clientDistribution_->getLocalDataIndexOnClient().size());
+     int nbStoreIndex = storeIndex_client.numElements();
+     for (int idx = 0; idx < nbStoreIndex; ++idx) storeIndex_client(idx) = (clientDistribution_->getLocalDataIndexOnClient())[idx];
+
+     if (0 == serverDistribution_) isDataDistributed_= clientDistribution_->isDataDistributed();
+     else
+     {
+        // Mapping global index received from clients to the storeIndex_client
+        CDistributionClient::GlobalLocalDataMap& globalDataIndex = clientDistribution_->getGlobalDataIndexOnClient();
+        CDistributionClient::GlobalLocalDataMap::const_iterator itGloe = globalDataIndex.end();
+        map<int, CArray<size_t, 1> >::iterator itb = outGlobalIndexFromClient.begin(),
+                                               ite = outGlobalIndexFromClient.end(), it;
+
+        for (it = itb; it != ite; ++it)
+        {
+          int rank = it->first;
+          CArray<size_t,1>& globalIndex = outGlobalIndexFromClient[rank];
+          outLocalIndexStoreOnClient.insert(make_pair(rank, CArray<size_t,1>(globalIndex.numElements())));
+          CArray<size_t,1>& localIndex = outLocalIndexStoreOnClient[rank];
+          size_t nbIndex = 0;
+
+          // Keep this code for this moment but it should be removed (or moved to DEBUG) to improve performance
+          for (size_t idx = 0; idx < globalIndex.numElements(); ++idx)
+          {
+            if (itGloe != globalDataIndex.find(globalIndex(idx)))
+            {
+              ++nbIndex;
+            }
+          }
+
+          if (doGridHaveDataDistributed(client) && (nbIndex != localIndex.numElements()))
+               ERROR("void CGrid::computeClientIndex()",
+                  << "Number of local index on client is different from number of received global index"
+                  << "Rank of sent client " << rank <<"."
+                  << "Number of local index " << nbIndex << ". "
+                  << "Number of received global index " << localIndex.numElements() << ".");
+
+          nbIndex = 0;
+          for (size_t idx = 0; idx < globalIndex.numElements(); ++idx)
+          {
+            if (itGloe != globalDataIndex.find(globalIndex(idx)))
+            {
+              localIndex(idx) = globalDataIndex[globalIndex(idx)];
+            }
+          }
+        }
+      }
+   }
+
+   /*!
+     Compute connected receivers and indexes to be sent to these receivers.
+   */
+   void CGrid::computeConnectedClients()
+   {
+     CContext* context = CContext::getCurrent();
+     int nbSrvPools = (context->clientPrimServer.size() == 0) ? 1 : context->clientPrimServer.size();
+     connectedServerRank_.clear();
+     connectedDataSize_.clear();
+     globalIndexOnServer_.clear();
+     nbSenders.clear();
+
+     for (int p = 0; p < nbSrvPools; ++p)
+     {
+       CContextClient* client = (context->clientPrimServer.size() == 0) ? context->client : context->clientPrimServer[p];
+       int receiverSize = client->serverSize;
+//       connectedServerRank_[client].clear();
+
+       if (connectedServerRank_.find(receiverSize) == connectedServerRank_.end())
+       {
+        if (!doGridHaveDataDistributed(client))
+         {
+            if (client->isServerLeader())
+            {
+              size_t ssize = clientDistribution_->getLocalDataIndexOnClient().size();
+              const std::list<int>& ranks = client->getRanksServerLeader();
+              for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+              {
+                connectedServerRank_[receiverSize].push_back(*itRank);
+                connectedDataSize_[receiverSize][*itRank] = ssize;
+              }
+            }
+            return;
+         }
+
+         // Compute mapping between client and server
+         std::vector<boost::unordered_map<size_t,std::vector<int> > > indexServerOnElement;
+         CServerDistributionDescription serverDistributionDescription(getGlobalDimension(), client->serverSize);
+         std::vector<int> serverZeroIndex = serverDistributionDescription.computeServerGlobalByElement(indexServerOnElement,
+                                                                                                    client->clientRank,
+                                                                                                    client->clientSize,
+                                                                                                    axis_domain_order,
+                                                                                                    getDistributedDimension());
+
+         // Even if servers have no index, they must received something from client
+         // We only use several client to send "empty" message to these servers
+         std::list<int> serverZeroIndexLeader;
+         std::list<int> serverZeroIndexNotLeader;
+         CContextClient::computeLeader(client->clientRank, client->clientSize, serverZeroIndex.size(), serverZeroIndexLeader, serverZeroIndexNotLeader);
+         for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+           *it = serverZeroIndex[*it];
+
+         if (globalIndexOnServer_.find(receiverSize) == globalIndexOnServer_.end())
+           computeIndexByElement(indexServerOnElement, client, globalIndexOnServer_[receiverSize]);
+
+         const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer();
+         CDistributionClient::GlobalLocalDataMap::const_iterator iteGlobalLocalIndexMap = globalLocalIndexSendToServer.end(), itGlobalLocalIndexMap;
+         CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itbGlobalMap, itGlobalMap;
+         itbGlobalMap = globalIndexOnServer_[receiverSize].begin();
+         iteGlobalMap = globalIndexOnServer_[receiverSize].end();
+
+         for (itGlobalMap  = itbGlobalMap; itGlobalMap != iteGlobalMap; ++itGlobalMap)
+         {
+           int serverRank = itGlobalMap->first;
+           int indexSize = itGlobalMap->second.size();
+           const std::vector<size_t>& indexVec = itGlobalMap->second;
+           for (int idx = 0; idx < indexSize; ++idx)
+           {
+              itGlobalLocalIndexMap = globalLocalIndexSendToServer.find(indexVec[idx]);
+              if (iteGlobalLocalIndexMap != itGlobalLocalIndexMap)
+              {
+                if (connectedDataSize_[receiverSize].end() == connectedDataSize_[receiverSize].find(serverRank))
+                  connectedDataSize_[receiverSize][serverRank] = 1;
+                else
+                  ++connectedDataSize_[receiverSize][serverRank];
+              }
+           }
+         }
+
+         // Connected servers which really have index
+         for (itGlobalMap = itbGlobalMap; itGlobalMap != iteGlobalMap; ++itGlobalMap) {
+           connectedServerRank_[receiverSize].push_back(itGlobalMap->first);
+         }
+
+         // Connected servers which have no index at all
+         for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+           connectedServerRank_[receiverSize].push_back(*it);
+
+         // Even if a client has no index, it must connect to at least one server and
+         // send an "empty" data to this server
+         if (connectedServerRank_[receiverSize].empty())
+          connectedServerRank_[receiverSize].push_back(client->clientRank % client->serverSize);
+
+         nbSenders[receiverSize] = clientServerMap_->computeConnectedClients(receiverSize, client->clientSize, client->intraComm, connectedServerRank_[receiverSize]);
+       }
+     }
+   }
 
    /*!
@@ -503 +855 @@
    {
      CContext* context = CContext::getCurrent();
-     CContextClient* client = context->client;
-
-     // First of all, compute distribution on client side
-     clientDistribution_ = new CDistributionClient(client->clientRank, this);
-     // Get local data index on client
-     storeIndex_client.resize(clientDistribution_->getLocalDataIndexOnClient().size());
-     int nbStoreIndex = storeIndex_client.numElements();
-     for (int idx = 0; idx < nbStoreIndex; ++idx) storeIndex_client(idx) = (clientDistribution_->getLocalDataIndexOnClient())[idx];
-     isDataDistributed_= clientDistribution_->isDataDistributed();
-
-     connectedServerRank_.clear();
-
-     if (!doGridHaveDataDistributed())
+     if (isScalarGrid())
      {
-        if (client->isServerLeader())
-        {
-          size_t ssize = clientDistribution_->getLocalDataIndexOnClient().size();
-          const std::list<int>& ranks = client->getRanksServerLeader();
-          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-          {
-            connectedServerRank_.push_back(*itRank);
-            connectedDataSize_[*itRank] = ssize;
-          }
-        }
-        return;
-     }
-
-     // Compute mapping between client and server
-     std::vector<boost::unordered_map<size_t,std::vector<int> > > indexServerOnElement;
-     CServerDistributionDescription serverDistributionDescription(getGlobalDimension(), client->serverSize);
-     serverDistributionDescription.computeServerGlobalByElement(indexServerOnElement,
-                                                                client->clientRank,
-                                                                client->clientSize,
-                                                                axis_domain_order,
-                                                                getDistributedDimension());
-     computeIndexByElement(indexServerOnElement, globalIndexOnServer_);
-
-     const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer();
-     CDistributionClient::GlobalLocalDataMap::const_iterator iteGlobalLocalIndexMap = globalLocalIndexSendToServer.end(), itGlobalLocalIndexMap;
-     CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itbGlobalMap, itGlobalMap;
-     itGlobalMap  = itbGlobalMap = globalIndexOnServer_.begin();
-     iteGlobalMap = globalIndexOnServer_.end();
-
-     for (; itGlobalMap != iteGlobalMap; ++itGlobalMap)
-     {
-       int serverRank = itGlobalMap->first;
-       int indexSize = itGlobalMap->second.size();
-       const std::vector<size_t>& indexVec = itGlobalMap->second;
-       for (int idx = 0; idx < indexSize; ++idx)
+       computeClientIndexScalarGrid();
+       if (context->hasClient)
        {
-          itGlobalLocalIndexMap = globalLocalIndexSendToServer.find(indexVec[idx]);
-          if (iteGlobalLocalIndexMap != itGlobalLocalIndexMap)
-          {
-             if (connectedDataSize_.end() == connectedDataSize_.find(serverRank))
-               connectedDataSize_[serverRank] = 1;
-             else
-               ++connectedDataSize_[serverRank];
-          }
+         computeConnectedClientsScalarGrid();
        }
      }
-
-     for (itGlobalMap = itbGlobalMap; itGlobalMap != iteGlobalMap; ++itGlobalMap) {
-       connectedServerRank_.push_back(itGlobalMap->first);
+     else
+     {
+       computeClientIndex();
+       if (context->hasClient)
+       {
+         computeConnectedClients();
+       }
      }
-
-     nbSenders = clientServerMap_->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_);
+     if (CServer::serverLevel==2)
+     {
+       computeWrittenIndex() ;
+       if (serverDistribution_!=0) serverDistribution_->partialClear() ;
+       if (clientDistribution_!=0) clientDistribution_->partialClear() ;
+       outGlobalIndexFromClient.clear() ;
+     }
    }
 
@@ -577 +886 @@
       on each element whose size is much smaller than one of whole grid.
       \param [in] indexServerOnElement global index of each element and the rank of server associated with these index
+      \param [in] client contextClient
       \param [out] globalIndexOnServer global index of grid and its corresponding rank of server.
    */
    void CGrid::computeIndexByElement(const std::vector<boost::unordered_map<size_t,std::vector<int> > >& indexServerOnElement,
+                                     const CContextClient* client,
                                      CClientServerMapping::GlobalIndexMap& globalIndexOnServer)
    {
-     CContext* context = CContext::getCurrent();
-     CContextClient* client = context->client;
      int serverSize = client->serverSize;
+
      std::vector<CDomain*> domList = getDomains();
      std::vector<CAxis*> axisList = getAxis();
@@ -667 +977 @@
        }
 
-       nbIndexOnServer = 0;
-       for (it = itb; it != ite; ++it)
+     nbIndexOnServer = 0;
+     for (size_t j = 0; j < globalIndexElementOnServerMap.size(); ++j)
+     {
+       it = globalIndexElementOnServerMap.find(globalIndexElementOnClient(j));
+       if (it != ite)
        {
          const std::vector<int>& tmp = it->second;
@@ -683 +996 @@
        }
      }
+   }
 
     // Determine server which contain global source index
@@ -931 +1245 @@
    }
 
+/*
    void CGrid::outputField(int rank, const CArray<double, 1>& stored, double* field)
    {
@@ -960 +1275 @@
      }
    }
-
+*/
    //----------------------------------------------------------------
 
@@ -978 +1293 @@
    }
 
-  void CGrid::computeIndexScalarGrid()
+   void CGrid::uncompressField_arr(const double* const data, CArray<double, 1>& out) const
+   {
+      const std::vector<int>& localMaskedDataIndex = clientDistribution_->getLocalMaskedDataIndexOnClient();
+      const int size = localMaskedDataIndex.size();
+      
+      for(int i = 0; i < size; ++i) out(localMaskedDataIndex[i]) = data[i];
+   }
+
+
+  void CGrid::computeClientIndexScalarGrid()
+  {
+    CContext* context = CContext::getCurrent();    
+//    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1; // This should be changed soon
+//    for (int p = 0; p < nbSrvPools; ++p)
+    {
+//      CContextClient* client = (context->hasServer) ? (context->hasClient ? context->clientPrimServer[p] : context->client)
+//                                                    : context->client;
+      CContextClient* client = context->client;
+
+      int rank = client->clientRank;
+
+      clientDistribution_ = new CDistributionClient(rank, this);
+
+      storeIndex_client.resize(1);
+      storeIndex_client(0) = 0;      
+
+      if (0 != serverDistribution_)
+      {
+        map<int, CArray<size_t, 1> >::iterator itb = outGlobalIndexFromClient.begin(),
+                                               ite = outGlobalIndexFromClient.end(), it;
+        for (it = itb; it != ite; ++it)
+        {
+          int rank = it->first;
+          CArray<size_t,1>& globalIndex = outGlobalIndexFromClient[rank];
+          outLocalIndexStoreOnClient.insert(make_pair(rank, CArray<size_t,1>(globalIndex.numElements())));
+          CArray<size_t,1>& localIndex = outLocalIndexStoreOnClient[rank];
+          if (1 != globalIndex.numElements())
+            ERROR("void CGrid::computeClientIndexScalarGrid()",
+              << "Something wrong happened. "
+              << "Number of received global index on scalar grid should equal to 1" 
+              << "Number of received global index " << globalIndex.numElements() << ".");
+
+          localIndex(0) = globalIndex(0);
+        }
+      }
+    }
+  }
+
+  void CGrid::computeConnectedClientsScalarGrid()
+  {
+    CContext* context = CContext::getCurrent();    
+    int nbSrvPools = (context->clientPrimServer.size()==0) ? 1 : context->clientPrimServer.size();
+    connectedServerRank_.clear();
+    connectedDataSize_.clear();
+    nbSenders.clear();
+
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (context->clientPrimServer.size()==0) ? context->client : context->clientPrimServer[p];
+      int receiverSize = client->serverSize;
+
+//      connectedServerRank_[client].clear();
+
+      if (connectedServerRank_.find(receiverSize)==connectedServerRank_.end())
+      {
+        if (client->isServerLeader())
+        {
+          const std::list<int>& ranks = client->getRanksServerLeader();
+          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+          {
+            int rank = *itRank;
+            int nb = 1;
+            connectedServerRank_[receiverSize].push_back(rank);
+            connectedDataSize_[receiverSize][rank] = nb;
+            nbSenders[receiverSize][rank] = nb;
+          }
+        }
+        else
+        {
+          const std::list<int>& ranks = client->getRanksServerNotLeader();
+          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+          {
+            int rank = *itRank;
+            int nb = 1;
+            connectedServerRank_[receiverSize].push_back(rank);
+            connectedDataSize_[receiverSize][rank] = nb;
+            nbSenders[receiverSize][rank] = nb;
+          }
+        }
+      }
+      isDataDistributed_ = false;
+    }
+  }
+
+  void CGrid::sendIndexScalarGrid()
   {
     CContext* context = CContext::getCurrent();
-    CContextClient* client=context->client;
-
-    storeIndex_client.resize(1);
-    storeIndex_client(0) = 0;
-
-    connectedServerRank_.clear();
-
-    if (0 == client->clientRank)
-    {
-      for (int rank = 0; rank < client->serverSize; ++rank)
-      {
-        connectedServerRank_.push_back(rank);
-        connectedDataSize_[rank] = 1;
-        nbSenders[rank] = 1;
-      }
-    }
-    isDataDistributed_ = false;
-  }
-
-  void CGrid::computeCompressedIndex()
-  {
-    std::map<size_t, size_t> indexes;
-
-    {
-      std::map<int, CArray<size_t,1> >::const_iterator it = outIndexFromClient.begin();
-      std::map<int, CArray<size_t,1> >::const_iterator itEnd = outIndexFromClient.end();
-      for (; it != itEnd; ++it)
-      {
-        for (int i = 0; i < it->second.numElements(); ++i)
-          indexes.insert(std::make_pair(it->second(i), 0));
-
-        compressedOutIndexFromClient[it->first].resize(it->second.numElements());
-      }
-    }
-
-    {
-      std::map<size_t, size_t>::iterator it = indexes.begin();
-      std::map<size_t, size_t>::iterator itEnd = indexes.end();
-      for (size_t i = 0; it != itEnd; ++it, ++i)
-        it->second = i;
-    }
-
-    {
-      std::map<int, CArray<size_t,1> >::iterator it = compressedOutIndexFromClient.begin();
-      std::map<int, CArray<size_t,1> >::iterator itEnd = compressedOutIndexFromClient.end();
-      for (; it != itEnd; ++it)
-      {
-        const CArray<size_t,1>& outIndex = outIndexFromClient[it->first];
-        for (int i = 0; i < it->second.numElements(); ++i)
-          it->second(i) = indexes[outIndex(i)];
-      }
-    }
-  }
-
-  void CGrid::sendIndexScalarGrid()
-  {
-    CContext* context = CContext::getCurrent();
-    CContextClient* client = context->client;
-
-    CEventClient event(getType(), EVENT_ID_INDEX);
-    list<CMessage> listMsg;
-    list<CArray<size_t,1> > listOutIndex;
-
-    if (client->isServerLeader())
-    {
-      const std::list<int>& ranks = client->getRanksServerLeader();
-      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-      {
-        int rank = *itRank;
-        int nb = 1;
-        storeIndex_toSrv.insert(std::make_pair(rank, CArray<int,1>(nb)));        
-        listOutIndex.push_back(CArray<size_t,1>(nb));
-
-        CArray<int, 1>& outLocalIndexToServer = storeIndex_toSrv[rank];
-        CArray<size_t, 1>& outGlobalIndexOnServer = listOutIndex.back();
-
-        for (int k = 0; k < nb; ++k)
-        {
-          outGlobalIndexOnServer(k) = 0;
-          outLocalIndexToServer(k)  = 0;
-        }
-
-        storeIndex_fromSrv.insert(std::make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
-        listMsg.push_back(CMessage());
-        listMsg.back() << getId( )<< isDataDistributed_ << isCompressible_ << listOutIndex.back();
-
-        event.push(rank, 1, listMsg.back());
-      }
-      client->sendEvent(event);
-    }
-    else
-    {
-      const std::list<int>& ranks = client->getRanksServerNotLeader();
-      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-      {
-        int rank = *itRank;
-        int nb = 1;        
-        storeIndex_fromSrv.insert(std::make_pair(rank, CArray<int,1>(nb)));
-        CArray<int, 1>& outLocalIndexToServer = storeIndex_fromSrv[rank];
-        for (int k = 0; k < nb; ++k)
-        {          
-          outLocalIndexToServer(k)  = 0;
-        }
-      }
-      client->sendEvent(event);
-    }
-  }
-
-  void CGrid::sendIndex(void)
-  {
-    CContext* context = CContext::getCurrent();
-    CContextClient* client = context->client;
-
-    CEventClient event(getType(), EVENT_ID_INDEX);
-    int rank;
-    list<CMessage> listMsg;
-    list<CArray<size_t,1> > listOutIndex;
-    const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer();
-    CDistributionClient::GlobalLocalDataMap::const_iterator itIndex = globalLocalIndexSendToServer.begin(),
-                                                           iteIndex = globalLocalIndexSendToServer.end();
-
-    if (!doGridHaveDataDistributed())
-    {
+    storeIndex_toSrv.clear();
+    std::list<CContextClient*>::iterator it;
+
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      CContextClient* client = *it;
+      int receiverSize = client->serverSize;
+
+      CEventClient event(getType(), EVENT_ID_INDEX);
+      list<CMessage> listMsg;
+      list<CArray<size_t,1> > listOutIndex;
+
       if (client->isServerLeader())
       {
-        int indexSize = globalLocalIndexSendToServer.size();
-        CArray<size_t,1> outGlobalIndexOnServer(indexSize);
-        CArray<int,1> outLocalIndexToServer(indexSize);
-        for (int idx = 0; itIndex != iteIndex; ++itIndex, ++idx)
-        {
-          outGlobalIndexOnServer(idx) = itIndex->first;
-          outLocalIndexToServer(idx) = itIndex->second;
-        }
-
         const std::list<int>& ranks = client->getRanksServerLeader();
         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
         {
-          storeIndex_toSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
-          storeIndex_fromSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
-          listOutIndex.push_back(CArray<size_t,1>(outGlobalIndexOnServer));
-
+          int rank = *itRank;
+          int nb = 1;
+          storeIndex_toSrv[client].insert(std::make_pair(rank, CArray<int,1>(nb)));
+          listOutIndex.push_back(CArray<size_t,1>(nb));
+
+          CArray<int, 1>& outLocalIndexToServer = storeIndex_toSrv[client][rank];
+          CArray<size_t, 1>& outGlobalIndexOnServer = listOutIndex.back();
+
+          for (int k = 0; k < nb; ++k)
+          {
+            outGlobalIndexOnServer(k) = 0;
+            outLocalIndexToServer(k)  = 0;
+          }
+
+          if (context->hasClient && !context->hasServer)
+            storeIndex_fromSrv.insert(std::make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
+
+          listMsg.push_back(CMessage());
+          listMsg.back() << getId( )<< isDataDistributed_ << isCompressible_ << listOutIndex.back();
+
+          event.push(rank, 1, listMsg.back());
+        }
+        client->sendEvent(event);
+      }
+      else
+      {
+        const std::list<int>& ranks = client->getRanksServerNotLeader();
+        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        {
+          int rank = *itRank;
+          int nb = 1;          
+          CArray<int, 1> outLocalIndexToServer(nb);
+          for (int k = 0; k < nb; ++k)
+          {
+            outLocalIndexToServer(k)  = 0;
+          }
+
+          if (context->hasClient && !context->hasServer)
+            storeIndex_fromSrv.insert(std::make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
+        }
+        client->sendEvent(event);
+      }
+    }
+  }
+
+  void CGrid::sendIndex(void)
+  {
+    CContext* context = CContext::getCurrent();
+    storeIndex_toSrv.clear();
+    std::list<CContextClient*>::iterator it;
+
+    for (it=clients.begin(); it!=clients.end(); ++it)
+    {
+      CContextClient* client = *it;
+      int receiverSize = client->serverSize;
+
+      CEventClient event(getType(), EVENT_ID_INDEX);
+      int rank;
+      list<CMessage> listMsg;
+      list<CArray<size_t,1> > listOutIndex;
+      const CDistributionClient::GlobalLocalDataMap& globalLocalIndexSendToServer = clientDistribution_->getGlobalLocalDataSendToServer();
+      CDistributionClient::GlobalLocalDataMap::const_iterator itbIndex = globalLocalIndexSendToServer.begin(), itIndex,
+                                                              iteIndex = globalLocalIndexSendToServer.end();
+      itIndex = itbIndex;                                                              
+
+      if (!doGridHaveDataDistributed(client))
+      {
+        if (client->isServerLeader())
+        {
+          int indexSize = globalLocalIndexSendToServer.size();
+          CArray<size_t,1> outGlobalIndexOnServer(indexSize);
+          CArray<int,1> outLocalIndexToServer(indexSize);
+          for (int idx = 0; itIndex != iteIndex; ++itIndex, ++idx)
+          {
+            outGlobalIndexOnServer(idx) = itIndex->first;
+            outLocalIndexToServer(idx) = itIndex->second;
+          }
+
+          const std::list<int>& ranks = client->getRanksServerLeader();
+          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+          {
+            storeIndex_toSrv[client].insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
+            if (context->hasClient && !context->hasServer)
+              storeIndex_fromSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
+            
+            listOutIndex.push_back(CArray<size_t,1>(outGlobalIndexOnServer));
+
+            listMsg.push_back(CMessage());
+            listMsg.back() << getId() << isDataDistributed_ << isCompressible_ << listOutIndex.back();
+
+            event.push(*itRank, 1, listMsg.back());
+          }
+          client->sendEvent(event);
+        }
+        else
+        {
+           int indexSize = globalLocalIndexSendToServer.size();
+           CArray<int,1> outLocalIndexToServer(indexSize);
+           for (int idx = 0; itIndex != iteIndex; ++itIndex, ++idx)
+           {
+             outLocalIndexToServer(idx) = itIndex->second;
+           }
+
+           const std::list<int>& ranks = client->getRanksServerNotLeader();
+           for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           {
+             storeIndex_fromSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
+           }
+           client->sendEvent(event);
+         }
+      }
+      else
+      {
+        CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itGlobalMap;
+        itGlobalMap = globalIndexOnServer_[receiverSize].begin();
+        iteGlobalMap = globalIndexOnServer_[receiverSize].end();
+
+        std::map<int,std::vector<int> >localIndexTmp;
+        std::map<int,std::vector<size_t> > globalIndexTmp;
+        for (; itGlobalMap != iteGlobalMap; ++itGlobalMap)
+        {
+          int serverRank = itGlobalMap->first;
+          int indexSize = itGlobalMap->second.size();
+          const std::vector<size_t>& indexVec = itGlobalMap->second;
+          for (int idx = 0; idx < indexSize; ++idx)
+          {
+            itIndex = globalLocalIndexSendToServer.find(indexVec[idx]);
+            if (iteIndex != itIndex)
+            {
+              globalIndexTmp[serverRank].push_back(itIndex->first);
+              localIndexTmp[serverRank].push_back(itIndex->second);
+            }
+          }
+        }
+
+        for (int ns = 0; ns < connectedServerRank_[receiverSize].size(); ++ns)
+        {
+          rank = connectedServerRank_[receiverSize][ns];
+          int nb = 0;
+          if (globalIndexTmp.end() != globalIndexTmp.find(rank))
+            nb = globalIndexTmp[rank].size();
+
+          storeIndex_toSrv[client].insert(make_pair(rank, CArray<int,1>(nb)));
+          listOutIndex.push_back(CArray<size_t,1>(nb));
+
+          CArray<int, 1>& outLocalIndexToServer = storeIndex_toSrv[client][rank];
+          CArray<size_t, 1>& outGlobalIndexOnServer = listOutIndex.back();
+
+          for (int k = 0; k < nb; ++k)
+          {
+            outGlobalIndexOnServer(k) = globalIndexTmp[rank].at(k);
+            outLocalIndexToServer(k)  = localIndexTmp[rank].at(k);
+          }
+
+          storeIndex_fromSrv.insert(make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
           listMsg.push_back(CMessage());
           listMsg.back() << getId() << isDataDistributed_ << isCompressible_ << listOutIndex.back();
 
-          event.push(*itRank, 1, listMsg.back());
-        }
+          event.push(rank, nbSenders[receiverSize][rank], listMsg.back());
+        }
+
         client->sendEvent(event);
       }
-      else 
-      {
-        int indexSize = globalLocalIndexSendToServer.size();        
-        CArray<int,1> outLocalIndexToServer(indexSize);
-        for (int idx = 0; itIndex != iteIndex; ++itIndex, ++idx)
-        {          
-          outLocalIndexToServer(idx) = itIndex->second;
-        }
-        
-        const std::list<int>& ranks = client->getRanksServerNotLeader();
-        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-        {          
-          storeIndex_fromSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
-        }
-        client->sendEvent(event);
-      }
-    }
-    else
-    {
-      CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itGlobalMap;
-      itGlobalMap = globalIndexOnServer_.begin();
-      iteGlobalMap = globalIndexOnServer_.end();
-
-      std::map<int,std::vector<int> >localIndexTmp;
-      std::map<int,std::vector<size_t> > globalIndexTmp;
-      for (; itGlobalMap != iteGlobalMap; ++itGlobalMap)
-      {
-        int serverRank = itGlobalMap->first;
-        int indexSize = itGlobalMap->second.size();
-        const std::vector<size_t>& indexVec = itGlobalMap->second;
-        for (int idx = 0; idx < indexSize; ++idx)
-        {
-          itIndex = globalLocalIndexSendToServer.find(indexVec[idx]);
-          if (iteIndex != itIndex)
-          {
-            globalIndexTmp[serverRank].push_back(itIndex->first);
-            localIndexTmp[serverRank].push_back(itIndex->second);
-          }
-        }
-      }
-
-      for (int ns = 0; ns < connectedServerRank_.size(); ++ns)
-      {
-        rank = connectedServerRank_[ns];
-        int nb = 0;
-        if (globalIndexTmp.end() != globalIndexTmp.find(rank))
-          nb = globalIndexTmp[rank].size();
-
-        storeIndex_toSrv.insert(make_pair(rank, CArray<int,1>(nb)));        
-        listOutIndex.push_back(CArray<size_t,1>(nb));
-
-        CArray<int, 1>& outLocalIndexToServer = storeIndex_toSrv[rank];
-        CArray<size_t, 1>& outGlobalIndexOnServer = listOutIndex.back();
-
-        for (int k = 0; k < nb; ++k)
-        {
-          outGlobalIndexOnServer(k) = globalIndexTmp[rank].at(k);
-          outLocalIndexToServer(k)  = localIndexTmp[rank].at(k);
-        }
-
-        storeIndex_fromSrv.insert(make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
-        listMsg.push_back(CMessage());
-        listMsg.back() << getId() << isDataDistributed_ << isCompressible_ << listOutIndex.back();
-
-        event.push(rank, nbSenders[rank], listMsg.back());
-      }
-
-      client->sendEvent(event);
     }
   }
@@ -1220 +1593 @@
   {
     CContext* context = CContext::getCurrent();
-    CContextServer* server = context->server;
-    numberWrittenIndexes_ = totalNumberWrittenIndexes_ = offsetWrittenIndexes_ = 0;
-    connectedServerRank_ = ranks;
-
-    for (int n = 0; n < ranks.size(); n++)
-    {
-      int rank = ranks[n];
-      CBufferIn& buffer = *buffers[n];
-
-      buffer >> isDataDistributed_ >> isCompressible_;
-      size_t dataSize = 0;
-
-      if (0 == serverDistribution_)
-      {
-        int idx = 0, numElement = axis_domain_order.numElements();
-        int ssize = numElement;
-        std::vector<int> indexMap(numElement);
-        for (int i = 0; i < numElement; ++i)
-        {
-          indexMap[i] = idx;
-          if (2 == axis_domain_order(i))
+    connectedServerRankRead_ = ranks;
+
+    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
+    nbSrvPools = 1;    
+    nbReadSenders.clear();
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextServer* server = (!context->hasClient) ? context->server : context->serverPrimServer[p];
+      CContextClient* client = context->client;   //(!context->hasClient) ? context->client : context->clientPrimServer[p];
+      
+      int idx = 0, numElement = axis_domain_order.numElements();
+      int ssize = numElement;
+      std::vector<int> indexMap(numElement);
+      for (int i = 0; i < numElement; ++i)
+      {
+        indexMap[i] = idx;
+        if (2 == axis_domain_order(i))
+        {
+          ++ssize;
+          idx += 2;
+        }
+        else
+          ++idx;
+      }
+
+      for (int n = 0; n < ranks.size(); n++)
+      {
+        int rank = ranks[n];
+        CBufferIn& buffer = *buffers[n];
+
+        buffer >> isDataDistributed_ >> isCompressible_;
+        size_t dataSize = 0;
+
+        if (0 == serverDistribution_)
+        {
+          int axisId = 0, domainId = 0, scalarId = 0, globalSize = 1;
+          std::vector<CDomain*> domainList = getDomains();
+          std::vector<CAxis*> axisList = getAxis();
+          std::vector<int> nZoomBegin(ssize), nZoomSize(ssize), nGlob(ssize), nZoomBeginGlobal(ssize), nGlobElement(numElement);
+          std::vector<CArray<int,1> > globalZoomIndex(numElement);
+          for (int i = 0; i < numElement; ++i)
           {
-            ++ssize;
-            idx += 2;
+            nGlobElement[i] = globalSize;
+            if (2 == axis_domain_order(i)) //domain
+            {
+              nZoomBegin[indexMap[i]] = domainList[domainId]->zoom_ibegin;
+              nZoomSize[indexMap[i]]  = domainList[domainId]->zoom_ni;
+              nZoomBeginGlobal[indexMap[i]] = domainList[domainId]->global_zoom_ibegin;              
+              nGlob[indexMap[i]] = domainList[domainId]->ni_glo;
+
+              nZoomBegin[indexMap[i] + 1] = domainList[domainId]->zoom_jbegin;
+              nZoomSize[indexMap[i] + 1] = domainList[domainId]->zoom_nj;
+              nZoomBeginGlobal[indexMap[i] + 1] = domainList[domainId]->global_zoom_jbegin;              
+              nGlob[indexMap[i] + 1] = domainList[domainId]->nj_glo;
+
+              {
+                int count = 0;
+                globalZoomIndex[i].resize(nZoomSize[indexMap[i]]*nZoomSize[indexMap[i]+1]);
+                for (int jdx = 0; jdx < nZoomSize[indexMap[i]+1]; ++jdx)
+                  for (int idx = 0; idx < nZoomSize[indexMap[i]]; ++idx)                
+                  {
+                    globalZoomIndex[i](count) = (nZoomBegin[indexMap[i]] + idx) + (nZoomBegin[indexMap[i]+1] + jdx) * nGlob[indexMap[i]];
+                    ++count;
+                  }
+              }
+
+              ++domainId;
+            }
+            else if (1 == axis_domain_order(i)) // axis
+            {
+              nZoomBegin[indexMap[i]] = axisList[axisId]->zoom_begin;
+              nZoomSize[indexMap[i]]  = axisList[axisId]->zoom_n;
+              nZoomBeginGlobal[indexMap[i]] = axisList[axisId]->global_zoom_begin;              
+              nGlob[indexMap[i]] = axisList[axisId]->n_glo;     
+              if (axisList[axisId]->zoomByIndex())
+              {
+                globalZoomIndex[i].reference(axisList[axisId]->zoom_index);                
+              }
+              else
+              {
+                globalZoomIndex[i].resize(nZoomSize[indexMap[i]]);
+                for (int idx = 0; idx < nZoomSize[indexMap[i]]; ++idx)
+                  globalZoomIndex[i](idx) = nZoomBegin[indexMap[i]] + idx;
+              }
+
+              ++axisId;
+            }
+            else // scalar
+            { 
+              nZoomBegin[indexMap[i]] = 0;
+              nZoomSize[indexMap[i]]  = 1;
+              nZoomBeginGlobal[indexMap[i]] = 0;              
+              nGlob[indexMap[i]] = 1;
+              globalZoomIndex[i].resize(1);
+              globalZoomIndex[i](0) = 0;
+              ++scalarId;
+            }
           }
-          else
-            ++idx;
-        }
-
-        int axisId = 0, domainId = 0, scalarId = 0;
+          dataSize = 1;
+
+          for (int i = 0; i < nZoomSize.size(); ++i)
+            dataSize *= nZoomSize[i];
+          serverDistribution_ = new CDistributionServer(server->intraCommRank, 
+                                                        globalZoomIndex, axis_domain_order,
+                                                        nZoomBegin, nZoomSize, nZoomBeginGlobal, nGlob);
+        }
+
+        CArray<size_t,1> outIndex;
+        buffer >> outIndex;
+        outGlobalIndexFromClient.insert(std::make_pair(rank, outIndex));
+        connectedDataSizeRead_[rank] = outIndex.numElements();
+
+        if (doGridHaveDataDistributed(client))
+        {}
+        else
+        {
+          // THE PROBLEM HERE IS THAT DATA CAN BE NONDISTRIBUTED ON CLIENT AND DISTRIBUTED ON SERVER
+          // BELOW IS THE TEMPORARY FIX only for a single type of element (domain, asix, scalar)
+          dataSize = serverDistribution_->getGridSize();
+        }
+        writtenDataSize_ += dataSize;
+      }
+
+
+      // Compute mask of the current grid
+      {
+        int axisId = 0, domainId = 0, scalarId = 0, globalSize = 1;
         std::vector<CDomain*> domainList = getDomains();
         std::vector<CAxis*> axisList = getAxis();
-        std::vector<int> nZoomBegin(ssize), nZoomSize(ssize), nGlob(ssize), nZoomBeginGlobal(ssize);
-        std::vector<CArray<int,1> > globalZoomIndex(numElement);
+        int dimSize = 2 * domainList.size() + axisList.size();
+        std::vector<int> nBegin(dimSize), nSize(dimSize), nGlob(dimSize), nBeginGlobal(dimSize);        
         for (int i = 0; i < numElement; ++i)
-        {
+        {          
           if (2 == axis_domain_order(i)) //domain
           {
-            nZoomBegin[indexMap[i]] = domainList[domainId]->zoom_ibegin_srv;
-            nZoomSize[indexMap[i]]  = domainList[domainId]->zoom_ni_srv;
-            nZoomBeginGlobal[indexMap[i]] = domainList[domainId]->global_zoom_ibegin;
+            nBegin[indexMap[i]] = domainList[domainId]->ibegin;
+            nSize[indexMap[i]]  = domainList[domainId]->ni;
+            nBeginGlobal[indexMap[i]] = 0;              
             nGlob[indexMap[i]] = domainList[domainId]->ni_glo;
 
-            nZoomBegin[indexMap[i] + 1] = domainList[domainId]->zoom_jbegin_srv;
-            nZoomSize[indexMap[i] + 1] = domainList[domainId]->zoom_nj_srv;
-            nZoomBeginGlobal[indexMap[i] + 1] = domainList[domainId]->global_zoom_jbegin;
+            nBegin[indexMap[i] + 1] = domainList[domainId]->jbegin;
+            nSize[indexMap[i] + 1] = domainList[domainId]->nj;
+            nBeginGlobal[indexMap[i] + 1] = 0;              
             nGlob[indexMap[i] + 1] = domainList[domainId]->nj_glo;
 
-            {
-              int count = 0;
-              globalZoomIndex[i].resize(nZoomSize[indexMap[i]]*nZoomSize[indexMap[i]+1]);
-              for (int jdx = 0; jdx < nZoomSize[indexMap[i]+1]; ++jdx)
-                for (int idx = 0; idx < nZoomSize[indexMap[i]]; ++idx)                
-                {
-                  globalZoomIndex[i](count) = (nZoomBegin[indexMap[i]] + idx) + (nZoomBegin[indexMap[i]+1] + jdx) * nGlob[indexMap[i]];
-                  ++count;
-                }
-            }
             ++domainId;
           }
           else if (1 == axis_domain_order(i)) // axis
           {
-            nZoomBegin[indexMap[i]] = axisList[axisId]->zoom_begin_srv;
-            nZoomSize[indexMap[i]]  = axisList[axisId]->zoom_size_srv;
-            nZoomBeginGlobal[indexMap[i]] = axisList[axisId]->global_zoom_begin_srv;
-            nGlob[indexMap[i]] = axisList[axisId]->n_glo;
-            if (!axisList[axisId]->global_zoom_index.isEmpty())
-            {
-              globalZoomIndex[i].reference(axisList[axisId]->zoom_index_srv);                
-            }
-            else
-            {
-              globalZoomIndex[i].resize(nZoomSize[indexMap[i]]);
-              for (int idx = 0; idx < nZoomSize[indexMap[i]]; ++idx)
-                globalZoomIndex[i](idx) = nZoomBegin[indexMap[i]] + idx;
-            }
-            
+            nBegin[indexMap[i]] = axisList[axisId]->begin;
+            nSize[indexMap[i]]  = axisList[axisId]->n;
+            nBeginGlobal[indexMap[i]] = 0;              
+            nGlob[indexMap[i]] = axisList[axisId]->n_glo;              
             ++axisId;
           }
           else // scalar
+          {  
+          }
+        }
+        
+        if (nSize.empty()) // Scalar grid
+        {
+          nBegin.push_back(0);
+          nSize.push_back(1);
+          nBeginGlobal.push_back(0);              
+          nGlob.push_back(1);  
+        }
+
+        modifyMaskSize(nSize, false);
+
+        // These below codes are reserved for future
+        CDistributionServer srvDist(server->intraCommRank, nBegin, nSize, nBeginGlobal, nGlob); 
+        map<int, CArray<size_t, 1> >::iterator itb = outGlobalIndexFromClient.begin(),
+                                               ite = outGlobalIndexFromClient.end(), it;  
+        const CDistributionServer::GlobalLocalMap&  globalLocalMask = srvDist.getGlobalLocalIndex();
+        CDistributionServer::GlobalLocalMap::const_iterator itSrv;
+        size_t nb = 0;
+        for (it = itb; it != ite; ++it)
+        {
+          CArray<size_t,1>& globalInd = it->second;
+          for (size_t idx = 0; idx < globalInd.numElements(); ++idx)
           {
-            nZoomBegin[indexMap[i]] = 0;
-            nZoomSize[indexMap[i]]  = 1;
-            nZoomBeginGlobal[indexMap[i]] = 0;
-            nGlob[indexMap[i]] = 1;
-            globalZoomIndex[i].resize(1);
-            globalZoomIndex[i](0) = 0;
-            ++scalarId;
+            if (globalLocalMask.end() != globalLocalMask.find(globalInd(idx))) ++nb;
           }
         }
-        dataSize = 1;
-        for (int i = 0; i < nZoomSize.size(); ++i)
-          dataSize *= nZoomSize[i];
-
-/*        serverDistribution_ = new CDistributionServer(server->intraCommRank, nZoomBegin, nZoomSize,
-                                                      nZoomBeginGlobal, nGlob);*/
-        serverDistribution_ = new CDistributionServer(server->intraCommRank, 
-                                                      globalZoomIndex, axis_domain_order,
-                                                      nZoomBegin, nZoomSize, nZoomBeginGlobal, nGlob);
-      }
-
-      CArray<size_t,1> outIndex;
-      buffer >> outIndex;
-      if (isDataDistributed_)
-        serverDistribution_->computeLocalIndex(outIndex);
-      else
-      {
-        dataSize = outIndex.numElements();
-        for (int i = 0; i < outIndex.numElements(); ++i) outIndex(i) = i;
-      }
-      writtenDataSize_ += dataSize;
-      
-      outIndexFromClient.insert(std::make_pair(rank, outIndex));
-      connectedDataSize_[rank] = outIndex.numElements();
-      numberWrittenIndexes_ += outIndex.numElements();
-    }
-
-    // if (isScalarGrid()) return;
-
-    if (isDataDistributed_)
-    {
-      MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
-      MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
-      offsetWrittenIndexes_ -= numberWrittenIndexes_;
-    }
-    else
-      totalNumberWrittenIndexes_ = numberWrittenIndexes_;
-
-    nbSenders = CClientServerMappingDistributed::computeConnectedClients(context->client->serverSize, context->client->clientSize, context->client->intraComm, ranks);
+        
+        CArray<int,1> indexToModify(nb);
+        nb = 0;    
+        for (it = itb; it != ite; ++it)
+        {
+          CArray<size_t,1>& globalInd = it->second;
+          for (size_t idx = 0; idx < globalInd.numElements(); ++idx)
+          {
+            itSrv = globalLocalMask.find(globalInd(idx));
+            if (globalLocalMask.end() != itSrv) 
+            {
+              indexToModify(nb) = itSrv->second;
+              ++nb;
+            }
+          }
+        }
+
+        modifyMask(indexToModify, true);
+      }
+
+      if (isScalarGrid()) return;
+
+      nbReadSenders[client] = CClientServerMappingDistributed::computeConnectedClients(context->client->serverSize, context->client->clientSize, context->client->intraComm, ranks);
+    }
   }
 
@@ -1366 +1819 @@
                                         const CArray<int,1>& axisDomainOrder)
   {
-    globalDim.resize(domains.size()*2+axis.size()+scalars.size());
+ //   globalDim.resize(domains.size()*2+axis.size()+scalars.size());
+    globalDim.resize(domains.size()*2+axis.size());
     int positionDimensionDistributed = 1;
     int idx = 0, idxDomain = 0, idxAxis = 0, idxScalar = 0;
@@ -1396 +1850 @@
       else
       {
-        globalDim[idx] = 1;
+//        globalDim[idx] = 1;
         ++idxScalar;
-        ++idx;
+//        ++idx;
       }
     }
@@ -1484 +1938 @@
   }
 
-  bool CGrid::doGridHaveDataDistributed()
+  bool CGrid::doGridHaveDataDistributed(CContextClient* client)
   {
     if (isScalarGrid()) return false;
+    else if (0 != client)
+    {
+      return  (isDataDistributed_ ||  (1 != client->clientSize) || (1 != client->serverSize));
+    }
     else
-      return isDataDistributed_;
+      return isDataDistributed_;    
   }
 
@@ -1583 +2041 @@
    void CGrid::sendAddDomain(const string& id)
    {
-    CContext* context=CContext::getCurrent();
-
-    if (! context->hasServer )
-    {
-       CContextClient* client=context->client;
-
-       CEventClient event(this->getType(),EVENT_ID_ADD_DOMAIN);
-       if (client->isServerLeader())
-       {
-         CMessage msg;
-         msg<<this->getId();
-         msg<<id;
-         const std::list<int>& ranks = client->getRanksServerLeader();
-         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-           event.push(*itRank,1,msg);
-         client->sendEvent(event);
-       }
-       else client->sendEvent(event);
-    }
+      sendAddItem(id, (int)EVENT_ID_ADD_DOMAIN);
    }
 
@@ -1610 +2050 @@
    void CGrid::sendAddAxis(const string& id)
    {
-    CContext* context=CContext::getCurrent();
-
-    if (! context->hasServer )
-    {
-       CContextClient* client=context->client;
-
-       CEventClient event(this->getType(),EVENT_ID_ADD_AXIS);
-       if (client->isServerLeader())
-       {
-         CMessage msg;
-         msg<<this->getId();
-         msg<<id;
-         const std::list<int>& ranks = client->getRanksServerLeader();
-         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-           event.push(*itRank,1,msg);
-         client->sendEvent(event);
-       }
-       else client->sendEvent(event);
-    }
+      sendAddItem(id, (int)EVENT_ID_ADD_AXIS);
    }
 
@@ -1637 +2059 @@
    void CGrid::sendAddScalar(const string& id)
    {
-    CContext* context=CContext::getCurrent();
-
-    if (! context->hasServer )
-    {
-       CContextClient* client=context->client;
-
-       CEventClient event(this->getType(),EVENT_ID_ADD_SCALAR);
-       if (client->isServerLeader())
-       {
-         CMessage msg;
-         msg<<this->getId();
-         msg<<id;
-         const std::list<int>& ranks = client->getRanksServerLeader();
-         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-           event.push(*itRank,1,msg);
-         client->sendEvent(event);
-       }
-       else client->sendEvent(event);
-    }
+      sendAddItem(id, (int)EVENT_ID_ADD_SCALAR);
    }
 
@@ -1746 +2150 @@
     {
       CDomain* pDom = CDomain::get(*it);
-      if (context->hasClient)
+      if (context->hasClient && !context->hasServer)      
       {
         pDom->solveRefInheritance(apply);
@@ -1758 +2162 @@
     {
       CAxis* pAxis = CAxis::get(*it);
-      if (context->hasClient)
+      if (context->hasClient && !context->hasServer)
       {
         pAxis->solveRefInheritance(apply);
@@ -1770 +2174 @@
     {
       CScalar* pScalar = CScalar::get(*it);
-      if (context->hasClient)
+      if (context->hasClient && !context->hasServer)
       {
         pScalar->solveRefInheritance(apply);
@@ -2145 +2549 @@
   }
 
+  void CGrid::setContextClient(CContextClient* contextClient)
+  {
+    if (clientsSet.find(contextClient)==clientsSet.end())
+    {
+      clients.push_back(contextClient) ;
+      clientsSet.insert(contextClient);
+    }
+    for (int i=0; i<this->getDomains().size(); i++)
+        this->getDomains()[i]->setContextClient(contextClient);
+    for (int i=0; i<this->getAxis().size(); i++)
+        this->getAxis()[i]->setContextClient(contextClient);
+  }
+
   /*!
     Parse a grid, for now, it contains only domain, axis and scalar