Changeset 1232

Timestamp:

07/31/17 17:59:25 (7 years ago)

Author:

mhnguyen

Message:

Fixing the blocking problem where there are more servers than the number of grid band distribution

+) Correct this problem not only for writing but also for reading
+) Allow "zero-size" domain, axis (i.e: domain, axis with ni = 0, and/or nj=0)

Test
+) On Curie
+) Work in both cases: Read and Write data

Location:

XIOS/dev/XIOS_DEV_CMIP6/src

Files:

• context_client.cpp (modified) (4 diffs)
• context_client.hpp (modified) (1 diff)
• node/context.cpp (modified) (19 diffs)
• node/context.hpp (modified) (3 diffs)
• node/domain.cpp (modified) (4 diffs)
• node/field.cpp (modified) (12 diffs)
• node/field.hpp (modified) (3 diffs)
• node/file.cpp (modified) (12 diffs)
• node/file.hpp (modified) (5 diffs)
• node/grid.cpp (modified) (5 diffs)
• server_distribution_description.cpp (modified) (9 diffs)
• server_distribution_description.hpp (modified) (2 diffs)

XIOS/dev/XIOS_DEV_CMIP6/src/context_client.cpp

@@ -35 +35 @@
       else  MPI_Comm_size(interComm, &serverSize);
 
+      computeLeader(clientRank, clientSize, serverSize, ranksServerLeader, ranksServerNotLeader);
+
+      timeLine = 0;
+    }
+
+    void CContextClient::computeLeader(int clientRank, int clientSize, int serverSize,
+                                       std::list<int>& rankRecvLeader,
+                                       std::list<int>& rankRecvNotLeader)
+    {
+      if ((0 == clientSize) || (0 == serverSize)) return;
+
       if (clientSize < serverSize)
       {
@@ -50 +61 @@
 
         for (int i = 0; i < serverByClient; i++)
-          ranksServerLeader.push_back(rankStart + i);
-
-        ranksServerNotLeader.resize(0);
+          rankRecvLeader.push_back(rankStart + i);
+
+        rankRecvNotLeader.resize(0);
       }
       else
@@ -62 +73 @@
         {
           if (clientRank % (clientByServer + 1) == 0)
-            ranksServerLeader.push_back(clientRank / (clientByServer + 1));
+            rankRecvLeader.push_back(clientRank / (clientByServer + 1));
           else
-            ranksServerNotLeader.push_back(clientRank / (clientByServer + 1));
+            rankRecvNotLeader.push_back(clientRank / (clientByServer + 1));
         }
         else
@@ -70 +81 @@
           int rank = clientRank - (clientByServer + 1) * remain;
           if (rank % clientByServer == 0)
-            ranksServerLeader.push_back(remain + rank / clientByServer);
+            rankRecvLeader.push_back(remain + rank / clientByServer);
           else
-            ranksServerNotLeader.push_back(remain + rank / clientByServer);
-        }
-      }
-
-      timeLine = 0;
+            rankRecvNotLeader.push_back(remain + rank / clientByServer);
+        }
+      }
     }
 

XIOS/dev/XIOS_DEV_CMIP6/src/context_client.hpp

@@ -49 +49 @@
       bool isAttachedModeEnabled() const;
       bool hasTemporarilyBufferedEvent() const { return !tmpBufferedEvent.isEmpty(); };
+
+      static void computeLeader(int clientRank, int clientSize, int serverSize,
+                                std::list<int>& rankRecvLeader,
+                                std::list<int>& rankRecvNotLeader);
 
       // Close and finalize context client

XIOS/dev/XIOS_DEV_CMIP6/src/node/context.cpp

@@ -477 +477 @@
          finalized = true;
 
+         closeAllFile(); // Just move to here to make sure that server-level 1 can close files
          if (hasServer && !hasClient)
-         {
-           closeAllFile();
+         {           
            registryOut->hierarchicalGatherRegistry() ;
            if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
@@ -525 +525 @@
 
      // Check grid and calculate its distribution
-     checkGridEnabledFields();    
-
+     checkGridEnabledFields();
+ 
      // Distribute files between secondary servers according to the data size
      distributeFiles();
@@ -548 +548 @@
       // We have enough information to send to server
       // First of all, send all enabled files
-      sendEnabledFiles();
-
-      // Then, send all enabled fields
-      sendEnabledFields();
+      sendEnabledFiles(this->enabledWriteModeFiles);
+      // We only use server-level 1 (for now) to read data
+      if (!hasServer)
+        sendEnabledFiles(this->enabledReadModeFiles);
+
+      // Then, send all enabled fields      
+      sendEnabledFieldsInFiles(this->enabledWriteModeFiles);
+      if (!hasServer)
+        sendEnabledFieldsInFiles(this->enabledReadModeFiles);
 
       // At last, we have all info of domain and axis, then send them
-       sendRefDomainsAxis();
+       sendRefDomainsAxisScalars(this->enabledWriteModeFiles);
+      if (!hasServer)
+        sendRefDomainsAxisScalars(this->enabledReadModeFiles);        
 
        // After that, send all grid (if any)
-       sendRefGrid();
+      sendRefGrid(this->enabledWriteModeFiles);
+      if (!hasServer)
+        sendRefGrid(this->enabledReadModeFiles);
 
        // We have a xml tree on the server side and now, it should be also processed
        sendPostProcessing();
-
-       sendGridEnabledFields();       
+       
+       sendGridEnabledFieldsInFiles(this->enabledWriteModeFiles);       
+       if (!hasServer)
+        sendGridEnabledFieldsInFiles(this->enabledReadModeFiles);       
      }
      allProcessed = true;
@@ -630 +641 @@
     }
     
-    checkGridEnabledFields();    
+    checkGridEnabledFields();   
 
     if (hasClient) this->sendProcessingGridOfEnabledFields();
@@ -646 +657 @@
    }
 
-   void CContext::findAllEnabledFields(void)
-   {
-     for (unsigned int i = 0; i < this->enabledFiles.size(); i++)
-     (void)this->enabledFiles[i]->getEnabledFields();
-   }
-
-   void CContext::findAllEnabledFieldsInReadModeFiles(void)
-   {
-     for (unsigned int i = 0; i < this->enabledReadModeFiles.size(); ++i)
-     (void)this->enabledReadModeFiles[i]->getEnabledFields();
+   void CContext::findAllEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     for (unsigned int i = 0; i < activeFiles.size(); i++)
+     (void)activeFiles[i]->getEnabledFields();
    }
 
@@ -664 +669 @@
    }
 
-   void CContext::sendGridEnabledFields()
+   /*!
+      Send active (enabled) fields in file from a client to others
+      \param [in] activeFiles files contains enabled fields to send
+   */
+   void CContext::sendGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
+     for (int i = 0; i < size; ++i)
+     {       
+       activeFiles[i]->sendGridOfEnabledFields();
+     }
+   }
+
+   void CContext::checkGridEnabledFields()
+   {
+     int size = enabledFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       enabledFiles[i]->checkGridOfEnabledFields();       
+     }
+   }
+
+   /*!
+      Check grid of active (enabled) fields in file 
+      \param [in] activeFiles files contains enabled fields whose grid needs checking
+   */
+   void CContext::checkGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       activeFiles[i]->checkGridOfEnabledFields();       
+     }
+   }
+
+    /*!
+      Go up the hierachical tree via field_ref and do check of attributes of fields
+      This can be done in a client then all computed information will be sent from this client to others
+      \param [in] sendToServer Flag to indicate whether calculated information will be sent
+   */
+   void CContext::solveOnlyRefOfEnabledFields(bool sendToServer)
    {
      int size = this->enabledFiles.size();
      for (int i = 0; i < size; ++i)
-     {       
-       this->enabledFiles[i]->sendGridOfEnabledFields();
-     }
-   }
-
-   void CContext::checkGridEnabledFields()
-   {
-     int size = this->enabledFiles.size();
+     {
+       this->enabledFiles[i]->solveOnlyRefOfEnabledFields(sendToServer);
+     }
+
      for (int i = 0; i < size; ++i)
      {
-       this->enabledFiles[i]->checkGridOfEnabledFields();       
-     }
-   }
-
-   void CContext::solveOnlyRefOfEnabledFields(bool sendToServer)
-   {
-     int size = this->enabledFiles.size();
-     for (int i = 0; i < size; ++i)
-     {
-       this->enabledFiles[i]->solveOnlyRefOfEnabledFields(sendToServer);
-     }
-
-     for (int i = 0; i < size; ++i)
-     {
        this->enabledFiles[i]->generateNewTransformationGridDest();
      }
    }
 
+    /*!
+      Go up the hierachical tree via field_ref and do check of attributes of fields.
+      The transformation can be done in this step.
+      All computed information will be sent from this client to others.
+      \param [in] sendToServer Flag to indicate whether calculated information will be sent
+   */
    void CContext::solveAllRefOfEnabledFieldsAndTransform(bool sendToServer)
    {
@@ -831 +864 @@
 
        // (1) Find all enabled files in write mode
-       for (int i = 0; i < this->enabledFiles.size(); ++i)
-       {
-         if (enabledFiles[i]->mode.isEmpty() || (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
-          enabledWriteModeFiles.push_back(enabledFiles[i]);
-       }
+       // for (int i = 0; i < this->enabledFiles.size(); ++i)
+       // {
+       //   if (enabledFiles[i]->mode.isEmpty() || (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
+       //    enabledWriteModeFiles.push_back(enabledFiles[i]);
+       // }
 
        // (2) Estimate the data volume for each file
@@ -882 +915 @@
  
        for (int i = 0; i < this->enabledReadModeFiles.size(); ++i)
-         enabledReadModeFiles[i]->setContextClient(client);
+       {
+         enabledReadModeFiles[i]->setContextClient(client);          
+       }
      }
      else
@@ -891 +926 @@
    }
 
+   /*!
+      Find all files in write mode
+   */
+   void CContext::findEnabledWriteModeFiles(void)
+   {
+     int size = this->enabledFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       if (enabledFiles[i]->mode.isEmpty() || 
+          (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
+        enabledWriteModeFiles.push_back(enabledFiles[i]);
+     }
+   }
+
+   /*!
+      Find all files in read mode
+   */
    void CContext::findEnabledReadModeFiles(void)
    {
@@ -1217 +1269 @@
 
       //Initialisation du vecteur 'enabledFiles' contenant la liste des fichiers à sortir.
-      this->findEnabledFiles();
-      
+      findEnabledFiles();
+      findEnabledWriteModeFiles();
+      findEnabledReadModeFiles();
+
       // For now, only read files with client and only one level server
-      if (hasClient && !hasServer) this->findEnabledReadModeFiles();
-
-      // Find all enabled fields of each file
-      this->findAllEnabledFields();
+      // if (hasClient && !hasServer) findEnabledReadModeFiles();      
+
+      // Find all enabled fields of each file      
+      findAllEnabledFieldsInFiles(this->enabledWriteModeFiles);
+      findAllEnabledFieldsInFiles(this->enabledReadModeFiles);
+
       // For now, only read files with client and only one level server
-      if (hasClient && !hasServer) this->findAllEnabledFieldsInReadModeFiles();
-
+      // if (hasClient && !hasServer) 
+      //   findAllEnabledFieldsInFiles(this->enabledReadModeFiles);      
 
       if (hasClient && !hasServer)
       {
-       // Try to read attributes of fields in file then fill in corresponding grid (or domain, axis)
-       this->readAttributesOfEnabledFieldsInReadModeFiles();
+        initReadFiles();
+        // Try to read attributes of fields in file then fill in corresponding grid (or domain, axis)
+        this->readAttributesOfEnabledFieldsInReadModeFiles();
       }
 
@@ -1259 +1316 @@
      if (hasClient)
      {
-       size_t numEnabledFiles = this->enabledFiles.size();
+       size_t numEnabledFiles = this->enabledWriteModeFiles.size();
        for (size_t i = 0; i < numEnabledFiles; ++i)
        {
-         CFile* file = this->enabledFiles[i];
+         CFile* file = this->enabledWriteModeFiles[i];
 //         if (file->getContextClient() == contextClient)
          {
@@ -1280 +1337 @@
                if (maxEventSize[it->first] < it->second)
                  maxEventSize[it->first] = it->second;
+             }
+           }
+         }
+       }
+
+      // Not a good approach here, duplicate code
+       if (!hasServer)
+       {
+         size_t numEnabledFiles = this->enabledReadModeFiles.size();
+         for (size_t i = 0; i < numEnabledFiles; ++i)
+         {
+           CFile* file = this->enabledReadModeFiles[i];  
+           {
+             std::vector<CField*> enabledFields = file->getEnabledFields();
+             size_t numEnabledFields = enabledFields.size();
+             for (size_t j = 0; j < numEnabledFields; ++j)
+             {
+               const std::map<int, StdSize> mapSize = enabledFields[j]->getGridAttributesBufferSize();
+               std::map<int, StdSize>::const_iterator it = mapSize.begin(), itE = mapSize.end();
+               for (; it != itE; ++it)
+               {
+                 // If attributesSize[it->first] does not exist, it will be zero-initialized
+                 // so we can use it safely without checking for its existance
+                 if (attributesSize[it->first] < it->second)
+                   attributesSize[it->first] = it->second;
+
+                 if (maxEventSize[it->first] < it->second)
+                   maxEventSize[it->first] = it->second;
+               }
              }
            }
@@ -1341 +1427 @@
 
    //! Client side: Send infomation of active files (files are enabled to write out)
-   void CContext::sendEnabledFiles()
-   {
-     int size = this->enabledFiles.size();
+   void CContext::sendEnabledFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
 
      // In a context, each type has a root definition, e.g: axis, domain, field.
@@ -1353 +1439 @@
      for (int i = 0; i < size; ++i)
      {
-       cfgrpPtr->sendCreateChild(this->enabledFiles[i]->getId(),enabledFiles[i]->getContextClient());
-       this->enabledFiles[i]->sendAllAttributesToServer(enabledFiles[i]->getContextClient());
-       this->enabledFiles[i]->sendAddAllVariables(enabledFiles[i]->getContextClient());
+       CFile* f = activeFiles[i];
+       cfgrpPtr->sendCreateChild(f->getId(),f->getContextClient());
+       f->sendAllAttributesToServer(f->getContextClient());
+       f->sendAddAllVariables(f->getContextClient());
      }
    }
 
    //! Client side: Send information of active fields (ones are written onto files)
-   void CContext::sendEnabledFields()
-   {
-     int size = this->enabledFiles.size();
+   void CContext::sendEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
      for (int i = 0; i < size; ++i)
      {
-       this->enabledFiles[i]->sendEnabledFields(enabledFiles[i]->getContextClient());
+       activeFiles[i]->sendEnabledFields(activeFiles[i]->getContextClient());
      }
    }
@@ -1488 +1575 @@
 
    //! Client side: Send information of reference grid of active fields
-   void CContext::sendRefGrid()
+   void CContext::sendRefGrid(const std::vector<CFile*>& activeFiles)
    {
      std::set<StdString> gridIds;
-     int sizeFile = this->enabledFiles.size();
+     int sizeFile = activeFiles.size();
      CFile* filePtr(NULL);
 
@@ -1497 +1584 @@
      for (int i = 0; i < sizeFile; ++i)
      {
-       filePtr = this->enabledFiles[i];
+       filePtr = activeFiles[i];
        std::vector<CField*> enabledFields = filePtr->getEnabledFields();
        int sizeField = enabledFields.size();
@@ -1521 +1608 @@
    }
 
-
-   //! Client side: Send information of reference domain and axis of active fields
-   void CContext::sendRefDomainsAxis()
+   //! Client side: Send information of reference domain, axis and scalar of active fields
+   void CContext::sendRefDomainsAxisScalars(const std::vector<CFile*>& activeFiles)
    {
      std::set<StdString> domainIds, axisIds, scalarIds;
 
      // Find all reference domain and axis of all active fields
-     int numEnabledFiles = this->enabledFiles.size();
+     int numEnabledFiles = activeFiles.size();
      for (int i = 0; i < numEnabledFiles; ++i)
      {
-       std::vector<CField*> enabledFields = this->enabledFiles[i]->getEnabledFields();
+       std::vector<CField*> enabledFields = activeFiles[i]->getEnabledFields();
        int numEnabledFields = enabledFields.size();
        for (int j = 0; j < numEnabledFields; ++j)
@@ -1592 +1678 @@
       info(50) << " Current memory used by XIOS : "<<  MemTrack::getCurrentMemorySize()*1.0/(1024*1024)<<" Mbyte, at timestep "<<step<<" of context "<<this->getId()<<endl ;
 #endif
-      if (hasClient)
+      //if (hasClient) 
+      if (hasClient && !hasServer) // For now we only use server level 1 to read data
       {
         checkPrefetchingOfEnabledReadModeFiles();
@@ -1599 +1686 @@
    }
 
+   void CContext::initReadFiles(void)
+   {
+      vector<CFile*>::const_iterator it;
+
+      for (it=enabledReadModeFiles.begin(); it != enabledReadModeFiles.end(); it++)
+      {
+         (*it)->initRead();
+      }
+   }
+
    //! Server side: Create header of netcdf file
-   void CContext::createFileHeader(void )
+   void CContext::createFileHeader(void)
    {
       vector<CFile*>::const_iterator it;
 
       for (it=enabledFiles.begin(); it != enabledFiles.end(); it++)
+      // for (it=enabledWriteModeFiles.begin(); it != enabledWriteModeFiles.end(); it++)
       {
-         (*it)->initFile();
+         (*it)->initWrite();
       }
    }

XIOS/dev/XIOS_DEV_CMIP6/src/node/context.hpp

@@ -104 +104 @@
 
          // Some functions to process context
-         void findAllEnabledFields(void);
-         void findAllEnabledFieldsInReadModeFiles(void);
+         void findAllEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles);
+         // void findAllEnabledFields(void);
+         // void findAllEnabledFieldsInReadModeFiles(void);
          void readAttributesOfEnabledFieldsInReadModeFiles();
          void solveAllInheritance(bool apply=true);
          void findEnabledFiles(void);
+         void findEnabledWriteModeFiles(void);
          void findEnabledReadModeFiles(void);
          void closeAllFile(void);
          void updateCalendar(int step);
-         void createFileHeader(void );
+         void createFileHeader(void);
+         void initReadFiles(void);
          void checkAxisDomainsGridsEligibilityForCompressedOutput();
          void prepareTimeseries(void);
@@ -127 +130 @@
          void solveAllRefOfEnabledFieldsAndTransform(bool sendToServer);
          void checkGridEnabledFields();
-         void sendGridEnabledFields();
+         void checkGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles);
+         void sendGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles);         
 
 //         std::map<int, StdSize> getAttributesBufferSize(std::map<int, StdSize>& maxEventSize);
@@ -144 +148 @@
          void sendUpdateCalendar(int step);
          void sendCreateFileHeader(void);
-         void sendEnabledFiles();
-         void sendEnabledFields();
-         void sendRefDomainsAxis();
-         void sendRefGrid();
+         void sendEnabledFiles(const std::vector<CFile*>& activeFiles);
+         void sendEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles);
+         void sendRefDomainsAxisScalars(const std::vector<CFile*>& activeFiles);
+         void sendRefGrid(const std::vector<CFile*>& activeFiles);
          void sendPostProcessing();
          void sendPostProcessingGlobalAttributes();

XIOS/dev/XIOS_DEV_CMIP6/src/node/domain.cpp

@@ -1810 +1810 @@
       }
 
+       // Even if servers have no index, they must received something from client
+       // We only use several client to send "empty" message to these servers
       CServerDistributionDescription serverDescription(nGlobDomain, nbServer);
-      if (isUnstructed_) serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 0);
-      else serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 1);
+      std::vector<int> serverZeroIndex;
+      if (isUnstructed_) serverZeroIndex = serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 0);
+      else serverZeroIndex = serverDescription.computeServerGlobalIndexInRange(std::make_pair<size_t,size_t>(indexBegin, indexEnd), 1);
+
+       std::list<int> serverZeroIndexLeader;
+       std::list<int> serverZeroIndexNotLeader; 
+       CContextClient::computeLeader(client->clientRank, client->clientSize, serverZeroIndex.size(), serverZeroIndexLeader, serverZeroIndexNotLeader);
+       for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+         *it = serverZeroIndex[*it];
 
       CClientServerMapping* clientServerMap = new CClientServerMappingDistributed(serverDescription.getGlobalIndexRange(),
@@ -1821 +1830 @@
       CClientServerMapping::GlobalIndexMap::const_iterator it  = globalIndexDomainOnServer.begin(),
                                                            ite = globalIndexDomainOnServer.end();
+      indSrv_.swap(globalIndexDomainOnServer);
       connectedServerRank_.clear();
-      for (it = globalIndexDomainOnServer.begin(); it != ite; ++it) {
+      for (it = indSrv_.begin(); it != ite; ++it) 
         connectedServerRank_.push_back(it->first);
-      }
-
-      indSrv_.swap(globalIndexDomainOnServer);
+
+      for (std::list<int>::const_iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+        connectedServerRank_.push_back(*it);
+
+       // Even if a client has no index, it must connect to at least one server and 
+       // send an "empty" data to this server
+       if (connectedServerRank_.empty())
+        connectedServerRank_.push_back(client->clientRank % client->serverSize);
+
       nbConnectedClients_ = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_);
 
-      clientServerMap->computeServerIndexMapping(globalIndexDomainZoom);
-      CClientServerMapping::GlobalIndexMap& globalIndexDomainZoomOnServer = clientServerMap->getGlobalIndexOnServer();
-      indZoomSrv_.swap(globalIndexDomainZoomOnServer);
+     //  clientServerMap->computeServerIndexMapping(globalIndexDomainZoom);
+     //  CClientServerMapping::GlobalIndexMap& globalIndexDomainZoomOnServer = clientServerMap->getGlobalIndexOnServer();
+     //  indZoomSrv_.swap(globalIndexDomainZoomOnServer);
       
-     for (it = indZoomSrv_.begin(); it != indZoomSrv_.end(); ++it)
-       connectedServerZoomRank_.push_back(it->first);
-
-      nbConnectedClientsZoom_ = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerZoomRank_);
+     // for (it = indZoomSrv_.begin(); it != indZoomSrv_.end(); ++it)
+     //   connectedServerZoomRank_.push_back(it->first);
+
+     //  nbConnectedClientsZoom_ = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerZoomRank_);
 
       delete clientServerMap;
@@ -2007 +2023 @@
     This function can be used in the future???
   */
-  void CDomain::sendIndexZoom()
-  {
-    int ns, n, i, j, ind, nv, idx;
-    CContext* context = CContext::getCurrent();
-
-    // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
-    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
-    for (int p = 0; p < nbSrvPools; ++p)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
-      CEventClient eventIndexZoom(getType(), EVENT_ID_INDEX_ZOOM);
-
-      list<CMessage> list_msgsIndex;
-      list<CArray<int,1> > list_indZoom;
-
-      boost::unordered_map<int, vector<size_t> >::const_iterator itZoom, iteZoom;
-      iteZoom = indZoomSrv_.end();
-      for (int k = 0; k < connectedServerZoomRank_.size(); ++k)
-      {
-        int nbIndGlob = 0;
-        int rank = connectedServerZoomRank_[k];
-        int nbIndZoom = 0;
-        itZoom = indZoomSrv_.find(rank);
-        if (iteZoom != itZoom)
-          nbIndZoom = itZoom->second.size();
+  // void CDomain::sendIndexZoom()
+  // {
+  //   int ns, n, i, j, ind, nv, idx;
+  //   CContext* context = CContext::getCurrent();
+
+  //   // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
+  //   int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
+  //   for (int p = 0; p < nbSrvPools; ++p)
+  //   {
+  //     CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
+  //     CEventClient eventIndexZoom(getType(), EVENT_ID_INDEX_ZOOM);
+
+  //     list<CMessage> list_msgsIndex;
+  //     list<CArray<int,1> > list_indZoom;
+
+  //     boost::unordered_map<int, vector<size_t> >::const_iterator itZoom, iteZoom;
+  //     iteZoom = indZoomSrv_.end();
+  //     for (int k = 0; k < connectedServerZoomRank_.size(); ++k)
+  //     {
+  //       int nbIndGlob = 0;
+  //       int rank = connectedServerZoomRank_[k];
+  //       int nbIndZoom = 0;
+  //       itZoom = indZoomSrv_.find(rank);
+  //       if (iteZoom != itZoom)
+  //         nbIndZoom = itZoom->second.size();
         
-        list_indZoom.push_back(CArray<int,1>(nbIndZoom));
-        CArray<int,1>& indZoom = list_indZoom.back();
-        for (n = 0; n < nbIndZoom; ++n)
-        {
-          indZoom(n) = static_cast<int>(itZoom->second[n]);
-        }
-
-        list_msgsIndex.push_back(CMessage());
-        list_msgsIndex.back() << this->getId(); // enum ne fonctionne pour les message => ToFix        
-        list_msgsIndex.back() << list_indZoom.back() << doZoomByIndex_; //list_indi.back() << list_indj.back      
-
-        eventIndexZoom.push(rank, nbConnectedClientsZoom_[rank], list_msgsIndex.back());
-      }
-
-      client->sendEvent(eventIndexZoom);
-    }
-  }
+  //       list_indZoom.push_back(CArray<int,1>(nbIndZoom));
+  //       CArray<int,1>& indZoom = list_indZoom.back();
+  //       for (n = 0; n < nbIndZoom; ++n)
+  //       {
+  //         indZoom(n) = static_cast<int>(itZoom->second[n]);
+  //       }
+
+  //       list_msgsIndex.push_back(CMessage());
+  //       list_msgsIndex.back() << this->getId(); // enum ne fonctionne pour les message => ToFix        
+  //       list_msgsIndex.back() << list_indZoom.back() << doZoomByIndex_; //list_indi.back() << list_indj.back      
+
+  //       eventIndexZoom.push(rank, nbConnectedClientsZoom_[rank], list_msgsIndex.back());
+  //     }
+
+  //     client->sendEvent(eventIndexZoom);
+  //   }
+  // }
 
   /*!
@@ -2673 +2689 @@
       {
         lInd = globalLocalIndexMap_[size_t(tmpInd(ind))];
-        if (!mask_1d(lInd))
-         mask_1d(lInd) = tmp(ind);
+        if (!mask_1d(lInd)) // Only rewrite mask_1d if it's not true
+          mask_1d(lInd) = tmp(ind);
       }
     }    

XIOS/dev/XIOS_DEV_CMIP6/src/node/field.cpp

@@ -41 +41 @@
       , hasTimeCentered(false)
       , wasDataAlreadyReceivedFromServer(false)
-      , isEOF(false)
+      , isEOF(false), nstepMaxRead(false)
    { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
 
@@ -56 +56 @@
       , hasTimeCentered(false)
       , wasDataAlreadyReceivedFromServer(false)
-      , isEOF(false)
+      , isEOF(false), nstepMaxRead(false)
    { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
 
@@ -262 +262 @@
   void CField::writeField(void)
   {
-    if (!getRelFile()->allDomainEmpty)
+    if (!getRelFile()->isEmptyZone())
     {
       if (grid->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)
@@ -273 +273 @@
   }
 
+  /*
+    Send a request for reading data.
+    Client sends a request to server for demanding server to read data and send back to it.
+    For now, this function is called only by client
+    In the future, it can be called by level-1 servers
+    \param [in] tsDataRequested timestamp when the call is made
+  */
   bool CField::sendReadDataRequest(const CDate& tsDataRequested)
   {
     CContext* context = CContext::getCurrent();
-//    CContextClient* client = context->client;
+    // CContextClient* client = context->client;
+
+    // This code is for future: If we want to read file with level-2 servers
     CContextClient* client = (!context->hasServer) ? context->client : this->file->getContextClient();
 
@@ -332 +341 @@
   }
 
+  /*!
+    Receive data request sent from client and process it
+    Every time server receives this request, it will try to read data and sent read data back to client
+    At the moment, this function is called by server level 1
+    In the future, this should (only) be done by the last level servers.
+  */
   void CField::recvReadDataRequest(void)
   {
     CContext* context = CContext::getCurrent();
     CContextClient* client = context->client;
-//    CContextClient* client = (!context->hasServer) ? context->client : this->file->getContextClient();
 
     CEventClient event(getType(), EVENT_ID_READ_DATA_READY);
     std::list<CMessage> msgs;
 
-    bool hasData = readField();
+    EReadField hasData = readField();
 
     map<int, CArray<double,1> >::iterator it;
@@ -356 +370 @@
               CMessage& msg = msgs.back();
               msg << getId();
-              if (hasData)
-                msg << getNStep() - 1 << recvDataSrv;
-              else
-                msg << int(-1);
+              switch (hasData)
+              {
+                case RF_DATA:
+                  msg << getNStep() - 1 << recvDataSrv;
+                  break;
+                case RF_NODATA:
+                  msg << int(-2);
+                  break;
+                case RF_EOF:                  
+                default:
+                  msg << int(-1);
+                  break;
+              }
+
               event.push(*itRank, 1, msg);
             }
@@ -385 +409 @@
         CMessage& msg = msgs.back();
         msg << getId();
-        if (hasData)
-          msg << getNStep() - 1 << tmp;
-        else
-          msg << int(-1);
+        switch (hasData)
+        {
+          case RF_DATA:
+            msg << getNStep() - 1 << tmp;
+            break;
+          case RF_NODATA:
+            msg << int(-2) << tmp;
+            break;
+          case RF_EOF:                  
+          default:
+            msg << int(-1);
+            break;
+        }
+
         event.push(it->first, grid->nbReadSenders[0][it->first], msg);
       }
@@ -395 +429 @@
   }
 
-  bool CField::readField(void)
+  /*!
+    Read field from a file.
+    A field is read with the distribution of data on the server side
+    \return State of field can be read from a file
+  */
+  CField::EReadField CField::readField(void)
   {
+    CContext* context = CContext::getCurrent();
     grid->computeWrittenIndex();
-    if (!getRelFile()->allDomainEmpty)
-    {
-      if (grid->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)
+    getRelFile()->initRead();
+    EReadField readState = RF_DATA;
+
+    if (!getRelFile()->isEmptyZone())
+    {      
+      if (grid->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)      
       {
         if (0 == recvDataSrv.numElements())
@@ -407 +450 @@
           recvDataSrv.resize(storeClient.numElements());          
         }
-
+        
         getRelFile()->checkReadFile();
+
         if (!nstepMax)
         {
@@ -417 +461 @@
 
         if (getNStep() > nstepMax && (getRelFile()->cyclic.isEmpty() || !getRelFile()->cyclic) )
-          return false;
-
-        getRelFile()->getDataInput()->readFieldData(CField::get(this));
-      }
-    }
-
-    return true;
+          readState = RF_EOF;
+
+        if (RF_EOF != readState)
+          getRelFile()->getDataInput()->readFieldData(CField::get(this));
+      }
+    }
+    else
+    {
+      this->incrementNStep();
+      if (getNStep() > nstepMax && (getRelFile()->cyclic.isEmpty() || !getRelFile()->cyclic) )
+        readState = RF_EOF;
+      else
+        readState = RF_NODATA;
+
+      if (!nstepMaxRead) // This can be a bug if we try to read field from zero time record
+        readState = RF_NODATA;
+    }
+
+    if (!nstepMaxRead)
+    {
+       MPI_Allreduce(&nstepMax, &nstepMax, 1, MPI_INT, MPI_MAX, context->server->intraComm);
+       nstepMaxRead = true;
+    }
+
+    return readState;
   }
 
+  /*
+    Receive read data from server.
+    At the moment, this function is called in the client side.
+    In the future, this function can be called hiearachically (server n-1, server n -2, ..., client)
+    \param event event containing read data
+  */
   void CField::recvReadDataReady(CEventServer& event)
   {
@@ -443 +511 @@
   }
 
+  /*!
+    Receive read data from server
+    \param [in] ranks Ranks of sending processes
+    \param [in] buffers buffers containing read data
+  */
   void CField::recvReadDataReady(vector<int> ranks, vector<CBufferIn*> buffers)
   {
@@ -528 +601 @@
    {
       this->nstepMax = 0;
+      nstepMaxRead = false;
    }
 

XIOS/dev/XIOS_DEV_CMIP6/src/node/field.hpp

@@ -58 +58 @@
          typedef CFieldAttributes SuperClassAttribute;
 
+         enum EReadField
+         {
+           RF_NODATA, RF_EOF, RF_DATA
+         };
+         
       public:
 
@@ -157 +162 @@
         static void recvReadDataRequest(CEventServer& event);
         void recvReadDataRequest(void);
-        bool readField(void);
+        EReadField readField(void);
         static void recvReadDataReady(CEventServer& event);
         void recvReadDataReady(vector<int> ranks, vector<CBufferIn*> buffers);
@@ -230 +235 @@
          bool isReferenceSolved;
          bool isReferenceSolvedAndTransformed;
+         bool nstepMaxRead;
 
       private:

XIOS/dev/XIOS_DEV_CMIP6/src/node/file.cpp

@@ -25 +25 @@
       : CObjectTemplate<CFile>(), CFileAttributes()
       , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
-      , allDomainEmpty(false), isOpen(false)
+      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
    {
      setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
@@ -34 +34 @@
       : CObjectTemplate<CFile>(id), CFileAttributes()
       , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
-      , allDomainEmpty(false), isOpen(false)
+      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
     {
       setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
@@ -207 +207 @@
 
    //! Initialize a file in order to write into it
-   void CFile::initFile(void)
+   void CFile::initWrite(void)
    {
       CContext* context = CContext::getCurrent();
@@ -228 +228 @@
         }
       }
-      isOpen = false;
-
-      allDomainEmpty = true;
+      isOpen = false;      
 
 //      if (!record_offset.isEmpty() && record_offset < 0)
@@ -243 +241 @@
       for (it = this->enabledFields.begin(); it != end; it++)
       {
-         CField* field = *it;
-         allDomainEmpty &= !field->grid->doGridHaveDataToWrite();
+         CField* field = *it;         
          std::vector<CAxis*> vecAxis = field->grid->getAxis();
          for (size_t i = 0; i < vecAxis.size(); ++i)
            setAxis.insert(vecAxis[i]->getAxisOutputName());
-//            setAxis.insert(vecAxis[i]);
          std::vector<CDomain*> vecDomains = field->grid->getDomains();
          for (size_t i = 0; i < vecDomains.size(); ++i)
            setDomains.insert(vecDomains[i]->getDomainOutputName());
-//            setDomains.insert(vecDomains[i]);
 
          field->resetNStep(recordOffset);
@@ -260 +255 @@
 
       // create sub communicator for file
-      int color = allDomainEmpty ? 0 : 1;
-      MPI_Comm_split(server->intraComm, color, server->intraCommRank, &fileComm);
-      if (allDomainEmpty) MPI_Comm_free(&fileComm);
+      createSubComFile();
 
       // if (time_counter.isEmpty()) time_counter.setValue(time_counter_attr::centered);
       if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
+    }
+
+    //! Initialize a file in order to write into it
+    void CFile::initRead(void)
+    {
+      if (checkRead) return;
+      createSubComFile();
+      checkRead = true;
+    }
+
+    /*!
+      Create a sub communicator in which processes participate in reading/opening file
+    */
+    void CFile::createSubComFile()
+    {
+      CContext* context = CContext::getCurrent();
+      CContextServer* server = context->server;
+
+      // create sub communicator for file
+      allZoneEmpty = true;      
+      std::vector<CField*>::iterator it, end = this->enabledFields.end();
+      for (it = this->enabledFields.begin(); it != end; it++)
+      {
+         CField* field = *it;
+         bool nullGrid = (0 == field->grid);
+         allZoneEmpty &= nullGrid ? false : !field->grid->doGridHaveDataToWrite();
+      }
+
+      int color = allZoneEmpty ? 0 : 1;
+      MPI_Comm_split(server->intraComm, color, server->intraCommRank, &fileComm);
+      if (allZoneEmpty) MPI_Comm_free(&fileComm);
     }
 
@@ -307 +331 @@
         {
           CTimer::get("Files : open headers").resume();
-          if (!isOpen) openInReadMode(&(context->server->intraComm));
+          
+          if (!isOpen) openInReadMode();
+
           CTimer::get("Files : open headers").suspend();
         }
         //checkSplit(); // Really need for reading?
       }
+    }
+
+    /*!
+      Verify if a process participates in an opening-file communicator 
+      \return true if the process doesn't participate in opening file
+    */
+    bool CFile::isEmptyZone()
+    {
+      return allZoneEmpty;
     }
 
@@ -376 +411 @@
       CContextServer* server = context->server;
 
-      if (!allDomainEmpty)
+      if (!allZoneEmpty)
       {
          StdString filename = getFileOutputName();
@@ -558 +593 @@
   \brief Open an existing NetCDF file in read-only mode
   */
-  void CFile::openInReadMode(MPI_Comm* comm)
+  void CFile::openInReadMode()
   {
     CContext* context = CContext::getCurrent();
     CContextServer* server = context->server;
-    MPI_Comm readComm = *comm;
-
-    if (!allDomainEmpty)
+    MPI_Comm readComm = this->fileComm;
+
+    if (!allZoneEmpty)
     {
       StdString filename = getFileOutputName();
@@ -632 +667 @@
    void CFile::close(void)
    {
-     if (!allDomainEmpty)
+     if (!allZoneEmpty)
        if (isOpen)
        {
@@ -639 +674 @@
          else
           this->data_in->closeFile();
+        isOpen = false;
        }
       if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
@@ -905 +941 @@
    }
 
+   void CFile::setReadContextClient(CContextClient* readContextclient)
+   {
+     read_client = readContextclient;
+   }
+
+   CContextClient* CFile::getReadContextClient()
+   {
+     return read_client;
+   }
+
    /*!
    \brief Send a message to create a field on server side

XIOS/dev/XIOS_DEV_CMIP6/src/node/file.hpp

@@ -89 +89 @@
          void checkWriteFile(void);
          void checkReadFile(void);
-         void initFile(void);
+         void initWrite(void);
+         void initRead(void);
+         bool isEmptyZone();
 
          /// Mutateurs ///
@@ -97 +99 @@
 
          void createHeader(void);
-         void openInReadMode(MPI_Comm* readComm = NULL);
+         void openInReadMode(void);
          void close(void);
          void readAttributesOfEnabledFieldsInReadMode();
@@ -122 +124 @@
          void setContextClient(CContextClient* newContextClient);
          CContextClient* getContextClient();
+
+         void setReadContextClient(CContextClient* newContextClient);
+         CContextClient* getReadContextClient();
 
          // Send info to server         
@@ -162 +167 @@
          CDate lastSplit;
          int nbAxis, nbDomains;
-         bool isOpen;
-         bool allDomainEmpty;
+         bool isOpen;         
          MPI_Comm fileComm;
 
+      private:
+         void createSubComFile();
+         bool checkRead;
+         bool allZoneEmpty;
+         
       private :
          /// Propriétés privées ///
          CContextClient* client;
+         CContextClient* read_client; // Context client for reading (channel between server 1 and client)
          CFieldGroup* vFieldGroup;
          CVariableGroup* vVariableGroup;
@@ -175 +185 @@
          std::vector<CField*> enabledFields;
 
+
       public:
         //         virtual void toBinary  (StdOStream& os) const;

XIOS/dev/XIOS_DEV_CMIP6/src/node/grid.cpp

@@ -155 +155 @@
      // The record index is sometimes sent along with the data but we always
      // include it in the size calculation for the sake of simplicity
-     const size_t extraSize = CEventClient::headerSize + (id.empty() ? getId() : id).size() + 2 * sizeof(size_t);
+     const size_t extraSize = CEventClient::headerSize + (id.empty() ? getId() : id).size() 
+                                                       + 2 * sizeof(size_t) 
+                                                       + sizeof(size_t);
      CContext* context = CContext::getCurrent();
      int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
@@ -162 +164 @@
      {
        std::map<int, size_t>::const_iterator itEnd = connectedDataSize_[p].end();
-       for (size_t k = 0; k < connectedServerRank_[p].size(); ++k) // TODO: Should change connectedServerRank_[0] to something more general
+       for (size_t k = 0; k < connectedServerRank_[p].size(); ++k)
        {
          int rank = connectedServerRank_[p][k];
-         std::map<int, size_t>::const_iterator it = connectedDataSize_[0].find(rank);
+         std::map<int, size_t>::const_iterator it = connectedDataSize_[p].find(rank);
          size_t count = (it != itEnd) ? it->second : 0;
 
@@ -714 +716 @@
        std::vector<boost::unordered_map<size_t,std::vector<int> > > indexServerOnElement;
        CServerDistributionDescription serverDistributionDescription(getGlobalDimension(), client->serverSize);
-       serverDistributionDescription.computeServerGlobalByElement(indexServerOnElement,
-                                                                  client->clientRank,
-                                                                  client->clientSize,
-                                                                  axis_domain_order,
-                                                                  getDistributedDimension());
+       std::vector<int> serverZeroIndex = serverDistributionDescription.computeServerGlobalByElement(indexServerOnElement,
+                                                                                                  client->clientRank,
+                                                                                                  client->clientSize,
+                                                                                                  axis_domain_order,
+                                                                                                  getDistributedDimension());
+
+       // Even if servers have no index, they must received something from client
+       // We only use several client to send "empty" message to these servers
+       std::list<int> serverZeroIndexLeader;
+       std::list<int> serverZeroIndexNotLeader; 
+       CContextClient::computeLeader(client->clientRank, client->clientSize, serverZeroIndex.size(), serverZeroIndexLeader, serverZeroIndexNotLeader);
+       for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+         *it = serverZeroIndex[*it];
+
        computeIndexByElement(indexServerOnElement, globalIndexOnServer_);
 
@@ -724 +735 @@
        CDistributionClient::GlobalLocalDataMap::const_iterator iteGlobalLocalIndexMap = globalLocalIndexSendToServer.end(), itGlobalLocalIndexMap;
        CClientServerMapping::GlobalIndexMap::const_iterator iteGlobalMap, itbGlobalMap, itGlobalMap;
-       itGlobalMap  = itbGlobalMap = globalIndexOnServer_.begin();
+       itbGlobalMap = globalIndexOnServer_.begin();
        iteGlobalMap = globalIndexOnServer_.end();
 
-       for (; itGlobalMap != iteGlobalMap; ++itGlobalMap)
+       for (itGlobalMap  = itbGlobalMap; itGlobalMap != iteGlobalMap; ++itGlobalMap)
        {
          int serverRank = itGlobalMap->first;
@@ -745 +756 @@
        }
 
+       // Connected servers which really have index
        for (itGlobalMap = itbGlobalMap; itGlobalMap != iteGlobalMap; ++itGlobalMap) {
          connectedServerRank_[p].push_back(itGlobalMap->first);
        }
 
+       // Connected servers which have no index at all
+       for (std::list<int>::iterator it = serverZeroIndexLeader.begin(); it != serverZeroIndexLeader.end(); ++it)
+         connectedServerRank_[p].push_back(*it);
+
+       // Even if a client has no index, it must connect to at least one server and 
+       // send an "empty" data to this server
+       if (connectedServerRank_[p].empty())
+        connectedServerRank_[p].push_back(client->clientRank % client->serverSize);
+      
        nbSenders[p] = clientServerMap_->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[p]);
      }

XIOS/dev/XIOS_DEV_CMIP6/src/server_distribution_description.cpp

@@ -100 +100 @@
   \param [in] positionDimensionDistributed dimension of server on which we make the cut.
 */
-void CServerDistributionDescription::computeServerGlobalIndexInRange(const std::pair<size_t, size_t>& indexBeginEnd,
+std::vector<int> CServerDistributionDescription::computeServerGlobalIndexInRange(const std::pair<size_t, size_t>& indexBeginEnd,
                                                                      int positionDimensionDistributed)
 {
+  int nBand  = 0;
   switch (serverType_) {
     case BAND_DISTRIBUTION:
-      computeBandDistribution(nServer_, positionDimensionDistributed);
+      nBand = computeBandDistribution(nServer_, positionDimensionDistributed);
       break;
     default:
@@ -122 +123 @@
   std::vector<int> currentIndex(dim);
 
-  for (int idxServer = 0; idxServer < nServer_; ++idxServer)
+  for (int idxServer = 0; idxServer < nBand; ++idxServer)
   {
     size_t ssize = 1, idx = 0;
@@ -161 +162 @@
     }
   }
+
+    // List of servers without distribution (cause total number of server is greater than number of bands, for example)
+  std::vector<int> zeroIndexServer(nServer_-nBand); 
+  for (int idxServer = nBand; idxServer < nServer_; ++idxServer)
+    zeroIndexServer[idxServer-nBand] = idxServer;
+
+  return zeroIndexServer;
 }
 
@@ -172 +180 @@
   \param [in] positionDimensionDistributed dimension of server on which we make the cut.
 */
-void CServerDistributionDescription::computeServerGlobalByElement(std::vector<boost::unordered_map<size_t,std::vector<int> > >& indexServerOnElement,
-                                                                  int clientRank,
-                                                                  int clientSize,
-                                                                  const CArray<int,1>& axisDomainOrder,
-                                                                  int positionDimensionDistributed)
-{
+std::vector<int> CServerDistributionDescription::computeServerGlobalByElement(std::vector<boost::unordered_map<size_t,std::vector<int> > >& indexServerOnElement,
+                                                                              int clientRank,
+                                                                              int clientSize,
+                                                                              const CArray<int,1>& axisDomainOrder,
+                                                                              int positionDimensionDistributed)
+{
+  int nBand  = 0;
   switch (serverType_) {
     case BAND_DISTRIBUTION:
-      computeBandDistribution(nServer_, positionDimensionDistributed);
+      nBand = computeBandDistribution(nServer_, positionDimensionDistributed);
       break;
     default:
@@ -197 +206 @@
   }
 
-  for (int idxServer = 0; idxServer < nServer_; ++idxServer)
+  for (int idxServer = 0; idxServer < nBand; ++idxServer)
   {
     std::vector<int> elementDimension(4);
@@ -247 +256 @@
     }
   }
+
+  // List of servers without distribution (cause total number of server is greater than number of bands, for example)
+  std::vector<int> zeroIndexServer(nServer_-nBand); 
+  for (int idxServer = nBand; idxServer < nServer_; ++idxServer)
+    zeroIndexServer[idxServer-nBand] = idxServer;
+
+  return zeroIndexServer;
 }
 
@@ -297 +313 @@
   \param [in] nServer number of server
 */
-void CServerDistributionDescription::computeBandDistribution(int nServer, int positionDimensionDistributed)
+int CServerDistributionDescription::computeBandDistribution(int nServer, int positionDimensionDistributed)
 {
   int dim = nGlobal_.size();
@@ -324 +340 @@
   int positionDistributed = (1<dim) ? positionDimensionDistributed_ : 0;
   nGlobTemp = nGlobal_[positionDistributed];
-
-  for (int i = 0; i < nServer; ++i)
+  int nbBand = std::min(nGlobTemp, nServer);
+
+  for (int i = 0; i < nbBand; ++i)
   {
     if (0 < i) njRangeBegin[i] = njRangeEnd[i-1];
-    njRangeSize = nGlobTemp / nServer;
-    if (i < nGlobTemp%nServer) ++njRangeSize;
+    njRangeSize = nGlobTemp / nbBand;
+    if (i < nGlobTemp%nbBand) ++njRangeSize;
     njRangeEnd[i] = njRangeSize + njRangeBegin[i];
   }
-  njRangeEnd[nServer-1] = nGlobTemp;
+  njRangeEnd[nbBand-1] = nGlobTemp;
+
+  for (int i = nbBand; i < nServer; ++i)
+  {
+    njRangeBegin[i] = njRangeEnd[i] = 0;
+  }
 
   for (int i = 0; i < nServer; ++i)
@@ -358 +380 @@
     }
   }
+
+  return nbBand;
 }
 

XIOS/dev/XIOS_DEV_CMIP6/src/server_distribution_description.hpp

@@ -38 +38 @@
 
     void computeServerDistribution(bool doComputeGlobalIndex = false, int positionDimensionDistributed = 1);
-    void computeServerGlobalIndexInRange(const std::pair<size_t, size_t>& indexBeginEnd, int positionDimensionDistributed = 1);
-    void computeServerGlobalByElement(std::vector<boost::unordered_map<size_t,std::vector<int> > >& indexServerOnElement,
-                                      int rank,
-                                      int clientSize,
-                                      const CArray<int,1>& axisDomainOrder,
-                                      int positionDimensionDistributed = 1);
+    std::vector<int> computeServerGlobalIndexInRange(const std::pair<size_t, size_t>& indexBeginEnd, int positionDimensionDistributed = 1);
+    std::vector<int> computeServerGlobalByElement(std::vector<boost::unordered_map<size_t,std::vector<int> > >& indexServerOnElement,
+                                                  int rank,
+                                                  int clientSize,
+                                                  const CArray<int,1>& axisDomainOrder,
+                                                  int positionDimensionDistributed = 1);
 
     std::vector<std::vector<int> > getServerIndexBegin() const;
@@ -52 +52 @@
 
   protected:
-    void computeBandDistribution(int nServer, int positionDimensionDistributed = 1);
+    int computeBandDistribution(int nServer, int positionDimensionDistributed = 1);
     void computePlanDistribution(int nServer);
     void computeRangeProcIndex(int clientRank,