Changeset 1054 for XIOS/dev/dev_olga/src

Timestamp:

02/17/17 19:51:36 (7 years ago)

Author:

oabramkina

Message:

dev: intermediate commit.

Location:

XIOS/dev/dev_olga/src

Files:

• client.cpp (modified) (2 diffs)
• client.hpp (modified) (1 diff)
• client_server_mapping_distributed.cpp (modified) (1 diff)
• context_client.cpp (modified) (4 diffs)
• context_client.hpp (modified) (4 diffs)
• context_server.cpp (modified) (10 diffs)
• context_server.hpp (modified) (2 diffs)
• cxios.cpp (modified) (3 diffs)
• cxios.hpp (modified) (1 diff)
• distribution_server.hpp (modified) (1 diff)
• interface/c/icdata.cpp (modified) (1 diff)
• interface/fortran/idata.F90 (modified) (2 diffs)
• io/nc4_data_output.cpp (modified) (1 diff)
• node/axis.cpp (modified) (4 diffs)
• node/context.cpp (modified) (10 diffs)
• node/context.hpp (modified) (5 diffs)
• node/domain.cpp (modified) (17 diffs)
• node/field.cpp (modified) (3 diffs)
• node/field.hpp (modified) (1 diff)
• node/file.cpp (modified) (2 diffs)
• node/file.hpp (modified) (2 diffs)
• node/grid.cpp (modified) (6 diffs)
• object_template_impl.hpp (modified) (1 diff)
• server.cpp (modified) (20 diffs)
• server.hpp (modified) (4 diffs)
• xios_server.f90 (modified) (1 diff)
• xios_server1.f90 (deleted)
• xios_server2.f90 (deleted)

XIOS/dev/dev_olga/src/client.cpp

@@ -162 +162 @@
     }
 
-    void CClient::initializeClientOnServer(const int rank, const MPI_Comm& intraCommPrmSrv, const int srvSndLeader)
-    {
-      MPI_Comm_dup(intraCommPrmSrv, &intraComm) ;
-      serverLeader.push_back(srvSndLeader);
-      int intraCommSize, intraCommRank ;
-      MPI_Comm_size(intraComm,&intraCommSize) ;
-      MPI_Comm_rank(intraComm,&intraCommRank) ;
-      info(50)<<"intercommCreate::client "<<rank<<" intraCommSize : "<<intraCommSize
-          <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< srvSndLeader<<endl ;
-      MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvSndLeader, 0, &interComm) ;
-    }
-
-
 ///---------------------------------------------------------------
 /*!
@@ -257 +244 @@
   * The only difference with CClient::registerContext() is naming of contexts on servers (appearing of pool id at the end).
   */
-        void CClient::registerContextByClienOfServer(const string& id, MPI_Comm contextComm)
-        {
-          CContext::setCurrent(id) ;
-          CContext* context=CContext::create(id);
-          StdString idServer(id);
-          idServer += "_server_";
-
-          int size,rank,globalRank ;
-          size_t message_size ;
-          int leaderRank ;
-          MPI_Comm contextInterComm ;
-
-          MPI_Comm_size(contextComm,&size) ;
-          MPI_Comm_rank(contextComm,&rank) ;
-          MPI_Comm_rank(CXios::globalComm,&globalRank) ;
-          if (rank!=0) globalRank=0 ;
-
-          CMessage msg ;
-
-          int messageSize ;
-          void * buff ;
-
-          for (int i = 0; i < serverLeader.size(); ++i)
-          {
-            StdString str = idServer + boost::lexical_cast<string>(i);
-            msg<<str<<size<<globalRank ;
-            messageSize = msg.size() ;
-            buff = new char[messageSize] ;
-            CBufferOut buffer(buff,messageSize) ;
-            buffer<<msg ;
-
-            MPI_Send(buff, buffer.count(), MPI_CHAR, serverLeader[i], 1, CXios::globalComm) ;
-            MPI_Intercomm_create(contextComm, 0, CXios::globalComm, serverLeader[i], 10+globalRank, &contextInterComm) ;
-            info(10)<<"Register new Context : "<<id<<endl ;
-            MPI_Comm inter ;
-            MPI_Intercomm_merge(contextInterComm,0,&inter) ;
-            MPI_Barrier(inter) ;
-
-            context->initClient(contextComm,contextInterComm) ;
-
-            contextInterComms.push_back(contextInterComm);
-            MPI_Comm_free(&inter);
-            delete [] buff ;
-          }
-        }
+//        void CClient::registerContextByClientOfServer(const string& id, MPI_Comm contextComm)
+//        {
+//          CContext::setCurrent(id) ;
+//          CContext* context=CContext::create(id);
+//          StdString idServer(id);
+//          idServer += "_server_";
+//
+//          int size,rank,globalRank ;
+//          size_t message_size ;
+//          int leaderRank ;
+//          MPI_Comm contextInterComm ;
+//
+//          MPI_Comm_size(contextComm,&size) ;
+//          MPI_Comm_rank(contextComm,&rank) ;
+//          MPI_Comm_rank(CXios::globalComm,&globalRank) ;
+//          if (rank!=0) globalRank=0 ;
+//
+//          CMessage msg ;
+//
+//          int messageSize ;
+//          void * buff ;
+//
+//          for (int i = 0; i < serverLeader.size(); ++i)
+//          {
+//            StdString str = idServer + boost::lexical_cast<string>(i);
+//            msg<<str<<size<<globalRank ;
+//            messageSize = msg.size() ;
+//            buff = new char[messageSize] ;
+//            CBufferOut buffer(buff,messageSize) ;
+//            buffer<<msg ;
+//
+//            MPI_Send(buff, buffer.count(), MPI_CHAR, serverLeader[i], 1, CXios::globalComm) ;
+//            MPI_Intercomm_create(contextComm, 0, CXios::globalComm, serverLeader[i], 10+globalRank, &contextInterComm) ;
+//            info(10)<<"Register new Context : "<<id<<endl ;
+//            MPI_Comm inter ;
+//            MPI_Intercomm_merge(contextInterComm,0,&inter) ;
+//            MPI_Barrier(inter) ;
+//
+//            context->initClient(contextComm,contextInterComm) ;
+//
+////            contextInterComms.push_back(contextInterComm);
+//            MPI_Comm_free(&inter);
+//            delete [] buff ;
+//          }
+//        }
 
     void CClient::finalize(void)

XIOS/dev/dev_olga/src/client.hpp

@@ -11 +11 @@
       public:
         static void initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm);
-        static void initializeClientOnServer(const int rank, const MPI_Comm& localComm, const int srvSndLeader);
 
         static void finalize(void);
         static void registerContext(const string& id, MPI_Comm contextComm);
-        static void registerContextByClienOfServer(const string& id, MPI_Comm contextComm);
+//        static void registerContextByClientOfServer(const string& id, MPI_Comm contextComm);
 
         static MPI_Comm intraComm;

XIOS/dev/dev_olga/src/client_server_mapping_distributed.cpp

@@ -39 +39 @@
 {
   CContext* context=CContext::getCurrent() ;
-  CContextClient* client=context->client ;
+//  CContextClient* client=context->client ;
+  // For now the assumption is that secondary server pools consist of the same number of procs.
+  // CHANGE the line below if the assumption changes.
+  CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client;
   int nbServer=client->serverSize;
 

XIOS/dev/dev_olga/src/context_client.cpp

@@ -84 +84 @@
     {
       list<int> ranks = event.getRanks();
+
       if (!event.isEmpty())
       {
         list<int> sizes = event.getSizes();
 
-        list<CBufferOut*> buffList = getBuffers(ranks, sizes);
-
-        event.send(timeLine, sizes, buffList);
-
-        checkBuffers(ranks);
-      }
-
-      if (isAttachedModeEnabled())
-      {
-        waitEvent(ranks);
-        CContext::setCurrent(context->getId());
+        // We force the getBuffers call to be non-blocking on the servers
+        list<CBufferOut*> buffList;
+        bool couldBuffer = getBuffers(ranks, sizes, buffList, !CXios::isClient);
+//        bool couldBuffer = getBuffers(ranks, sizes, buffList, CXios::isServer);
+
+        if (couldBuffer)
+        {
+          event.send(timeLine, sizes, buffList);
+
+          checkBuffers(ranks);
+
+          if (isAttachedModeEnabled()) // couldBuffer is always true in attached mode
+          {
+            waitEvent(ranks);
+            CContext::setCurrent(context->getId());
+          }
+        }
+        else
+        {
+          tmpBufferedEvent.ranks = ranks;
+          tmpBufferedEvent.sizes = sizes;
+
+          for (list<int>::const_iterator it = sizes.begin(); it != sizes.end(); it++)
+            tmpBufferedEvent.buffers.push_back(new CBufferOut(*it));
+
+          event.send(timeLine, tmpBufferedEvent.sizes, tmpBufferedEvent.buffers);
+        }
       }
 
       timeLine++;
+    }
+
+    /*!
+     * Send the temporarily buffered event (if any).
+     *
+     * \return true if a temporarily buffered event could be sent, false otherwise
+     */
+    bool CContextClient::sendTemporarilyBufferedEvent()
+    {
+      bool couldSendTmpBufferedEvent = false;
+
+      if (hasTemporarilyBufferedEvent())
+      {
+        list<CBufferOut*> buffList;
+        if (getBuffers(tmpBufferedEvent.ranks, tmpBufferedEvent.sizes, buffList, true)) // Non-blocking call
+        {
+          list<CBufferOut*>::iterator it, itBuffer;
+
+          for (it = tmpBufferedEvent.buffers.begin(), itBuffer = buffList.begin(); it != tmpBufferedEvent.buffers.end(); it++, itBuffer++)
+            (*itBuffer)->put((char*)(*it)->start(), (*it)->count());
+
+          checkBuffers(tmpBufferedEvent.ranks);
+
+          tmpBufferedEvent.clear();
+
+          couldSendTmpBufferedEvent = true;
+        }
+      }
+
+      return couldSendTmpBufferedEvent;
     }
 
@@ -124 +171 @@
     }
 
-    /*!
-    Setup buffer for each connection to server and verify their state to put content into them
-    \param [in] serverList list of rank of connected server
-    \param [in] sizeList size of message corresponding to each connection
-    \return List of buffer input which event can be placed
-    */
-    list<CBufferOut*> CContextClient::getBuffers(list<int>& serverList, list<int>& sizeList)
-    {
-      list<int>::iterator itServer, itSize;
+
+    /*!
+     * Get buffers for each connection to the servers. This function blocks until there is enough room in the buffers unless
+     * it is explicitly requested to be non-blocking.
+     *
+     * \param [in] serverList list of rank of connected server
+     * \param [in] sizeList size of message corresponding to each connection
+     * \param [out] retBuffers list of buffers that can be used to store an event
+     * \param [in] nonBlocking whether this function should be non-blocking
+     * \return whether the already allocated buffers could be used
+    */
+    bool CContextClient::getBuffers(const list<int>& serverList, const list<int>& sizeList, list<CBufferOut*>& retBuffers, bool nonBlocking /*= false*/)
+    {
+      list<int>::const_iterator itServer, itSize;
       list<CClientBuffer*> bufferList;
-      map<int,CClientBuffer*>::iterator it;
+      map<int,CClientBuffer*>::const_iterator it;
       list<CClientBuffer*>::iterator itBuffer;
-      list<CBufferOut*>  retBuffer;
       bool areBuffersFree;
 
@@ -160 +211 @@
         {
           checkBuffers();
-          context->server->listen();
-        }
-      } while (!areBuffersFree);
+//          if (?)
+//          {
+//            for (int i = 0; i < context->serverPrimServer.size(); ++i)
+//              context->serverPrimServer[i]->listen();
+//          }
+//          else
+            context->server->listen();
+        }
+      } while (!areBuffersFree && !nonBlocking);
       CTimer::get("Blocking time").suspend();
 
-      for (itBuffer = bufferList.begin(), itSize = sizeList.begin(); itBuffer != bufferList.end(); itBuffer++, itSize++)
-      {
-        retBuffer.push_back((*itBuffer)->getBuffer(*itSize));
-      }
-      return retBuffer;
+      if (areBuffersFree)
+      {
+        for (itBuffer = bufferList.begin(), itSize = sizeList.begin(); itBuffer != bufferList.end(); itBuffer++, itSize++)
+          retBuffers.push_back((*itBuffer)->getBuffer(*itSize));
+      }
+
+      return areBuffersFree;
    }
 
@@ -299 +358 @@
    Finalize context client and do some reports
    */
-   void CContextClient::finalize(void)
-   {
-     map<int,CClientBuffer*>::iterator itBuff;
-     bool stop = true;
-
-     CEventClient event(CContext::GetType(), CContext::EVENT_ID_CONTEXT_FINALIZE);
-     if (isServerLeader())
-     {
-       CMessage msg;
-       const std::list<int>& ranks = getRanksServerLeader();
-       for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-         event.push(*itRank, 1, msg);
-       sendEvent(event);
-     }
-     else sendEvent(event);
-
-     CTimer::get("Blocking time").resume();
-     while (stop)
-     {
-       checkBuffers();
-       stop = false;
-       for (itBuff = buffers.begin(); itBuff != buffers.end(); itBuff++) stop |= itBuff->second->hasPendingRequest();
-     }
-     CTimer::get("Blocking time").suspend();
-
-     std::map<int,StdSize>::const_iterator itbMap = mapBufferSize_.begin(),
-                                           iteMap = mapBufferSize_.end(), itMap;
-     StdSize totalBuf = 0;
-     for (itMap = itbMap; itMap != iteMap; ++itMap)
-     {
-       report(10) << " Memory report : Context <" << context->getId() << "> : client side : memory used for buffer of each connection to server" << endl
-                  << "  +) To server with rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
-       totalBuf += itMap->second;
-     }
-     report(0) << " Memory report : Context <" << context->getId() << "> : client side : total memory used for buffer " << totalBuf << " bytes" << endl;
-
-     releaseBuffers();
-   }
+  void CContextClient::finalize(void)
+  {
+    map<int,CClientBuffer*>::iterator itBuff;
+    bool stop = false;
+
+    CTimer::get("Blocking time").resume();
+    while (hasTemporarilyBufferedEvent())
+    {
+      checkBuffers();
+      sendTemporarilyBufferedEvent();
+    }
+    CTimer::get("Blocking time").suspend();
+
+    CEventClient event(CContext::GetType(), CContext::EVENT_ID_CONTEXT_FINALIZE);
+    if (isServerLeader())
+    {
+      CMessage msg;
+      const std::list<int>& ranks = getRanksServerLeader();
+      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        event.push(*itRank, 1, msg);
+      sendEvent(event);
+    }
+    else sendEvent(event);
+
+    CTimer::get("Blocking time").resume();
+    while (!stop)
+    {
+      checkBuffers();
+      if (hasTemporarilyBufferedEvent())
+        sendTemporarilyBufferedEvent();
+
+      stop = true;
+      for (itBuff = buffers.begin(); itBuff != buffers.end(); itBuff++) stop &= !itBuff->second->hasPendingRequest();
+    }
+    CTimer::get("Blocking time").suspend();
+
+    std::map<int,StdSize>::const_iterator itbMap = mapBufferSize_.begin(),
+                                          iteMap = mapBufferSize_.end(), itMap;
+    StdSize totalBuf = 0;
+    for (itMap = itbMap; itMap != iteMap; ++itMap)
+    {
+      report(10) << " Memory report : Context <" << context->getId() << "> : client side : memory used for buffer of each connection to server" << endl
+                 << "  +) To server with rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
+      totalBuf += itMap->second;
+    }
+    report(0) << " Memory report : Context <" << context->getId() << "> : client side : total memory used for buffer " << totalBuf << " bytes" << endl;
+
+    releaseBuffers();
+  }
 }

XIOS/dev/dev_olga/src/context_client.hpp

@@ -31 +31 @@
       // Send event to server
       void sendEvent(CEventClient& event);
+      bool sendTemporarilyBufferedEvent();
       void waitEvent(list<int>& ranks);
 
-      // Functions relates to set/get buffers
-      list<CBufferOut*> getBuffers(list<int>& serverlist, list<int>& sizeList);
+      // Functions to set/get buffers
+      bool getBuffers(const list<int>& serverList, const list<int>& sizeList, list<CBufferOut*>& retBuffers, bool nonBlocking = false);
       void newBuffer(int rank);
       bool checkBuffers(list<int>& ranks);
@@ -46 +47 @@
 
       bool isAttachedModeEnabled() const;
+      bool hasTemporarilyBufferedEvent() const { return !tmpBufferedEvent.isEmpty(); };
 
       // Close and finalize context client
-      void closeContext(void);
+//      void closeContext(void);  Never been implemented.
       void finalize(void);
 
@@ -68 +70 @@
       MPI_Comm intraComm; //!< Communicator of client group
 
-      map<int,CClientBuffer*> buffers; //!< Buffers for connection to servers
+      map<int,CClientBuffer*> buffers;     //!< Buffers for connection to servers
 
     private:
@@ -75 +77 @@
       //! Maximum number of events that can be buffered
       StdSize maxBufferedEvents;
+
+      struct {
+        std::list<int> ranks, sizes;
+        std::list<CBufferOut*> buffers;
+
+        bool isEmpty() const { return ranks.empty(); };
+        void clear() {
+          ranks.clear();
+          sizes.clear();
+
+          for (std::list<CBufferOut*>::iterator it = buffers.begin(); it != buffers.end(); it++)
+            delete *it;
+
+          buffers.clear();
+        };
+      } tmpBufferedEvent; //! Event temporarily buffered (used only on the server)
 
       //! Context for server (Only used in attached mode)

XIOS/dev/dev_olga/src/context_server.cpp

@@ -22 +22 @@
 namespace xios
 {
+  StdSize CContextServer::totalBuf_ = 0;
 
   CContextServer::CContextServer(CContext* parent, MPI_Comm intraComm_,MPI_Comm interComm_)
@@ -29 +30 @@
     MPI_Comm_size(intraComm,&intraCommSize);
     MPI_Comm_rank(intraComm,&intraCommRank);
+
     interComm=interComm_;
     int flag;
@@ -38 +40 @@
     scheduled=false;
     finished=false;
-
     boost::hash<string> hashString;
     hashId=hashString(context->getId());
-
-  }
-
-//  CContextServer::CContextServer(CContext* parent, int srvLvl, MPI_Comm intraComm_,MPI_Comm interComm_)
-//  {
-//    context=parent;
-//    intraComm=intraComm_;
-//    MPI_Comm_size(intraComm,&intraCommSize);
-//    MPI_Comm_rank(intraComm,&intraCommRank);
-//    interComm=interComm_;
-//    int flag;
-//    MPI_Comm_test_inter(interComm,&flag);
-//    if (flag) MPI_Comm_remote_size(interComm,&commSize);
-//    else  MPI_Comm_size(interComm,&commSize);
-//
-//    currentTimeLine=0;
-//    scheduled=false;
-//    finished=false;
-//
-//    boost::hash<string> hashString;
-//    StdString contextId = context->getId();
-//    hashId=hashString(contextId);
-//
-//  }
+  }
+
   void CContextServer::setPendingEvent(void)
   {
@@ -70 +49 @@
   }
 
+
   bool CContextServer::hasPendingEvent(void)
   {
@@ -80 +60 @@
   }
 
-  bool CContextServer::eventLoop(void)
+  bool CContextServer::eventLoop(bool enableEventsProcessing /*= true*/)
   {
     listen();
     checkPendingRequest();
-    processEvents();
+    if (enableEventsProcessing)
+      processEvents();
     return finished;
   }
@@ -188 +169 @@
     map<size_t,CEventServer*>::iterator it;
     CEventServer* event;
+    boost::hash<string> hashString;
+    size_t hashId=hashString(context->getId());
 
     it=events.find(currentTimeLine);
@@ -224 +207 @@
   {
     map<int,CServerBuffer*>::iterator it;
-    for(it=buffers.begin();it!=buffers.end();++it) delete it->second;
-  }
-
+    for(it=buffers.begin();it!=buffers.end();++it)
+      delete it->second;
+  }
 
   void CContextServer::dispatchEvent(CEventServer& event)
@@ -235 +218 @@
     int rank;
     list<CEventServer::SSubEvent>::iterator it;
-    CContext::setCurrent(context->getId());
+//    CContext::setCurrent(context->getId());
+    StdString ctxId = context->getId();
+    CContext::setCurrent(ctxId);
 
     if (event.classId==CContext::GetType() && event.type==CContext::EVENT_ID_CONTEXT_FINALIZE)
@@ -243 +228 @@
       std::map<int, StdSize>::const_iterator itbMap = mapBufferSize_.begin(),
                            iteMap = mapBufferSize_.end(), itMap;
-      StdSize totalBuf = 0;
       for (itMap = itbMap; itMap != iteMap; ++itMap)
       {
-        report(10)<< " Memory report : Context <"<<context->getId()<<"> : server side : memory used for buffer of each connection to client" << endl
-            << "  +) With client of rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
-        totalBuf += itMap->second;
+        rank = itMap->first;
+        report(10)<< " Memory report : Context <"<<ctxId<<"> : server side : memory used for buffer of each connection to client" << endl
+            << "  +) With client of rank " << rank << " : " << itMap->second << " bytes " << endl;
+        totalBuf_ += itMap->second;
       }
       context->finalize();
-      report(0)<< " Memory report : Context <"<<context->getId()<<"> : server side : total memory used for buffer "<<totalBuf<<" bytes"<<endl;
+
+//      report(0)<< " Memory report : Context <"<<ctxId<<"> : server side : total memory used for buffer "<<totalBuf<<" bytes"<<endl;
     }
     else if (event.classId==CContext::GetType()) CContext::dispatchEvent(event);
@@ -274 +260 @@
     }
   }
+
+  size_t CContextServer::getTotalBuf(void)
+  {
+    return totalBuf_;
+  }
+
 }

XIOS/dev/dev_olga/src/context_server.hpp

@@ -15 +15 @@
 
     CContextServer(CContext* parent, MPI_Comm intraComm, MPI_Comm interComm) ;
-    CContextServer(CContext* parent, int srvLvl, MPI_Comm intraComm, MPI_Comm interComm) ;
-    bool eventLoop(void) ;
+
+    bool eventLoop(bool enableEventsProcessing = true);
     void listen(void) ;
     void checkPendingRequest(void) ;
-    void processRequest(int rank, char* buff,int count) ;
     void processEvents(void) ;
+    bool hasFinished(void);
     void dispatchEvent(CEventServer& event) ;
     void setPendingEvent(void) ;
     bool hasPendingEvent(void) ;
-    bool hasFinished(void);
+
+    void processRequest(int rank, char* buff,int count) ;
 
     MPI_Comm intraComm ;
@@ -45 +46 @@
     bool scheduled  ;    /*!< event of current timeline is alreading scheduled ? */
     size_t hashId ;
+
+    static size_t getTotalBuf(void);
+
     ~CContextServer() ;
 
     private:
-      std::map<int, StdSize> mapBufferSize_;
+    std::map<int, StdSize> mapBufferSize_;
+    static size_t totalBuf_ ;  /*!< Total memory allocated by servers per context.*/
 
   } ;

XIOS/dev/dev_olga/src/cxios.cpp

@@ -136 +136 @@
 
   //! Initialize server then put it into listening state
-  void CXios::initServerSide(int serverLvl)
+  void CXios::initServerSide(void)
   {
     initServer();
-
-//    if (serverLvl == 1)
-//      isClient = true;
-//    else
-//      isClient = false;
-//
-//    isServer = true;
-//    serverLevel = serverLvl;
-
-
 
     // Initialize all aspects MPI
@@ -163 +153 @@
     {
       if (CServer::serverLevel == 0)
-//      if (CXios::serverLevel == 0)
       {
         CServer::openInfoStream(serverFile);
@@ -169 +158 @@
       }
       else if (CServer::serverLevel == 1)
-//      else if (CXios::serverLevel == 1)
       {
         CServer::openInfoStream(serverPrmFile);

XIOS/dev/dev_olga/src/cxios.hpp

@@ -16 +16 @@
      static void initialize(void) ;
      static void initClientSide(const string & codeId, MPI_Comm& localComm, MPI_Comm& returnComm) ;
-     static void initServerSide(int serverLevel) ;
+     static void initServerSide(void) ;
      static void clientFinalize(void) ;
      static void parseFile(const string& filename) ;

XIOS/dev/dev_olga/src/distribution_server.hpp

@@ -41 +41 @@
     const std::vector<int>& getZoomSizeServer() const;
     const GlobalLocalMap& getGlobalLocalIndex() const { return globalLocalIndexMap_; }
-    const std::vector<CArray<int,1> >& getGlobalIndexEachDimension() const; 
+    const std::vector<CArray<int,1> >& getGlobalIndexEachDimension() const {return globalIndexEachDimension_;}
 
     virtual CArray<size_t,1> computeLocalIndex(const CArray<size_t,1>& globalIndex);

XIOS/dev/dev_olga/src/interface/c/icdata.cpp

@@ -47 +47 @@
    }
 
-   void cxios_init_server(int server_level)
-   {
-     CXios::initServerSide(server_level);
+   void cxios_init_server(void)
+   {
+     CXios::initServerSide();
    }
 

XIOS/dev/dev_olga/src/interface/fortran/idata.F90

@@ -7 +7 @@
    INTERFACE ! Ne pas appeler directement/Interface FORTRAN 2003 <-> C99
 
-      SUBROUTINE  cxios_init_server(server_level) BIND(C)
-         USE ISO_C_BINDING
-         INTEGER  (kind = C_INT)    , VALUE         :: server_level
+      SUBROUTINE  cxios_init_server() BIND(C)
+         USE ISO_C_BINDING
       END SUBROUTINE cxios_init_server
 
@@ -453 +452 @@
    CONTAINS ! Fonctions disponibles pour les utilisateurs.
 
-   SUBROUTINE  xios(init_server)(server_level)
-   IMPLICIT NONE
-       INTEGER  (kind = C_INT), INTENT(IN)         :: server_level
-       CALL cxios_init_server(server_level)
+   SUBROUTINE  xios(init_server)()
+   IMPLICIT NONE
+       CALL cxios_init_server()
    END SUBROUTINE xios(init_server)
 

XIOS/dev/dev_olga/src/io/nc4_data_output.cpp

@@ -231 +231 @@
                        SuperClassWriter::writeData(CArray<double,1>(lat.copy()), latid, isCollective, 0);
                        CArray<double,1> lon = domain->lonvalue(Range(0,local_size_write[1])) ;
+//                       CArray<double,1> lon = domain->lonvalue(Range(0,local_size_write[1]-1)) ;
                        SuperClassWriter::writeData(CArray<double,1>(lon.copy()), lonid, isCollective, 0);
                        break;

XIOS/dev/dev_olga/src/node/axis.cpp

@@ -174 +174 @@
    std::map<int, StdSize> CAxis::getAttributesBufferSize()
    {
-     CContextClient* client = CContext::getCurrent()->client;
+//     CContextClient* client = CContext::getCurrent()->client;
+     // For now the assumption is that secondary server pools consist of the same number of procs.
+     // CHANGE the line below if the assumption changes.
+     CContext* context = CContext::getCurrent();
+     CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client;
 
      std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes();
@@ -455 +459 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[i] : context->client;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
       int nbServer = client->serverSize;
       int range, clientSize = client->clientSize;
@@ -751 +755 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[i] : context->client;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
 
       CEventClient event(getType(), EVENT_ID_NON_DISTRIBUTED_ATTRIBUTES);
@@ -857 +861 @@
     //int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[i] : context->client;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
 
       CEventClient eventData(getType(), EVENT_ID_DISTRIBUTED_ATTRIBUTES);

XIOS/dev/dev_olga/src/node/context.cpp

@@ -249 +249 @@
 
      hasClient = true;
+
      if (CServer::serverLevel != 1)
-//     if (CXios::serverLevel != 1)
-     // initClient is called by client pool
+     // initClient is called by client
      {
        client = new CContextClient(this, intraComm, interComm, cxtServer);
        server = new CContextServer(this, intraComm, interComm);
-     }
+       MPI_Comm intraCommServer, interCommServer;
+       if (cxtServer) // Attached mode
+       {
+         intraCommServer = intraComm;
+         interCommServer = interComm;
+       }
+       else
+       {
+//         MPI_Comm_dup(intraComm, &intraCommServer);
+//         comms.push_back(intraCommServer);
+//         MPI_Comm_dup(interComm, &interCommServer);
+//         comms.push_back(interCommServer);
+       }
+     }
+
      else
-     // initClient is called by primary server pool
+     // initClient is called by primary server
      {
        clientPrimServer.push_back(new CContextClient(this, intraComm, interComm));
@@ -263 +277 @@
      }
 
-     registryIn=new CRegistry(intraComm);
-     registryIn->setPath(getId()) ;
-     if (client->clientRank==0) registryIn->fromFile("xios_registry.bin") ;
-     registryIn->bcastRegistry() ;
-
-     registryOut=new CRegistry(intraComm) ;
-     registryOut->setPath(getId()) ;
-
-     MPI_Comm intraCommServer, interCommServer;
-     if (cxtServer) // Attached mode
-     {
-       intraCommServer = intraComm;
-       interCommServer = interComm;
-     }
-     else
-     {
-       MPI_Comm_dup(intraComm, &intraCommServer);
-       comms.push_back(intraCommServer);
-       MPI_Comm_dup(interComm, &interCommServer);
-       comms.push_back(interCommServer);
-     }
+
+
+//     registryIn=new CRegistry(intraComm);
+//     registryIn->setPath(getId()) ;
+//     if (client->clientRank==0) registryIn->fromFile("xios_registry.bin") ;
+//     registryIn->bcastRegistry() ;
+//
+//     registryOut=new CRegistry(intraComm) ;
+//     registryOut->setPath(getId()) ;
+
+
    }
 
@@ -353 +357 @@
 //     client = new CContextClient(this,intraComm,interComm, cxtClient);
 
-     registryIn=new CRegistry(intraComm);
-     registryIn->setPath(getId()) ;
-     if (server->intraCommRank==0) registryIn->fromFile("xios_registry.bin") ;
-     registryIn->bcastRegistry() ;
-     registryOut=new CRegistry(intraComm) ;
-     registryOut->setPath(getId()) ;
+//     registryIn=new CRegistry(intraComm);
+//     registryIn->setPath(getId()) ;
+//     if (server->intraCommRank==0) registryIn->fromFile("xios_registry.bin") ;
+//     registryIn->bcastRegistry() ;
+//     registryOut=new CRegistry(intraComm) ;
+//     registryOut->setPath(getId()) ;
 
      MPI_Comm intraCommClient, interCommClient;
@@ -368 +372 @@
      else
      {
-       MPI_Comm_dup(intraComm, &intraCommClient);
-       comms.push_back(intraCommClient);
-       MPI_Comm_dup(interComm, &interCommClient);
-       comms.push_back(interCommClient);
+//       MPI_Comm_dup(intraComm, &intraCommClient);
+//       comms.push_back(intraCommClient);
+//       MPI_Comm_dup(interComm, &interCommClient);
+//       comms.push_back(interCommClient);
      }
 
@@ -377 +381 @@
 
    //! Server side: Put server into a loop in order to listen message from client
-   bool CContext::eventLoop(void)
-   {
-     if (CServer::serverLevel == 0)
-     {
-       return server->eventLoop();
-     }
-     else if (CServer::serverLevel == 1)
-     {
-       bool serverFinished = server->eventLoop();
-       bool serverPrimFinished = true;
-       for (int i = 0; i < serverPrimServer.size(); ++i)
-       {
-         serverPrimFinished *= serverPrimServer[i]->eventLoop();
-       }
-       return ( serverFinished && serverPrimFinished);
-     }
-     else
-     {
-       return server->eventLoop();
-     }
-   }
+//   bool CContext::eventLoop(void)
+//   {
+//     if (CServer::serverLevel == 0)
+//     {
+//       return server->eventLoop();
+//     }
+//     else if (CServer::serverLevel == 1)
+//     {
+//       bool serverFinished = server->eventLoop();
+//       bool serverPrimFinished = true;
+//       for (int i = 0; i < serverPrimServer.size(); ++i)
+//       {
+//         serverPrimFinished *= serverPrimServer[i]->eventLoop();
+//       }
+//       return ( serverFinished && serverPrimFinished);
+//     }
+//     else
+//     {
+//       return server->eventLoop();
+//     }
+//   }
 
    //! Try to send the buffers and receive possible answers
@@ -405 +409 @@
      {
        client->checkBuffers();
-       return server->eventLoop();
-     }
+       bool hasTmpBufferedEvent = client->hasTemporarilyBufferedEvent();
+       if (hasTmpBufferedEvent)
+         hasTmpBufferedEvent = !client->sendTemporarilyBufferedEvent();
+
+       // Don't process events if there is a temporarily buffered event
+       return server->eventLoop(!hasTmpBufferedEvent);
+     }
+
      else if (CServer::serverLevel == 1)
      {
        client->checkBuffers();
+       bool hasTmpBufferedEvent = client->hasTemporarilyBufferedEvent();
+       if (hasTmpBufferedEvent)
+         hasTmpBufferedEvent = !client->sendTemporarilyBufferedEvent();
+       bool serverFinished = server->eventLoop(!hasTmpBufferedEvent);
+
+       bool serverPrimFinished = true;
        for (int i = 0; i < clientPrimServer.size(); ++i)
+       {
          clientPrimServer[i]->checkBuffers();
-       bool serverFinished = server->eventLoop();
-       bool serverPrimFinished = true;
-       for (int i = 0; i < serverPrimServer.size(); ++i)
-       {
-         serverPrimFinished *= serverPrimServer[i]->eventLoop();
+         bool hasTmpBufferedEventPrim = clientPrimServer[i]->hasTemporarilyBufferedEvent();
+         if (hasTmpBufferedEventPrim)
+           hasTmpBufferedEventPrim = !clientPrimServer[i]->sendTemporarilyBufferedEvent();
+         serverPrimFinished *= serverPrimServer[i]->eventLoop(hasTmpBufferedEventPrim);
        }
        return ( serverFinished && serverPrimFinished);
      }
+
      else if (CServer::serverLevel == 2)
      {
        client->checkBuffers();
-       return server->eventLoop();
+       bool hasTmpBufferedEvent = client->hasTemporarilyBufferedEvent();
+       if (hasTmpBufferedEvent)
+         hasTmpBufferedEvent = !client->sendTemporarilyBufferedEvent();
+       return server->eventLoop(!hasTmpBufferedEvent);
      }
    }
@@ -433 +453 @@
      {
        finalized = true;
-       if (hasClient) sendRegistry() ;
+//       if (hasClient) sendRegistry() ;
+
+       client->finalize();
+       while (!server->hasFinished())
+       {
+         server->eventLoop();
+       }
 
        if ((hasClient) && (hasServer))
@@ -450 +476 @@
        }
 
-       client->finalize();
-       while (!server->hasFinished())
-       {
-         server->eventLoop();
-       }
-
-       if (hasServer)
+       report(0)<< " Memory report : Context <"<<getId()<<"> : server side : total memory used for buffers "<<CContextServer::getTotalBuf()<<" bytes"<<endl;
+
+//       if (hasServer)
+       if (hasServer && !hasClient)
        {
          closeAllFile();
-         registryOut->hierarchicalGatherRegistry() ;
-         if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
+//         registryOut->hierarchicalGatherRegistry() ;
+//         if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
        }
 
@@ -472 +495 @@
          MPI_Comm_free(&(*it));
        comms.clear();
-
 
       }
@@ -1627 +1649 @@
   }
 
+  bool CContext::isFinalized(void)
+  {
+    return finalized;
+  }
+
 } // namespace xios

XIOS/dev/dev_olga/src/node/context.hpp

@@ -95 +95 @@
 
          // Put sever or client into loop state
-         bool eventLoop(void);
-
          bool checkBuffersAndListen(void);
 
@@ -167 +165 @@
          void recvRegistry(CBufferIn& buffer) ; //!< registry is received by the servers
 
+         bool isFinalized(void);
+
          // dispatch event
          static bool dispatchEvent(CEventServer& event);
@@ -208 +208 @@
          virtual bool hasChild(void) const;
 
+
       public :
          // Calendar of context
@@ -238 +239 @@
          CContextServer* server;
 
-         // Client-server pair in case of secondary server pool
 //         CContextClient* clientPrimServer;
 //         CContextServer* serverPrimServer;
@@ -250 +250 @@
          StdString idServer_;
          CGarbageCollector garbageCollector;
-         std::list<MPI_Comm> comms; //!< Communicators allocated internally
+         std::list<MPI_Comm> comms; //!< Communicators allocated internally  --- significance??
 
       public: // Some function maybe removed in the near future

XIOS/dev/dev_olga/src/node/domain.cpp

@@ -156 +156 @@
    std::map<int, StdSize> CDomain::getAttributesBufferSize()
    {
-     CContextClient* client = CContext::getCurrent()->client;
+     CContext* context = CContext::getCurrent();
+     // For now the assumption is that secondary server pools consist of the same number of procs.
+     // CHANGE the line below if the assumption changes.
+     CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client;
 
      std::map<int, StdSize> attributesSizes = getMinimumBufferSizeForAttributes();
@@ -272 +275 @@
      this->isRedistributed_ = true;
      CContext* context = CContext::getCurrent();
-     CContextClient* client = context->client;
+     // For now the assumption is that secondary server pools consist of the same number of procs.
+     // CHANGE the line below if the assumption changes.
+     CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client;
      int rankClient = client->clientRank;
      int rankOnDomain = rankClient%nbLocalDomain;
@@ -508 +513 @@
    {
           CContext* context = CContext::getCurrent();
-      CContextClient* client = context->client;
+    // For now the assumption is that secondary server pools consist of the same number of procs.
+    // CHANGE the line below if the assumption changes.
+    CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client;
           lon_g.resize(ni_glo) ;
           lat_g.resize(nj_glo) ;
@@ -1351 +1358 @@
       this->computeConnectedClients();
        // if (hasLonLat || hasArea || isCompressible_) this->computeConnectedClients();
-       if (hasLonLat) this->completeLonLatClient();
+       if (hasLonLat)
+         if (!context->hasServer)
+           this->completeLonLatClient();
      }
 
@@ -1397 +1406 @@
      if (context->hasClient)
      {
+//       this->completeLonLatClient();
        sendAttributes();
      }
@@ -1444 +1454 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* contextClientTmp = (context->hasServer) ? context->clientPrimServer[i]
-                                                                         : context->client;    
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* contextClientTmp = (context->hasServer) ? context->clientPrimServer[p]
+                                                                       : context->client;
+
       int nbServer = contextClientTmp->serverSize;
       std::vector<int> nGlobDomain(2);
@@ -1498 +1509 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[i] : context->client;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
       int nbServer=client->serverSize;
       int rank = client->clientRank;
@@ -1668 +1679 @@
       for (it = globalIndexDomainOnServer.begin(); it != ite; ++it) {
         connectedServerRank_.push_back(it->first);
-//        std::vector<size_t> vec = it->second;
-//        std::sort(vec.begin(), vec.end());
-//        indSrv_[it->first] = vec;
-      }
-
-      indSrv_.swap(globalIndexDomainOnServer);
+        std::vector<size_t> vec = it->second;
+        std::sort(vec.begin(), vec.end());
+        indSrv_[it->first] = vec;
+      }
+
+//      indSrv_.swap(globalIndexDomainOnServer);
       nbConnectedClients_ = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_);
 
       clientServerMap->computeServerIndexMapping(globalIndexDomainZoom);
       CClientServerMapping::GlobalIndexMap& globalIndexDomainZoomOnServer = clientServerMap->getGlobalIndexOnServer();
-      indZoomSrv_.swap(globalIndexDomainZoomOnServer);
+//      indZoomSrv_.swap(globalIndexDomainZoomOnServer);
       std::vector<int> connectedServerZoomRank(indZoomSrv_.size());
-      for (it = indZoomSrv_.begin(); it != indZoomSrv_.end(); ++it)
+//      for (it = indZoomSrv_.begin(); it != indZoomSrv_.end(); ++it)
+//        connectedServerZoomRank.push_back(it->first);
+      for (it = globalIndexDomainZoomOnServer.begin(); it != globalIndexDomainZoomOnServer.end(); ++it)
+      {
         connectedServerZoomRank.push_back(it->first);
+        std::vector<size_t> vec = it->second;
+        std::sort(vec.begin(), vec.end());
+        indZoomSrv_[it->first] = vec;
+      }
       nbConnectedClientsZoom_ = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerZoomRank);
 
@@ -1699 +1717 @@
   void CDomain::sendAttributes()
   {
+    sendDistributionAttributes();
     sendIndex();
-    sendDistributionAttributes();
     sendMask();
     sendLonLat();
@@ -1717 +1735 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[i] : context->client;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
 
       CEventClient eventIndex(getType(), EVENT_ID_INDEX);
@@ -1788 +1806 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[i] : context->client;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
 
       // send area for each connected server
@@ -1835 +1853 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[i] : context->client;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
 
       // send area for each connected server
@@ -1885 +1903 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[i] : context->client;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
 
       // send lon lat for each connected server
@@ -1954 +1972 @@
         eventLat.push(rank, nbConnectedClients_[rank], list_msgsLat.back());
       }
-
       client->sendEvent(eventLon);
       client->sendEvent(eventLat);
@@ -1971 +1988 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[i] : context->client;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[p] : context->client;
 
       // send area for each connected server
@@ -2204 +2221 @@
         for (idx =0; idx < ni_zoom; ++idx)
         {
-           if ((ibegin <= zoom_i_index(idx)) && (zoom_i_index(idx) < ibegin+ni) && (nbIZoom < ni))
+          if ((ibegin <= zoom_i_index(idx)) && (zoom_i_index(idx) < ibegin+ni) && (nbIZoom < ni))
             ++nbIZoom;
-           if ((jbegin <= zoom_j_index(idx)) && (zoom_j_index(idx) < jbegin+nj) && (nbJZoom < nj))
+          if ((jbegin <= zoom_j_index(idx)) && (zoom_j_index(idx) < jbegin+nj) && (nbJZoom < nj))
             ++nbJZoom;
         }
@@ -2237 +2254 @@
       }
             
-      MPI_Scan(&count_write_index_[0], &start_write_index_[0], 2, MPI_INT, MPI_SUM, server->intraComm);      
-      start_write_index_[0] = 0; 
-      start_write_index_[1] -= count_write_index_[1];
+      MPI_Scan(&count_write_index_[0], &start_write_index_[0], 2, MPI_INT, MPI_SUM, server->intraComm);
+      if ((this->type) != CDomain::type_attr::unstructured)
+      {
+        start_write_index_[0] = 0;
+        start_write_index_[1] -= count_write_index_[1];
+      }
+      else
+      {
+        start_write_index_[0] -= count_write_index_[0];
+      }
       local_write_size_[0] = count_write_index_[0];
       local_write_size_[1] = count_write_index_[1];
       MPI_Allreduce(&count_write_index_[0], &global_write_size_[0], 2, MPI_INT, MPI_SUM, server->intraComm);
-      global_write_size_[0] = count_write_index_[0];
-      global_write_size_[1] = (global_write_size_[1] > nj_glo) ? nj_glo : global_write_size_[1];
-         
+      if ((this->type) != CDomain::type_attr::unstructured)
+      {
+        global_write_size_[0] = count_write_index_[0];
+        global_write_size_[1] = (global_write_size_[1] > nj_glo) ? nj_glo : global_write_size_[1];
+      }
     }
 

XIOS/dev/dev_olga/src/node/field.cpp

@@ -125 +125 @@
 //    CContextClient* client = context->client;
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
-    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = (!context->hasServer) ? context->client : context->clientPrimServer[i];
+//    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 0) : 1;
+//    for (int i = 0; i < nbSrvPools; ++i)
+//    {
+//      CContextClient* client = (!context->hasServer) ? context->client : context->clientPrimServer[i];
+      CContextClient* client = (!context->hasServer) ? context->client : this->file->getContextClient();
 
       CEventClient event(getType(), EVENT_ID_UPDATE_DATA);
@@ -176 +177 @@
         client->sendEvent(event);
       }
-    }
+//    }
 
     CTimer::get("XIOS Send Data").suspend();
   }
 
+/*
   void CField::sendUpdateData(const CArray<double,1>& data, CContextClient* client)
   {
@@ -233 +235 @@
     CTimer::get("XIOS Send Data").suspend();
   }
-
+*/
   void CField::recvUpdateData(CEventServer& event)
   {

XIOS/dev/dev_olga/src/node/field.hpp

@@ -146 +146 @@
         static bool dispatchEvent(CEventServer& event);
         void sendUpdateData(const CArray<double,1>& data);
-        void sendUpdateData(const CArray<double,1>& data, CContextClient* client);
+//        void sendUpdateData(const CArray<double,1>& data, CContextClient* client);
         static void recvUpdateData(CEventServer& event);
         void recvUpdateData(std::map<int,CBufferIn*>& rankBuffers);

XIOS/dev/dev_olga/src/node/file.cpp

@@ -270 +270 @@
     void CFile::checkFile(void)
     {
-      if (mode.isEmpty() || mode.getValue() == mode_attr::write)
+      CContext* context = CContext::getCurrent();
+      // Done by classical server or secondary server
+      if (!CXios::usingServer2 || (CXios::usingServer2 && !context->hasClient))
       {
-        if (!isOpen) createHeader();
-        checkSync();
-      }
-      else
-      {
-        if (!isOpen) openInReadMode();
-      }
-      checkSplit();
+        if (mode.isEmpty() || mode.getValue() == mode_attr::write)
+        {
+          if (!isOpen) createHeader();
+          checkSync();
+        }
+        else
+        {
+          if (!isOpen) openInReadMode();
+        }
+        checkSplit();
+      }
     }
 
@@ -590 +595 @@
      CContext* context = CContext::getCurrent();
      CContextClient* client=context->client;
+//     CContextClient* client = (0 != context->clientPrimServer.size()) ? context->clientPrimServer[0] : context->client;
 
      // It would probably be better to call initFile() somehow

XIOS/dev/dev_olga/src/node/file.hpp

@@ -170 +170 @@
       private :
          /// Propriétés privées ///
+         CContextClient* client;
          CFieldGroup* vFieldGroup;
          CVariableGroup* vVariableGroup;
@@ -175 +176 @@
          boost::shared_ptr<CDataInput> data_in;
          std::vector<CField*> enabledFields;
-         CContextClient* client;
 
       public:

XIOS/dev/dev_olga/src/node/grid.cpp

@@ -630 +630 @@
      computeClientIndex();
      if (context->hasClient)
-      computeConnectedClients();
+     {
+       computeConnectedClients();
+     }
 
 
@@ -792 +794 @@
          }
 
-         nbIndexOnServer = 0;
-         for (it = itb; it != ite; ++it)
+//         nbIndexOnServer = 0;
+//         for (it = itb; it != ite; ++it)
+//         {
+//           const std::vector<int>& tmp = it->second;
+//           nbIndexOnServerTmp = 0;
+//           for (int i = 0; i < tmp.size(); ++i)
+//           {
+//             if (0 == nbIndexOnServerTmp(tmp[i]))
+//             {
+//               globalElementIndexOnServer[idx][tmp[i]][nbIndexOnServer(tmp[i])] = it->first;
+//               ++nbIndexOnServerTmp(tmp[i]);
+//             }
+//           }
+//           nbIndexOnServer += nbIndexOnServerTmp;
+//         }
+//       }
+
+       nbIndexOnServer = 0;
+//       for (it = itb; it != ite; ++it)
+       for (size_t j = 0; j < globalIndexElementOnServerMap.size(); ++j)
+       {
+         it = globalIndexElementOnServerMap.find(globalIndexElementOnClient(j));
+         if (it != ite)
          {
            const std::vector<int>& tmp = it->second;
@@ -808 +831 @@
          }
        }
+     }
 
       // Determine server which contain global source index
@@ -1236 +1260 @@
     // int nbSrvPools = (context->hasServer) ? context->clientPrimServer.size() : 1;
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextClient* client = context->hasServer ? context->clientPrimServer[i] : context->client ;
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextClient* client = context->hasServer ? context->clientPrimServer[p] : context->client ;
 
       CEventClient event(getType(), EVENT_ID_INDEX);
@@ -1397 +1421 @@
     int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
     nbSrvPools = 1;
-    for (int i = 0; i < nbSrvPools; ++i)
-    {
-      CContextServer* server = (context->hasServer) ? context->server : context->serverPrimServer[i];
-      CContextClient* client = (context->hasServer) ? context->client : context->clientPrimServer[i];
+    for (int p = 0; p < nbSrvPools; ++p)
+    {
+      CContextServer* server = (!context->hasClient) ? context->server : context->serverPrimServer[p];
+      CContextClient* client = (!context->hasClient) ? context->client : context->clientPrimServer[p];
+//      CContextServer* server = (context->hasServer) ? context->server : context->serverPrimServer[p];
+//      CContextClient* client = (context->hasServer) ? context->client : context->clientPrimServer[p];
       numberWrittenIndexes_ = totalNumberWrittenIndexes_ = offsetWrittenIndexes_ = 0;
       connectedServerRank_ = ranks;
@@ -1503 +1529 @@
         else
         {
-          dataSize = outIndex.numElements();
-          for (int i = 0; i < outIndex.numElements(); ++i) outIndex(i) = i;
+//          dataSize = outIndex.numElements();
+//          for (int i = 0; i < outIndex.numElements(); ++i) outIndex(i) = i;
+          // THE PROBLEM HERE IS THAT DATA CAN BE NONDISTRIBUTED ON CLIENT AND DISTRIBUTED ON SERVER
+          // BELOW IS THE TEMPORARY FIX only for a single type of element (domain, asix, scalar)
+          dataSize = serverDistribution_->getGlobalIndexEachDimension()[0].numElements();
+          outIndex.resize(dataSize);
+          outIndex = serverDistribution_->getGlobalIndexEachDimension()[0];
+
         }
         writtenDataSize_ += dataSize;

XIOS/dev/dev_olga/src/object_template_impl.hpp

@@ -296 +296 @@
       }
     }
-//     // if (!context->hasServer)
-//     if (context->hasClient)
-//     {
-//        CContextClient* client=context->client;
-
-//        CEventClient event(getType(),EVENT_ID_SEND_ATTRIBUTE);
-//        if (client->isServerLeader())
-//        {
-//          CMessage msg;
-// //       msg << this->getId();
-//          msg << this->getIdServer();
-//          msg << attr.getName();
-//          msg << attr;
-//          const std::list<int>& ranks = client->getRanksServerLeader();
-//          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
-//            event.push(*itRank,1,msg);
-//          client->sendEvent(event);
-//        }
-//        else client->sendEvent(event);
-//     }
-
   }
 

XIOS/dev/dev_olga/src/server.cpp

@@ -20 +20 @@
     list<MPI_Comm> CServer::interCommLeft ;
     list<MPI_Comm> CServer::interCommRight ;
-    list<MPI_Comm> CServer::interComm ;
+//    list<MPI_Comm> CServer::interComm ;
     std::list<MPI_Comm> CServer::contextInterComms;
     int CServer::serverLevel = 0 ;
+    int CServer::serverLeader = 0;
+    int CServer::serverSize = 0;
     int CServer::nbPools = 0;
     int CServer::poolId = 0;
-    int CServer::serverSize = 0;
     bool CServer::isRoot = false ;
     int CServer::rank = INVALID_RANK;
@@ -38 +39 @@
 /*!
  * \fn void CServer::initialize(void)
- * Creates intraComm and interComm for a server pool (primary or secondary).
+ * Creates intraComm for each possible type of servers (classical, primary or secondary).
+ * In case of secondary servers intraComm is created for each secondary server pool.
+ * (For now the assumption is that there is one proc per pool.)
+ * Creates the following lists of interComms:
+ *   classical server -- interCommLeft
+ *   primary server -- interCommLeft and interCommRight
+ *   secondary server -- interComm for each pool.
  */
     void CServer::initialize(void)
@@ -64 +71 @@
 
         unsigned long* hashAll ;
-//        unsigned long* hashAllServers ;
 
 //        int rank ;
@@ -129 +135 @@
               MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
                interCommLeft.push_back(newComm) ;
-               interComm.push_back(newComm) ;
             }
           }
@@ -135 +140 @@
         else if (serverLevel == 1)
         {
-          int clientLeader, srvPrmLeader, srvSndLeader;
+          int clientLeader, srvSndLeader;
+          int srvPrmLeader ;
           for (it=leaders.begin();it!=leaders.end();it++)
           {
@@ -148 +154 @@
               MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
               interCommLeft.push_back(newComm) ;
-              interComm.push_back(newComm) ;
             }
             else
-              srvPrmLeader = it->second;
+              serverLeader = it->second;
           }
 
           for (int i = 0; i < nbPools; ++i)
           {
-            srvSndLeader = srvPrmLeader + serverSize - nbPools + i;
-//            CClient::initializeClientOnServer(rank, serversComm, srvSndLeader);
-            CClient::initializeClientOnServer(rank, intraComm, srvSndLeader);
-            interCommRight.push_back(CClient::getInterComm());
-            interComm.push_back(CClient::getInterComm());
+            srvSndLeader = serverLeader + serverSize - nbPools + i;
+            int intraCommSize, intraCommRank ;
+            MPI_Comm_size(intraComm, &intraCommSize) ;
+            MPI_Comm_rank(intraComm, &intraCommRank) ;
+            info(50)<<"intercommCreate::client "<<rank<<" intraCommSize : "<<intraCommSize
+                <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< srvSndLeader<<endl ;
+            MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvSndLeader, 0, &newComm) ;
+            interCommRight.push_back(newComm) ;
           }
         } // primary server
@@ -175 +183 @@
           MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
           interCommLeft.push_back(newComm) ;
-          interComm.push_back(newComm) ;
         } // secondary server
 
@@ -210 +217 @@
           if (rank==0) MPI_Send(&globalRank,1,MPI_INT,0,0,newComm) ;
           MPI_Comm_remote_size(newComm,&size);
-          interComm.push_back(newComm) ;
+//          interComm.push_back(newComm) ;
+          interCommLeft.push_back(newComm) ;
         }
               oasis_enddef() ;
       }
 
-      MPI_Comm_rank(intraComm, &rank) ;
-      if (rank==0) isRoot=true;
+      int rankServer;
+      MPI_Comm_rank(intraComm, &rankServer) ;
+      if (rankServer==0) isRoot=true;
       else isRoot=false;
       
@@ -235 +244 @@
 //        MPI_Comm_free(&(*it));
 
-      for (std::list<MPI_Comm>::iterator it = interCommLeft.begin(); it != interCommLeft.end(); it++)
-        MPI_Comm_free(&(*it));
-
-      for (std::list<MPI_Comm>::iterator it = interCommRight.begin(); it != interCommRight.end(); it++)
-        MPI_Comm_free(&(*it));
+        for (std::list<MPI_Comm>::iterator it = interCommLeft.begin(); it != interCommLeft.end(); it++)
+          MPI_Comm_free(&(*it));
+
+        for (std::list<MPI_Comm>::iterator it = interCommRight.begin(); it != interCommRight.end(); it++)
+          MPI_Comm_free(&(*it));
 
       MPI_Comm_free(&intraComm);
@@ -273 +282 @@
 
          contextEventLoop() ;
+//         if (finished && contextList.empty()) stop=true ;
          if (finished && contextList.empty()) stop=true ;
          eventScheduler->checkEvent() ;
@@ -294 +304 @@
            if (flag==true)
            {
-             MPI_Recv(&msg,1,MPI_INT,0,0,*it,&status) ;
-             info(20)<<" CServer : Receive client finalize"<<endl ;
-
-             // If primary server, send finalize to secondary server pool(s)
-             for(itr=interCommRight.begin(); itr!=interCommRight.end(); itr++)
-             {
-               MPI_Send(&msg,1,MPI_INT,0,0,*itr) ;
-//               MPI_Comm_free(&(*itr));
-//               interCommRight.erase(itr) ;
-             }
-
+              MPI_Recv(&msg,1,MPI_INT,0,0,*it,&status) ;
+              info(20)<<" CServer : Receive client finalize"<<endl ;
+              // Sending server finalize message to secondary servers (if any)
+              for(itr=interCommRight.begin();itr!=interCommRight.end();itr++)
+              {
+                MPI_Send(&msg,1,MPI_INT,0,0,*itr) ;
+//                itr = interCommRight.erase(itr) ;
+              }
               MPI_Comm_free(&(*it));
-//              interComm.erase(it) ;
               interCommLeft.erase(it) ;
               break ;
@@ -312 +318 @@
          }
 
-        if (interCommLeft.empty())
-//        if (interComm.empty())
+         if (interCommLeft.empty())
          {
            int i,size ;
@@ -381 +386 @@
            MPI_Get_count(&status,MPI_CHAR,&count) ;
            recvContextMessage(buffer,count) ;
-           delete [] buffer ;
+           delete [] buffer;
            recept=false ;
          }
@@ -390 +395 @@
      {
        static map<string,contextMessage> recvContextId;
-
        map<string,contextMessage>::iterator it ;
-
        CBufferIn buffer(buff,count) ;
        string id ;
@@ -434 +437 @@
      void CServer::listenRootContext(void)
      {
-
        MPI_Status status ;
        int flag ;
@@ -464 +466 @@
            MPI_Get_count(&status,MPI_CHAR,&count) ;
            registerContext(buffer,count) ;
-
            delete [] buffer ;
            recept=false ;
@@ -484 +485 @@
                << "Context '" << contextId << "' has already been registred");
 
-       MPI_Comm contextInterComm;
-       MPI_Intercomm_create(intraComm,0,CXios::globalComm,leaderRank,10+leaderRank,&contextInterComm);
-
-       MPI_Comm inter;
-       MPI_Intercomm_merge(contextInterComm,1,&inter);
-       MPI_Barrier(inter);
-
        context=CContext::create(contextId);
        contextList[contextId]=context;
-       context->initServer(intraComm,contextInterComm);
-       contextInterComms.push_back(contextInterComm);
-
+
+       // All type of servers initialize its own server (CContextServer)
+       if (serverLevel < 2)
+       {
+         MPI_Comm contextInterComm;
+         MPI_Intercomm_create(intraComm, 0, CXios::globalComm, leaderRank, 10+leaderRank, &contextInterComm);
+         MPI_Comm inter;
+         MPI_Intercomm_merge(contextInterComm,1,&inter);
+         MPI_Barrier(inter);
+         MPI_Comm_free(&inter);
+         context->initServer(intraComm,contextInterComm);
+         contextInterComms.push_back(contextInterComm);
+       }
+       else if (serverLevel == 2)
+       {
+         context->initServer(intraComm, interCommLeft.front());
+       }
+
+       // Primary server: send create context message to secondary servers and initialize its own client (CContextClient)
        if (serverLevel == 1)
        {
-//         CClient::registerContext(contextId, intraComm);
-         CClient::registerContextByClienOfServer(contextId, intraComm);
-       }
-
-       MPI_Comm_free(&inter);
-
+         int i = 0, size;
+         CMessage msg;
+         int messageSize;
+         MPI_Comm_size(intraComm, &size) ;
+         for (std::list<MPI_Comm>::iterator it = interCommRight.begin(); it != interCommRight.end(); it++, ++i)
+         {
+           StdString str = contextId +"_server_" + boost::lexical_cast<string>(i);
+           msg<<str<<size<<rank ;
+           messageSize = msg.size() ;
+           buff = new char[messageSize] ;
+           CBufferOut buffer(buff,messageSize) ;
+           buffer<<msg ;
+           int sndServerGloRanks = serverSize-nbPools+serverLeader +i;  // the assumption is that there is only one proc per secondary server pool
+           MPI_Send(buff, buffer.count(), MPI_CHAR, sndServerGloRanks, 1, CXios::globalComm) ;
+           context->initClient(intraComm, *it) ;
+           delete [] buff ;
+         }
+       }
      }
 
@@ -509 +531 @@
      {
        bool finished ;
+
        map<string,CContext*>::iterator it ;
 
        for(it=contextList.begin();it!=contextList.end();it++)
        {
-         finished=it->second->checkBuffersAndListen();
+         finished=it->second->isFinalized();
          if (finished)
          {
@@ -519 +542 @@
            break ;
          }
+         else
+           finished=it->second->checkBuffersAndListen();
        }
      }
@@ -554 +579 @@
       {
         if (serverLevel == 1)
-          id = getRank();
+          id = rank-serverLeader;
         else
           id = poolId;

XIOS/dev/dev_olga/src/server.hpp

@@ -14 +14 @@
       public:
         static void initialize(void);
-        static void initialize(const StdString& serverId);
         static void finalize(void);
         static void eventLoop(void);
@@ -23 +22 @@
         static void listenRootContext(void);
         static void listenRootFinalize(void);
-        static void registerContext(void* buff,int count, int leaderRank=0);        // context registered by the primary server
+        static void registerContext(void* buff,int count, int leaderRank=0);
 
-        // Communicators for the primary group of servers
         static MPI_Comm intraComm;
-        static list<MPI_Comm> interCommLeft;   // interComm between server (primary or secondary) and its client (client or primary server)
-        static list<MPI_Comm> interCommRight;  // interComm between primary server and secondary server (non-empty only for primary server pool)
-        static list<MPI_Comm> interComm;       // interCommLeft + interCommRight
-        static std::list<MPI_Comm> contextInterComms;
+        static list<MPI_Comm> interCommLeft;           // interComm between server (primary, classical or secondary) and its client (client or primary server)
+        static list<MPI_Comm> interCommRight;          // interComm between primary server and secondary server (non-empty only for primary server pool)
+        static std::list<MPI_Comm> contextInterComms;  // significance ??
         static CEventScheduler* eventScheduler;
 
         static int serverLevel ;
-
-//        static int nbSndSrvPools;   // number of secondary server pools
-//        static int poolNb;          // for secondary servers; stores the pool number
 
         struct contextMessage
@@ -46 +40 @@
         static bool isRoot;
 
-        static map<string,CContext*> contextList;       // contexts on the primary server
+        static map<string,CContext*> contextList;
         static bool finished;
         static bool is_MPI_Initialized;
@@ -70 +64 @@
       private:
         static int rank;
-        static int serverSize;  //!< Number of procs dedicated to server
-        static int nbPools;     //!< Number of secondary-server pools
-        static int poolId;      //!< ID of a secondary-server pool
+        static int serverLeader;  //!< Leader of the classical or primary server (needed in case of secondary servers)
+        static int serverSize;    //!< Number of procs dedicated to servers (primary and seconday (if any) combined)
+        static int nbPools;       //!< Number of secondary server pools
+        static int poolId;        //!< id of a secondary server pool starting from 1
         static StdOFStream m_infoStream;
         static StdOFStream m_errorStream;

XIOS/dev/dev_olga/src/xios_server.f90

@@ -4 +4 @@
   INCLUDE "mpif.h"
   INTEGER :: ierr
-  INTEGER            :: server_level = 0  
-  ! 0 in case of a single server pool 
-  ! 1 for primary server in case of two server pools
-  ! 2 for secondary server in case of two server pools
 
-  CALL xios_init_server(server_level)
+  CALL xios_init_server
 
 END PROGRAM server_main