Changeset 1761

Timestamp:

10/18/19 15:40:35 (5 years ago)

Author:

ymipsl

Message:

implementing first guess for service functionnalities.

YM

Location:

XIOS/dev/dev_ym/XIOS_SERVICES/src

Files:

• client.cpp (modified) (9 diffs)
• client.hpp (modified) (4 diffs)
• config/context_attribute.conf (modified) (1 diff)
• context_client.cpp (modified) (7 diffs)
• context_client.hpp (modified) (3 diffs)
• context_server.cpp (modified) (4 diffs)
• context_server.hpp (modified) (2 diffs)
• cxios.cpp (modified) (10 diffs)
• cxios.hpp (modified) (4 diffs)
• filter/store_filter.cpp (modified) (1 diff)
• interface/c/iccalendar.cpp (modified) (1 diff)
• interface/c/icdata.cpp (modified) (32 diffs)
• interface/fortran/ixios.F90 (modified) (1 diff)
• manager/contexts_manager.cpp (added)
• manager/contexts_manager.hpp (added)
• manager/daemons_manager.cpp (added)
• manager/daemons_manager.hpp (added)
• manager/pool_ressource.cpp (added)
• manager/pool_ressource.hpp (added)
• manager/register_context_info.hpp (added)
• manager/ressources_manager.cpp (added)
• manager/ressources_manager.hpp (added)
• manager/server_context.cpp (added)
• manager/server_context.hpp (added)
• manager/servers_ressource.cpp (added)
• manager/servers_ressource.hpp (added)
• manager/services.cpp (added)
• manager/services.hpp (added)
• manager/services_manager.cpp (added)
• manager/services_manager.hpp (added)
• manager/window_manager.cpp (added)
• manager/window_manager.hpp (added)
• node/axis.cpp (modified) (1 diff)
• node/context.cpp (modified) (16 diffs)
• node/context.hpp (modified) (8 diffs)
• node/domain.cpp (modified) (2 diffs)
• node/field.cpp (modified) (1 diff)
• node/file.cpp (modified) (2 diffs)
• node/grid.cpp (modified) (2 diffs)
• object.cpp (modified) (1 diff)
• object.hpp (modified) (1 diff)
• object_template_impl.hpp (modified) (2 diffs)
• server.cpp (modified) (8 diffs)
• server.hpp (modified) (4 diffs)
• test/generic_testcase.f90 (added)

XIOS/dev/dev_ym/XIOS_SERVICES/src/client.cpp

@@ -12 +12 @@
 #include "buffer_client.hpp"
 #include "string_tools.hpp"
+#include "ressources_manager.hpp"
+#include "services_manager.hpp"
+#include <functional>
+#include <cstdio>
+
 
 namespace xios
 {
 
+    const double serverPublishDefaultTimeout=10;
+
     MPI_Comm CClient::intraComm ;
     MPI_Comm CClient::interComm ;
+    MPI_Comm CClient::clientsComm_ ;
+
     std::list<MPI_Comm> CClient::contextInterComms;
     int CClient::serverLeader ;
@@ -24 +33 @@
     StdOFStream CClient::m_infoStream;
     StdOFStream CClient::m_errorStream;
+    CPoolRessource* CClient::poolRessource_=nullptr ;
+
     MPI_Comm& CClient::getInterComm(void)   { return (interComm); }
      
@@ -35 +46 @@
  */
 
+    void CClient::initRessources(void)
+    {
+
+ /*     
+      int commRank; 
+      MPI_Comm_rank(CXios::globalComm,&commRank) ;
+      if (commRank==0)
+      {
+        ressources.createPool("ioserver1",ressources.getRessourcesSize()/2) ;
+      }
+      else if (commRank==1)
+      {
+        ressources.createPool("ioserver2",ressources.getRessourcesSize()/2) ;
+      }
+  */
+    }
+
     void CClient::initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)
+    {
+    
+       MPI_Comm clientComm ;
+      // initialize MPI if not initialized
+      int initialized ;
+      MPI_Initialized(&initialized) ;
+      if (initialized) is_MPI_Initialized=true ;
+      else is_MPI_Initialized=false ;
+      
+      MPI_Comm globalComm=CXios::getGlobalComm() ;
+
+      /////////////////////////////////////////
+      ///////////// PART 1 ////////////////////
+      /////////////////////////////////////////
+      
+
+      // localComm isn't given
+      if (localComm == MPI_COMM_NULL)
+      {
+          
+        // don't use OASIS
+        if (!CXios::usingOasis)
+        {
+
+          if (!is_MPI_Initialized)
+          {
+            MPI_Init(NULL, NULL);
+          }
+          CTimer::get("XIOS").resume() ;
+          CTimer::get("XIOS init/finalize",false).resume() ;
+          
+          // split the global communicator
+          // get hash from all model to attribute a unique color (int) and then split to get client communicator
+          // every mpi process of globalComm (MPI_COMM_WORLD) must participate
+
+          int commRank, commSize ;
+          MPI_Comm_rank(globalComm,&commRank) ;
+          MPI_Comm_size(globalComm,&commSize) ;
+
+          std::hash<string> hashString ;
+          size_t hashClient=hashString(codeId) ;
+          
+          size_t* hashAll = new size_t[commSize] ;
+          MPI_Allgather(&hashClient,1,MPI_UNSIGNED_LONG,hashAll,1,MPI_LONG,globalComm) ;
+          
+          int color=0 ;
+          set<size_t> listHash ;
+          for(int i=0 ; i<=commRank ; i++) 
+            if (listHash.count(hashAll[i])==0)
+            {
+              listHash.insert(hashAll[i]) ;
+              color=color+1 ;
+            }
+          delete[] hashAll ;
+
+          MPI_Comm_split(globalComm, color, commRank, &clientComm) ;
+        }
+        else // using oasis to split communicator
+        {
+          if (!is_MPI_Initialized) oasis_init(codeId) ;
+          oasis_get_localcomm(clientComm) ;
+        }
+      }
+      else // localComm is given
+      {
+        MPI_Comm_dup(localComm,&clientComm) ;
+      }
+      
+     
+      /////////////////////////////////////////
+      ///////////// PART 2 ////////////////////
+      /////////////////////////////////////////
+      
+
+      // Create the XIOS communicator for every process which is related
+      // to XIOS, as well on client side as on server side
+      
+      MPI_Comm xiosGlobalComm ;
+      string strIds=CXios::getin<string>("clients_code_id","") ;
+      vector<string> clientsCodeId=splitRegex(strIds,"\\s*,\\s*") ;
+      if (strIds.empty())
+      {
+         // no code Ids given, suppose XIOS initialisation is global            
+         int commRank, commGlobalRank, serverLeader, clientLeader,serverRemoteLeader,clientRemoteLeader ;
+         MPI_Comm splitComm,interComm ;
+         MPI_Comm_rank(globalComm,&commGlobalRank) ;
+         MPI_Comm_split(globalComm, 0, commGlobalRank, &splitComm) ;
+         int splitCommSize, globalCommSize ;
+        
+         MPI_Comm_size(splitComm,&splitCommSize) ;
+         MPI_Comm_size(globalComm,&globalCommSize) ;
+         if (splitCommSize==globalCommSize) // no server
+         {
+           MPI_Comm_dup(globalComm,&xiosGlobalComm) ;
+           CXios::setXiosComm(xiosGlobalComm) ;
+         }
+         else
+         {
+           MPI_Comm_rank(splitComm,&commRank) ;
+           if (commRank==0) clientLeader=commGlobalRank ;
+           else clientLeader=0 ;
+           serverLeader=0 ;
+           MPI_Allreduce(&clientLeader,&clientRemoteLeader,1,MPI_INT,MPI_SUM,globalComm) ;
+           MPI_Allreduce(&serverLeader,&serverRemoteLeader,1,MPI_INT,MPI_SUM,globalComm) ;
+           MPI_Intercomm_create(splitComm, 0, globalComm, serverRemoteLeader,1341,&interComm) ;
+           MPI_Intercomm_merge(interComm,true,&xiosGlobalComm) ;
+           CXios::setXiosComm(xiosGlobalComm) ;
+         }
+      }
+      else
+      {
+
+        xiosGlobalCommByFileExchange(clientComm, codeId) ;
+      
+      }
+
+      int commRank ;
+      MPI_Comm_rank(CXios::getXiosComm(), &commRank) ;
+      MPI_Comm_split(CXios::getXiosComm(),false,commRank, &clientsComm_) ;
+      
+      // is using server or not ?
+      int xiosCommSize, clientsCommSize ; 
+      MPI_Comm_size(CXios::getXiosComm(), &xiosCommSize) ;
+      MPI_Comm_size(clientsComm_, &clientsCommSize) ;
+      if (xiosCommSize==clientsCommSize) CXios::setUsingServer() ;
+      else CXios::setNotUsingServer() ;
+
+
+      CXios::setGlobalRegistry(new CRegistry(clientsComm_)) ;
+      /////////////////////////////////////////
+      ///////////// PART 3 ////////////////////
+      /////////////////////////////////////////
+     
+      CXios::launchDaemonsManager(false) ;
+      poolRessource_ = new CPoolRessource(clientComm, codeId) ;
+
+      /////////////////////////////////////////
+      ///////////// PART 4 ////////////////////
+      /////////////////////////////////////////      
+      
+      // create the services
+/*
+      int commRank ;
+      MPI_Comm_rank(clientComm,&commRank) ;
+      auto contextsManager=CXios::getContextsManager() ;
+    
+      if (commRank==0) 
+      {
+        contextsManager->createServerContext(CXios::defaultPoolId, CXios::defaultGathererId, 0, codeId) ;
+      }
+      
+      MPI_Comm interComm ;
+
+      contextsManager->createServerContextIntercomm(CXios::defaultPoolId, CXios::defaultGathererId, 0, codeId, clientComm, interComm) ;
+*/  
+/*      while (true) 
+      {
+
+      }
+*/      
+      returnComm = clientComm ;
+    }
+
+
+    void CClient::xiosGlobalCommByFileExchange(MPI_Comm clientComm, const string& codeId)
+    {
+ 
+      MPI_Comm globalComm=CXios::getGlobalComm() ;
+      MPI_Comm xiosGlobalComm ;
+
+      string strIds=CXios::getin<string>("clients_code_id","") ;
+      vector<string> clientsCodeId=splitRegex(strIds,"\\s*,\\s*") ;
+
+      int commRank, globalRank, clientRank, serverRank ;
+      MPI_Comm_rank(clientComm, &commRank) ;
+      MPI_Comm_rank(globalComm, &globalRank) ;
+      string clientFileName("__xios_publisher::"+codeId+"__to_remove__") ;
+           
+      int error ;
+
+      if (commRank==0) // if root process publish name
+      {  
+        std::ofstream ofs (clientFileName, std::ofstream::out);
+        ofs<<globalRank ;
+        ofs.close();
+        
+  // get server root rank
+
+        std::ifstream ifs ;
+        string fileName=("__xios_publisher::"+CXios::xiosCodeId+"__to_remove__") ;
+      
+        double timeout = CXios::getin<double>("server_puplish_timeout",serverPublishDefaultTimeout) ;
+        double time ;
+          
+        do
+        {
+          CTimer::get("server_publish_timeout").resume() ;  
+          ifs.clear() ;
+          ifs.open(fileName, std::ifstream::in) ;
+          CTimer::get("server_publish_timeout").suspend() ;
+        } while (ifs.fail() && CTimer::get("server_publish_timeout").getCumulatedTime()<timeout) ;
+        
+        if (CTimer::get("server_publish_timeout").getCumulatedTime()>=timeout || ifs.fail())
+        {
+          ifs.clear() ;
+          ifs.close() ;
+          ifs.clear() ;
+          error=true ;            
+        }
+        else 
+        {
+          ifs>>serverRank ;
+          ifs.close() ;
+          error=false ;
+        } 
+
+      } 
+      
+      MPI_Bcast(&error,1,MPI_INT,0,clientComm) ;
+      
+      if (error==false)  // you have a server
+      {
+        MPI_Comm intraComm ;
+        MPI_Comm_dup(clientComm,&intraComm) ;
+        MPI_Comm interComm ;
+        
+        int pos=0 ;
+        for(int i=0 ; codeId!=clientsCodeId[i]; i++) pos=pos+1 ;
+
+        bool high=true ;
+        for(int i=pos ; i<clientsCodeId.size(); i++)
+        {  
+          MPI_Intercomm_create(intraComm, 0, globalComm, serverRank, 3141, &interComm);
+          MPI_Comm_free(&intraComm) ;
+          MPI_Intercomm_merge(interComm,high, &intraComm ) ;
+          high=false ;
+        }
+        xiosGlobalComm=intraComm ;
+      }
+      else  // no server detected
+      {
+        vector<int> clientsRank(clientsCodeId.size()) ;
+        
+        if (commRank==0)
+        {  
+          for(int i=0;i<clientsRank.size();i++)
+          {
+            std::ifstream ifs ;
+            string fileName=("__xios_publisher::"+clientsCodeId[i]+"__to_remove__") ;
+            do
+            {
+              ifs.clear() ;
+              ifs.open(fileName, std::ifstream::in) ;
+            } while (ifs.fail()) ;
+            ifs>>clientsRank[i] ;
+            ifs.close() ;
+          }
+        }
+         
+        int client ;
+        MPI_Comm intraComm ;
+        MPI_Comm_dup(clientComm,&intraComm) ;
+        MPI_Comm interComm ;
+        
+        int pos=0 ;
+        for(int i=0 ; codeId!=clientsCodeId[i]; i++) pos=pos+1 ;
+        
+        bool high=true ;
+        for(int i=pos+1 ; i<clientsCodeId.size(); i++)
+        {  
+          if (codeId==clientsCodeId[0])   // first model play the server rule
+          {          
+            MPI_Intercomm_create(intraComm, 0, globalComm, clientsRank[i], 3141, &interComm);
+            MPI_Intercomm_merge(interComm,false, &intraComm ) ;
+          }
+          else
+          {          
+            MPI_Intercomm_create(intraComm, 0, globalComm, clientsRank[0], 3141, &interComm);
+            MPI_Intercomm_merge(interComm,high, &intraComm ) ;
+            high=false ;
+          }
+        }
+        xiosGlobalComm=intraComm ;
+      }
+
+      MPI_Barrier(xiosGlobalComm);
+      if (commRank==0) std::remove(clientFileName.c_str()) ;         
+      MPI_Barrier(xiosGlobalComm);
+  
+      CXios::setXiosComm(xiosGlobalComm) ;
+
+    }
+
+    void CClient::xiosGlobalCommByPublishing(MPI_Comm clientComm, const string& codeId)
+    {
+
+      // untested. need to be developped an a true MPI compliant library
+
+/*
+        // try to discover other client/server
+        // do you have a xios server ?
+        char portName[MPI_MAX_PORT_NAME];
+        int ierr ;
+        int commRank ;
+        MPI_Comm_rank(clientComm,&commRank) ;
+
+        MPI_Barrier(globalComm) ;
+        if (commRank==0)
+        {
+             
+          MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN );
+          const char* serviceName=CXios::xiosCodeId.c_str() ;
+          ierr=MPI_Lookup_name(CXios::xiosCodeId.c_str(), MPI_INFO_NULL, portName);
+          MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL );
+        }
+        ierr=MPI_SUCCESS ;
+        MPI_Bcast(&ierr,1,MPI_INT,0,clientComm) ;
+
+        if (ierr==MPI_SUCCESS) // you have a server
+        {  
+          MPI_Comm intraComm=clientComm ;
+          MPI_Comm interComm ;
+          for(int i=0 ; i<clientsCodeId.size(); i++)
+          {  
+            MPI_Comm_connect(portName, MPI_INFO_NULL, 0, intraComm, &interComm);
+            MPI_Intercomm_merge(interComm, true, &intraComm ) ;
+          }
+          xiosGlobalComm=intraComm ;
+        }
+        else  // you don't have any server
+        {
+          if (codeId==clientsCodeId[0]) // first code will publish his name
+          {
+
+            if (commRank==0) // if root process publish name
+            {  
+              MPI_Open_port(MPI_INFO_NULL, portName);
+              MPI_Publish_name(CXios::xiosCodeId.c_str(), MPI_INFO_NULL, portName);
+            }
+
+            MPI_Comm intraComm=clientComm ;
+            MPI_Comm interComm ;
+            for(int i=0 ; i<clientsCodeId.size()-1; i++)
+            {  
+              MPI_Comm_accept(portName, MPI_INFO_NULL, 0, intraComm, &interComm);
+              MPI_Intercomm_merge(interComm,false, &intraComm ) ;
+            }
+          }
+          else  // other clients are connecting to the first one
+          {
+            if (commRank==0)
+            {
+
+              MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN );
+              ierr=MPI_Lookup_name(CXios::xiosCodeId.c_str(), MPI_INFO_NULL, portName);
+              MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL );
+             }
+
+            MPI_Bcast(&ierr,1,MPI_INT,0,clientComm) ;
+
+            if (ierr==MPI_SUCCESS) // you can connect
+            {  
+              MPI_Comm intraComm=clientComm ;
+              MPI_Comm interComm ;
+              for(int i=0 ; i<clientsCodeId.size()-1; i++)
+              {  
+                MPI_Comm_connect(portName, MPI_INFO_NULL, 0, intraComm, &interComm);
+                MPI_Intercomm_merge(interComm, true, &intraComm ) ;
+              }
+              xiosGlobalComm=intraComm ;
+            }
+          }
+        }  
+      */
+    }
+
+    void CClient::initialize_old(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)
     {
       int initialized ;
@@ -42 +447 @@
       else is_MPI_Initialized=false ;
       int rank ;
-
+      
+      CXios::launchRessourcesManager(false) ;
+      CXios::launchServicesManager( false) ;
+      CXios::launchContextsManager(false) ;
+
+      initRessources() ;
 // don't use OASIS
       if (!CXios::usingOasis)
@@ -160 +570 @@
     }
 
+
+
+    void CClient::registerContext(const string& id, MPI_Comm contextComm)
+    {
+      int commRank, commSize ;
+      MPI_Comm_rank(contextComm,&commRank) ;
+      MPI_Comm_size(contextComm,&commSize) ;
+
+      getPoolRessource()->createService(contextComm, id, 0, CServicesManager::CLIENT, 1) ;
+      getPoolRessource()->createService(contextComm, CXios::defaultServerId, 0, CServicesManager::IO_SERVER, 1) ;
+
+      if (commRank==0) while (!CXios::getServicesManager()->hasService(getPoolRessource()->getId(), id, 0)) { CXios::getDaemonsManager()->eventLoop();}
+
+      if (commRank==0) CXios::getContextsManager()->createServerContext(getPoolRessource()->getId(), id, 0, id) ;
+      int type=CServicesManager::CLIENT ;
+      string name = CXios::getContextsManager()->getServerContextName(getPoolRessource()->getId(), id, 0, type, id) ;
+      while (!CXios::getContextsManager()->hasContext(name, contextComm) )
+      {
+        CXios::getDaemonsManager()->eventLoop() ;
+      }
+
+/*      
+
+      CContext::setCurrent(id) ;
+      CContext* context=CContext::create(id);
+      
+      // register the new client side context to the contexts manager
+      if (commRank==0)
+      {
+        MPI_Comm_rank(CXios::getXiosComm(),&commRank) ;
+        SRegisterContextInfo contextInfo ;
+        contextInfo.serviceType=CServicesManager::CLIENT ;
+        contextInfo.partitionId=0 ;
+        contextInfo.leader=commRank ;
+        contextInfo.size=commSize ;
+        CXios::getContextsManager()->registerContext(id, contextInfo) ;
+      }
+      context->initClient(contextComm) ;
+*/  
+    }
+
+
 ///---------------------------------------------------------------
 /*!
@@ -168 +620 @@
  * Function is only called by client.
  */
-    void CClient::registerContext(const string& id, MPI_Comm contextComm)
+    void CClient::registerContext_old(const string& id, MPI_Comm contextComm)
     {
       CContext::setCurrent(id) ;
@@ -260 +712 @@
     }
 
-
     void CClient::finalize(void)
+    {
+     
+      MPI_Barrier(clientsComm_) ;
+      int commRank ;
+      MPI_Comm_rank(clientsComm_, &commRank) ;
+      if (commRank==0) CXios::getRessourcesManager()->finalize() ;
+      
+      auto globalRegistry=CXios::getGlobalRegistry() ;
+      globalRegistry->hierarchicalGatherRegistry() ;
+
+      if (commRank==0)
+      {
+        info(80)<<"Write data base Registry"<<endl<<globalRegistry->toString()<<endl ;
+        globalRegistry->toFile("xios_registry.bin") ;
+      }
+      delete globalRegistry ;
+
+      CTimer::get("XIOS init/finalize",false).suspend() ;
+      CTimer::get("XIOS").suspend() ;
+      if (!is_MPI_Initialized)
+      {
+        if (CXios::usingOasis) oasis_finalize();
+        else MPI_Finalize() ;
+      }
+      
+      info(20) << "Client side context is finalized"<<endl ;
+      report(0) <<" Performance report : Whole time from XIOS init and finalize: "<< CTimer::get("XIOS init/finalize").getCumulatedTime()<<" s"<<endl ;
+      report(0) <<" Performance report : total time spent for XIOS : "<< CTimer::get("XIOS").getCumulatedTime()<<" s"<<endl ;
+      report(0)<< " Performance report : time spent for waiting free buffer : "<< CTimer::get("Blocking time").getCumulatedTime()<<" s"<<endl ;
+      report(0)<< " Performance report : Ratio : "<< CTimer::get("Blocking time").getCumulatedTime()/CTimer::get("XIOS init/finalize").getCumulatedTime()*100.<<" %"<<endl ;
+      report(0)<< " Performance report : This ratio must be close to zero. Otherwise it may be usefull to increase buffer size or numbers of server"<<endl ;
+//      report(0)<< " Memory report : Current buffer_size : "<<CXios::bufferSize<<endl ;
+      report(0)<< " Memory report : Minimum buffer size required : " << CClientBuffer::maxRequestSize << " bytes" << endl ;
+      report(0)<< " Memory report : increasing it by a factor will increase performance, depending of the volume of data wrote in file at each time step of the file"<<endl ;
+      report(100)<<CTimer::getAllCumulatedTime()<<endl ;
+    }
+    
+
+    void CClient::finalize_old(void)
     {
       int rank ;
@@ -325 +815 @@
       int size = 0;
       int rank;
-      MPI_Comm_size(CXios::globalComm, &size);
+      MPI_Comm_size(CXios::getGlobalComm(), &size);
+      MPI_Comm_rank(CXios::getGlobalComm(),&rank);
       while (size)
       {
@@ -332 +823 @@
       }
 
-      if (CXios::usingOasis)
-      {
-        MPI_Comm_rank(CXios::globalComm,&rank);
-        fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << rank << ext;
-      }
-      else
-        fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << getRank() << ext;
-
+      fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << rank << ext;
 
       fb->open(fileNameClient.str().c_str(), std::ios::out);

XIOS/dev/dev_ym/XIOS_SERVICES/src/client.hpp

@@ -7 +7 @@
 namespace xios
 {
+    class CPoolRessource ;
     class CClient
     {
       public:
         static void initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm);
+        static void initialize_old(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm);
+        static void xiosGlobalCommByFileExchange(MPI_Comm clientComm, const string& codeId) ;
+        static void xiosGlobalCommByPublishing(MPI_Comm clientComm, const string& codeId) ;
         static void finalize(void);
+        static void finalize_old(void);
         static void registerContext(const string& id, MPI_Comm contextComm);
-        static void callOasisEnddef(void) ;
-
+        static void registerContext_old(const string& id, MPI_Comm contextComm);
+        static void callOasisEnddef(void) ; 
+        static void initRessources(void) ;
+        
         static MPI_Comm intraComm;
         static MPI_Comm interComm;
@@ -20 +27 @@
         static int serverLeader;
         static bool is_MPI_Initialized ;
+        static MPI_Comm clientsComm_ ;
 
         static MPI_Comm& getInterComm();
@@ -39 +47 @@
         //! Close the error log file if it opens
         static void closeErrorStream();
+        static CPoolRessource* getPoolRessource(void) { return poolRessource_ ; }
 
       protected:
@@ -46 +55 @@
 
         static void openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb);
+        static CPoolRessource* poolRessource_ ;
     };
 }

XIOS/dev/dev_ym/XIOS_SERVICES/src/config/context_attribute.conf

@@ -1 +1 @@
 DECLARE_ATTRIBUTE(StdString, output_dir)
+DECLARE_ATTRIBUTE(bool, attached_mode)

XIOS/dev/dev_ym/XIOS_SERVICES/src/context_client.cpp

@@ -24 +24 @@
      : mapBufferSize_(), parentServer(cxtSer), maxBufferedEvents(4)
     {
-      pureOneSided=CXios::getin<bool>("pure_one_sided",false); // pure one sided communication (for test)
-      if (isAttachedModeEnabled()) pureOneSided=false ; // no one sided in attach mode
       
       context = parent;
@@ -35 +33 @@
       int flag;
       MPI_Comm_test_inter(interComm, &flag);
+      if (flag) isAttached_=false ;
+      else  isAttached_=true ;
+
+      pureOneSided=CXios::getin<bool>("pure_one_sided",false); // pure one sided communication (for test)
+      if (isAttachedModeEnabled()) pureOneSided=false ; // no one sided in attach mode
+      
+
+
       if (flag) MPI_Comm_remote_size(interComm, &serverSize);
       else  MPI_Comm_size(interComm, &serverSize);
@@ -151 +157 @@
           
         checkBuffers(ranks);
-
-        if (isAttachedModeEnabled()) // couldBuffer is always true in attached mode
-        {
-          waitEvent(ranks);
-          CContext::setCurrent(context->getId());
-        }
-      }
-
+      }
+      
+      if (isAttachedModeEnabled()) // couldBuffer is always true in attached mode
+      {
+        waitEvent(ranks);
+        CContext::setCurrent(context->getId());
+      }
+      
       timeLine++;
     }
@@ -168 +174 @@
     */
     void CContextClient::waitEvent(list<int>& ranks)
+    {
+      while (checkBuffers(ranks))
+      {
+        CXios::getDaemonsManager()->eventLoop() ;
+      }
+
+      MPI_Request req ;
+      MPI_Status status ;
+
+      MPI_Ibarrier(intraComm,&req) ;
+      int flag=false ;
+
+      do  
+      {
+        CXios::getDaemonsManager()->eventLoop() ;
+        MPI_Test(&req,&flag,&status) ;
+      } while (!flag) ;
+
+
+    }
+
+
+    void CContextClient::waitEvent_old(list<int>& ranks)
     {
       parentServer->server->setPendingEvent();
@@ -227 +256 @@
           for (itBuffer = bufferList.begin(); itBuffer != bufferList.end(); itBuffer++) (*itBuffer)->unlockBuffer();
           checkBuffers();
-          if (CServer::serverLevel == 0)  context->server->listen();
-          else if (CServer::serverLevel == 1)
+          
+          context->server->listen();
+
+          if (context->serverPrimServer.size()>0)
           {
-            context->server->listen();
             for (int i = 0; i < context->serverPrimServer.size(); ++i)  context->serverPrimServer[i]->listen();
             CServer::contextEventLoop(false) ; // avoid dead-lock at finalize...
           }
-
-          else if (CServer::serverLevel == 2) context->server->listen();
 
         }
@@ -320 +348 @@
         }
       } 
-   }
 */
+   }
+
       
   /*!
@@ -402 +431 @@
   {
     return !ranksServerLeader.empty();
-  }
-
-  /*!
-   * Check if the attached mode is used.
-   *
-   * \return true if and only if attached mode is used
-   */
-  bool CContextClient::isAttachedModeEnabled() const
-  {
-    return (parentServer != 0);
   }
 

XIOS/dev/dev_ym/XIOS_SERVICES/src/context_client.hpp

@@ -32 +32 @@
       void sendEvent(CEventClient& event);
       void waitEvent(list<int>& ranks);
+      void waitEvent_old(list<int>& ranks);
 
       // Functions to set/get buffers
@@ -46 +47 @@
       const std::list<int>& getRanksServerNotLeader(void) const;
 
-      bool isAttachedModeEnabled() const;
+  /*!
+   * Check if the attached mode is used.
+   *
+   * \return true if and only if attached mode is used
+   */
+      bool isAttachedModeEnabled() const { return isAttached_ ; } 
 
       static void computeLeader(int clientRank, int clientSize, int serverSize,
@@ -103 +109 @@
 
       std::vector<std::vector<MPI_Win> >windows ; //! one sided mpi windows to expose client buffers to servers == windows[nbServers][2]
-
+      bool isAttached_ ;
 
   };

XIOS/dev/dev_ym/XIOS_SERVICES/src/context_server.cpp

@@ -16 +16 @@
 #include "event_scheduler.hpp"
 #include "server.hpp"
+#include "servers_ressource.hpp"
+#include "pool_ressource.hpp"
+#include "services.hpp"
+#include "contexts_manager.hpp"
+
 #include <boost/functional/hash.hpp>
-
+#include <random>
+#include <chrono>
 
 
 namespace xios
 {
+  using namespace std ;
 
   CContextServer::CContextServer(CContext* parent,MPI_Comm intraComm_,MPI_Comm interComm_)
@@ -40 +47 @@
     else  MPI_Comm_size(interComm,&commSize);
 
-     
+   
+    SRegisterContextInfo contextInfo ;
+    CXios::getContextsManager()->getContextInfo(context->getId(), contextInfo, intraComm) ;
+
+    if (contextInfo.serviceType != CServicesManager::CLIENT) // we must have an event scheduler => to be retrieve from the associated services
+    {
+      eventScheduler_=CXios::getPoolRessource()->getService(contextInfo.serviceId,contextInfo.partitionId)->getEventScheduler() ;
+    }
+
+
     currentTimeLine=1;
     scheduled=false;
     finished=false;
+
+    // generate unique hash for server
+    auto time=chrono::system_clock::now().time_since_epoch().count() ;
+    std::default_random_engine rd(time); // not reproducible from a run to another
+    std::uniform_int_distribution<size_t> dist;
+    hashId=dist(rd) ;
+    MPI_Bcast(&hashId,1,MPI_SIZE_T,0,intraComm) ; // Bcast to all server of the context
+
+    /*
     boost::hash<string> hashString;
+
     if (CServer::serverLevel == 1)
       hashId=hashString(context->getId() + boost::lexical_cast<string>(context->clientPrimServer.size()));
     else
       hashId=hashString(context->getId());
+*/
 
     if (!isAttachedModeEnabled())
@@ -293 +320 @@
     map<size_t,CEventServer*>::iterator it;
     CEventServer* event;
+    
+    if (context->isProcessingEvent()) return ;
 
     it=events.find(currentTimeLine);
@@ -313 +342 @@
          if (!CServer::eventScheduler && CXios::isServer) MPI_Barrier(intraComm) ;
 
+         context->setProcessingEvent() ;
          CTimer::get("Process events").resume();
          dispatchEvent(*event);
          CTimer::get("Process events").suspend();
+         context->unsetProcessingEvent() ;
          pendingEvent=false;
          delete event;

XIOS/dev/dev_ym/XIOS_SERVICES/src/context_server.hpp

@@ -5 +5 @@
 #include "buffer_server.hpp"
 #include "mpi.hpp"
+#include "event_scheduler.hpp"
 
 namespace xios
@@ -64 +65 @@
       std::map<int, StdSize> mapBufferSize_;
       vector<MPI_Win> windows ; //! one sided mpi windows to expose client buffers to servers ; No memory will be attached on server side.
-
+      CEventScheduler* eventScheduler_ ;
   } ;
 

XIOS/dev/dev_ym/XIOS_SERVICES/src/cxios.cpp

@@ -11 +11 @@
 #include "memtrack.hpp"
 #include "registry.hpp"
+#include "ressources_manager.hpp"
+#include "services_manager.hpp"
+#include "servers_ressource.hpp"
 
 namespace xios
@@ -20 +23 @@
   string CXios::serverPrmFile="./xios_server1";
   string CXios::serverSndFile="./xios_server2";
-
+  const string CXios::defaultPoolId="default_pool_id" ;
+  const string CXios::defaultServerId="default_server_id" ;
+  const string CXios::defaultGathererId="default_gatherer_id" ;
+  
   bool CXios::xiosStack = true;
   bool CXios::systemStack = false;
@@ -26 +32 @@
   bool CXios::isClient ;
   bool CXios::isServer ;
+  
   MPI_Comm CXios::globalComm ;
+  MPI_Comm CXios::xiosComm ;
+
   bool CXios::usingOasis ;
   bool CXios::usingServer = false;
@@ -41 +50 @@
   double CXios::recvFieldTimeout = 300.0;
   bool CXios::checkEventSync=false ;
- 
+
+  CDaemonsManager*    CXios::daemonsManager_=nullptr ;
+  CRessourcesManager* CXios::ressourcesManager_=nullptr ;
+  CServicesManager*   CXios::servicesManager_=nullptr ;
+  CContextsManager*   CXios::contextsManager_=nullptr ;
+
   //! Parse configuration file and create some objects from it
   void CXios::initialize()
@@ -107 +121 @@
     isClient = true;
 
+    //CClient::initialize(codeId,localComm,returnComm) ;
     CClient::initialize(codeId,localComm,returnComm) ;
-    if (CClient::getRank()==0) globalRegistry = new CRegistry(returnComm) ;
-
+    
     // If there are no server processes then we are in attached mode
     // and the clients are also servers
@@ -130 +144 @@
   {
      CClient::finalize() ;
-     if (CClient::getRank()==0)
-     {
-       info(80)<<"Write data base Registry"<<endl<<globalRegistry->toString()<<endl ;
-       globalRegistry->toFile("xios_registry.bin") ;
-       delete globalRegistry ;
-     }
-
+          
 #ifdef XIOS_MEMTRACK
 
@@ -172 +180 @@
 
     // Initialize all aspects MPI
+   // CServer::initialize();
+
     CServer::initialize();
-    if (CServer::getRank()==0 && CServer::serverLevel != 1) globalRegistry = new CRegistry(CServer::intraComm) ;
-    
+
+    //if (CServer::getRank()==0 && CServer::serverLevel != 1) globalRegistry = new CRegistry(CServer::intraComm) ;
+
+/*    
     if (printLogs2Files)
     {
@@ -201 +213 @@
     // Enter the loop to listen message from Client
     CServer::eventLoop();
-
+*/
     // Finalize
+/*    
     if (CServer::serverLevel == 0)
     {
@@ -271 +284 @@
       delete globalRegistry;
     }
+  */
+
     CServer::finalize();
 
@@ -305 +320 @@
     usingServer = false;
   }
+
+  void CXios::launchRessourcesManager(bool isXiosServer)
+  {
+    ressourcesManager_ = new CRessourcesManager(isXiosServer) ;
+  }
+
+  void CXios::launchServicesManager(bool isXiosServer)
+  {
+    servicesManager_ = new CServicesManager(isXiosServer) ;
+  }
+
+  void CXios::launchContextsManager(bool isXiosServer)
+  {
+    contextsManager_ = new CContextsManager(isXiosServer) ;
+  }
+  
+  void CXios::launchDaemonsManager(bool isXiosServer)
+  {
+    daemonsManager_ = new CDaemonsManager(isXiosServer) ;
+  }
+
+  CPoolRessource* CXios::getPoolRessource(void)
+  {
+    if (isClient) return CClient::getPoolRessource() ;
+    else if (isServer) return CServer::getServersRessource()->getPoolRessource() ;
+  }
 }
+

XIOS/dev/dev_ym/XIOS_SERVICES/src/cxios.hpp

@@ -5 +5 @@
 #include "mpi.hpp"
 #include "registry.hpp"
+#include "ressources_manager.hpp"
+#include "services_manager.hpp"
+#include "contexts_manager.hpp"
+#include "daemons_manager.hpp"
 
 namespace xios
@@ -41 +45 @@
 
      static MPI_Comm globalComm ; //!< Global communicator
+     static MPI_Comm xiosComm ; //!< Global communicator
 
      static bool printLogs2Files; //!< Printing out logs into files
@@ -56 +61 @@
      static double recvFieldTimeout; //!< Time to wait for data before issuing an error when receiving a field
      static bool checkEventSync; //!< For debuuging, check if event are coherent and synchrone on client side
+     
+     static const string defaultPoolId ;
+     static const string defaultServerId ;
+     static const string defaultGathererId ;
+
+
+     static CRessourcesManager* ressourcesManager_ ;
+     static CServicesManager* servicesManager_ ;
+     static CContextsManager* contextsManager_ ;
+     static CDaemonsManager* daemonsManager_ ;
+     
 
     public:
@@ -63 +79 @@
      //! Setting xios NOT to use server mode
      static void setNotUsingServer();
+     
+     //! is using server mode
+     static bool isUsingServer() {return usingServer;}
 
      //! Initialize server (if any)
      static  void initServer();
+
+     static void launchServicesManager( bool isXiosServer) ;
+     static void launchContextsManager(bool isXiosServer) ;
+     static void launchDaemonsManager(bool isXiosServer) ;
+     static void launchRessourcesManager(bool isXiosServer) ;
+     
+     static CRessourcesManager* getRessourcesManager(void) { return ressourcesManager_ ;}
+     static CServicesManager*   getServicesManager(void) { return servicesManager_ ;}
+     static CContextsManager*   getContextsManager(void) { return contextsManager_ ;}
+     static CDaemonsManager*    getDaemonsManager(void) { return daemonsManager_ ;}
+     static CPoolRessource*     getPoolRessource(void) ;
+
+     static MPI_Comm getGlobalComm(void) { return globalComm ;}
+     static MPI_Comm getXiosComm(void) { return xiosComm ;}
+     static void setXiosComm(MPI_Comm comm) { xiosComm=comm ;}
+     static CRegistry* getGlobalRegistry(void) { return globalRegistry ;}
+     static void setGlobalRegistry(CRegistry* registry) { globalRegistry=registry ;}
 
     private:

XIOS/dev/dev_ym/XIOS_SERVICES/src/filter/store_filter.cpp

@@ -45 +45 @@
         packet = it->second;
       else // if the packet is not available yet, check if it can be received
-        context->checkBuffersAndListen();
+//ym        context->checkBuffersAndListen();
+        context->eventLoop();
 
       timer.suspend();

XIOS/dev/dev_ym/XIOS_SERVICES/src/interface/c/iccalendar.cpp

@@ -16 +16 @@
     xios::CContext* context = CContext::getCurrent();
     if (!context->hasServer && !context->client->isAttachedModeEnabled())
-      context->checkBuffersAndListen();
+      context->eventLoop();
+//ym      context->checkBuffersAndListen();
+      
     context->updateCalendar(step);
     context->sendUpdateCalendar(step);

XIOS/dev/dev_ym/XIOS_SERVICES/src/interface/c/icdata.cpp

@@ -423 +423 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
       CArray<double, 1> data(data_k8, shape(data_Xsize), neverDeleteData);
       CField::get(fieldid_str)->setData(data);
@@ -442 +443 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 1> data(data_k8, shape(data_Xsize), neverDeleteData);
@@ -463 +465 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 2>data(data_k8, shape(data_Xsize, data_Ysize), neverDeleteData);
@@ -484 +487 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 3>data(data_k8, shape(data_Xsize, data_Ysize, data_Zsize), neverDeleteData);
@@ -505 +509 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 4>data(data_k8, shape(data_0size, data_1size, data_2size, data_3size), neverDeleteData);
@@ -527 +532 @@
 
       CContext* context = CContext::getCurrent();
-      if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 5>data(data_k8, shape(data_0size, data_1size, data_2size, data_3size, data_4size), neverDeleteData);
@@ -551 +556 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//        context->checkBuffersAndListen();
 
       CArray<double, 6>data(data_k8, shape(data_0size, data_1size, data_2size, data_3size, data_4size, data_5size), neverDeleteData);
@@ -575 +581 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//        context->checkBuffersAndListen();
 
       CArray<double, 7>data(data_k8, shape(data_0size, data_1size, data_2size, data_3size, data_4size, data_5size, data_6size), neverDeleteData);
@@ -595 +602 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<float, 1> data_tmp(data_k4, shape(data_Xsize), neverDeleteData);
@@ -617 +625 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<float, 1> data_tmp(data_k4, shape(data_Xsize), neverDeleteData);
@@ -640 +649 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<float, 2> data_tmp(data_k4, shape(data_Xsize, data_Ysize), neverDeleteData);
@@ -663 +673 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<float, 3> data_tmp(data_k4, shape(data_Xsize, data_Ysize, data_Zsize), neverDeleteData);
@@ -688 +699 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<float, 4> data_tmp(data_k4, shape(data_0size, data_1size, data_2size, data_3size), neverDeleteData);
@@ -713 +725 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<float, 5> data_tmp(data_k4, shape(data_0size, data_1size, data_2size, data_3size, data_4size), neverDeleteData);
@@ -738 +751 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<float, 6> data_tmp(data_k4, shape(data_0size, data_1size, data_2size, data_3size, data_4size, data_5size), neverDeleteData);
@@ -764 +778 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<float, 7> data_tmp(data_k4, shape(data_0size, data_1size, data_2size, data_3size, data_4size, data_5size, data_6size), neverDeleteData);
@@ -789 +804 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 1> data(data_k8, shape(data_Xsize), neverDeleteData);
@@ -810 +826 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 1> data(data_k8, shape(data_Xsize), neverDeleteData);
@@ -831 +848 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 2>data(data_k8, shape(data_Xsize, data_Ysize), neverDeleteData);
@@ -852 +870 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 3>data(data_k8, shape(data_Xsize, data_Ysize, data_Zsize), neverDeleteData);
@@ -875 +894 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 4>data(data_k8, shape(data_0size, data_1size, data_2size, data_3size), neverDeleteData);
@@ -898 +918 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 5>data(data_k8, shape(data_0size, data_1size, data_2size, data_3size, data_4size), neverDeleteData);
@@ -921 +942 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 6>data(data_k8, shape(data_0size, data_1size, data_2size, data_3size, data_4size, data_5size), neverDeleteData);
@@ -945 +967 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 7>data(data_k8, shape(data_0size, data_1size, data_2size, data_3size, data_4size, data_5size, data_6size), neverDeleteData);
@@ -966 +989 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 1> data(data_Xsize);
@@ -989 +1013 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 1> data(data_Xsize);
@@ -1012 +1037 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 2> data(data_Xsize, data_Ysize);
@@ -1035 +1061 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 3> data(data_Xsize, data_Ysize, data_Zsize);
@@ -1060 +1087 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 4> data(data_0size, data_1size, data_2size, data_3size);
@@ -1085 +1113 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 5> data(data_0size, data_1size, data_2size, data_3size, data_4size);
@@ -1110 +1139 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 6> data(data_0size, data_1size, data_2size, data_3size, data_4size, data_5size);
@@ -1136 +1166 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+        context->eventLoop();
+//ym        context->checkBuffersAndListen();
 
       CArray<double, 7> data(data_0size, data_1size, data_2size, data_3size, data_4size, data_5size, data_6size);

XIOS/dev/dev_ym/XIOS_SERVICES/src/interface/fortran/ixios.F90

@@ -8 +8 @@
                       xios(get_year_length_in_seconds), xios(get_day_length_in_seconds)
 
-USE icontext, ONLY : txios(context), xios(set_current_context), xios(is_valid_context)
+USE icontext, ONLY : txios(context), xios(set_current_context), xios(get_current_context), xios(is_valid_context)
 
 USE icontext_attr, ONLY : xios(set_context_attr), xios(get_context_attr), xios(is_defined_context_attr)

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/axis.cpp

@@ -847 +847 @@
 
       numberWrittenIndexes_[writtenCommSize] = nbWritten;
-      if (isDistributed())
+
+      bool distributed_glo, distributed=isDistributed() ;
+      MPI_Allreduce(&distributed,&distributed_glo, 1, MPI_INT, MPI_LOR, writtenComm) ;
+      if (distributed_glo)
       {
              

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/context.cpp

@@ -20 +20 @@
 #include "server.hpp"
 #include "distribute_file_server2.hpp"
+#include "services_manager.hpp"
+#include "contexts_manager.hpp"
+#include "cxios.hpp"
+#include "client.hpp"
 
 namespace xios {
@@ -31 +35 @@
       , calendar(), hasClient(false), hasServer(false)
       , isPostProcessed(false), finalized(false)
-      , idServer_(), client(0), server(0)
-      , allProcessed(false), countChildCtx_(0)
+      , idServer_(), client(nullptr), server(nullptr)
+      , allProcessed(false), countChildCtx_(0), isProcessingEvent_(false)
 
    { /* Ne rien faire de plus */ }
@@ -40 +44 @@
       , calendar(), hasClient(false), hasServer(false)
       , isPostProcessed(false), finalized(false)
-      , idServer_(), client(0), server(0)
-      , allProcessed(false), countChildCtx_(0)
+      , idServer_(), client(nullptr), server(nullptr)
+      , allProcessed(false), countChildCtx_(0), isProcessingEvent_(false)
    { /* Ne rien faire de plus */ }
 
@@ -264 +268 @@
    ///---------------------------------------------------------------
 
-   //! Initialize client side
+
+   //! Initialize client side : old interface to be removed
    void CContext::initClient(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtServer /*= 0*/)
    TRY
@@ -270 +275 @@
 
      hasClient = true;
-     MPI_Comm intraCommServer, interCommServer;
+     MPI_Comm intraCommServer, interCommServer; 
      
 
@@ -383 +388 @@
    CATCH_DUMP_ATTR
 
+
+   void CContext::init(CServerContext* parentServerContext, MPI_Comm intraComm, int serviceType)
+   TRY
+   {
+     parentServerContext_ = parentServerContext ;
+     if (serviceType==CServicesManager::CLIENT) 
+       initClient(intraComm, serviceType) ;
+     else
+       initServer(intraComm, serviceType) ;
+    }
+    CATCH_DUMP_ATTR
+
+
+
+//! Initialize client side
+   void CContext::initClient(MPI_Comm intraComm, int serviceType)
+   TRY
+   {
+      intraComm_=intraComm ;
+      serviceType_ = CServicesManager::CLIENT ;
+      if (serviceType_==CServicesManager::CLIENT)
+      {
+        hasClient=true ;
+        hasServer=false ;
+      }
+      contextId_ = getId() ;
+      
+      attached_mode=true ;
+      if (!CXios::isUsingServer()) attached_mode=false ;
+
+
+      string contextRegistryId=getId() ;
+      registryIn=new CRegistry(intraComm);
+      registryIn->setPath(contextRegistryId) ;
+      
+      int commRank ;
+      MPI_Comm_rank(intraComm_,&commRank) ;
+      if (commRank==0) registryIn->fromFile("xios_registry.bin") ;
+      registryIn->bcastRegistry() ;
+      registryOut=new CRegistry(intraComm_) ;
+      registryOut->setPath(contextRegistryId) ;
+     
+   }
+   CATCH_DUMP_ATTR
+
+   
+   void CContext::initServer(MPI_Comm intraComm, int serviceType)
+   TRY
+   {
+     hasServer=true;
+     intraComm_=intraComm ;
+     serviceType_=serviceType ;
+
+     if (serviceType_==CServicesManager::GATHERER)
+     {
+       hasClient=true ;
+       hasServer=true ;
+     }
+     else if (serviceType_==CServicesManager::IO_SERVER || serviceType_==CServicesManager::OUT_SERVER)
+     {
+       hasClient=false ;
+       hasServer=true ;
+     }
+
+     CXios::getContextsManager()->getContextId(getId(), contextId_, intraComm) ;
+     
+     registryIn=new CRegistry(intraComm);
+     registryIn->setPath(contextId_) ;
+     
+     int commRank ;
+     MPI_Comm_rank(intraComm_,&commRank) ;
+     if (commRank==0) registryIn->fromFile("xios_registry.bin") ;
+    
+     registryIn->bcastRegistry() ;
+     registryOut=new CRegistry(intraComm) ;
+     registryOut->setPath(contextId_) ;
+
+   }
+   CATCH_DUMP_ATTR
+
+
+  void CContext::createClientInterComm(MPI_Comm interCommClient, MPI_Comm interCommServer) // for servers
+  TRY
+  {
+    MPI_Comm intraCommClient ;
+    MPI_Comm_dup(intraComm_, &intraCommClient);
+    comms.push_back(intraCommClient);
+    
+    server = new CContextServer(this,intraComm_, interCommServer); // check if we need to dupl. intraComm_ ?
+    client = new CContextClient(this,intraCommClient,interCommClient);
+
+  }
+  CATCH_DUMP_ATTR
+
+  void CContext::createServerInterComm(void) 
+  TRY
+  {
+   
+    MPI_Comm interCommClient, interCommServer ;
+
+    if (serviceType_ == CServicesManager::CLIENT)
+    {
+
+      int commRank ;
+      MPI_Comm_rank(intraComm_,&commRank) ;
+      if (commRank==0)
+      {
+        if (attached_mode) CXios::getContextsManager()->createServerContext(CClient::getPoolRessource()->getId(), CXios::defaultServerId, 0, getContextId()) ;
+        else if (CXios::usingServer2) CXios::getContextsManager()->createServerContext(CXios::defaultPoolId, CXios::defaultGathererId, 0, getContextId()) ;
+        else  CXios::getContextsManager()->createServerContext(CXios::defaultPoolId, CXios::defaultServerId, 0, getContextId()) ;
+      }
+
+      MPI_Comm interComm ;
+      
+      if (attached_mode)
+      {
+        parentServerContext_->createIntercomm(CClient::getPoolRessource()->getId(), CXios::defaultServerId, 0, getContextId(), intraComm_, 
+                                              interCommClient, interCommServer) ;
+        int type ; 
+        if (commRank==0) CXios::getServicesManager()->getServiceType(CClient::getPoolRessource()->getId(), CXios::defaultServerId, 0, type) ;
+        MPI_Bcast(&type,1,MPI_INT,0,intraComm_) ;
+        setIdServer(CXios::getContextsManager()->getServerContextName(CClient::getPoolRessource()->getId(), CXios::defaultServerId, 0, type, getContextId())) ;
+        setCurrent(getId()) ; // getCurrent/setCurrent may be supress, it can cause a lot of trouble
+      }
+      else if (CXios::usingServer2)
+      { 
+//      CXios::getContextsManager()->createServerContextIntercomm(CXios::defaultPoolId, CXios::defaultGathererId, 0, getContextId(), intraComm_, interComm) ;
+        parentServerContext_->createIntercomm(CXios::defaultPoolId, CXios::defaultGathererId, 0, getContextId(), intraComm_,
+                                              interCommClient, interCommServer) ;
+        int type ; 
+        if (commRank==0) CXios::getServicesManager()->getServiceType(CXios::defaultPoolId, CXios::defaultGathererId, 0, type) ;
+        MPI_Bcast(&type,1,MPI_INT,0,intraComm_) ;
+        setIdServer(CXios::getContextsManager()->getServerContextName(CXios::defaultPoolId, CXios::defaultGathererId, 0, type, getContextId())) ;
+      }
+      else
+      {
+        //CXios::getContextsManager()->createServerContextIntercomm(CXios::defaultPoolId, CXios::defaultServerId, 0, getContextId(), intraComm_, interComm) ;
+        parentServerContext_->createIntercomm(CXios::defaultPoolId, CXios::defaultServerId, 0, getContextId(), intraComm_,
+                                              interCommClient, interCommServer) ;
+        int type ; 
+        if (commRank==0) CXios::getServicesManager()->getServiceType(CXios::defaultPoolId, CXios::defaultServerId, 0, type) ;
+        MPI_Bcast(&type,1,MPI_INT,0,intraComm_) ;
+        setIdServer(CXios::getContextsManager()->getServerContextName(CXios::defaultPoolId, CXios::defaultServerId, 0, type, getContextId())) ;
+      }
+
+        // intraComm client is not duplicated. In all the code we use client->intraComm for MPI
+        // in future better to replace it by intracommuncator associated to the context
+      
+/*        MPI_Comm intraCommClient, intraCommServer ;
+        MPI_Comm interCommClient, interCommServer ;
+
+        intraCommClient=intraComm_ ;
+        MPI_Comm_dup(intraComm_, &intraCommServer) ;
+
+        interCommClient=interComm ;               
+        MPI_Comm_dup(interComm, &interCommServer) ; */
+      
+      MPI_Comm intraCommClient, intraCommServer ;
+      intraCommClient=intraComm_ ;
+      MPI_Comm_dup(intraComm_, &intraCommServer) ;
+      client = new CContextClient(this, intraCommClient, interCommClient);
+      server = new CContextServer(this, intraCommServer, interCommServer);
+    
+    }
+    
+    if (serviceType_ == CServicesManager::GATHERER)
+    {
+      int commRank ;
+      MPI_Comm_rank(intraComm_,&commRank) ;
+      
+      int nbPartitions ;
+      if (commRank==0) 
+      { 
+        CXios::getServicesManager()->getServiceNbPartitions(CXios::defaultPoolId, CXios::defaultServerId, 0, nbPartitions) ;
+        for(int i=0 ; i<nbPartitions; i++)
+          CXios::getContextsManager()->createServerContext(CXios::defaultPoolId, CXios::defaultServerId, i, getContextId()) ;
+      }      
+      MPI_Bcast(&nbPartitions, 1, MPI_INT, 0, intraComm_) ;
+      
+      MPI_Comm interComm ;
+      for(int i=0 ; i<nbPartitions; i++)
+      {
+//        CXios::getContextsManager()->createServerContextIntercomm(CXios::defaultPoolId, CXios::defaultServerId, i, getContextId(), intraComm_, interComm) ;
+        parentServerContext_->createIntercomm(CXios::defaultPoolId, CXios::defaultServerId, i, getContextId(), intraComm_, interCommClient, interCommServer) ;
+        int type ; 
+        if (commRank==0) CXios::getServicesManager()->getServiceType(CXios::defaultPoolId, CXios::defaultServerId, 0, type) ;
+        MPI_Bcast(&type,1,MPI_INT,0,intraComm_) ;
+        primServerId_.push_back(CXios::getContextsManager()->getServerContextName(CXios::defaultPoolId, CXios::defaultServerId, i, type, getContextId())) ;
+
+        // intraComm client is not duplicated. In all the code we use client->intraComm for MPI
+        // in future better to replace it by intracommuncator associated to the context
+      
+        MPI_Comm intraCommClient, intraCommServer ;
+//        MPI_Comm interCommClient, interCommServer ;
+
+        intraCommClient=intraComm_ ;
+        MPI_Comm_dup(intraComm_, &intraCommServer) ;
+
+//        interCommClient=interComm ;               
+//        MPI_Comm_dup(interComm, &interCommServer) ;
+
+        clientPrimServer.push_back(new CContextClient(this, intraCommClient, interCommClient));
+        serverPrimServer.push_back(new CContextServer(this, intraCommServer, interCommServer));  
+      
+      }
+    }
+  }
+  CATCH_DUMP_ATTR
+
+ 
    void CContext::initServer(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtClient /*= 0*/)
    TRY
@@ -418 +633 @@
    }
    CATCH_DUMP_ATTR
+
+ 
+
+  bool CContext::eventLoop(bool enableEventsProcessing)
+  {
+    bool finished=true; 
+
+    if (client!=nullptr && !finalized) client->checkBuffers();
+    
+    for (int i = 0; i < clientPrimServer.size(); ++i)
+    {
+      if (!finalized) clientPrimServer[i]->checkBuffers();
+      if (!finalized) finished &= serverPrimServer[i]->eventLoop(enableEventsProcessing);
+    }
+
+    if (server!=nullptr) if (!finalized) finished &= server->eventLoop(enableEventsProcessing);
+  
+    return finalized && finished ;
+  }
 
    //! Try to send the buffers and receive possible answers
@@ -451 +685 @@
     }
   }
-   CATCH_DUMP_ATTR
-
-   //! Terminate a context
+  CATCH_DUMP_ATTR
+
+
+
+
    void CContext::finalize(void)
    TRY
@@ -462 +698 @@
       }
      // Send registry upon calling the function the first time
-     if (countChildCtx_ == 0)
-       if (hasClient) sendRegistry() ;
+     if (countChildCtx_ == 0) if (hasClient) sendRegistry() ;
 
      // Client:
@@ -562 +797 @@
          
          closeAllFile(); // Just move to here to make sure that server-level 1 can close files
+        
+        /*  ym
          if (hasServer && !hasClient)
          {           
@@ -567 +804 @@
            if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
          }
+        */
 
          //! Deallocate client buffers
@@ -589 +827 @@
    CATCH_DUMP_ATTR
 
+
+
+   //! Terminate a context
+   void CContext::finalize_old(void)
+   TRY
+   {
+      if (hasClient && !hasServer) // For now we only use server level 1 to read data
+      {
+        doPreTimestepOperationsForEnabledReadModeFiles();
+      }
+     // Send registry upon calling the function the first time
+     if (countChildCtx_ == 0) if (hasClient) sendRegistry() ;
+
+     // Client:
+     // (1) blocking send context finalize to its server
+     // (2) blocking receive context finalize from its server
+     // (3) some memory deallocations
+     if (CXios::isClient)
+     {
+       // Make sure that client (model) enters the loop only once
+       if (countChildCtx_ < 1)
+       {
+         ++countChildCtx_;
+
+         info(100)<<"DEBUG: context "<<getId()<<" Send client finalize"<<endl ;
+         client->finalize();
+         info(100)<<"DEBUG: context "<<getId()<<" Client finalize sent"<<endl ;
+         while (client->havePendingRequests()) client->checkBuffers();
+         
+         info(100)<<"DEBUG: context "<<getId()<<" no pending request ok"<<endl ;
+         while (!server->hasFinished())
+           server->eventLoop();
+        info(100)<<"DEBUG: context "<<getId()<<" server has finished"<<endl ;
+        
+        bool notifiedFinalized=false ;
+        do
+        {
+          notifiedFinalized=client->isNotifiedFinalized() ;
+        } while (!notifiedFinalized) ;
+        client->releaseBuffers();
+
+         if (hasServer) // Mode attache
+         {
+           closeAllFile();
+           registryOut->hierarchicalGatherRegistry() ;
+           if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
+         }
+
+         //! Deallocate client buffers
+//         client->releaseBuffers();
+        info(100)<<"DEBUG: context "<<getId()<<" release client ok"<<endl ;
+         //! Free internally allocated communicators
+         for (std::list<MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
+           MPI_Comm_free(&(*it));
+         comms.clear();
+
+         info(20)<<"CContext: Context <"<<getId()<<"> is finalized."<<endl;
+       }
+     }
+     else if (CXios::isServer)
+     {
+       // First context finalize message received from a model
+       // Send context finalize to its child contexts (if any)
+       if (countChildCtx_ == 0)
+         for (int i = 0; i < clientPrimServer.size(); ++i)
+         {
+           clientPrimServer[i]->finalize();
+           bool bufferReleased;
+           do
+           {
+             clientPrimServer[i]->checkBuffers();
+             bufferReleased = !clientPrimServer[i]->havePendingRequests();
+           } while (!bufferReleased);
+           
+           bool notifiedFinalized=false ;
+           do
+           {
+//             clientPrimServer[i]->checkBuffers();
+             notifiedFinalized=clientPrimServer[i]->isNotifiedFinalized() ;
+           } while (!notifiedFinalized) ;
+           clientPrimServer[i]->releaseBuffers();
+         }
+           
+
+       // (Last) context finalized message received
+       if (countChildCtx_ == clientPrimServer.size())
+       {
+         // Blocking send of context finalize message to its client (e.g. primary server or model)
+         info(100)<<"DEBUG: context "<<getId()<<" Send client finalize"<<endl ;
+         client->finalize();
+         info(100)<<"DEBUG: context "<<getId()<<" Client finalize sent"<<endl ;
+         bool bufferReleased;
+         do
+         {
+           client->checkBuffers();
+           bufferReleased = !client->havePendingRequests();
+         } while (!bufferReleased);
+         
+         bool notifiedFinalized=false ;
+         do
+         {
+  //         client->checkBuffers();
+           notifiedFinalized=client->isNotifiedFinalized() ;
+         } while (!notifiedFinalized) ;
+         client->releaseBuffers();
+         
+         finalized = true;
+         info(100)<<"DEBUG: context "<<getId()<<" bufferRelease OK"<<endl ;
+         
+         closeAllFile(); // Just move to here to make sure that server-level 1 can close files
+        
+        /*  ym
+         if (hasServer && !hasClient)
+         {           
+           registryOut->hierarchicalGatherRegistry() ;
+           if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
+         }
+        */
+
+         //! Deallocate client buffers
+//         client->releaseBuffers();
+         info(100)<<"DEBUG: context "<<getId()<<" client release"<<endl ;
+
+/*         
+         for (int i = 0; i < clientPrimServer.size(); ++i)
+           clientPrimServer[i]->releaseBuffers();
+*/
+         //! Free internally allocated communicators
+         for (std::list<MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
+           MPI_Comm_free(&(*it));
+         comms.clear();
+
+         info(20)<<"CContext: Context <"<<getId()<<"> is finalized."<<endl;
+       }
+
+       ++countChildCtx_;
+     }
+   }
+   CATCH_DUMP_ATTR
+
    //! Free internally allocated communicators
    void CContext::freeComms(void)
@@ -614 +992 @@
      if (allProcessed) return;  
      
+    // create intercommunicator with servers. 
+    // not sure it is the good place to be called here 
+    createServerInterComm() ;
+
+
      // After xml is parsed, there are some more works with post processing
      postProcessing();
@@ -734 +1117 @@
    TRY
    {
-    CTimer::get("Context : close definition").resume() ;
+     CTimer::get("Context : close definition").resume() ;
+    
+    //
     postProcessingGlobalAttributes();
 
@@ -1548 +1933 @@
    CATCH_DUMP_ATTR
 
+   void CContext::setIdServer(const StdString& idServer)
+   TRY
+   {
+      idServer_=idServer ;
+   }
+   CATCH_DUMP_ATTR
+
+   
+   const StdString& CContext::getIdServer()
+   TRY
+   {
+      return idServer_;
+   }
+   CATCH_DUMP_ATTR
+
+   const StdString& CContext::getIdServer(const int i)
+   TRY
+   {
+//     return idServer_ + std::to_string(static_cast<unsigned long long>(i));
+      return primServerId_[i] ;
+   }
+   CATCH_DUMP_ATTR
+
+/*
    const StdString& CContext::getIdServer()
    TRY
@@ -1570 +1979 @@
    }
    CATCH_DUMP_ATTR
+*/
+
 
    /*!

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/context.hpp

@@ -12 +12 @@
 #include "registry.hpp"
 #include "mpi.hpp"
+#include "services_manager.hpp"
+#include "server_context.hpp"
 
 
@@ -89 +91 @@
          // Initialize server or client
          void initClient(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtServer = 0);
+         void init(CServerContext* parentServerContext, MPI_Comm intraComm, int serviceType);
+         void initClient(MPI_Comm intraComm, int serviceType);
+         
          void initServer(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtClient = 0);
+         void initServer(MPI_Comm intraComm, int serviceType );
+         void createClientInterComm(MPI_Comm interCommClient, MPI_Comm interCommServer)  ;
+         void createServerInterComm(void)  ;
+
          bool isInitialized(void);
 
@@ -96 +105 @@
          // Put sever or client into loop state
          bool checkBuffersAndListen(bool enableEventsProcessing=true);
+         bool eventLoop(bool enableEventsProcessing=true);
 
          // Finalize a context
          void finalize(void);
+         void finalize_old(void);
          bool isFinalized(void);
 
@@ -161 +172 @@
 
          const StdString& getIdServer();
+         void setIdServer(const StdString& idServer);
          const StdString& getIdServer(const int srvPoolNb);
+         std::string getContextId() {return contextId_;}
 
          // Client side: Receive and process messages
@@ -208 +221 @@
         static void ShowTree(StdOStream & out = std::clog);
         static void CleanTree(void);
+        int getServiceType(void) {return serviceType_;}
 
       public :
@@ -223 +237 @@
          virtual bool hasChild(void) const;
 
+         bool isProcessingEvent(void) {return isProcessingEvent_;}
+         bool setProcessingEvent(void) {isProcessingEvent_=true ;}
+         bool unsetProcessingEvent(void) {isProcessingEvent_=false ;}
 
       public :
@@ -252 +269 @@
          std::vector<CContextServer*> serverPrimServer;
          std::vector<CContextClient*> clientPrimServer;
+         std::vector<std::string> primServerId_;
 
          CRegistry* registryIn ;    //!< input registry which is read from file
          CRegistry* registryOut ;   //!< output registry which will be written into file at the finalize
+
+
+        MPI_Comm intraComm_ ; //! context intra communicator
 
       private:
@@ -265 +286 @@
          std::list<MPI_Comm> comms; //!< Communicators allocated internally
 
+         int serviceType_;  //!< service associated to the context
+         string contextId_ ; //!< context client id for the servers. For clients this is same as getId() 
+         bool isProcessingEvent_ ;
+         CServerContext* parentServerContext_ ;
+
       public: // Some function maybe removed in the near future
         // virtual void toBinary  (StdOStream & os) const;

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/domain.cpp

@@ -226 +226 @@
       bool distributed =  !((!ni.isEmpty() && (ni == ni_glo) && !nj.isEmpty() && (nj == nj_glo)) ||
               (!i_index.isEmpty() && i_index.numElements() == ni_glo*nj_glo));
+      bool distributed_glo ;
       distributed |= (1 == CContext::getCurrent()->client->clientSize);
 
@@ -2061 +2062 @@
 
       numberWrittenIndexes_[writtenCommSize] = nbWritten;
-      if (isDistributed())
+      bool distributed_glo, distributed=isDistributed() ;
+      MPI_Allreduce(&distributed,&distributed_glo, 1, MPI_INT, MPI_LOR, writtenComm) ;
+      
+      if (distributed_glo)
       {
              

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/field.cpp

@@ -634 +634 @@
           timer.resume();
 
-          context->checkBuffersAndListen();
+//ym          context->checkBuffersAndListen();
+            context->eventLoop();
 
           timer.suspend();

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/file.cpp

@@ -323 +323 @@
       // Done by classical server or secondary server
       // This condition should be changed soon
-      if (CServer::serverLevel == 0 || CServer::serverLevel == 2)
-      {
+//ym      if (CServer::serverLevel == 0 || CServer::serverLevel == 2)
+       if (context->getServiceType()==CServicesManager::IO_SERVER || context->getServiceType()==CServicesManager::OUT_SERVER)
+       {
         if (mode.isEmpty() || mode.getValue() == mode_attr::write)
         {
@@ -349 +350 @@
       // Done by classical server or secondary server
       // TODO: This condition should be changed soon. It only works with maximum number of level as 2
-      if (CServer::serverLevel == 0 || CServer::serverLevel == 1)
+
+//ym      if (CServer::serverLevel == 0 || CServer::serverLevel == 1)
+      if (context->getServiceType()==CServicesManager::IO_SERVER || context->getServiceType()==CServicesManager::GATHERER)
       {
         if (!mode.isEmpty() && mode.getValue() == mode_attr::read)

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/grid.cpp

@@ -907 +907 @@
        }
      }
-     if (CServer::serverLevel==2)
+//ym     if (CServer::serverLevel==2)
+     if (context->getServiceType()==CServicesManager::OUT_SERVER)
      {
        computeWrittenIndex() ;
@@ -1679 +1680 @@
     connectedServerRankRead_ = ranks;
 
-    int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
-    nbSrvPools = 1;    
+// ym something is bad here, I comment some line, to be checked in future
+   // int nbSrvPools = (context->hasServer) ? (context->hasClient ? context->clientPrimServer.size() : 1) : 1;
+    int nbSrvPools = 1;    
     nbReadSenders.clear();
     for (int p = 0; p < nbSrvPools; ++p)
     {
-      CContextServer* server = (!context->hasClient) ? context->server : context->serverPrimServer[p];
+     // CContextServer* server = (!context->hasClient) ? context->server : context->serverPrimServer[p];
+      CContextServer* server = context->server  ;
       CContextClient* client = context->client;   //(!context->hasClient) ? context->client : context->clientPrimServer[p];
       

XIOS/dev/dev_ym/XIOS_SERVICES/src/object.cpp

@@ -29 +29 @@
   }
 
-  const StdString& CObject::getIdServer() const
+  const StdString& CObject::getIdServer() 
+  {
+    return this->id;
+  }
+
+  const StdString& CObject::getIdServer(int nSrvpool) 
   {
     return this->id;

XIOS/dev/dev_ym/XIOS_SERVICES/src/object.hpp

@@ -17 +17 @@
       /// Accesseurs ///
       const StdString& getId(void) const;
-      virtual const StdString& getIdServer() const;
+      virtual const StdString& getIdServer() ;
+      virtual const StdString& getIdServer(int nsrvPool) ;
 
       virtual StdString dumpClassAttributes(void);

XIOS/dev/dev_ym/XIOS_SERVICES/src/object_template_impl.hpp

@@ -270 +270 @@
         {
           CMessage msg;
-          msg<<this->getIdServer();
+          if (context->hasServer)  msg<<this->getIdServer(i);
+          else  msg<<this->getIdServer(); 
+
           msg << attr.getName();
           msg << attr;
@@ -290 +292 @@
     {
       CMessage msg;
-      msg<<this->getIdServer();
+      msg<<this->getIdServer(); // pb with context attribute -> to check : for now seem to be never used for context...
       msg << attr.getName();
       msg << attr;

XIOS/dev/dev_ym/XIOS_SERVICES/src/server.cpp

@@ -15 +15 @@
 #include "event_scheduler.hpp"
 #include "string_tools.hpp"
+#include "ressources_manager.hpp"
+#include "services_manager.hpp"
+#include "contexts_manager.hpp"
+#include "servers_ressource.hpp"
+#include <cstdio>
+
+
 
 namespace xios
 {
     MPI_Comm CServer::intraComm ;
+    MPI_Comm CServer::serversComm_ ;
     std::list<MPI_Comm> CServer::interCommLeft ;
     std::list<MPI_Comm> CServer::interCommRight ;
@@ -34 +42 @@
     bool CServer::is_MPI_Initialized ;
     CEventScheduler* CServer::eventScheduler = 0;
+    CServersRessource* CServer::serversRessource_=nullptr ;
+
+    void CServer::initRessources(void)
+    { 
+      auto ressourcesManager=CXios::getRessourcesManager() ;
+      auto servicesManager=CXios::getServicesManager() ;
+      auto contextsManager=CXios::getContextsManager() ;
+      auto daemonsManager=CXios::getDaemonsManager() ;
+      auto serversRessource=CServer::getServersRessource() ;
+
+      if (serversRessource->isServerLeader())
+      {
+ //        ressourcesManager->createPool("LMDZ",ressourcesManager->getRessourcesSize()/2) ;
+ //        ressourcesManager->createPool("NEMO",ressourcesManager->getRessourcesSize()/2) ;
+          ressourcesManager->createPool("LMDZ",ressourcesManager->getRessourcesSize()) ;
+          servicesManager->createServices("LMDZ", "ioserver", CServicesManager::IO_SERVER, 8, 5) ;
+          for(int i=0 ; i<5;i++)
+          {
+            contextsManager->createServerContext("LMDZ","ioserver",i,"lmdz") ;
+          }
+      }
+
+
+
+      while (true)
+      {
+        daemonsManager->eventLoop() ;
+      }
+
+
+    }
+    
+    void CServer::initialize(void)
+    {
+      
+      MPI_Comm serverComm ;
+      int initialized ;
+      MPI_Initialized(&initialized) ;
+      if (initialized) is_MPI_Initialized=true ;
+      else is_MPI_Initialized=false ;
+      MPI_Comm globalComm=CXios::getGlobalComm() ;
+
+      /////////////////////////////////////////
+      ///////////// PART 1 ////////////////////
+      /////////////////////////////////////////
+
+      // don't use OASIS
+      if (!CXios::usingOasis)
+      {
+        if (!is_MPI_Initialized) MPI_Init(NULL, NULL);
+       
+        // split the global communicator
+        // get hash from all model to attribute a unique color (int) and then split to get client communicator
+        // every mpi process of globalComm (MPI_COMM_WORLD) must participate
+         
+        int commRank, commSize ;
+        MPI_Comm_rank(globalComm,&commRank) ;
+        MPI_Comm_size(globalComm,&commSize) ;
+
+        std::hash<string> hashString ;
+        size_t hashServer=hashString(CXios::xiosCodeId) ;
+          
+        size_t* hashAll = new size_t[commSize] ;
+        MPI_Allgather(&hashServer,1,MPI_UNSIGNED_LONG,hashAll,1,MPI_LONG,globalComm) ;
+          
+        int color=0 ;
+        set<size_t> listHash ;
+        for(int i=0 ; i<=commRank ; i++) 
+          if (listHash.count(hashAll[i])==1)
+          {
+            listHash.insert(hashAll[i]) ;
+            color=color+1 ;
+          }
+        delete[] hashAll ;
+
+        MPI_Comm_split(globalComm, color, commRank, &serverComm) ;
+      }
+      else // using OASIS
+      {
+        if (!is_MPI_Initialized) oasis_init(CXios::xiosCodeId);
+
+        CTimer::get("XIOS").resume() ;
+        oasis_get_localcomm(serverComm);
+      }
+ 
+      /////////////////////////////////////////
+      ///////////// PART 2 ////////////////////
+      /////////////////////////////////////////
+      
+
+      // Create the XIOS communicator for every process which is related
+      // to XIOS, as well on client side as on server side
+      MPI_Comm xiosGlobalComm ;
+      string strIds=CXios::getin<string>("clients_code_id","") ;
+      vector<string> clientsCodeId=splitRegex(strIds,"\\s*,\\s*") ;
+      if (strIds.empty())
+      {
+        // no code Ids given, suppose XIOS initialisation is global            
+        int commRank, commGlobalRank, serverLeader, clientLeader,serverRemoteLeader,clientRemoteLeader ;
+        MPI_Comm splitComm,interComm ;
+        MPI_Comm_rank(globalComm,&commGlobalRank) ;
+        MPI_Comm_split(globalComm, 1, commGlobalRank, &splitComm) ;
+        MPI_Comm_rank(splitComm,&commRank) ;
+        if (commRank==0) serverLeader=commGlobalRank ;
+        else serverLeader=0 ;
+        clientLeader=0 ;
+        MPI_Allreduce(&clientLeader,&clientRemoteLeader,1,MPI_INT,MPI_SUM,globalComm) ;
+        MPI_Allreduce(&serverLeader,&serverRemoteLeader,1,MPI_INT,MPI_SUM,globalComm) ;
+        MPI_Intercomm_create(splitComm, 0, globalComm, clientRemoteLeader,1341,&interComm) ;
+        MPI_Intercomm_merge(interComm,false,&xiosGlobalComm) ;
+        CXios::setXiosComm(xiosGlobalComm) ;
+      }
+      else
+      {
+
+        xiosGlobalCommByFileExchange(serverComm) ;
+
+      }
+      
+      /////////////////////////////////////////
+      ///////////// PART 4 ////////////////////
+      //  create servers intra communicator  //
+      ///////////////////////////////////////// 
+      
+      int commRank ;
+      MPI_Comm_rank(CXios::getXiosComm(), &commRank) ;
+      MPI_Comm_split(CXios::getXiosComm(),true,commRank,&serversComm_) ;
+      
+      CXios::setUsingServer() ;
+
+      /////////////////////////////////////////
+      ///////////// PART 5 ////////////////////
+      //       redirect files output         //
+      ///////////////////////////////////////// 
+      
+      CServer::openInfoStream(CXios::serverFile);
+      CServer::openErrorStream(CXios::serverFile);
+
+      /////////////////////////////////////////
+      ///////////// PART 4 ////////////////////
+      /////////////////////////////////////////
+
+      CXios::launchDaemonsManager(true) ;
+     
+      /////////////////////////////////////////
+      ///////////// PART 5 ////////////////////
+      /////////////////////////////////////////
+
+      // create the services
+
+      auto ressourcesManager=CXios::getRessourcesManager() ;
+      auto servicesManager=CXios::getServicesManager() ;
+      auto contextsManager=CXios::getContextsManager() ;
+      auto daemonsManager=CXios::getDaemonsManager() ;
+      auto serversRessource=CServer::getServersRessource() ;
+
+      if (serversRessource->isServerLeader())
+      {
+        int nbRessources = ressourcesManager->getRessourcesSize() ;
+        if (!CXios::usingServer2)
+        {
+          ressourcesManager->createPool(CXios::defaultPoolId, nbRessources) ;
+          servicesManager->createServices(CXios::defaultPoolId, CXios::defaultServerId, CServicesManager::IO_SERVER,nbRessources,1) ;
+        }
+        else
+        {
+          int nprocsServer = nbRessources*CXios::ratioServer2/100.;
+          int nprocsGatherer = nbRessources - nprocsServer ;
+          
+          int nbPoolsServer2 = CXios::nbPoolsServer2 ;
+          if (nbPoolsServer2 == 0) nbPoolsServer2 = nprocsServer;
+          ressourcesManager->createPool(CXios::defaultPoolId, nbRessources) ;
+          servicesManager->createServices(CXios::defaultPoolId,  CXios::defaultGathererId, CServicesManager::GATHERER, nprocsGatherer, 1) ;
+          servicesManager->createServices(CXios::defaultPoolId,  CXios::defaultServerId, CServicesManager::OUT_SERVER, nprocsServer, nbPoolsServer2) ;
+        }
+      }
+
+      /////////////////////////////////////////
+      ///////////// PART 5 ////////////////////
+      /////////////////////////////////////////
+      // loop on event loop
+
+      bool finished=false ;
+      while (!finished)
+      {
+        finished=daemonsManager->eventLoop() ;
+      }
+
+    }
+
+
+
+
+
+    void  CServer::xiosGlobalCommByFileExchange(MPI_Comm serverComm)
+    {
+        
+      MPI_Comm globalComm=CXios::getGlobalComm() ;
+      MPI_Comm xiosGlobalComm ;
+      
+      string strIds=CXios::getin<string>("clients_code_id","") ;
+      vector<string> clientsCodeId=splitRegex(strIds,"\\s*,\\s*") ;
+      
+      int commRank, globalRank ;
+      MPI_Comm_rank(serverComm, &commRank) ;
+      MPI_Comm_rank(globalComm, &globalRank) ;
+      string serverFileName("__xios_publisher::"+CXios::xiosCodeId+"__to_remove__") ;
+
+      if (commRank==0) // if root process publish name
+      {  
+        std::ofstream ofs (serverFileName, std::ofstream::out);
+        ofs<<globalRank ;
+        ofs.close();
+      }
+        
+      vector<int> clientsRank(clientsCodeId.size()) ;
+      for(int i=0;i<clientsRank.size();i++)
+      {
+        std::ifstream ifs ;
+        string fileName=("__xios_publisher::"+clientsCodeId[i]+"__to_remove__") ;
+        do
+        {
+          ifs.clear() ;
+          ifs.open(fileName, std::ifstream::in) ;
+        } while (ifs.fail()) ;
+        ifs>>clientsRank[i] ;
+        ifs.close() ; 
+      }
+
+      MPI_Comm intraComm ;
+      MPI_Comm_dup(serverComm,&intraComm) ;
+      MPI_Comm interComm ;
+      for(int i=0 ; i<clientsRank.size(); i++)
+      {  
+        MPI_Intercomm_create(intraComm, 0, globalComm, clientsRank[i], 3141, &interComm);
+        MPI_Comm_free(&intraComm) ;
+        MPI_Intercomm_merge(interComm,false, &intraComm ) ;
+      }
+      xiosGlobalComm=intraComm ;  
+      MPI_Barrier(xiosGlobalComm);
+      if (commRank==0) std::remove(serverFileName.c_str()) ;
+      MPI_Barrier(xiosGlobalComm);
+
+      CXios::setXiosComm(xiosGlobalComm) ;
+      
+    }
+
+
+    void  CServer::xiosGlobalCommByPublishing(MPI_Comm serverComm)
+    {
+        // untested, need to be tested on a true MPI-2 compliant library
+
+        // try to discover other client/server
+/*
+        // publish server name
+        char portName[MPI_MAX_PORT_NAME];
+        int ierr ;
+        int commRank ;
+        MPI_Comm_rank(serverComm, &commRank) ;
+        
+        if (commRank==0) // if root process publish name
+        {  
+          MPI_Open_port(MPI_INFO_NULL, portName);
+          MPI_Publish_name(CXios::xiosCodeId.c_str(), MPI_INFO_NULL, portName);
+        }
+
+        MPI_Comm intraComm=serverComm ;
+        MPI_Comm interComm ;
+        for(int i=0 ; i<clientsCodeId.size(); i++)
+        {  
+          MPI_Comm_accept(portName, MPI_INFO_NULL, 0, intraComm, &interComm);
+          MPI_Intercomm_merge(interComm,false, &intraComm ) ;
+        }
+*/      
+    }
 
 //---------------------------------------------------------------
@@ -45 +328 @@
  *   IMPORTANT: CXios::usingServer2 should NOT be used beyond this function. Use CServer::serverLevel instead.
  */
-    void CServer::initialize(void)
+    void CServer::initialize_old(void)
     {
       int initialized ;
@@ -53 +336 @@
       int rank ;
 
+      CXios::launchRessourcesManager(true) ;
+      CXios::launchServicesManager(true) ;
+      CXios::launchContextsManager(true) ;
+      
+      initRessources() ;
       // Not using OASIS
       if (!CXios::usingOasis)
@@ -421 +709 @@
           MPI_Comm_free(&(*it));
 
-      MPI_Comm_free(&intraComm);
+//      MPI_Comm_free(&intraComm);
 
       if (!is_MPI_Initialized)
@@ -854 +1142 @@
          }
          else
-           it->second->checkBuffersAndListen(enableEventsProcessing);
+          it->second->eventLoop(enableEventsProcessing);
+//ym          it->second->checkBuffersAndListen(enableEventsProcessing);
        }
      }
@@ -881 +1170 @@
     void CServer::openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb)
     {
-      StdStringStream fileNameClient;
+      StdStringStream fileNameServer;
       int numDigit = 0;
-      int size = 0;
+      int commSize = 0;
+      int commRank ;
       int id;
-      MPI_Comm_size(CXios::globalComm, &size);
-      while (size)
+      
+      MPI_Comm_size(CXios::getGlobalComm(), &commSize);
+      MPI_Comm_rank(CXios::getGlobalComm(), &commRank);
+
+      while (commSize)
       {
-        size /= 10;
+        commSize /= 10;
         ++numDigit;
       }
-      id = rank_; //getRank();
-
-      fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << id << ext;
-      fb->open(fileNameClient.str().c_str(), std::ios::out);
+      id = commRank;
+
+      fileNameServer << fileName << "_" << std::setfill('0') << std::setw(numDigit) << id << ext;
+      fb->open(fileNameServer.str().c_str(), std::ios::out);
       if (!fb->is_open())
         ERROR("void CServer::openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb)",
-              << std::endl << "Can not open <" << fileNameClient.str() << "> file to write the server log(s).");
+              << std::endl << "Can not open <" << fileNameServer.str() << "> file to write the server log(s).");
     }
 
@@ -953 +1246 @@
       if (m_errorStream.is_open()) m_errorStream.close();
     }
+
+    void CServer::launchServersRessource(MPI_Comm serverComm)
+    {
+      serversRessource_ = new CServersRessource(serverComm) ;
+    }
 }

XIOS/dev/dev_ym/XIOS_SERVICES/src/server.hpp

@@ -10 +10 @@
 namespace xios
 {
+    class CServersRessource ;
+
     class CServer
     {
       public:
         static void initialize(void);
+        static void initialize_old(void);
+        static void xiosGlobalCommByFileExchange(MPI_Comm serverComm) ;
+        static void xiosGlobalCommByPublishing(MPI_Comm serverComm) ;
+
         static void finalize(void);
         static void eventLoop(void);
@@ -25 +31 @@
         static void listenOasisEnddef(void);
         static void registerContext(void* buff,int count, int leaderRank=0);
+        static void initRessources() ;
 
         static MPI_Comm intraComm;
+        static MPI_Comm serversComm_;
         static std::list<MPI_Comm> interCommLeft;           // interComm between server (primary, classical or secondary) and its client (client or primary server)
         static std::list<MPI_Comm> interCommRight;          // interComm between primary server and secondary server (non-empty only for primary server pool)
@@ -68 +76 @@
         static void closeErrorStream();
 
+        static CServersRessource* getServersRessource(void) { return serversRessource_;}
+        static void launchServersRessource(MPI_Comm commServer) ;
+      
       private:
         static vector<int> sndServerGlobalRanks;  //!< Global ranks of pool leaders on the secondary server
@@ -75 +86 @@
         static StdOFStream m_errorStream;
         static void openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb);
+        static CServersRessource* serversRessource_ ;
     };
 }