Changeset 1764

Timestamp:

11/05/19 16:02:34 (3 years ago)

Author:

ymipsl

Message:

Some Update on XIOS services
Seems to work on Irène for :

• first level of servers
• fisrt + second level of servers
• attached mode

YM

Location:

XIOS/dev/dev_ym/XIOS_SERVICES/src

Files:

• client.cpp (modified) (3 diffs)
• context_client.cpp (modified) (1 diff)
• context_server.cpp (modified) (4 diffs)
• context_server.hpp (modified) (1 diff)
• cxios.cpp (modified) (1 diff)
• cxios.hpp (modified) (1 diff)
• manager/contexts_manager.cpp (modified) (3 diffs)
• manager/contexts_manager.hpp (modified) (1 diff)
• manager/daemons_manager.cpp (modified) (2 diffs)
• manager/daemons_manager.hpp (modified) (1 diff)
• manager/pool_ressource.cpp (modified) (1 diff)
• manager/pool_ressource.hpp (modified) (1 diff)
• manager/ressources_manager.cpp (modified) (1 diff)
• manager/ressources_manager.hpp (modified) (1 diff)
• manager/server_context.cpp (modified) (14 diffs)
• manager/server_context.hpp (modified) (3 diffs)
• manager/servers_ressource.cpp (modified) (4 diffs)
• manager/servers_ressource.hpp (modified) (2 diffs)
• manager/services.cpp (modified) (6 diffs)
• manager/services.hpp (modified) (2 diffs)
• manager/services_manager.cpp (modified) (1 diff)
• manager/services_manager.hpp (modified) (1 diff)
• manager/window_manager.hpp (modified) (1 diff)
• node/context.cpp (modified) (9 diffs)
• node/context.hpp (modified) (5 diffs)
• node/field.cpp (modified) (1 diff)
• server.cpp (modified) (1 diff)

XIOS/dev/dev_ym/XIOS_SERVICES/src/client.cpp

@@ -355 +355 @@
       CXios::setXiosComm(xiosGlobalComm) ;
 
+      MPI_Comm commUnfree ;
+      MPI_Comm_dup(clientComm, &commUnfree ) ;
+ 
     }
 
@@ -732 +735 @@
       CTimer::get("XIOS init/finalize",false).suspend() ;
       CTimer::get("XIOS").suspend() ;
+      
+      CXios::finalizeDaemonsManager() ;
+
       if (!is_MPI_Initialized)
       {
@@ -748 +754 @@
       report(0)<< " Memory report : increasing it by a factor will increase performance, depending of the volume of data wrote in file at each time step of the file"<<endl ;
       report(100)<<CTimer::getAllCumulatedTime()<<endl ;
+    
+      
     }
     

XIOS/dev/dev_ym/XIOS_SERVICES/src/context_client.cpp

@@ -262 +262 @@
           {
             for (int i = 0; i < context->serverPrimServer.size(); ++i)  context->serverPrimServer[i]->listen();
-            CServer::contextEventLoop(false) ; // avoid dead-lock at finalize...
+ //ym           CServer::contextEventLoop(false) ; // avoid dead-lock at finalize...
+            context->globalEventLoop() ;
           }
 

XIOS/dev/dev_ym/XIOS_SERVICES/src/context_server.cpp

@@ -30 +30 @@
   using namespace std ;
 
-  CContextServer::CContextServer(CContext* parent,MPI_Comm intraComm_,MPI_Comm interComm_)
+  CContextServer::CContextServer(CContext* parent,MPI_Comm intraComm_,MPI_Comm interComm_) : eventScheduler_(nullptr), isProcessingEvent_(false)
   {
     context=parent;
@@ -53 +53 @@
     if (contextInfo.serviceType != CServicesManager::CLIENT) // we must have an event scheduler => to be retrieve from the associated services
     {
-      eventScheduler_=CXios::getPoolRessource()->getService(contextInfo.serviceId,contextInfo.partitionId)->getEventScheduler() ;
+      if (!isAttachedModeEnabled()) eventScheduler_=CXios::getPoolRessource()->getService(contextInfo.serviceId,contextInfo.partitionId)->getEventScheduler() ;
     }
 
@@ -321 +321 @@
     CEventServer* event;
     
-    if (context->isProcessingEvent()) return ;
+//    if (context->isProcessingEvent()) return ;
+    if (isProcessingEvent_) return ;
 
     it=events.find(currentTimeLine);
@@ -330 +331 @@
       if (event->isFull())
       {
-        if (!scheduled && CServer::eventScheduler) // Skip event scheduling for attached mode and reception on client side
+        if (!scheduled && eventScheduler_) // Skip event scheduling for attached mode and reception on client side
         {
-          CServer::eventScheduler->registerEvent(currentTimeLine,hashId);
+          eventScheduler_->registerEvent(currentTimeLine,hashId);
           scheduled=true;
         }
-        else if (!CServer::eventScheduler || CServer::eventScheduler->queryEvent(currentTimeLine,hashId) )
+        else if (!eventScheduler_ || eventScheduler_->queryEvent(currentTimeLine,hashId) )
         {
          // When using attached mode, synchronise the processes to avoid that differents event be scheduled by differents processes
          // The best way to properly solve this problem will be to use the event scheduler also in attached mode
          // for now just set up a MPI barrier
-         if (!CServer::eventScheduler && CXios::isServer) MPI_Barrier(intraComm) ;
-
-         context->setProcessingEvent() ;
+         if (!eventScheduler_ && CXios::isServer) MPI_Barrier(intraComm) ;
+
+//         context->setProcessingEvent() ;
+         isProcessingEvent_=true ;
          CTimer::get("Process events").resume();
          dispatchEvent(*event);
          CTimer::get("Process events").suspend();
-         context->unsetProcessingEvent() ;
+         isProcessingEvent_=false ;
+//         context->unsetProcessingEvent() ;
          pendingEvent=false;
          delete event;

XIOS/dev/dev_ym/XIOS_SERVICES/src/context_server.hpp

@@ -66 +66 @@
       vector<MPI_Win> windows ; //! one sided mpi windows to expose client buffers to servers ; No memory will be attached on server side.
       CEventScheduler* eventScheduler_ ;
+      bool isProcessingEvent_ ;
   } ;
 

XIOS/dev/dev_ym/XIOS_SERVICES/src/cxios.cpp

@@ -341 +341 @@
   }
 
+  
+  void CXios::finalizeRessourcesManager()
+  {
+    delete ressourcesManager_;
+  }
+
+  void CXios::finalizeServicesManager()
+  {
+    delete servicesManager_  ;
+  }
+
+  void CXios::finalizeContextsManager()
+  {
+    delete contextsManager_  ;
+  }
+  
+  void CXios::finalizeDaemonsManager()
+  {
+    delete daemonsManager_  ;
+  }
+  
+
   CPoolRessource* CXios::getPoolRessource(void)
   {

XIOS/dev/dev_ym/XIOS_SERVICES/src/cxios.hpp

@@ -91 +91 @@
      static void launchRessourcesManager(bool isXiosServer) ;
      
+     static void finalizeServicesManager() ;
+     static void finalizeContextsManager() ;
+     static void finalizeDaemonsManager() ;
+     static void finalizeRessourcesManager() ;
+
      static CRessourcesManager* getRessourcesManager(void) { return ressourcesManager_ ;}
      static CServicesManager*   getServicesManager(void) { return servicesManager_ ;}

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/contexts_manager.cpp

@@ -36 +36 @@
   }
 
+
+  CContextsManager::~CContextsManager()
+  {
+    delete winNotify_ ;
+    delete winContexts_ ;
+  }
+
   bool CContextsManager::createServerContext(const std::string& poolId, const std::string& serviceId, const int& partitionId,
                                              const string& contextId, bool wait)
@@ -48 +55 @@
       while (!ok) 
       {
-        CXios::getDaemonsManager()->eventLoop() ;
+        CXios::getDaemonsManager()->servicesEventLoop() ;
         ok=servicesManager->getServiceLeader(poolId, serviceId, partitionId, serviceLeader) ;
       }
@@ -79 +86 @@
       while (!ok) 
       {
-        CXios::getDaemonsManager()->eventLoop() ;
+        CXios::getDaemonsManager()->servicesEventLoop() ;
         ok=CXios::getServicesManager()->getServiceType(poolId,serviceId, 0, type) ;
         if (ok) ok=getContextLeader(getServerContextName(poolId, serviceId, partitionId, type, contextId), contextLeader) ;

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/contexts_manager.hpp

@@ -28 +28 @@
  
     CContextsManager(bool isXiosServer) ;
+    ~CContextsManager() ;
+    
     bool createServerContext(const std::string& poolId, const std::string& serviceId, const int& partitionId, const string& contextId, bool wait=true) ;
 /*    bool createServerContextIntercomm(const std::string& poolId, const std::string& serviceId, const int& partitionId, const std::string& contextId,

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/daemons_manager.cpp

@@ -26 +26 @@
   }
 
+  CDaemonsManager::~CDaemonsManager()
+  {
+    CXios::finalizeContextsManager() ;
+    CXios::finalizeServicesManager() ;
+    CXios::finalizeRessourcesManager() ;
+  }
+
   bool CDaemonsManager::eventLoop(void)
   {
@@ -31 +38 @@
     CXios::getServicesManager()->eventLoop() ;
     CXios::getContextsManager()->eventLoop() ;
-    if (isServer_) return CServer::getServersRessource()->eventLoop() ;
-    else  return CXios::getPoolRessource()->eventLoop() ;
+    if (isServer_) return CServer::getServersRessource()->eventLoop(false) ;
+    else  return CXios::getPoolRessource()->eventLoop(false) ;
   }
+  
+  bool CDaemonsManager::servicesEventLoop(void)
+  {
+    CXios::getRessourcesManager()->eventLoop() ;
+    CXios::getServicesManager()->eventLoop() ;
+    CXios::getContextsManager()->eventLoop() ;
+    if (isServer_) return CServer::getServersRessource()->eventLoop(true) ;
+    else  return CXios::getPoolRessource()->eventLoop(true) ;
+  } 
 
 }

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/daemons_manager.hpp

@@ -11 +11 @@
 
     CDaemonsManager(bool isXiosServer) ;
+    ~CDaemonsManager() ;
 
     bool eventLoop(void) ;
+    bool servicesEventLoop(void) ;
     
     private:

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/pool_ressource.cpp

@@ -100 +100 @@
   }
 
-  bool CPoolRessource::eventLoop(void)
+  bool CPoolRessource::eventLoop(bool serviceOnly)
   {
     checkCreateServiceNotification() ;
     for (auto it=services_.begin(); it!=services_.end() ; ++it) 
     {
-      if (it->second->eventLoop())
+      if (it->second->eventLoop(serviceOnly))
       {
         services_.erase(it) ;

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/pool_ressource.hpp

@@ -31 +31 @@
     void checkCreateServiceNotification(void) ;
     void createNewService(const std::string& serviceId, int type, int size, int nbPartitions, bool in) ;
-     bool eventLoop(void) ;
+     bool eventLoop(bool serviceOnly=false) ;
     CService* getService(const std::string serviceId, int partitionId) { return services_[make_tuple(serviceId,partitionId)]; }
     void finalizeSignal(void) ;

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/ressources_manager.cpp

@@ -34 +34 @@
     MPI_Barrier(xiosComm_)  ;    
   }
-
+  
+  CRessourcesManager::~CRessourcesManager()
+  {
+    delete winNotify_ ;
+    delete winRessources_ ;
+  } 
 
   void CRessourcesManager::createPool(const string& poolId, int size)

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/ressources_manager.hpp

@@ -28 +28 @@
     
     CRessourcesManager(bool isXiosServer) ;
+    ~CRessourcesManager() ;
 
     void eventLoop(void) ;

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/server_context.cpp

@@ -15 +15 @@
 
   CServerContext::CServerContext(CService* parentService, MPI_Comm contextComm, const std::string& poolId, const std::string& serviceId, 
-                                 const int& partitionId, const std::string& contextId) : finalizeSignal_(false), parentService_(parentService)
+                                 const int& partitionId, const std::string& contextId) : finalizeSignal_(false), parentService_(parentService),
+                                 hasNotification_(false)
   {
    int localRank, globalRank, commSize ;
@@ -48 +49 @@
   }
 
-  
+  CServerContext::~CServerContext()
+  {
+
+  } 
+
   bool CServerContext::createIntercomm(const string& poolId, const string& serviceId, const int& partitionId, const string& contextId, 
                                        const MPI_Comm& intraComm, MPI_Comm& interCommClient, MPI_Comm& interCommServer, bool wait)
@@ -75 +80 @@
     }
     
-    auto eventScheduler=parentService_->getEventScheduler() ;
-    std::hash<string> hashString ;
-    size_t hashId = hashString(name_) ;
-    size_t currentTimeLine=0 ;
-    eventScheduler->registerEvent(currentTimeLine,hashId); 
-
-    while (!eventScheduler->queryEvent(currentTimeLine,hashId))
-    {
-       CXios::getDaemonsManager()->eventLoop() ;
-    }  
+    MPI_Request req ;
+    MPI_Status status ;
+    MPI_Ibarrier(intraComm,&req) ;
+    
+    int flag=false ;
+    while(!flag) 
+    {
+      CXios::getDaemonsManager()->servicesEventLoop() ;
+      MPI_Test(&req,&flag,&status) ;
+    }
+//    auto eventScheduler=parentService_->getEventScheduler() ;
+//    std::hash<string> hashString ;
+//    size_t hashId = hashString(name_) ;
+//    size_t currentTimeLine=0 ;
+//    eventScheduler->registerEvent(currentTimeLine,hashId); 
+//
+//    while (!eventScheduler->queryEvent(currentTimeLine,hashId))
+//    {
+//       CXios::getDaemonsManager()->servicesEventLoop() ;
+//       eventScheduler->checkEvent() ;
+//    }  
     
     MPI_Bcast(&ok, 1, MPI_INT, 0, intraComm) ;
@@ -90 +106 @@
     if (ok)  
     {
+      int globalRank ;
+      MPI_Comm_rank(xiosComm_,&globalRank) ;
+      MPI_Bcast(&contextLeader, 1, MPI_INT, 0, intraComm) ;
+      
+      int overlap, nOverlap ;
+      if (contextLeader==globalRank) overlap=1 ;
+      else overlap=0 ;
+      MPI_Allreduce(&overlap, &nOverlap, 1, MPI_INT, MPI_SUM, contextComm_) ;
+/*
       int overlap  ;
       if (get<0>(overlapedComm_[name_])) overlap=1 ;
@@ -98 +123 @@
       int commSize ;
       MPI_Comm_size(contextComm_,&commSize ) ;
-      if (nOverlap==commSize)
-      {
+*/
+      if (nOverlap> 0 )
+      {
+        while (get<0>(overlapedComm_[name_])==false) CXios::getDaemonsManager()->servicesEventLoop() ;
+        isAttachedMode_=true ;
         cout<<"CServerContext::createIntercomm : total overlap ==> context in attached mode"<<endl ;
         interCommClient=newInterCommClient ;
@@ -107 +135 @@
       { 
         cout<<"CServerContext::createIntercomm : No overlap ==> context in server mode"<<endl ;
+        isAttachedMode_=false ;
         MPI_Intercomm_create(intraComm, 0, xiosComm_, contextLeader, 3141, &interCommClient) ;
         MPI_Comm_dup(interCommClient, &interCommServer) ;
@@ -128 +157 @@
      for(int rank=0; rank<commSize; rank++)
      {
-       notifyType_=NOTIFY_CREATE_INTERCOMM ;
-       notifyCreateIntercomm_ = make_tuple(remoteLeader, sourceContext) ;
+       notifyOutType_=NOTIFY_CREATE_INTERCOMM ;
+       notifyOutCreateIntercomm_ = make_tuple(remoteLeader, sourceContext) ;
        sendNotification(rank) ;
      }
@@ -147 +176 @@
     buffer.realloc(maxBufferSize_) ;
     
-    if (notifyType_==NOTIFY_CREATE_INTERCOMM)
-    {
-      auto& arg=notifyCreateIntercomm_ ;
-      buffer << notifyType_ << std::get<0>(arg)<<std::get<1>(arg) ;
+    if (notifyOutType_==NOTIFY_CREATE_INTERCOMM)
+    {
+      auto& arg=notifyOutCreateIntercomm_ ;
+      buffer << notifyOutType_ << std::get<0>(arg)<<std::get<1>(arg) ;
     }
   }
@@ -156 +185 @@
   void CServerContext::notificationsDumpIn(CBufferIn& buffer)
   {
-    if (buffer.bufferSize() == 0) notifyType_= NOTIFY_NOTHING ;
+    if (buffer.bufferSize() == 0) notifyInType_= NOTIFY_NOTHING ;
     else
     {
-      buffer>>notifyType_;
-      if (notifyType_==NOTIFY_CREATE_INTERCOMM)
-      {
-        auto& arg=notifyCreateIntercomm_ ;
+      buffer>>notifyInType_;
+      if (notifyInType_==NOTIFY_CREATE_INTERCOMM)
+      {
+        auto& arg=notifyInCreateIntercomm_ ;
         buffer >> std::get<0>(arg)>> std::get<1>(arg) ;
       }
@@ -170 +199 @@
   void CServerContext::checkNotifications(void)
   {
-    int commRank ;
-    MPI_Comm_rank(contextComm_, &commRank) ;
-    winNotify_->lockWindow(commRank,0) ;
-    winNotify_->popFromWindow(commRank, this, &CServerContext::notificationsDumpIn) ;
-    winNotify_->unlockWindow(commRank,0) ;
-    if (notifyType_==NOTIFY_CREATE_INTERCOMM) createIntercomm() ;
-  }
-
-  bool CServerContext::eventLoop(void)
+    if (!hasNotification_)
+    {
+      int commRank ;
+      MPI_Comm_rank(contextComm_, &commRank) ;
+      winNotify_->lockWindow(commRank,0) ;
+      winNotify_->popFromWindow(commRank, this, &CServerContext::notificationsDumpIn) ;
+      winNotify_->unlockWindow(commRank,0) ;
+      
+      if (notifyInType_!= NOTIFY_NOTHING)
+      {
+        hasNotification_=true ;
+        auto eventScheduler=parentService_->getEventScheduler() ;
+        std::hash<string> hashString ;
+        size_t hashId = hashString(name_) ;
+        size_t currentTimeLine=0 ;
+        eventScheduler->registerEvent(currentTimeLine,hashId); 
+      }
+    }
+    
+    if (hasNotification_)
+    {
+      auto eventScheduler=parentService_->getEventScheduler() ;
+      std::hash<string> hashString ;
+      size_t hashId = hashString(name_) ;
+      size_t currentTimeLine=0 ;
+      if (eventScheduler->queryEvent(currentTimeLine,hashId))
+      {
+        if (notifyInType_==NOTIFY_CREATE_INTERCOMM) createIntercomm() ;
+        hasNotification_=false ;
+      }
+    }
+  }
+
+  bool CServerContext::eventLoop(bool serviceOnly)
   {
     bool finished=false ;
-    checkNotifications() ;
-    if (context_!=nullptr)  
+    if (winNotify_!=nullptr) checkNotifications() ;
+    if (!serviceOnly && context_!=nullptr)  
     {
       if (context_->eventLoop())
@@ -198 +252 @@
   {
      MPI_Comm interCommServer, interCommClient ;
-     int remoteLeader=get<0>(notifyCreateIntercomm_) ;
-     string sourceContext=get<1>(notifyCreateIntercomm_) ;
+     auto& arg=notifyInCreateIntercomm_ ;
+     int remoteLeader=get<0>(arg) ;
+     string sourceContext=get<1>(arg) ;
 
      auto it=overlapedComm_.find(sourceContext) ;
@@ -215 +270 @@
     if (nOverlap==commSize)
     {
-      cout<<"CServerContext::createIntercomm : total overlap ==> context in attached mode"<<endl ;
+      info(10)<<"CServerContext::createIntercomm : total overlap ==> context in attached mode"<<endl ;
+      isAttachedMode_=true ;
       interCommClient=get<2>(it->second) ;
       interCommServer=get<1>(it->second) ;
@@ -224 +280 @@
     else if (nOverlap==0)
     { 
-      cout<<"CServerContext::createIntercomm : No overlap ==> context in server mode"<<endl ;
+      info(10)<<"CServerContext::createIntercomm : No overlap ==> context in server mode"<<endl ;
+      isAttachedMode_=false ;
       MPI_Intercomm_create(contextComm_, 0, xiosComm_, remoteLeader, 3141, &interCommServer) ;
       MPI_Comm_dup(interCommServer,&interCommClient) ;
@@ -233 +290 @@
     else
     {
-      cout<<"CServerContext::createIntercomm : partial overlap ==> not managed"<<endl ;
+      ERROR("void CServerContext::createIntercomm(void)",<<"CServerContext::createIntercomm : partial overlap ==> not managed") ;
     }
    
   }
 
+  void CServerContext::freeComm(void)
+  {
+    delete winNotify_ ;
+    winNotify_=nullptr ;
+    MPI_Comm_free(&contextComm_) ;
+    // don't forget intercomm -> later
+  }
+  
   void CServerContext::finalizeSignal(void)
   {
     finalizeSignal_=true ;
   }
-   
 
 }

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/server_context.hpp

@@ -18 +18 @@
     CServerContext(CService* parentService, MPI_Comm contextComm, const std::string& poolId, const std::string& serviceId, 
                   const int& partitionId, const std::string& contextId) ;
+    ~CServerContext() ;
 
     bool createIntercomm(const string& poolId, const string& serviceId, const int& partitionId, const string& contextId, 
@@ -26 +27 @@
     void checkNotifications(void) ;
 
-    bool eventLoop(void) ;
+    bool eventLoop(bool serviceOnly=false) ;
     void notificationsDumpOut(CBufferOut& buffer) ;
     void notificationsDumpIn(CBufferIn& buffer) ;
     void finalizeSignal(void) ;
+    void freeComm(void) ;
+    bool isAttachedMode(void) { return isAttachedMode_ ;}
     private:
     void createIntercomm(void) ;
@@ -47 +50 @@
     const size_t maxBufferSize_=1024*1024 ;
     CWindowManager* winNotify_ ;
-    int notifyType_ ;
-    tuple<int, std::string> notifyCreateIntercomm_ ;
+    int notifyInType_, notifyOutType_ ;
+    tuple<int, std::string> notifyInCreateIntercomm_, notifyOutCreateIntercomm_ ;
 
     const int localLeader_=0 ;
     int globalLeader_ ;
     bool finalizeSignal_ ;
+    bool hasNotification_ ;
+    bool isAttachedMode_ ;
 
     friend class CWindowManager ;

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/servers_ressource.cpp

@@ -57 +57 @@
        }
        
-       notifyType_=NOTIFY_CREATE_POOL ;
-       notifyCreatePool_ = make_tuple(poolId, isPartOf) ;
+       notifyOutType_=NOTIFY_CREATE_POOL ;
+       notifyOutCreatePool_ = make_tuple(poolId, isPartOf) ;
        sendNotification(freeRessourcesRank_[i]) ;
     }
@@ -71 +71 @@
     for(int rank=0; rank<commSize;rank++)
     { 
-      notifyType_=NOTIFY_FINALIZE ;
+      notifyOutType_=NOTIFY_FINALIZE ;
       sendNotification(rank) ;
     }
@@ -89 +89 @@
     buffer.realloc(maxBufferSize_) ;
     
-    if (notifyType_==NOTIFY_CREATE_POOL)
+    if (notifyOutType_==NOTIFY_CREATE_POOL)
     {
-      auto& arg=notifyCreatePool_ ;
-      buffer << notifyType_ << std::get<0>(arg) << std::get<1>(arg) ;
+      auto& arg=notifyOutCreatePool_ ;
+      buffer << notifyOutType_ << std::get<0>(arg) << std::get<1>(arg) ;
     }
-    else if (notifyType_==NOTIFY_FINALIZE) buffer << notifyType_ ;
+    else if (notifyOutType_==NOTIFY_FINALIZE) buffer << notifyOutType_ ;
   }
 
   void CServersRessource::notificationsDumpIn(CBufferIn& buffer)
   {
-    if (buffer.bufferSize() == 0) notifyType_= NOTIFY_NOTHING ;
+    if (buffer.bufferSize() == 0) notifyInType_= NOTIFY_NOTHING ;
     else
     {
-      buffer>>notifyType_;
-      if (notifyType_==NOTIFY_CREATE_POOL)
+      buffer>>notifyInType_;
+      if (notifyInType_==NOTIFY_CREATE_POOL)
       {
-        auto& arg=notifyCreatePool_ ;
+        auto& arg=notifyInCreatePool_ ;
         buffer >> std::get<0>(arg) >> std::get<1>(arg)  ;
       }
-      else if (notifyType_==NOTIFY_FINALIZE) { /*nothing to do*/}
+      else if (notifyInType_==NOTIFY_FINALIZE) { /*nothing to do*/}
     }
   }
 
-  bool CServersRessource::eventLoop()
+  bool CServersRessource::eventLoop(bool serviceOnly)
   {
     checkNotifications() ;
     if (poolRessource_!=nullptr) 
     {
-      if (poolRessource_->eventLoop())
+      if (poolRessource_->eventLoop(serviceOnly))
       {
         poolRessource_=nullptr ;
@@ -135 +135 @@
     winNotify_->popFromWindow(commRank, this, &CServersRessource::notificationsDumpIn) ;
     winNotify_->unlockWindow(commRank,0) ;
-    if (notifyType_==NOTIFY_CREATE_POOL) createPool() ;
-    else if (notifyType_==NOTIFY_FINALIZE) finalizeSignal() ;
+    if (notifyInType_==NOTIFY_CREATE_POOL) createPool() ;
+    else if (notifyInType_==NOTIFY_FINALIZE) finalizeSignal() ;
   }
 
   void CServersRessource::createPool(void)
   {
-    auto& arg=notifyCreatePool_ ;
+    auto& arg=notifyInCreatePool_ ;
     string poolId=get<0>(arg) ;
     bool isPartOf=get<1>(arg) ;

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/servers_ressource.hpp

@@ -26 +26 @@
     void createPool(const string& poolId, const int size) ;
     void createPool(void) ;
-    bool eventLoop(void) ;
+    bool eventLoop(bool serviceOnly=false) ;
     void sendNotification(int rank) ;
     void notificationsDumpOut(CBufferOut& buffer) ;
@@ -44 +44 @@
     const size_t maxBufferSize_=1024*1024 ;
     CWindowManager* winNotify_ ;
-    int notifyType_ ;
-    std::tuple<std::string, bool> notifyCreatePool_ ;
+    
+    int notifyInType_,notifyOutType_ ;
+    std::tuple<std::string, bool> notifyInCreatePool_,notifyOutCreatePool_ ;
     CPoolRessource* poolRessource_ ;
     bool finalizeSignal_ ;

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/services.cpp

@@ -10 +10 @@
   CService::CService(MPI_Comm serviceComm, const std::string& poolId, const std::string& serviceId, const int& partitionId, 
                      int type, int nbPartitions) : finalizeSignal_(false), eventScheduler_(nullptr), poolId_(poolId), serviceId_(serviceId),
-                                                   partitionId_(partitionId), type_(type), nbPartitions_(nbPartitions)
+                                                   partitionId_(partitionId), type_(type), nbPartitions_(nbPartitions), hasNotification_(false)
 
 
@@ -34 +34 @@
     }
     eventScheduler_ = new CEventScheduler(serviceComm_) ;
+
+    ostringstream oss;
+    oss<<partitionId;
+    name_= poolId+"::"+serviceId+"_"+oss.str();
   }
 
@@ -41 +45 @@
     MPI_Comm_size(serviceComm_, &commSize) ;
     
-    for(int rank=0; rank<commSize; rank++) createContextNotify(rank, poolId, serviceId, partitionId, contextId) ;
+    for(int rank=0; rank<commSize; rank++) 
+    {
+      notifyOutType_=NOTIFY_CREATE_CONTEXT ;
+      notifyOutCreateContext_ = make_tuple(poolId, serviceId, partitionId, contextId) ;
+      sendNotification(rank) ;
+    }
   }
 /*
@@ -87 +96 @@
   }
 
-  bool CService::eventLoop(void)
-  {
-    checkCreateContextNotification() ;
+  bool CService::eventLoop(bool serviceOnly)
+  {
+    //checkCreateContextNotification() ;
+    checkNotifications() ;
+
     eventScheduler_->checkEvent() ;
     for(auto it=contexts_.begin();it!=contexts_.end();++it) 
     {
-      if (it->second->eventLoop())
+      if (it->second->eventLoop(serviceOnly))
       {
         contexts_.erase(it) ;
@@ -105 +116 @@
   }
 
+  void CService::sendNotification(int rank)
+  {
+    winNotify_->lockWindow(rank,0) ;
+    winNotify_->pushToWindow(rank, this, &CService::notificationsDumpOut) ;
+    winNotify_->unlockWindow(rank,0) ;
+  }
+
+  
+  void CService::notificationsDumpOut(CBufferOut& buffer)
+  {
+    
+    buffer.realloc(maxBufferSize_) ;
+    
+    if (notifyOutType_==NOTIFY_CREATE_CONTEXT)
+    {
+      auto& arg=notifyOutCreateContext_ ;
+      buffer << notifyOutType_ << std::get<0>(arg)<<std::get<1>(arg) << std::get<2>(arg)<<std::get<3>(arg) ;
+    }
+  }
+
+  void CService::notificationsDumpIn(CBufferIn& buffer)
+  {
+    if (buffer.bufferSize() == 0) notifyInType_= NOTIFY_NOTHING ;
+    else
+    {
+      buffer>>notifyInType_;
+      if (notifyInType_==NOTIFY_CREATE_CONTEXT)
+      {
+        info(10)<<"NotifyDumpOut"<<endl ;
+        auto& arg=notifyInCreateContext_ ;
+        buffer >> std::get<0>(arg)>> std::get<1>(arg) >> std::get<2>(arg)>> std::get<3>(arg);
+      }
+    }
+  }
+
+
+
+
+  void CService::checkNotifications(void)
+  {
+    if (!hasNotification_)
+    {
+      int commRank ;
+      MPI_Comm_rank(serviceComm_, &commRank) ;
+      winNotify_->lockWindow(commRank,0) ;
+      winNotify_->popFromWindow(commRank, this, &CService::notificationsDumpIn) ;
+      winNotify_->unlockWindow(commRank,0) ;
+      
+      if (notifyInType_!= NOTIFY_NOTHING)
+      {
+        hasNotification_=true ;
+        std::hash<string> hashString ;
+        size_t hashId = hashString(name_) ;
+        size_t currentTimeLine=0 ;
+        eventScheduler_->registerEvent(currentTimeLine,hashId); 
+      }
+    }
+    
+    if (hasNotification_)
+    {
+      std::hash<string> hashString ;
+      size_t hashId = hashString(name_) ;
+      size_t currentTimeLine=0 ;
+      if (eventScheduler_->queryEvent(currentTimeLine,hashId))
+      {
+        if (notifyInType_==NOTIFY_CREATE_CONTEXT) createContext() ;
+        hasNotification_=false ;
+      }
+    }
+  }
+
+
+
+
   void CService::checkCreateContextNotification(void)
   {
@@ -122 +207 @@
   }
 
+  void CService::createContext(void)
+   {
+     auto& arg=notifyInCreateContext_ ;
+     string poolId = get<0>(arg) ;
+     string& serviceId = get<1>(arg) ;
+     int partitionId = get<2>(arg) ;
+     string contextId = get<3>(arg) ;
+     contexts_[contextId] = new CServerContext(this, serviceComm_, poolId, serviceId, partitionId, contextId) ; 
+   }
+
+   //to remove
    void CService::createNewContext(const std::string& poolId, const std::string& serviceId, const int& partitionId, const std::string& contextId)
    {

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/services.hpp

@@ -16 +16 @@
     public:
     
+    const int NOTIFY_NOTHING=0 ;
+    const int NOTIFY_CREATE_CONTEXT=1 ;
+
     CService(MPI_Comm serviceComm, const std::string& poolId, const std::string& serviceId, const int& partitionId, 
              int type, int nbPartitions) ;
-    bool eventLoop(void) ;
+    bool eventLoop(bool serviceOnly=false) ;
     void createContext(const std::string& poolId, const std::string& serviceId, const int& partitionId, const std::string& contextId) ;
     void checkCreateContextNotification(void) ;
@@ -36 +39 @@
 
     private:
-    
+    void sendNotification(int rank) ;
+    void notificationsDumpOut(CBufferOut& buffer) ;
+    void notificationsDumpIn(CBufferIn& buffer) ;
+    void checkNotifications(void) ;
+    void createContext(void) ;
+
     MPI_Comm serviceComm_ ;
     MPI_Comm globalComm_ ;
-   
+
     const size_t maxBufferSize_=1024*1024 ;
     const int localLeader_=0 ;
     int globalLeader_ ;
     CWindowManager* winNotify_ ;
+    
+    std::string name_ ;
+
     std::list<std::tuple<std::string, std::string, int, std::string>> notifications_;
+    
+    bool hasNotification_ ;
+    int notifyInType_,notifyOutType_ ;
+    std::tuple<std::string, std::string, int, std::string> notifyInCreateContext_, notifyOutCreateContext_ ;
+
     std::map<std::string, CServerContext*> contexts_ ;
     bool finalizeSignal_ ;

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/services_manager.cpp

@@ -42 +42 @@
   }
 
+  CServicesManager::~CServicesManager()
+  {
+    delete winNotify_ ;
+    delete winServices_ ;
+  }
+
   bool CServicesManager::createServices(const std::string& poolId, const std::string& serviceId, 
                                         int type, int size, int nbPartitions, bool wait) 

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/services_manager.hpp

@@ -27 +27 @@
     
     CServicesManager(bool isXiosServer) ;
+    ~CServicesManager() ;
+    
     bool createServices(const std::string& poolId, const std::string& serviceId, int type, int size, int nbPartition, bool wait=true) ;
     void createServicesNotify(int rank, const string& serviceId, int type, int size, int nbPartitions) ;

XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/window_manager.hpp

@@ -126 +126 @@
     }
 
+    ~CWindowManager()
+    {
+      MPI_Win_free(&window_) ;
+    }
   } ;
 }

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/context.cpp

@@ -36 +36 @@
       , isPostProcessed(false), finalized(false)
       , idServer_(), client(nullptr), server(nullptr)
-      , allProcessed(false), countChildCtx_(0), isProcessingEvent_(false)
+      , allProcessed(false), countChildContextFinalized_(0), isProcessingEvent_(false)
 
    { /* Ne rien faire de plus */ }
@@ -45 +45 @@
       , isPostProcessed(false), finalized(false)
       , idServer_(), client(nullptr), server(nullptr)
-      , allProcessed(false), countChildCtx_(0), isProcessingEvent_(false)
+      , allProcessed(false), countChildContextFinalized_(0), isProcessingEvent_(false)
    { /* Ne rien faire de plus */ }
 
@@ -475 +475 @@
     MPI_Comm_dup(intraComm_, &intraCommClient);
     comms.push_back(intraCommClient);
-    
+    // attached_mode=parentServerContext_->isAttachedMode() ; //ym probably inherited from source context
     server = new CContextServer(this,intraComm_, interCommServer); // check if we need to dupl. intraComm_ ?
     client = new CContextClient(this,intraCommClient,interCommClient);
@@ -687 +687 @@
   CATCH_DUMP_ATTR
 
-
+  
+  void CContext::globalEventLoop(void)
+  {
+    CXios::getDaemonsManager()->eventLoop() ;
+    setCurrent(getId()) ;
+  }
 
 
@@ -693 +698 @@
    TRY
    {
-      if (hasClient && !hasServer) // For now we only use server level 1 to read data
+      registryOut->hierarchicalGatherRegistry() ;
+      if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
+
+      if (hasClient && !hasServer)
       {
-        doPreTimestepOperationsForEnabledReadModeFiles();
-      }
-     // Send registry upon calling the function the first time
-     if (countChildCtx_ == 0) if (hasClient) sendRegistry() ;
-
-     // Client:
-     // (1) blocking send context finalize to its server
-     // (2) blocking receive context finalize from its server
-     // (3) some memory deallocations
-     if (CXios::isClient)
-     {
-       // Make sure that client (model) enters the loop only once
-       if (countChildCtx_ < 1)
-       {
-         ++countChildCtx_;
-
-         info(100)<<"DEBUG: context "<<getId()<<" Send client finalize"<<endl ;
-         client->finalize();
-         info(100)<<"DEBUG: context "<<getId()<<" Client finalize sent"<<endl ;
-         while (client->havePendingRequests()) client->checkBuffers();
-         
-         info(100)<<"DEBUG: context "<<getId()<<" no pending request ok"<<endl ;
-         while (!server->hasFinished())
-           server->eventLoop();
-        info(100)<<"DEBUG: context "<<getId()<<" server has finished"<<endl ;
-        
+        doPreTimestepOperationsForEnabledReadModeFiles(); // For now we only use server level 1 to read data
+
+        info(100)<<"DEBUG: context "<<getId()<<" Send client finalize"<<endl ;
+        client->finalize();
+        info(100)<<"DEBUG: context "<<getId()<<" Client finalize sent"<<endl ;
+        while (client->havePendingRequests()) client->checkBuffers();
+        info(100)<<"DEBUG: context "<<getId()<<" no pending request ok"<<endl ;
         bool notifiedFinalized=false ;
         do
@@ -727 +716 @@
         } while (!notifiedFinalized) ;
         client->releaseBuffers();
-
-         if (hasServer) // Mode attache
-         {
-           closeAllFile();
-           registryOut->hierarchicalGatherRegistry() ;
-           if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
-         }
-
-         //! Deallocate client buffers
-//         client->releaseBuffers();
         info(100)<<"DEBUG: context "<<getId()<<" release client ok"<<endl ;
-         //! Free internally allocated communicators
-         for (std::list<MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
-           MPI_Comm_free(&(*it));
-         comms.clear();
-
-         info(20)<<"CContext: Context <"<<getId()<<"> is finalized."<<endl;
-       }
-     }
-     else if (CXios::isServer)
-     {
-       // First context finalize message received from a model
-       // Send context finalize to its child contexts (if any)
-       if (countChildCtx_ == 0)
+      }
+      else if (hasClient && hasServer)
+      {
          for (int i = 0; i < clientPrimServer.size(); ++i)
          {
@@ -764 +733 @@
            do
            {
-//             clientPrimServer[i]->checkBuffers();
              notifiedFinalized=clientPrimServer[i]->isNotifiedFinalized() ;
            } while (!notifiedFinalized) ;
            clientPrimServer[i]->releaseBuffers();
          }
-           
-
-       // (Last) context finalized message received
-       if (countChildCtx_ == clientPrimServer.size())
-       {
-         // Blocking send of context finalize message to its client (e.g. primary server or model)
-         info(100)<<"DEBUG: context "<<getId()<<" Send client finalize"<<endl ;
-         client->finalize();
-         info(100)<<"DEBUG: context "<<getId()<<" Client finalize sent"<<endl ;
-         bool bufferReleased;
-         do
-         {
-           client->checkBuffers();
-           bufferReleased = !client->havePendingRequests();
-         } while (!bufferReleased);
-         
-         bool notifiedFinalized=false ;
-         do
-         {
-  //         client->checkBuffers();
-           notifiedFinalized=client->isNotifiedFinalized() ;
-         } while (!notifiedFinalized) ;
-         client->releaseBuffers();
-         
-         finalized = true;
-         info(100)<<"DEBUG: context "<<getId()<<" bufferRelease OK"<<endl ;
-         
-         closeAllFile(); // Just move to here to make sure that server-level 1 can close files
+         closeAllFile();
+
+      }
+      else if (!hasClient && hasServer)
+      {
+        closeAllFile();
+      }
+
+      freeComms() ;
         
-        /*  ym
-         if (hasServer && !hasClient)
-         {           
-           registryOut->hierarchicalGatherRegistry() ;
-           if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
-         }
-        */
-
-         //! Deallocate client buffers
-//         client->releaseBuffers();
-         info(100)<<"DEBUG: context "<<getId()<<" client release"<<endl ;
-
-/*         
-         for (int i = 0; i < clientPrimServer.size(); ++i)
-           clientPrimServer[i]->releaseBuffers();
-*/
-         //! Free internally allocated communicators
-         for (std::list<MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
-           MPI_Comm_free(&(*it));
-         comms.clear();
-
-         info(20)<<"CContext: Context <"<<getId()<<"> is finalized."<<endl;
-       }
-
-       ++countChildCtx_;
-     }
+      parentServerContext_->freeComm() ;
+      finalized = true;
+      info(20)<<"CContext: Context <"<<getId()<<"> is finalized."<<endl;
    }
    CATCH_DUMP_ATTR
@@ -833 +759 @@
    TRY
    {
+      int countChildCtx_ ; // ym temporary
+
       if (hasClient && !hasServer) // For now we only use server level 1 to read data
       {
@@ -2555 +2483 @@
   CATCH_DUMP_ATTR
 
+  
+  void CContext::sendFinalizeClient(CContextClient* contextClient, const string& contextClientId)
+  TRY
+  {
+    CEventClient event(getType(),EVENT_ID_CONTEXT_FINALIZE_CLIENT);
+    if (contextClient->isServerLeader())
+    {
+      CMessage msg;
+      msg<<contextClientId ;
+      const std::list<int>& ranks = contextClient->getRanksServerLeader();
+      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+           event.push(*itRank,1,msg);
+      contextClient->sendEvent(event);
+    }
+    else contextClient->sendEvent(event);
+  }
+  CATCH_DUMP_ATTR
+
+ 
+  void CContext::recvFinalizeClient(CEventServer& event)
+  TRY
+  {
+    CBufferIn* buffer=event.subEvents.begin()->buffer;
+    string id;
+    *buffer>>id;
+    get(id)->recvFinalizeClient(*buffer);
+  }
+  CATCH
+
+  void CContext::recvFinalizeClient(CBufferIn& buffer)
+  TRY
+  {
+    countChildContextFinalized_++ ;
+  }
+  CATCH_DUMP_ATTR
+
+
+
+
   /*!
   * \fn bool CContext::isFinalized(void)

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/context.hpp

@@ -53 +53 @@
            EVENT_ID_POST_PROCESS, EVENT_ID_SEND_REGISTRY,
            EVENT_ID_POST_PROCESS_GLOBAL_ATTRIBUTES,
-           EVENT_ID_PROCESS_GRID_ENABLED_FIELDS
+           EVENT_ID_PROCESS_GRID_ENABLED_FIELDS,
+           EVENT_ID_CONTEXT_FINALIZE_CLIENT,
          };
 
@@ -106 +107 @@
          bool checkBuffersAndListen(bool enableEventsProcessing=true);
          bool eventLoop(bool enableEventsProcessing=true);
+         void globalEventLoop(void);
 
          // Finalize a context
          void finalize(void);
+
          void finalize_old(void);
          bool isFinalized(void);
@@ -170 +173 @@
          //!< after be gathered to the root process of the context, merged registry is sent to the root process of the servers
          void sendRegistry(void) ;
+         void sendFinalizeClient(CContextClient* contextClient, const string& contextClientId);
+
 
          const StdString& getIdServer();
@@ -191 +196 @@
          static void recvRegistry(CEventServer& event) ;
          void recvRegistry(CBufferIn& buffer) ; //!< registry is received by the servers
-
+         static void recvFinalizeClient(CEventServer& event) ;
+         void recvFinalizeClient(CBufferIn& buffer);
+         
          void freeComms(void);                  //!< Free internally allcoated communicators
          void releaseClientBuffers(void);       //! Deallocate buffers allocated by clientContexts
@@ -281 +288 @@
          bool allProcessed;
          bool finalized;
-         int countChildCtx_;        //!< Counter of child contexts (for now it is the number of secondary server pools)
+         int countChildContextFinalized_;        //!< Counter of child contexts (for now it is the number of secondary server pools)
          StdString idServer_;
          CGarbageCollector garbageCollector;

XIOS/dev/dev_ym/XIOS_SERVICES/src/node/field.cpp

@@ -635 +635 @@
 
 //ym          context->checkBuffersAndListen();
-            context->eventLoop();
+//ym            context->eventLoop();
+          context->globalEventLoop();
 
           timer.suspend();

XIOS/dev/dev_ym/XIOS_SERVICES/src/server.cpp

@@ -711 +711 @@
 //      MPI_Comm_free(&intraComm);
 
+      CXios::finalizeDaemonsManager();
+      
       if (!is_MPI_Initialized)
       {