Changeset 2523 for XIOS3/trunk/src/manager

Timestamp:

06/23/23 14:46:51 (13 months ago)

Author:

ymipsl

Message:

Adaptation to new hyper event scheduler.
YM

Location:

XIOS3/trunk/src/manager

Files:

• contexts_manager.hpp (modified) (1 diff)
• pool_ressource.cpp (modified) (6 diffs)
• pool_ressource.hpp (modified) (4 diffs)
• ressources_manager.hpp (modified) (1 diff)
• servers_ressource.cpp (modified) (4 diffs)
• servers_ressource.hpp (modified) (3 diffs)
• services.cpp (modified) (3 diffs)
• services.hpp (modified) (1 diff)
• services_manager.cpp (modified) (1 diff)
• services_manager.hpp (modified) (2 diffs)

XIOS3/trunk/src/manager/contexts_manager.hpp

@@ -75 +75 @@
     MPI_Comm xiosComm_ ;
 
-    int notifyType_ ;
+    int notifyType_ = NOTIFY_NOTHING ;
     tuple<std::string, std::string, int, std::string> notifyCreateContext_ ;
     tuple<std::string, std::string, int, std::string, int, std::string> notifyCreateIntercomm_ ;

XIOS3/trunk/src/manager/pool_ressource.cpp

@@ -7 +7 @@
 #include "cxios.hpp"
 #include "timer.hpp"
+#include "event_scheduler.hpp"
 
 namespace xios
 {
-  CPoolRessource::CPoolRessource(MPI_Comm poolComm, const std::string& Id, bool isServer) : Id_(Id), finalizeSignal_(false)
+  CPoolRessource::CPoolRessource(MPI_Comm poolComm, shared_ptr<CEventScheduler> eventScheduler, const std::string& Id, bool isServer) : Id_(Id), finalizeSignal_(false)
   {
     int commRank, commSize ;
@@ -29 +30 @@
     winNotify_->updateToExclusiveWindow(commRank, this, &CPoolRessource::notificationsDumpOut) ;
     MPI_Barrier(poolComm_) ;
+    if (eventScheduler) eventScheduler_=eventScheduler ;
+    else eventScheduler_= make_shared<CEventScheduler>(poolComm) ;
+    freeRessourceEventScheduler_ = eventScheduler_ ;
   }
 
@@ -36 +40 @@
     
     auto it=occupancy_.begin() ;
+
+    // ym obsolete, service cannot overlap, only created on separate ressource or matching excatly existing service
+    // occupancy management must not be used anymore => simplification
+    // for now raise a message error when no ressources are availables
+    
     int commSize ;
     MPI_Comm_size(poolComm_, &commSize) ;
@@ -43 +52 @@
     for(int i=0; i<size; i++) 
     {
+      if (it->first != 0) ERROR("void CPoolRessource::createService(const std::string& serviceId, int type, int size, int nbPartitions)",
+                                 << "No enough free ressources on pool id="<<getId()<<" to launch service id="<<serviceId);
       procs_in[it->second]=true ;
       procs_update.push_back(std::pair<int,int>(it->first+1,it->second)) ;
       ++it ;
     }
-    
+
+
     occupancy_.erase(occupancy_.begin(),it) ;
     occupancy_.insert(procs_update.begin(),procs_update.end()) ;
@@ -247 +259 @@
   {
      
-     info(40)<<"CPoolRessource::createNewService  : receive createService notification ; serviceId : "<<serviceId<<endl ;
-     MPI_Comm serviceComm, newServiceComm ;
-     int commRank ;
-     MPI_Comm_rank(poolComm_,&commRank) ;
-     MPI_Comm_split(poolComm_, in, commRank, &serviceComm) ;
-     if (in)
-     {
-       int serviceCommSize ;
-       int serviceCommRank ;
-       MPI_Comm_size(serviceComm,&serviceCommSize) ;
-       MPI_Comm_rank(serviceComm,&serviceCommRank) ;
-
-       info(10)<<"Service  "<<serviceId<<" created "<<"  service size : "<<serviceCommSize<< "   service rank : "<<serviceCommRank 
-                            <<" on rank pool "<<commRank<<endl ;
+    info(40)<<"CPoolRessource::createNewService  : receive createService notification ; serviceId : "<<serviceId<<endl ;
+    MPI_Comm serviceComm, newServiceComm, freeComm ;
+    int commRank ;
+     
+    int color;
+    if (!services_.empty()) color = 0 ;
+    else color=1 ; 
+    MPI_Comm_rank(poolComm_,&commRank) ;
+    MPI_Comm_split(poolComm_, color, commRank, &freeComm) ;  // workaround
+    
+    if (services_.empty()) 
+    {
+      MPI_Comm_rank(freeComm,&commRank) ;
+      MPI_Comm_split(freeComm, in, commRank, &serviceComm) ;
+
+      // temporary for event scheduler, we must using hierarchical split of free ressources communicator.
+      // we hope for now that spliting using occupancy make this match
+
+      if (in)
+      {
+        int serviceCommSize ;
+        int serviceCommRank ;
+        MPI_Comm_size(serviceComm,&serviceCommSize) ;
+        MPI_Comm_rank(serviceComm,&serviceCommRank) ;
+
+        info(10)<<"Service  "<<serviceId<<" created "<<"  service size : "<<serviceCommSize<< "   service rank : "<<serviceCommRank 
+                              <<" on rank pool "<<commRank<<endl ;
        
-       int partitionId ; 
-       if ( serviceCommRank >= (serviceCommSize/nbPartitions+1)*(serviceCommSize%nbPartitions) )
-       {
-         int rank =  serviceCommRank - (serviceCommSize/nbPartitions+1)*(serviceCommSize%nbPartitions) ;
-         partitionId = serviceCommSize%nbPartitions +  rank / (serviceCommSize/nbPartitions) ;
-       }
-       else  partitionId = serviceCommRank / (serviceCommSize/nbPartitions + 1) ;
-
-       MPI_Comm_split(serviceComm, partitionId, commRank, &newServiceComm) ;
+        int partitionId ; 
+        if ( serviceCommRank >= (serviceCommSize/nbPartitions+1)*(serviceCommSize%nbPartitions) )
+        {
+          int rank =  serviceCommRank - (serviceCommSize/nbPartitions+1)*(serviceCommSize%nbPartitions) ;
+          partitionId = serviceCommSize%nbPartitions +  rank / (serviceCommSize/nbPartitions) ;
+        }
+        else  partitionId = serviceCommRank / (serviceCommSize/nbPartitions + 1) ;
+
+        MPI_Comm_split(serviceComm, partitionId, commRank, &newServiceComm) ;
+
+        MPI_Comm_size(newServiceComm,&serviceCommSize) ;
+        MPI_Comm_rank(newServiceComm,&serviceCommRank) ;
+        info(10)<<"Service  "<<serviceId<<" created "<<"  partition : " <<partitionId<<" service size : "<<serviceCommSize
+                << " service rank : "<<serviceCommRank <<" on rank pool "<<commRank<<endl ;
+      
+        shared_ptr<CEventScheduler> parentScheduler, childScheduler ;
+        freeRessourceEventScheduler_->splitScheduler(newServiceComm, parentScheduler, childScheduler) ;
+        if (isFirstSplit_) eventScheduler_ = parentScheduler ;
+        isFirstSplit_=false ;
+
+        services_[std::make_tuple(serviceId,partitionId)] = new CService(newServiceComm, childScheduler, Id_, serviceId, partitionId, type, nbPartitions) ;
        
-       MPI_Comm_size(newServiceComm,&serviceCommSize) ;
-       MPI_Comm_rank(newServiceComm,&serviceCommRank) ;
-       info(10)<<"Service  "<<serviceId<<" created "<<"  partition : " <<partitionId<<" service size : "<<serviceCommSize
-               << " service rank : "<<serviceCommRank <<" on rank pool "<<commRank<<endl ;
-      
-       services_[std::make_tuple(serviceId,partitionId)] = new CService(newServiceComm, Id_, serviceId, partitionId, type, nbPartitions) ;
-       
-       MPI_Comm_free(&newServiceComm) ;
-     }
-     MPI_Comm_free(&serviceComm) ;
+        MPI_Comm_free(&newServiceComm) ;
+      }
+      else
+      {
+        shared_ptr<CEventScheduler> parentScheduler, childScheduler ;
+        freeRessourceEventScheduler_->splitScheduler(serviceComm, parentScheduler, childScheduler) ;
+        if (isFirstSplit_) eventScheduler_ = parentScheduler ;
+        freeRessourceEventScheduler_ = childScheduler ;
+        isFirstSplit_=false ;
+      }
+      MPI_Comm_free(&serviceComm) ;
+    }
+    MPI_Comm_free(&freeComm) ;
   }
   
@@ -300 +340 @@
         shared_ptr<CEventScheduler>  eventScheduler = service.second->getEventScheduler() ;
         info(40)<<"CPoolRessource::createNewServiceOnto ; found onServiceId : "<<onServiceId<<endl  ;
-        services_[std::make_tuple(serviceId,partitionId)] = new CService(newServiceComm, Id_, serviceId, partitionId, type,
-                                                                         nbPartitions, eventScheduler) ;       
-      }
-    }
-    
-  }
-
-  void CPoolRessource::createService(MPI_Comm serviceComm, const std::string& serviceId, int partitionId, int type, int nbPartitions) // for clients & attached
-  {
-    services_[std::make_tuple(serviceId,partitionId)] = new CService(serviceComm, Id_, serviceId, partitionId, type, nbPartitions) ;
+        services_[std::make_tuple(serviceId,partitionId)] = new CService(newServiceComm, eventScheduler, Id_, serviceId, partitionId, type,
+                                                                         nbPartitions) ;       
+      }
+    }
+    
+  }
+
+  void CPoolRessource::createService(MPI_Comm serviceComm, shared_ptr<CEventScheduler> eventScheduler, const std::string& serviceId, int partitionId, int type, int nbPartitions) // for clients & attached
+  {
+    services_[std::make_tuple(serviceId,partitionId)] = new CService(serviceComm, eventScheduler, Id_, serviceId, partitionId, type, nbPartitions) ;
   }
 

XIOS3/trunk/src/manager/pool_ressource.hpp

@@ -5 +5 @@
 #include "window_manager.hpp"
 #include "services_manager.hpp"
+#include "event_scheduler.hpp"
 
 
@@ -27 +28 @@
 
     public:
-    CPoolRessource(MPI_Comm poolComm, const std::string& Id, bool isServer) ;
+    CPoolRessource(MPI_Comm poolComm, shared_ptr<CEventScheduler> eventScheduler, const std::string& Id, bool isServer) ;
     ~CPoolRessource() ;
     
     void createService(const std::string& serviceId, int type, int size, int nbPartition) ;
-    void createService(MPI_Comm serviceComm, const std::string& serviceId, int partitionId, int type, int nbPartitions) ; 
+    void createService(MPI_Comm serviceComm, shared_ptr<CEventScheduler> eventScheduler, const std::string& serviceId, int partitionId, int type, int nbPartitions) ; 
     void createServiceOnto(const std::string& serviceId, int type, const std::string& OnServiceId) ;
     bool eventLoop(bool serviceOnly=false) ;
+    bool hasService(const std::string serviceId, int partitionId) {return services_.count(make_tuple(serviceId,partitionId))>0 ;}
     CService* getService(const std::string serviceId, int partitionId) { return services_[make_tuple(serviceId,partitionId)]; }
     void finalizeSignal(void) ;
     string getId(void) { return Id_; }
+    const MPI_Comm& getCommunicator(void) { return poolComm_ ;}
 
   private:
@@ -53 +56 @@
 //    void checkCreateServiceNotification(void) ;
     void createNewService(const std::string& serviceId, int type, int size, int nbPartitions, bool in) ;
+  public:
     void createNewServiceOnto(const std::string& serviceId, int type, const string& onServiceId) ;
     
@@ -73 +77 @@
     const double eventLoopLatency_=0; 
     double lastEventLoop_=0. ;
+
+    private:
+      shared_ptr<CEventScheduler> eventScheduler_ ;
+      shared_ptr<CEventScheduler> freeRessourceEventScheduler_ ;
+      bool isFirstSplit_=true ;
+    public:
+      shared_ptr<CEventScheduler> getEventScheduler(void) { return eventScheduler_ ;}
+
   };
 

XIOS3/trunk/src/manager/ressources_manager.hpp

@@ -72 +72 @@
     MPI_Comm xiosComm_ ;
 
-    int notifyType_ ;
+    int notifyType_ = NOTIFY_NOTHING;
     tuple<std::string, int> notifyCreatePool_ ;
 

XIOS3/trunk/src/manager/servers_ressource.cpp

@@ -3 +3 @@
 #include "ressources_manager.hpp"
 #include "pool_ressource.hpp"
+#include "event_scheduler.hpp"
 #include "cxios.hpp"
 #include "mpi.hpp"
@@ -8 +9 @@
 #include <vector>
 #include <string>
+
 
 
@@ -39 +41 @@
 
     MPI_Comm_dup(serverComm_, &freeRessourcesComm_) ; 
-
+    eventScheduler_ = make_shared<CEventScheduler>(freeRessourcesComm_) ;
+    freeRessourceEventScheduler_ = eventScheduler_ ;
   }
 
@@ -160 +163 @@
     MPI_Comm_split(freeRessourcesComm_, isPartOf, commRank, &poolComm) ;
     
+    shared_ptr<CEventScheduler> parentScheduler, childScheduler ;
+    freeRessourceEventScheduler_->splitScheduler(poolComm, parentScheduler, childScheduler) ;
+    
+    if (isFirstSplit_) eventScheduler_ = parentScheduler ; 
+    isFirstSplit_ = false ;
+
     if (isPartOf)
     {  
-      poolRessource_ = new CPoolRessource(poolComm, poolId, true) ;
+      poolRessource_ = new CPoolRessource(poolComm, childScheduler, poolId, true) ;
       MPI_Comm_free(&poolComm) ;
     }
     else 
     {
+      freeRessourceEventScheduler_ = childScheduler ;
       MPI_Comm_free(&freeRessourcesComm_) ;
       freeRessourcesComm_=poolComm ;

XIOS3/trunk/src/manager/servers_ressource.hpp

@@ -3 +3 @@
 
 #include "window_manager.hpp"
+#include "event_scheduler.hpp"
 #include "mpi.hpp"
 #include <vector>
@@ -33 +34 @@
     void checkNotifications(void) ;
     CPoolRessource* getPoolRessource(void) { return poolRessource_; } 
+    const MPI_Comm& getCommunicator(void) { return serverComm_ ;}
     bool isServerLeader(void) ;
     void finalize(void) ;
@@ -54 +56 @@
     double lastEventLoop_=0. ;
 
+    private:
+      shared_ptr<CEventScheduler> eventScheduler_ ;
+      shared_ptr<CEventScheduler> freeRessourceEventScheduler_ ;
+      bool isFirstSplit_=true ;
+    public:
+      shared_ptr<CEventScheduler> getEventScheduler(void) { return eventScheduler_ ;}
     friend class CWindowManager ;
   } ;

XIOS3/trunk/src/manager/services.cpp

@@ -9 +9 @@
 namespace xios
 {
-  CService::CService(MPI_Comm serviceComm, const std::string& poolId, const std::string& serviceId, const int& partitionId, 
-                     int type, int nbPartitions, shared_ptr<CEventScheduler> eventScheduler) 
+  CService::CService(MPI_Comm serviceComm, shared_ptr<CEventScheduler> eventScheduler, const std::string& poolId, const std::string& serviceId, const int& partitionId, 
+                     int type, int nbPartitions) 
                          : finalizeSignal_(false), eventScheduler_(nullptr), poolId_(poolId), serviceId_(serviceId),
                            partitionId_(partitionId), type_(type), nbPartitions_(nbPartitions), hasNotification_(false)
@@ -191 +191 @@
           size_t hashId = hashString(name_) ;
           size_t currentTimeLine=0 ;
-          info(40)<<"CService::checkNotifications(void) : receive notification => event scheduler"<<endl ;
+          info(40)<<"CService::checkNotifications(void) : receive notification => event scheduler : timeLine : "<<currentTimeLine<<"  hashId : "<<hashId<<endl ;
           eventScheduler_->registerEvent(currentTimeLine,hashId); 
         }
@@ -203 +203 @@
       size_t hashId = hashString(name_) ;
       size_t currentTimeLine=0 ;
-      info(40)<<"CService::checkNotifications(void) : receive notification => event scheduler : eventIsReceived ?"<<endl ;
+//      info(40)<<"CService::checkNotifications(void) : receive notification => event scheduler : eventIsReceived ?"<<endl ;
       if (eventScheduler_->queryEvent(currentTimeLine,hashId))
       {

XIOS3/trunk/src/manager/services.hpp

@@ -19 +19 @@
     const int NOTIFY_CREATE_CONTEXT=1 ;
 
-    CService(MPI_Comm serviceComm, const std::string& poolId, const std::string& serviceId, const int& partitionId, 
-             int type, int nbPartitions, shared_ptr<CEventScheduler> = nullptr) ;
+    CService(MPI_Comm serviceComm, shared_ptr<CEventScheduler>, const std::string& poolId, const std::string& serviceId, const int& partitionId, 
+             int type, int nbPartitions) ;
     ~CService() ;
 

XIOS3/trunk/src/manager/services_manager.cpp

@@ -304 +304 @@
   }
 
+  void CServicesManager::waitServiceRegistration(const std::string& poolId, const std::string& serviceId)
+  {
+    int nbPartition ;
+    getServiceNbPartitions(poolId,serviceId,0,nbPartition, true) ;
+    for(int n=1;n<nbPartition;n++) waitServiceRegistration(poolId, serviceId, n) ;
+
+  }
 }

XIOS3/trunk/src/manager/services_manager.hpp

@@ -49 +49 @@
     bool hasService(const std::string& poolId, const std::string& serviceId, const int& partitionId) ;
     void waitServiceRegistration(const std::string& poolId, const std::string& serviceId, const int& partitionId);
+    void waitServiceRegistration(const std::string& poolId, const std::string& serviceId);
     void servicesDumpOut(CBufferOut& buffer) ;
     void servicesDumpIn(CBufferIn& buffer) ;
@@ -74 +75 @@
     MPI_Comm xiosComm_ ;
 
-    int notifyType_ ;
+    int notifyType_ = NOTIFY_NOTHING ;
     tuple<std::string, int, int, int> notifyCreateService_ ;
     tuple<std::string, int, std::string> notifyCreateServiceOnto_ ;