Changeset 1234 for XIOS

Timestamp:

08/03/17 15:16:56 (7 years ago)

Author:

oabramkina

Message:

Minor modifications for secondary server:

• adding a check of the number of processes requested for the secondary server
• making process distribution between two server levels in case of oasis similar to that wihtout oasis.

Location:

XIOS/dev/XIOS_DEV_CMIP6/src

Files:

• cxios.cpp (modified) (1 diff)
• cxios.hpp (modified) (1 diff)
• node/file.cpp (modified) (3 diffs)
• server.cpp (modified) (7 diffs)

XIOS/dev/XIOS_DEV_CMIP6/src/cxios.cpp

@@ -186 +186 @@
 
     // Finalize
-    if (!usingServer2)
+    if (CServer::serverLevel == 0)
     {
       if (CServer::getRank()==0)

XIOS/dev/XIOS_DEV_CMIP6/src/cxios.hpp

@@ -42 +42 @@
      static bool usingOasis ; //!< Using Oasis
      static bool usingServer ; //!< Using server (server mode)
-     static bool usingServer2 ; //!< Using secondary server (server mode)
+     static bool usingServer2 ; //!< Using secondary server (server mode). IMPORTANT: Use this variable ONLY in CServer::initialize().
      static int ratioServer2 ;  //!< Percentage of server processors dedicated to secondary server
      static double bufferSizeFactor; //!< Factor used to tune the buffer size

XIOS/dev/XIOS_DEV_CMIP6/src/node/file.cpp

@@ -17 +17 @@
 #include "mpi.hpp"
 #include "timer.hpp"
+#include "server.hpp"
 
 namespace xios {
@@ -302 +303 @@
       // Done by classical server or secondary server
       // This condition should be changed soon
-      if (!CXios::usingServer2 || (CXios::usingServer2 && !context->hasClient))          
+      if (CServer::serverLevel == 0 || CServer::serverLevel == 2)
       {
         if (mode.isEmpty() || mode.getValue() == mode_attr::write)
@@ -326 +327 @@
       // Done by classical server or secondary server
       // TODO: This condition should be changed soon. It only works with maximum number of level as 2
-      if (!CXios::usingServer2 || (CXios::usingServer2 && context->hasClient))
+      if (CServer::serverLevel == 0 || CServer::serverLevel == 2)
       {
         if (!mode.isEmpty() && mode.getValue() == mode_attr::read)

XIOS/dev/XIOS_DEV_CMIP6/src/server.cpp

@@ -38 +38 @@
  * \fn void CServer::initialize(void)
  * Creates intraComm for each possible type of servers (classical, primary or secondary).
- * In case of secondary servers intraComm is created for each secondary server pool.
- * (For now the assumption is that there is one proc per pool.)
+ * (For now the assumption is that there is one proc per secondary server pool.)
  * Creates interComm and stores them into the following lists:
  *   classical server -- interCommLeft
  *   primary server -- interCommLeft and interCommRight
  *   secondary server -- interCommLeft for each pool.
+ *   IMPORTANT: CXios::usingServer2 should NOT be used beyond this function. Use CServer::serverLevel instead.
  */
     void CServer::initialize(void)
@@ -85 +85 @@
 
         // (1) Establish client leaders, distribute processes between two server levels
-        vector<int> srvRanks;
+        std::vector<int> srvRanks;
         for(i=0,c=0;i<size;i++)
         {
@@ -98 +98 @@
               srvRanks.push_back(i);
         }
-        for (i=0; i<srvRanks.size(); i++)
-        {
-          if (i >= srvRanks.size()*(100.-CXios::ratioServer2)/100.)
-          {
-            sndServerGlobalRanks.push_back(srvRanks[i]);
-            if (rank_ == srvRanks[i]) serverLevel=2;
+
+        if (CXios::usingServer2)
+        {
+          int reqNbProc = srvRanks.size()*CXios::ratioServer2/100.;
+          if (reqNbProc<1 || reqNbProc==srvRanks.size())
+          {
+            error(0)<<"WARNING: void CServer::initialize(void)"<<endl
+                << "It is impossible to dedicate the requested number of processes = "<<reqNbProc
+                <<" to secondary server. XIOS will run in the classical server mode."<<endl;
           }
           else
           {
-            if (rank_ == srvRanks[i]) serverLevel=1;
-          }
-        }
-
+            for (i=0; i<srvRanks.size(); i++)
+            {
+              if (i >= srvRanks.size()*(100.-CXios::ratioServer2)/100.)
+              {
+                sndServerGlobalRanks.push_back(srvRanks[i]);
+                if (rank_ == srvRanks[i]) serverLevel=2;
+              }
+              else
+              {
+                if (rank_ == srvRanks[i]) serverLevel=1;
+              }
+            }
+          }
+        }
         // (2) Create intraComm
         myColor = (serverLevel == 2) ? rank_ : colors[hashServer];
@@ -188 +201 @@
         int size;
         int myColor;
-        unsigned long* srvGlobalRanks;
+        int* srvGlobalRanks;
         if (!is_MPI_Initialized) oasis_init(CXios::xiosCodeId);
 
@@ -194 +207 @@
         MPI_Comm localComm;
         oasis_get_localcomm(localComm);
-
-        // Create server intraComm
+        MPI_Comm_rank(localComm,&rank_) ;
+
+//      (1) Create server intraComm
         if (!CXios::usingServer2)
         {
           MPI_Comm_dup(localComm, &intraComm);
-          MPI_Comm_rank(localComm,&rank_) ;
         }
         else
         {
           int globalRank;
-          MPI_Comm_rank(localComm,&rank_) ;
           MPI_Comm_size(localComm,&size) ;
           MPI_Comm_rank(CXios::globalComm,&globalRank) ;
-          srvGlobalRanks = new unsigned long[size] ;
-          MPI_Allgather(&globalRank, 1, MPI_LONG, srvGlobalRanks, 1, MPI_LONG, localComm) ;
-
-          for (int i=size*(100.-CXios::ratioServer2)/100.; i<size; i++)
-            sndServerGlobalRanks.push_back(srvGlobalRanks[i]);
-
-          if ( rank_ < (size - sndServerGlobalRanks.size()) )
-          {
-            serverLevel = 1;
-            myColor = 0;
+          srvGlobalRanks = new int[size] ;
+          MPI_Allgather(&globalRank, 1, MPI_INT, srvGlobalRanks, 1, MPI_INT, localComm) ;
+
+          int reqNbProc = size*CXios::ratioServer2/100.;
+          if (reqNbProc < 1 || reqNbProc == size)
+          {
+            error(0)<<"WARNING: void CServer::initialize(void)"<<endl
+                << "It is impossible to dedicate the requested number of processes = "<<reqNbProc
+                <<" to secondary server. XIOS will run in the classical server mode."<<endl;
+            MPI_Comm_dup(localComm, &intraComm);
           }
           else
           {
-            serverLevel = 2;
-            myColor = rank_;
-          }
-          MPI_Comm_split(localComm, myColor, rank_, &intraComm) ;
-
-        }
-        MPI_Comm_size(intraComm,&size) ;
+            for (int i=0; i<size; i++)
+            {
+              if (i >= size*(100.-CXios::ratioServer2)/100.)
+              {
+                sndServerGlobalRanks.push_back(srvGlobalRanks[i]);
+                if (globalRank == srvGlobalRanks[i]) serverLevel=2;
+              }
+              else
+              {
+                if (globalRank == srvGlobalRanks[i]) serverLevel=1;
+              }
+            }
+            myColor = (serverLevel == 2) ? globalRank : 0;
+            MPI_Comm_split(localComm, myColor, rank_, &intraComm) ;
+          }
+        }
+
         string codesId=CXios::getin<string>("oasis_codes_id") ;
-
         vector<string> splitted ;
         boost::split( splitted, codesId, boost::is_any_of(","), boost::token_compress_on ) ;
@@ -237 +258 @@
         MPI_Comm_rank(CXios::globalComm,&globalRank);
 
-//      (1) Create interComms with models
+//      (2) Create interComms with models
         for(it=splitted.begin();it!=splitted.end();it++)
         {
           oasis_get_intercomm(newComm,*it) ;
-          if ( !CXios::usingServer2 || serverLevel == 1)
+          if ( serverLevel == 0 || serverLevel == 1)
           {
             interCommLeft.push_back(newComm) ;
@@ -248 +269 @@
         }
 
-//      (2) Create interComms between primary and secondary servers
+//      (3) Create interComms between primary and secondary servers
+        MPI_Comm_size(intraComm,&size) ;
         if (serverLevel == 1)
         {