Changeset 1152 for XIOS/dev/dev_olga/src/server.cpp

Timestamp:

06/02/17 16:56:46 (7 years ago)

Author:

oabramkina

Message:

Two servel levels: correcting initialization in case of non-contiguous server ranks.
Tests on Curie: test_complete.

File:

• XIOS/dev/dev_olga/src/server.cpp (modified) (16 diffs)

XIOS/dev/dev_olga/src/server.cpp

@@ -20 +20 @@
     list<MPI_Comm> CServer::interCommLeft ;
     list<MPI_Comm> CServer::interCommRight ;
-//    list<MPI_Comm> CServer::interComm ;
     std::list<MPI_Comm> CServer::contextInterComms;
     std::list<MPI_Comm> CServer::contextIntraComms;
     int CServer::serverLevel = 0 ;
-    int CServer::serverLeader_ = 0;
-    int CServer::serverSize_ = 0;
-    int CServer::nbPools = 0;
-    int CServer::poolId = 0;
     int CServer::nbContexts = 0;
     bool CServer::isRoot = false ;
@@ -34 +29 @@
     StdOFStream CServer::m_errorStream;
     map<string,CContext*> CServer::contextList ;
+    vector<int> CServer::sndServerGlobalRanks;
     bool CServer::finished=false ;
     bool CServer::is_MPI_Initialized ;
@@ -55 +51 @@
       if (initialized) is_MPI_Initialized=true ;
       else is_MPI_Initialized=false ;
+      int rank ;
 
       // Not using OASIS
@@ -70 +67 @@
 
         unsigned long* hashAll ;
-
-//        int rank ;
+        unsigned long* srvLevelAll ;
+
         int size ;
         int myColor ;
         int i,c ;
-        MPI_Comm newComm, serversComm;
+        MPI_Comm newComm;
 
         MPI_Comm_size(CXios::globalComm, &size) ;
@@ -88 +85 @@
 
         int nbSrv = 0;
+        int srvNodeSize = 1, srvNodeLeader = 0;
+
+        // (1) Establish client leaders, distribute processes between two server levels
         for(i=0,c=0;i<size;i++)
         {
@@ -96 +96 @@
             c++ ;
           }
-          if (hashAll[i] == hashServer) ++serverSize_;
+          if (CXios::usingServer2)
+          {
+            if (hashAll[i] == hashServer)
+            {
+              if (hashAll[i-1] != hashServer || i == 0)
+              {
+                srvNodeLeader = i;
+              }
+              if (hashAll[i+1] == hashServer)
+              {
+                ++srvNodeSize;
+              }
+              else
+              {
+                if ( (rank_-srvNodeLeader) >= 0 &&
+                     (rank_-srvNodeLeader) < (1.- CXios::ratioServer2/100.)*srvNodeSize )
+                {
+                  serverLevel = 1;
+                }
+                if ( (rank_-srvNodeLeader) >= (1.- CXios::ratioServer2/100.)*srvNodeSize &&
+                     (rank_ - srvNodeLeader) < srvNodeSize )
+                {
+                  serverLevel = 2;
+                }
+                srvNodeSize = 1;
+              }
+            }
+          }
         }
 
-        // Setting the number of secondary pools
-        myColor = colors[hashServer];
+        // (2) Create intraComm
+        myColor = (serverLevel == 2) ? rank_ : colors[hashServer];
+        MPI_Comm_split(CXios::globalComm, myColor, rank_, &intraComm) ;
+
+        // (3) Create interComm
         if (CXios::usingServer2)
         {
-          int serverRank = rank_ - leaders[hashServer]; // server proc rank starting 0
-          nbPools = serverSize_ * CXios::ratioServer2 / 100;
-          if ( serverRank < (serverSize_ - nbPools) )
-          {
-            serverLevel = 1;
-          }
-          else
-          {
-            serverLevel = 2;
-            poolId = serverRank - serverSize_ + nbPools;
-            myColor = rank_ + size; // + size to make sure that myColor is unique among not only servers but also clients. It's only a temporary solution
-          }
+          MPI_Allgather(&serverLevel, 1, MPI_LONG, hashAll, 1, MPI_LONG, CXios::globalComm) ;
+          for (i=0; i<size; i++)
+            if (hashAll[i] == 2)
+              sndServerGlobalRanks.push_back(i);
         }
-
-        MPI_Comm_split(CXios::globalComm, myColor, rank_, &intraComm) ;
 
         if (serverLevel == 0)
@@ -142 +162 @@
           int clientLeader, srvSndLeader;
           int srvPrmLeader ;
+
           for (it=leaders.begin();it!=leaders.end();it++)
           {
@@ -155 +176 @@
               interCommLeft.push_back(newComm) ;
             }
-            else
-              serverLeader_ = it->second;
-          }
-
-          for (int i = 0; i < nbPools; ++i)
-          {
-            srvSndLeader = serverLeader_ + serverSize_ - nbPools + i;
+          }
+
+          for (int i = 0; i < sndServerGlobalRanks.size(); ++i)
+          {
             int intraCommSize, intraCommRank ;
             MPI_Comm_size(intraComm, &intraCommSize) ;
             MPI_Comm_rank(intraComm, &intraCommRank) ;
             info(50)<<"intercommCreate::client (server level 1) "<<rank_<<" intraCommSize : "<<intraCommSize
-                <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< srvSndLeader<<endl ;
-            MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvSndLeader, 1, &newComm) ;
+                <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< sndServerGlobalRanks[i]<<endl ;
+            MPI_Intercomm_create(intraComm, 0, CXios::globalComm, sndServerGlobalRanks[i], 1, &newComm) ;
             interCommRight.push_back(newComm) ;
           }
@@ -191 +209 @@
       else
       {
-        int size, rank;
+        int size;
         int myColor;
         if (!is_MPI_Initialized) oasis_init(CXios::xiosCodeId);
@@ -204 +222 @@
         else
         {
-          MPI_Comm_rank(localComm,&rank) ;
-          MPI_Comm_size(localComm,&serverSize_) ;
-          nbPools = serverSize_ * CXios::ratioServer2 / 100;
-          if ( rank < (serverSize_ - nbPools) )
+          MPI_Comm_rank(localComm,&rank_) ;
+          MPI_Comm_size(localComm,&size) ;
+
+          for (int i=size*CXios::ratioServer2/100; i<size; i++)
+            sndServerGlobalRanks.push_back(i);
+
+          if ( rank_ < (size - sndServerGlobalRanks.size()) )
           {
             serverLevel = 1;
@@ -215 +236 @@
           {
             serverLevel = 2;
-            poolId = rank - serverSize_ + nbPools;
-            myColor = rank;
-          }
-          MPI_Comm_split(localComm, myColor, rank, &intraComm) ;
+            myColor = rank_;
+          }
+          MPI_Comm_split(localComm, myColor, rank_, &intraComm) ;
 
         }
-        MPI_Comm_rank(intraComm,&rank_) ;
         MPI_Comm_size(intraComm,&size) ;
-
         string codesId=CXios::getin<string>("oasis_codes_id") ;
 
@@ -233 +251 @@
         int globalRank ;
         MPI_Comm_rank(CXios::globalComm,&globalRank);
-        if (rank_ == 0)
-          serverLeader_ = globalRank;
 
         for(it=splitted.begin();it!=splitted.end();it++)
@@ -251 +267 @@
               interCommLeft.push_back(newComm) ;
               if (rank_==0) MPI_Send(&globalRank,1,MPI_INT,0,0,newComm) ;
-              for (int i = 0; i < nbPools; ++i)
+              for (int i = 0; i < sndServerGlobalRanks.size(); ++i)
               {
-                int srvSndLeader = serverLeader_ + serverSize_ - nbPools + i;
+                int srvSndLeader = sndServerGlobalRanks[i];
                 info(50)<<"intercommCreate::client (server level 1) "<<globalRank<<" intraCommSize : "<<size
                     <<" intraCommRank :"<<rank_<<"  clientLeader "<< srvSndLeader<<endl ;
@@ -273 +289 @@
       }
 
-      MPI_Comm_rank(intraComm, &rank_) ;
-      if (rank_==0) isRoot=true;
+      MPI_Comm_rank(intraComm, &rank) ;
+      if (rank==0) isRoot=true;
       else isRoot=false;
       
@@ -612 +628 @@
            CBufferOut buffer(buff,messageSize) ;
            buffer<<msg ;
-           int sndServerGloRanks = serverSize_-nbPools+serverLeader_ +i;  // the assumption is that there is only one proc per secondary server pool
-           MPI_Send(buff, buffer.count(), MPI_CHAR, sndServerGloRanks, 1, CXios::globalComm) ;
+           MPI_Send(buff, buffer.count(), MPI_CHAR, sndServerGlobalRanks[i], 1, CXios::globalComm) ;
            MPI_Comm_dup(*it, &inter);
            contextInterComms.push_back(inter);
@@ -668 +683 @@
         ++numDigit;
       }
-
-      if (!CXios::usingServer2)
-        id = getRank();
-      else
-      {
-        if (serverLevel == 1)
-          id = rank_;
-        else
-          id = poolId;
-      }
+      id = getRank();
+
+//      if (!CXios::usingServer2)
+//        id = getRank();
+//      else
+//      {
+//        if (serverLevel == 1)
+//          id = rank_;
+//        else
+//          id = poolId;
+//      }
       fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << id << ext;
       fb->open(fileNameClient.str().c_str(), std::ios::out);