Changeset 1021 for XIOS/dev/dev_olga/src/server.cpp

Timestamp:

01/10/17 14:36:29 (7 years ago)

Author:

oabramkina

Message:

Intermeadiate version for merging with new server functionalities.

File:

• XIOS/dev/dev_olga/src/server.cpp (modified) (12 diffs)

XIOS/dev/dev_olga/src/server.cpp

@@ -22 +22 @@
     list<MPI_Comm> CServer::interComm ;
     std::list<MPI_Comm> CServer::contextInterComms;
-    int CServer::nbSndSrvPools = (CXios::serverLevel == 0) ? 0 : 1;
-    int CServer::poolNb = 0;
+    int CServer::serverLevel = 0 ;
+    int CServer::nbPools = 0;
+    int CServer::poolId = 0;
+    int CServer::serverSize = 0;
     bool CServer::isRoot = false ;
     int CServer::rank = INVALID_RANK;
-    int CServer::rankSndServers = 0;
     StdOFStream CServer::m_infoStream;
     StdOFStream CServer::m_errorStream;
@@ -37 +38 @@
 /*!
  * \fn void CServer::initialize(void)
- * Function creates intra and inter comm for each initialized server pool
+ * Creates intraComm and interComm for a server pool (primary or secondary).
  */
     void CServer::initialize(void)
@@ -46 +47 @@
       else is_MPI_Initialized=false ;
 
-
       // Not using OASIS
       if (!CXios::usingOasis)
@@ -58 +58 @@
 
         boost::hash<string> hashString ;
-        unsigned long hashServer1 = hashString(CXios::xiosCodeIdPrm);
-        unsigned long hashServer2 = hashString(CXios::xiosCodeIdSnd);
-        unsigned long hashServer = (CXios::serverLevel < 2)  ? hashServer1 : hashServer2;
+//        unsigned long hashServer1 = hashString(CXios::xiosCodeIdPrm);
+//        unsigned long hashServer2 = hashString(CXios::xiosCodeIdSnd);
+//        unsigned long hashServer = (CXios::serverLevel < 2)  ? hashServer1 : hashServer2;
+        unsigned long hashServer = hashString(CXios::xiosCodeId);
 
         unsigned long* hashAll ;
-        unsigned long* hashAllServers ;
+//        unsigned long* hashAllServers ;
 
 //        int rank ;
@@ -69 +70 @@
         int myColor ;
         int i,c ;
-        MPI_Comm newComm, serversInterComm;
+        MPI_Comm newComm, serversComm;
 
         MPI_Comm_size(CXios::globalComm, &size) ;
@@ -77 +78 @@
         MPI_Allgather(&hashServer, 1, MPI_LONG, hashAll, 1, MPI_LONG, CXios::globalComm) ;
 
-        map<unsigned long, int> colors, colorsServers ;
+        map<unsigned long, int> colors ;
         map<unsigned long, int> leaders ;
         map<unsigned long, int>::iterator it ;
-//        map<unsigned long, int> leadersServers ;
-        vector<int> leadersServers;
 
         for(i=0,c=0;i<size;i++)
@@ -93 +92 @@
         }
 
-        nbSndSrvPools = size - leaders[hashServer2];    // one proc per secondary-server pool
-
-        // Taking into account multiple pools on secondary server
-        if (nbSndSrvPools > 1)
-        {
-          if (CXios::serverLevel > 1)
+        // Setting the number of secondary pools
+        myColor = colors[hashServer];
+        if (CXios::usingServer2)
+        {
+          int serverRank = rank - leaders[hashServer]; // server proc rank starting 0
+          serverSize = size - leaders[hashServer];
+          nbPools = serverSize * CXios::ratioServer2 / 100;
+          if ( serverRank < (serverSize - nbPools) )
           {
-            int nbProcs = size - leaders[hashServer2];
-            int remain = nbProcs % nbSndSrvPools;
-            int procsPerPool = nbProcs / nbSndSrvPools;
-            rankSndServers = rank - leaders[hashServer2];
-            StdString strTmp = CXios::xiosCodeIdSnd;
-
-            if (remain == 0)
-            {
-              poolNb = rankSndServers/procsPerPool;
-            }
-            else
-            {
-              if (rankSndServers <= (procsPerPool + 1) * remain)
-                poolNb = rankSndServers/(procsPerPool+1);
-              else
-              {
-                poolNb = remain + 1;
-                rankSndServers -= (procsPerPool + 1) * remain;
-                rankSndServers -= procsPerPool;
-                poolNb += rankSndServers/procsPerPool;
-              }
-            }
-            strTmp += boost::lexical_cast<string>(poolNb+1);  // add 1 to avoid hashing zero
-            hashServer = hashString(strTmp);
-            hashServer2 = hashString(strTmp);
+            serverLevel = 1;
+          }
+          else
+          {
+            serverLevel = 2;
+            poolId = serverRank - serverSize + nbPools;
+            myColor = rank;
           }
         }
 
-        if (nbSndSrvPools > 1)
-        {
-          myColor = size;
-          MPI_Comm_split(MPI_COMM_WORLD, myColor, rank, &serversInterComm) ;
-        }
-        else
-        {
-          myColor=colors[hashServer] ;
-          MPI_Comm_split(MPI_COMM_WORLD, myColor, rank, &intraComm) ;
-        }
-
-        if (nbSndSrvPools > 1)
-        {
-          int sizeServers;
-//          int rankServers;
-//          MPI_Comm_rank(serversInterComm, &rankServers) ;
-          MPI_Comm_size(serversInterComm, &sizeServers) ;
-          hashAllServers=new unsigned long[sizeServers] ;
-          MPI_Allgather(&hashServer, 1, MPI_LONG, hashAllServers, 1, MPI_LONG, serversInterComm) ;
-
-          for(i=0, c=0; i<sizeServers; i++)
-          {
-            if (colorsServers.find(hashAllServers[i])==colorsServers.end())
-            {
-              colorsServers[hashAllServers[i]]=c ;
-//              leadersServers[hashAllServers[i]]= leaders[hashServer1] + i ;
-              leadersServers.push_back( leaders[hashServer1] + i );
-              c++ ;
-            }
-          }
-          myColor=colorsServers[hashServer] ;
-          MPI_Comm_split(serversInterComm, myColor, rank, &intraComm) ;
-        }
-
-
-        if (CXios::serverLevel == 0)
+        MPI_Comm_split(CXios::globalComm, myColor, rank, &intraComm) ;
+
+        if (serverLevel == 0)
         {
           int clientLeader;
@@ -183 +133 @@
           }
         }
-        else
-        {
-          if ((CXios::serverLevel == 1))
+        else if (serverLevel == 1)
+        {
+          int clientLeader, srvPrmLeader, srvSndLeader;
+          for (it=leaders.begin();it!=leaders.end();it++)
           {
-            // Creating interComm with client (interCommLeft)
-            int clientLeader;
-            int srvSndLeader;
-            for(it=leaders.begin();it!=leaders.end();it++)
+            if (it->first != hashServer)
             {
-              if (it->first != hashServer2)
-              {
-                if (it->first != hashServer1)
-                {
-                  clientLeader=it->second ;
-                  int intraCommSize, intraCommRank ;
-                  MPI_Comm_size(intraComm, &intraCommSize) ;
-                  MPI_Comm_rank(intraComm, &intraCommRank) ;
-                  info(50)<<"intercommCreate::server "<<rank<<" intraCommSize : "<<intraCommSize
-                           <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
-                  MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
-                  interCommLeft.push_back(newComm) ;
-                  interComm.push_back(newComm) ;
-                }
-              }
-              else
-              {
-                srvSndLeader = it->second;
-              }
-            }
-
-            // Creating interComm with secondary server pool(s) (interCommRight)
-//            if (nbSndSrvPools < 1)
-            if (nbSndSrvPools < 2)
-            {
-              CClient::initializeClientOnServer(rank, intraComm, srvSndLeader);
-              interCommRight.push_back(CClient::getInterComm());
-              interComm.push_back(CClient::getInterComm());
+              clientLeader=it->second ;
+              int intraCommSize, intraCommRank ;
+              MPI_Comm_size(intraComm, &intraCommSize) ;
+              MPI_Comm_rank(intraComm, &intraCommRank) ;
+              info(50)<<"intercommCreate::server "<<rank<<" intraCommSize : "<<intraCommSize
+                       <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
+              MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
+              interCommLeft.push_back(newComm) ;
+              interComm.push_back(newComm) ;
             }
             else
-            {
-//              for(it=leadersServers.begin();it!=leadersServers.end();it++)
-//              {
-//                if (it->first != hashServer)
-//                {
-//                  srvSndLeader = it->second;
-//                  CClient::initializeClientOnServer(rank, intraComm, srvSndLeader);
-//                  interCommRight.push_back(CClient::getInterComm());
-//                  interComm.push_back(CClient::getInterComm());
-//                }
-//              }
-
-              for(int i = 1; i < leadersServers.size(); ++i)
-              {
-                srvSndLeader = leadersServers[i];
-                CClient::initializeClientOnServer(rank, intraComm, srvSndLeader);
-                interCommRight.push_back(CClient::getInterComm());
-                interComm.push_back(CClient::getInterComm());
-              }
-            }
-          } // primary server
-
-          else // secondary server pool(s)
+              srvPrmLeader = it->second;
+          }
+
+          for (int i = 0; i < nbPools; ++i)
           {
-            int clientLeader;
-            if (nbSndSrvPools < 2)
-//            if (nbSndSrvPools < 1)
-            {
-              for(it=leaders.begin();it!=leaders.end();it++)
-              {
-                if (it->first == hashServer1)
-                {
-                  clientLeader=it->second ;
-                  int intraCommSize, intraCommRank ;
-                  MPI_Comm_size(intraComm, &intraCommSize) ;
-                  MPI_Comm_rank(intraComm, &intraCommRank) ;
-                  info(50)<<"intercommCreate::server "<<rank<<" intraCommSize : "<<intraCommSize
-                           <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
-
-                  MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
-                  interCommLeft.push_back(newComm) ;
-                  interComm.push_back(newComm) ;
-
-                  break;
-                }
-              }
-            }
-            else
-            {
-//              for(it=leadersServers.begin();it!=leadersServers.end();it++)
-              {
- //               if (it->first == hashServer1)
-                {
-//                  clientLeader=it->second ;
-                  clientLeader = leadersServers[0];
-                  int intraCommSize, intraCommRank ;
-                  MPI_Comm_size(intraComm, &intraCommSize) ;
-                  MPI_Comm_rank(intraComm, &intraCommRank) ;
-                  info(50)<<"intercommCreate::server "<<rank<<" intraCommSize : "<<intraCommSize
-                           <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
-
-                  MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
-                  interCommLeft.push_back(newComm) ;
-                  interComm.push_back(newComm) ;
-                }
-              }
-            }
-          } // secondary server
-        } // CXios::serverLevel != 0
-
-        if (nbSndSrvPools > 1) delete [] hashAllServers;
+            srvSndLeader = srvPrmLeader + serverSize - nbPools + i;
+//            CClient::initializeClientOnServer(rank, serversComm, srvSndLeader);
+            CClient::initializeClientOnServer(rank, intraComm, srvSndLeader);
+            interCommRight.push_back(CClient::getInterComm());
+            interComm.push_back(CClient::getInterComm());
+          }
+        } // primary server
+        else
+        {
+          int clientLeader;
+          clientLeader = leaders[hashString(CXios::xiosCodeId)];
+          int intraCommSize, intraCommRank ;
+          MPI_Comm_size(intraComm, &intraCommSize) ;
+          MPI_Comm_rank(intraComm, &intraCommRank) ;
+          info(50)<<"intercommCreate::server "<<rank<<" intraCommSize : "<<intraCommSize
+                   <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
+
+          MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
+          interCommLeft.push_back(newComm) ;
+          interComm.push_back(newComm) ;
+        } // secondary server
+
         delete [] hashAll ;
 
@@ -416 +301 @@
              {
                MPI_Send(&msg,1,MPI_INT,0,0,*itr) ;
-
 //               MPI_Comm_free(&(*itr));
 //               interCommRight.erase(itr) ;
@@ -612 +496 @@
        contextInterComms.push_back(contextInterComm);
 
-       if (CXios::serverLevel == 1)
+       if (serverLevel == 1)
        {
 //         CClient::registerContext(contextId, intraComm);
-         CClient::registerContextOnSrvPools(contextId, intraComm);
+         CClient::registerContextByClienOfServer(contextId, intraComm);
        }
 
@@ -657 +541 @@
       int numDigit = 0;
       int size = 0;
-      int mpiRank;
+      int id;
       MPI_Comm_size(CXios::globalComm, &size);
       while (size)
@@ -665 +549 @@
       }
 
-      if (nbSndSrvPools < 2)
-        mpiRank = getRank();
+      if (!CXios::usingServer2)
+        id = getRank();
       else
-        mpiRank = rankSndServers;
-      fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << mpiRank << ext;
+      {
+        if (serverLevel == 1)
+          id = getRank();
+        else
+          id = poolId;
+      }
+      fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << id << ext;
       fb->open(fileNameClient.str().c_str(), std::ios::out);
       if (!fb->is_open())