Changeset 1180

Timestamp:

06/22/17 17:03:07 (7 years ago)

Author:

oabramkina

Message:

Two server levels:
(1) oasis works now with two server levels
(2) corrected assignment of processes between two server levels in case of non-contiguous processes
(3) corrected a bug during context finalization.

Location:

XIOS/dev/dev_olga/src

Files:

• client.cpp (modified) (1 diff)
• node/context.cpp (modified) (1 diff)
• server.cpp (modified) (8 diffs)
• server.hpp (modified) (3 diffs)

XIOS/dev/dev_olga/src/client.cpp

@@ -100 +100 @@
           MPI_Comm_split(CXios::globalComm,myColor,rank_,&intraComm) ;
 
-          if (CXios::usingServer2)
-            MPI_Allgather(&hashClient, 1, MPI_LONG, hashAll, 1, MPI_LONG, CXios::globalComm) ;
-
           if (CXios::usingServer)
           {

XIOS/dev/dev_olga/src/node/context.cpp

@@ -491 +491 @@
      info(20)<<"Context <"<<getId()<<"> is finalized."<<endl;
 
-     //     if (hasServer && !hasClient)
+     if (hasServer && !hasClient)
      {
        closeAllFile();

XIOS/dev/dev_olga/src/server.cpp

@@ -18 +18 @@
 {
     MPI_Comm CServer::intraComm ;
-    list<MPI_Comm> CServer::interCommLeft ;
-    list<MPI_Comm> CServer::interCommRight ;
+    std::list<MPI_Comm> CServer::interCommLeft ;
+    std::list<MPI_Comm> CServer::interCommRight ;
     std::list<MPI_Comm> CServer::contextInterComms;
     std::list<MPI_Comm> CServer::contextIntraComms;
@@ -84 +84 @@
         map<unsigned long, int>::iterator it ;
 
-        int srvNodeSize = 1, srvNodeLeader = 0;
-
         // (1) Establish client leaders, distribute processes between two server levels
+        vector<int> srvRanks;
         for(i=0,c=0;i<size;i++)
         {
@@ -96 +95 @@
           }
           if (CXios::usingServer2)
-          {
             if (hashAll[i] == hashServer)
-            {
-              if (hashAll[i-1] != hashServer || i == 0)
-              {
-                srvNodeLeader = i;
-              }
-              if (hashAll[i+1] == hashServer)
-              {
-                ++srvNodeSize;
-              }
-              else
-              {
-                if ( (rank_-srvNodeLeader) >= 0 &&
-                     (rank_-srvNodeLeader) < (1.- CXios::ratioServer2/100.)*srvNodeSize )
-                {
-                  serverLevel = 1;
-                }
-                if ( (rank_-srvNodeLeader) >= (1.- CXios::ratioServer2/100.)*srvNodeSize &&
-                     (rank_ - srvNodeLeader) < srvNodeSize )
-                {
-                  serverLevel = 2;
-                }
-                srvNodeSize = 1;
-              }
-            }
+              srvRanks.push_back(i);
+        }
+        for (i=0; i<srvRanks.size(); i++)
+        {
+          if (i >= srvRanks.size()*CXios::ratioServer2/100)
+          {
+            sndServerGlobalRanks.push_back(srvRanks[i]);
+            if (rank_ == srvRanks[i]) serverLevel=2;
+          }
+          else
+          {
+            if (rank_ == srvRanks[i]) serverLevel=1;
           }
         }
@@ -130 +116 @@
 
         // (3) Create interComm
-        if (CXios::usingServer2)
-        {
-          MPI_Allgather(&serverLevel, 1, MPI_LONG, hashAll, 1, MPI_LONG, CXios::globalComm) ;
-
-          for (i=0; i<size; i++)
-            if (hashAll[i] == 2)
-              sndServerGlobalRanks.push_back(i);
-        }
-
         if (serverLevel == 0)
         {
@@ -154 +131 @@
 
               MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
-               interCommLeft.push_back(newComm) ;
+              interCommLeft.push_back(newComm) ;
             }
           }
@@ -211 +188 @@
         int size;
         int myColor;
+        unsigned long* srvGlobalRanks;
         if (!is_MPI_Initialized) oasis_init(CXios::xiosCodeId);
 
@@ -225 +203 @@
         else
         {
+          int globalRank;
           MPI_Comm_rank(localComm,&rank_) ;
           MPI_Comm_size(localComm,&size) ;
+          MPI_Comm_rank(CXios::globalComm,&globalRank) ;
+          srvGlobalRanks = new unsigned long[size] ;
+          MPI_Allgather(&globalRank, 1, MPI_LONG, srvGlobalRanks, 1, MPI_LONG, localComm) ;
 
           for (int i=size*CXios::ratioServer2/100; i<size; i++)
-            sndServerGlobalRanks.push_back(i);
+            sndServerGlobalRanks.push_back(srvGlobalRanks[i]);
 
           if ( rank_ < (size - sndServerGlobalRanks.size()) )
@@ -255 +237 @@
         MPI_Comm_rank(CXios::globalComm,&globalRank);
 
+//      (1) Create interComms with models
         for(it=splitted.begin();it!=splitted.end();it++)
         {
           oasis_get_intercomm(newComm,*it) ;
-//        interComm.push_back(newComm) ;
-          if ( !CXios::usingServer2)
+          if ( !CXios::usingServer2 || serverLevel == 1)
           {
             interCommLeft.push_back(newComm) ;
             if (rank_==0) MPI_Send(&globalRank,1,MPI_INT,0,0,newComm) ;
           }
-          else
-          {
-            if (serverLevel == 1)
-            {
-              interCommLeft.push_back(newComm) ;
-              if (rank_==0) MPI_Send(&globalRank,1,MPI_INT,0,0,newComm) ;
-              for (int i = 0; i < sndServerGlobalRanks.size(); ++i)
-              {
-                int srvSndLeader = sndServerGlobalRanks[i];
-                info(50)<<"intercommCreate::client (server level 1) "<<globalRank<<" intraCommSize : "<<size
-                    <<" intraCommRank :"<<rank_<<"  clientLeader "<< srvSndLeader<<endl ;
-                MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvSndLeader, 0, &newComm) ;
-                interCommRight.push_back(newComm) ;
-              }
-            }
-            else if (serverLevel == 2)
-            {
-              info(50)<<"intercommCreate::server "<<rank_<<" intraCommSize : "<<size
-                       <<" intraCommRank :"<<rank_<<"  clientLeader "<< 0<<endl ;
-              MPI_Intercomm_create(intraComm, 0, localComm, 0, 0, &newComm) ;
-              interCommLeft.push_back(newComm) ;
-            }
-          }
-//          MPI_Comm_remote_size(newComm,&size);
-        }
+        }
+
+//      (2) Create interComms between primary and secondary servers
+        if (serverLevel == 1)
+        {
+          for (int i = 0; i < sndServerGlobalRanks.size(); ++i)
+          {
+            int srvSndLeader = sndServerGlobalRanks[i];
+            info(50)<<"intercommCreate::client (server level 1) "<<globalRank<<" intraCommSize : "<<size
+                <<" intraCommRank :"<<rank_<<"  clientLeader "<< srvSndLeader<<endl ;
+            MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvSndLeader, 0, &newComm) ;
+            interCommRight.push_back(newComm) ;
+          }
+        }
+        else if (serverLevel == 2)
+        {
+          info(50)<<"intercommCreate::server "<<rank_<<" intraCommSize : "<<size
+                   <<" intraCommRank :"<<rank_<<"  clientLeader "<< srvGlobalRanks[0] <<endl ;
+          MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvGlobalRanks[0], 0, &newComm) ;
+          interCommLeft.push_back(newComm) ;
+        }
+        delete [] srvGlobalRanks ;
+
               oasis_enddef() ;
       }

XIOS/dev/dev_olga/src/server.hpp

@@ -25 +25 @@
 
         static MPI_Comm intraComm;
-        static list<MPI_Comm> interCommLeft;           // interComm between server (primary, classical or secondary) and its client (client or primary server)
-        static list<MPI_Comm> interCommRight;          // interComm between primary server and secondary server (non-empty only for primary server pool)
+        static std::list<MPI_Comm> interCommLeft;           // interComm between server (primary, classical or secondary) and its client (client or primary server)
+        static std::list<MPI_Comm> interCommRight;          // interComm between primary server and secondary server (non-empty only for primary server pool)
         static std::list<MPI_Comm> contextInterComms;  // list of context intercomms
         static std::list<MPI_Comm> contextIntraComms;  // list of context intercomms (needed only in case of secondary servers)
@@ -49 +49 @@
         static int getRank();
 
-        //!< Get global ranks of processes dedicated to the secondary server
+        //!< Get global ranks of secondary server processes
         static vector<int>& getSecondaryServerGlobalRanks();
 
@@ -68 +68 @@
       private:
         static vector<int> sndServerGlobalRanks;  //!< Global ranks of secondary server processes
-        static int rank_;             //!< If (!oasis) global rank, else rank in the intraComm returned by oasis
-        static int nbContexts;        //!< Number of contexts registered by server
+        static int rank_;                   //!< If (!oasis) global rank, else rank in the intraComm returned by oasis
+        static int nbContexts;              //!< Number of contexts registered by server
         static StdOFStream m_infoStream;
         static StdOFStream m_errorStream;