Changeset 1168

Timestamp:

06/14/17 15:24:43 (7 years ago)

Author:

oabramkina

Message:

Two server levels: merging registries of secondary server pools into one and then writing the merged one.

Tested on curie with toy_cmip6.

Location:

XIOS/dev/dev_olga/src

Files:

• cxios.cpp (modified) (1 diff)
• server.cpp (modified) (5 diffs)
• server.hpp (modified) (1 diff)

XIOS/dev/dev_olga/src/cxios.cpp

@@ -184 +184 @@
 
     // Finalize
-     if (CServer::getRank()==0)
-     {
-       info(80)<<"Write data base Registry"<<endl<<globalRegistry->toString()<<endl ;
-       globalRegistry->toFile("xios_registry.bin") ;
-       delete globalRegistry ;
-     }
+    if (!usingServer2)
+    {
+      if (CServer::getRank()==0)
+      {
+        info(80)<<"Write data base Registry"<<endl<<globalRegistry->toString()<<endl ;
+        globalRegistry->toFile("xios_registry.bin") ;
+        delete globalRegistry ;
+      }
+    }
+    else
+    {
+      // If using two server levels, first merge globalRegistry of each server pool into one registry on the first pool
+      if (CServer::serverLevel == 2)
+      {
+        vector<int>& secondaryServerGlobalRanks = CServer::getSecondaryServerGlobalRanks();
+        int firstPoolGlobalRank = secondaryServerGlobalRanks[0];
+        int rankGlobal;
+        MPI_Comm_rank(globalComm, &rankGlobal);
+
+        // All pools (except the first): send globalRegistry to the first pool
+        if (rankGlobal != firstPoolGlobalRank)
+        {
+          int registrySize = globalRegistry->size();
+          MPI_Send(&registrySize,1,MPI_LONG,firstPoolGlobalRank,15,CXios::globalComm) ;
+          CBufferOut buffer(registrySize) ;
+          globalRegistry->toBuffer(buffer) ;
+          MPI_Send(buffer.start(),registrySize,MPI_CHAR,firstPoolGlobalRank,15,CXios::globalComm) ;
+        }
+
+        // First pool: receive globalRegistry of all secondary server pools, merge and write the resultant registry
+        else
+        {
+          MPI_Status status;
+          char* recvBuff;
+
+          CRegistry globalRegistrySndServers (CServer::intraComm);
+          globalRegistrySndServers.mergeRegistry(*globalRegistry) ;
+
+          for (int i=1; i< secondaryServerGlobalRanks.size(); i++)
+          {
+            int rank = secondaryServerGlobalRanks[i];
+            int registrySize = 0;
+            MPI_Recv(&registrySize, 1, MPI_LONG, rank, 15, CXios::globalComm, &status);
+            recvBuff = new char[registrySize];
+            MPI_Recv(recvBuff, registrySize, MPI_CHAR, rank, 15, CXios::globalComm, &status);
+            CBufferIn buffer(recvBuff, registrySize) ;
+            CRegistry recvRegistry;
+            recvRegistry.fromBuffer(buffer) ;
+            globalRegistrySndServers.mergeRegistry(recvRegistry) ;
+            delete[] recvBuff;
+          }
+
+          info(80)<<"Write data base Registry"<<endl<<globalRegistrySndServers.toString()<<endl ;
+          globalRegistrySndServers.toFile("xios_registry.bin") ;
+
+        }
+      }
+      delete globalRegistry;
+    }
     CServer::finalize();
 

XIOS/dev/dev_olga/src/server.cpp

@@ -84 +84 @@
         map<unsigned long, int>::iterator it ;
 
-        int nbSrv = 0;
         int srvNodeSize = 1, srvNodeLeader = 0;
 
@@ -134 +133 @@
         {
           MPI_Allgather(&serverLevel, 1, MPI_LONG, hashAll, 1, MPI_LONG, CXios::globalComm) ;
+
           for (i=0; i<size; i++)
             if (hashAll[i] == 2)
@@ -188 +188 @@
             interCommRight.push_back(newComm) ;
           }
-        } // primary server
+        }
         else
         {
@@ -201 +201 @@
           MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 1, &newComm) ;
           interCommLeft.push_back(newComm) ;
-        } // secondary server
+        }
 
         delete [] hashAll ;
@@ -667 +667 @@
      }
 
+     vector<int>& CServer::getSecondaryServerGlobalRanks()
+     {
+       return sndServerGlobalRanks;
+     }
+
     /*!
     * Open a file specified by a suffix and an extension and use it for the given file buffer.

XIOS/dev/dev_olga/src/server.hpp

@@ -49 +49 @@
         static int getRank();
 
+        //!< Get global ranks of processes dedicated to the secondary server
+        static vector<int>& getSecondaryServerGlobalRanks();
+
         //! Open a file stream to write the info logs
         static void openInfoStream(const StdString& fileName);