Changeset 1460 for XIOS/dev/branch_openmp/src/server.cpp

Timestamp:

03/22/18 10:43:20 (6 years ago)

Author:

yushan

Message:

branch_openmp merged with XIOS_DEV_CMIP6@1459

File:

• XIOS/dev/branch_openmp/src/server.cpp (modified) (18 diffs)

XIOS/dev/branch_openmp/src/server.cpp

@@ -3 +3 @@
 #include "cxios.hpp"
 #include "server.hpp"
+#include "client.hpp"
 #include "type.hpp"
 #include "context.hpp"
@@ -18 +19 @@
 {
     MPI_Comm CServer::intraComm ;
-    list<MPI_Comm> CServer::interComm ;
+    std::list<MPI_Comm> CServer::interCommLeft ;
+    std::list<MPI_Comm> CServer::interCommRight ;
     std::list<MPI_Comm> CServer::contextInterComms;
-    bool CServer::isRoot ;
-    int CServer::rank = INVALID_RANK;
+    std::list<MPI_Comm> CServer::contextIntraComms;
+    int CServer::serverLevel = 0 ;
+    int CServer::nbContexts = 0;
+    bool CServer::isRoot = false ;
+    int CServer::rank_ = INVALID_RANK;
     StdOFStream CServer::m_infoStream;
     StdOFStream CServer::m_errorStream;
     map<string,CContext*> CServer::contextList ;
+    vector<int> CServer::sndServerGlobalRanks;
     bool CServer::finished=false ;
     bool CServer::is_MPI_Initialized ;
     CEventScheduler* CServer::eventScheduler = 0;
-   
+
+//---------------------------------------------------------------
+/*!
+ * \fn void CServer::initialize(void)
+ * Creates intraComm for each possible type of servers (classical, primary or secondary).
+ * (For now the assumption is that there is one proc per secondary server pool.)
+ * Creates interComm and stores them into the following lists:
+ *   classical server -- interCommLeft
+ *   primary server -- interCommLeft and interCommRight
+ *   secondary server -- interCommLeft for each pool.
+ *   IMPORTANT: CXios::usingServer2 should NOT be used beyond this function. Use CServer::serverLevel instead.
+ */
     void CServer::initialize(void)
     {
-     // int initialized ;
-     // MPI_Initialized(&initialized) ;
-     // if (initialized) is_MPI_Initialized=true ;
-     // else is_MPI_Initialized=false ;
+      //int initialized ;
+      //MPI_Initialized(&initialized) ;
+      //if (initialized) is_MPI_Initialized=true ;
+      //else is_MPI_Initialized=false ;
+      int rank ;
 
       // Not using OASIS
@@ -40 +58 @@
       {
 
-       // if (!is_MPI_Initialized)
-       // {
-       //   MPI_Init(NULL, NULL);
-       // }
+        //if (!is_MPI_Initialized)
+        //{
+        //  MPI_Init(NULL, NULL);
+        //}
         CTimer::get("XIOS").resume() ;
 
         boost::hash<string> hashString ;
-
-        unsigned long hashServer=hashString(CXios::xiosCodeId) ;
+        unsigned long hashServer = hashString(CXios::xiosCodeId);
+
         unsigned long* hashAll ;
-
-//        int rank ;
+        unsigned long* srvLevelAll ;
+
         int size ;
         int myColor ;
         int i,c ;
-        MPI_Comm newComm ;
-
-        MPI_Comm_size(CXios::globalComm,&size) ;
-        MPI_Comm_rank(CXios::globalComm,&rank);
+        MPI_Comm newComm;
+
+        MPI_Comm_size(CXios::globalComm, &size) ;
+        MPI_Comm_rank(CXios::globalComm, &rank_);
+
         hashAll=new unsigned long[size] ;
-
-        MPI_Allgather(&hashServer,1,MPI_LONG,hashAll,1,MPI_LONG,CXios::globalComm) ;
+        MPI_Allgather(&hashServer, 1, MPI_LONG, hashAll, 1, MPI_LONG, CXios::globalComm) ;
 
         map<unsigned long, int> colors ;
@@ -67 +85 @@
         map<unsigned long, int>::iterator it ;
 
+        // (1) Establish client leaders, distribute processes between two server levels
+        std::vector<int> srvRanks;
         for(i=0,c=0;i<size;i++)
         {
@@ -75 +95 @@
             c++ ;
           }
-        }
-
-        myColor=colors[hashServer] ;
-        MPI_Comm_split(CXios::globalComm,myColor,rank,&intraComm) ;
-
-        int serverLeader=leaders[hashServer] ;
-        int clientLeader;
-
-         serverLeader=leaders[hashServer] ;
-         for(it=leaders.begin();it!=leaders.end();it++)
-         {
-           if (it->first!=hashServer)
-           {
-             clientLeader=it->second ;
-             int intraCommSize, intraCommRank ;
-             MPI_Comm_size(intraComm,&intraCommSize) ;
-             MPI_Comm_rank(intraComm,&intraCommRank) ;
-             #pragma omp critical (_output)
-             {
-               info(50)<<"intercommCreate::server "<<rank<<" intraCommSize : "<<intraCommSize
+          if (CXios::usingServer2)
+            if (hashAll[i] == hashServer)
+              srvRanks.push_back(i);
+        }
+
+        if (CXios::usingServer2)
+        {
+          int reqNbProc = srvRanks.size()*CXios::ratioServer2/100.;
+          if (reqNbProc<1 || reqNbProc==srvRanks.size())
+          {
+            error(0)<<"WARNING: void CServer::initialize(void)"<<endl
+                << "It is impossible to dedicate the requested number of processes = "<<reqNbProc
+                <<" to secondary server. XIOS will run in the classical server mode."<<endl;
+          }
+          else
+          {
+            int firstSndSrvRank = srvRanks.size()*(100.-CXios::ratioServer2)/100. ;
+            int poolLeader = firstSndSrvRank;
+//*********** (1) Comment out the line below to set one process per pool
+//            sndServerGlobalRanks.push_back(srvRanks[poolLeader]);
+            int nbPools = CXios::nbPoolsServer2;
+            if ( nbPools > reqNbProc || nbPools < 1)
+            {
+              error(0)<<"WARNING: void CServer::initialize(void)"<<endl
+                  << "It is impossible to allocate the requested number of pools = "<<nbPools
+                  <<" on the secondary server. It will be set so that there is one process per pool."<<endl;
+              nbPools = reqNbProc;
+            }
+            int remainder = ((int) (srvRanks.size()*CXios::ratioServer2/100.)) % nbPools;
+            int procsPerPool = ((int) (srvRanks.size()*CXios::ratioServer2/100.)) / nbPools;
+            for (i=0; i<srvRanks.size(); i++)
+            {
+              if (i >= firstSndSrvRank)
+              {
+                if (rank_ == srvRanks[i])
+                {
+                  serverLevel=2;
+                }
+                poolLeader += procsPerPool;
+                if (remainder != 0)
+                {
+                  ++poolLeader;
+                  --remainder;
+                }
+//*********** (2) Comment out the two lines below to set one process per pool
+//                if (poolLeader < srvRanks.size())
+//                  sndServerGlobalRanks.push_back(srvRanks[poolLeader]);
+//*********** (3) Uncomment the line below to set one process per pool
+                sndServerGlobalRanks.push_back(srvRanks[i]);
+              }
+              else
+              {
+                if (rank_ == srvRanks[i]) serverLevel=1;
+              }
+            }
+            if (serverLevel==2)
+            {
+              #pragma omp critical (_output)
+              info(50)<<"The number of secondary server pools is "<< sndServerGlobalRanks.size() <<endl ;
+              for (i=0; i<sndServerGlobalRanks.size(); i++)
+              {
+                if (rank_>= sndServerGlobalRanks[i])
+                {
+                  if ( i == sndServerGlobalRanks.size()-1)
+                  {
+                    myColor = colors.size() + sndServerGlobalRanks[i];
+                  }
+                  else if (rank_< sndServerGlobalRanks[i+1])
+                  {
+                    myColor = colors.size() + sndServerGlobalRanks[i];
+                    break;
+                  }
+                }
+              }
+            }
+          }
+        }
+
+        // (2) Create intraComm
+        if (serverLevel != 2) myColor=colors[hashServer];
+        MPI_Comm_split(CXios::globalComm, myColor, rank_, &intraComm) ;
+
+        // (3) Create interComm
+        if (serverLevel == 0)
+        {
+          int clientLeader;
+          for(it=leaders.begin();it!=leaders.end();it++)
+          {
+            if (it->first!=hashServer)
+            {
+              clientLeader=it->second ;
+              int intraCommSize, intraCommRank ;
+              MPI_Comm_size(intraComm,&intraCommSize) ;
+              MPI_Comm_rank(intraComm,&intraCommRank) ;
+              #pragma omp critical (_output)
+              {
+                info(50)<<"intercommCreate::server (classical mode) "<<rank_<<" intraCommSize : "<<intraCommSize
                        <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
-             }
-             MPI_Intercomm_create(intraComm,0,CXios::globalComm,clientLeader,0,&newComm) ;
-             interComm.push_back(newComm) ;
-           }
-         }
-
-         delete [] hashAll ;
+              }
+
+              MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
+              interCommLeft.push_back(newComm) ;
+            }
+          }
+        }
+        else if (serverLevel == 1)
+        {
+          int clientLeader, srvSndLeader;
+          int srvPrmLeader ;
+
+          for (it=leaders.begin();it!=leaders.end();it++)
+          {
+            if (it->first != hashServer)
+            {
+              clientLeader=it->second ;
+              int intraCommSize, intraCommRank ;
+              MPI_Comm_size(intraComm, &intraCommSize) ;
+              MPI_Comm_rank(intraComm, &intraCommRank) ;
+              #pragma omp critical (_output)
+              {
+                info(50)<<"intercommCreate::server (server level 1) "<<rank_<<" intraCommSize : "<<intraCommSize
+                       <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
+              }
+              MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 0, &newComm) ;
+              interCommLeft.push_back(newComm) ;
+            }
+          }
+
+          for (int i = 0; i < sndServerGlobalRanks.size(); ++i)
+          {
+            int intraCommSize, intraCommRank ;
+            MPI_Comm_size(intraComm, &intraCommSize) ;
+            MPI_Comm_rank(intraComm, &intraCommRank) ;
+            #pragma omp critical (_output)
+            {
+              info(50)<<"intercommCreate::client (server level 1) "<<rank_<<" intraCommSize : "<<intraCommSize
+                <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< sndServerGlobalRanks[i]<<endl ;
+            }
+            MPI_Intercomm_create(intraComm, 0, CXios::globalComm, sndServerGlobalRanks[i], 1, &newComm) ;
+            interCommRight.push_back(newComm) ;
+          }
+        }
+        else
+        {
+          int clientLeader;
+          clientLeader = leaders[hashString(CXios::xiosCodeId)];
+          int intraCommSize, intraCommRank ;
+          MPI_Comm_size(intraComm, &intraCommSize) ;
+          MPI_Comm_rank(intraComm, &intraCommRank) ;
+          #pragma omp critical (_output)
+          {
+            info(50)<<"intercommCreate::server (server level 2) "<<rank_<<" intraCommSize : "<<intraCommSize
+                   <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
+          }
+
+          MPI_Intercomm_create(intraComm, 0, CXios::globalComm, clientLeader, 1, &newComm) ;
+          interCommLeft.push_back(newComm) ;
+        }
+
+        delete [] hashAll ;
+
       }
       // using OASIS
-/*      else
+      else
       {
-        int rank ,size;
         int size;
+        int myColor;
+        int* srvGlobalRanks;
         if (!is_MPI_Initialized) oasis_init(CXios::xiosCodeId);
 
@@ -114 +268 @@
         MPI_Comm localComm;
         oasis_get_localcomm(localComm);
-        MPI_Comm_dup(localComm, &intraComm);
-
-        MPI_Comm_rank(intraComm,&rank) ;
-        MPI_Comm_size(intraComm,&size) ;
+        MPI_Comm_rank(localComm,&rank_) ;
+
+//      (1) Create server intraComm
+        if (!CXios::usingServer2)
+        {
+          MPI_Comm_dup(localComm, &intraComm);
+        }
+        else
+        {
+          int globalRank;
+          MPI_Comm_size(localComm,&size) ;
+          MPI_Comm_rank(CXios::globalComm,&globalRank) ;
+          srvGlobalRanks = new int[size] ;
+          MPI_Allgather(&globalRank, 1, MPI_INT, srvGlobalRanks, 1, MPI_INT, localComm) ;
+
+          int reqNbProc = size*CXios::ratioServer2/100.;
+          if (reqNbProc < 1 || reqNbProc == size)
+          {
+            error(0)<<"WARNING: void CServer::initialize(void)"<<endl
+                << "It is impossible to dedicate the requested number of processes = "<<reqNbProc
+                <<" to secondary server. XIOS will run in the classical server mode."<<endl;
+            MPI_Comm_dup(localComm, &intraComm);
+          }
+          else
+          {
+            int firstSndSrvRank = size*(100.-CXios::ratioServer2)/100. ;
+            int poolLeader = firstSndSrvRank;
+//*********** (1) Comment out the line below to set one process per pool
+//            sndServerGlobalRanks.push_back(srvGlobalRanks[poolLeader]);
+            int nbPools = CXios::nbPoolsServer2;
+            if ( nbPools > reqNbProc || nbPools < 1)
+            {
+              error(0)<<"WARNING: void CServer::initialize(void)"<<endl
+                  << "It is impossible to allocate the requested number of pools = "<<nbPools
+                  <<" on the secondary server. It will be set so that there is one process per pool."<<endl;
+              nbPools = reqNbProc;
+            }
+            int remainder = ((int) (size*CXios::ratioServer2/100.)) % nbPools;
+            int procsPerPool = ((int) (size*CXios::ratioServer2/100.)) / nbPools;
+            for (int i=0; i<size; i++)
+            {
+              if (i >= firstSndSrvRank)
+              {
+                if (globalRank == srvGlobalRanks[i])
+                {
+                  serverLevel=2;
+                }
+                poolLeader += procsPerPool;
+                if (remainder != 0)
+                {
+                  ++poolLeader;
+                  --remainder;
+                }
+//*********** (2) Comment out the two lines below to set one process per pool
+//                if (poolLeader < size)
+//                  sndServerGlobalRanks.push_back(srvGlobalRanks[poolLeader]);
+//*********** (3) Uncomment the line below to set one process per pool
+                sndServerGlobalRanks.push_back(srvGlobalRanks[i]);
+              }
+              else
+              {
+                if (globalRank == srvGlobalRanks[i]) serverLevel=1;
+              }
+            }
+            if (serverLevel==2)
+            {
+              info(50)<<"The number of secondary server pools is "<< sndServerGlobalRanks.size() <<endl ;
+              for (int i=0; i<sndServerGlobalRanks.size(); i++)
+              {
+                if (globalRank>= sndServerGlobalRanks[i])
+                {
+                  if (i == sndServerGlobalRanks.size()-1)
+                  {
+                    myColor = sndServerGlobalRanks[i];
+                  }
+                  else if (globalRank< sndServerGlobalRanks[i+1])
+                  {
+                    myColor = sndServerGlobalRanks[i];
+                    break;
+                  }
+                }
+              }
+            }
+            if (serverLevel != 2) myColor=0;
+            MPI_Comm_split(localComm, myColor, rank_, &intraComm) ;
+          }
+        }
+
         string codesId=CXios::getin<string>("oasis_codes_id") ;
-
         vector<string> splitted ;
         boost::split( splitted, codesId, boost::is_any_of(","), boost::token_compress_on ) ;
@@ -128 +365 @@
         MPI_Comm_rank(CXios::globalComm,&globalRank);
 
+//      (2) Create interComms with models
         for(it=splitted.begin();it!=splitted.end();it++)
         {
           oasis_get_intercomm(newComm,*it) ;
-          if (rank==0) MPI_Send(&globalRank,1,MPI_INT,0,0,newComm) ;
-          MPI_Comm_remote_size(newComm,&size);
-          interComm.push_back(newComm) ;
-        }
-              oasis_enddef() ;
+          if ( serverLevel == 0 || serverLevel == 1)
+          {
+            interCommLeft.push_back(newComm) ;
+            if (rank_==0) MPI_Send(&globalRank,1,MPI_INT,0,0,newComm) ;
+          }
+        }
+
+//      (3) Create interComms between primary and secondary servers
+        int intraCommSize, intraCommRank ;
+        MPI_Comm_size(intraComm,&intraCommSize) ;
+        MPI_Comm_rank(intraComm, &intraCommRank) ;
+
+        if (serverLevel == 1)
+        {
+          for (int i = 0; i < sndServerGlobalRanks.size(); ++i)
+          {
+            int srvSndLeader = sndServerGlobalRanks[i];
+            info(50)<<"intercommCreate::client (server level 1) "<<globalRank<<" intraCommSize : "<<intraCommSize
+                <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< srvSndLeader<<endl ;
+            MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvSndLeader, 0, &newComm) ;
+            interCommRight.push_back(newComm) ;
+          }
+        }
+        else if (serverLevel == 2)
+        {
+          info(50)<<"intercommCreate::server (server level 2)"<<globalRank<<" intraCommSize : "<<intraCommSize
+                   <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< srvGlobalRanks[0] <<endl ;
+          MPI_Intercomm_create(intraComm, 0, CXios::globalComm, srvGlobalRanks[0], 0, &newComm) ;
+          interCommLeft.push_back(newComm) ;
+        }
+        if (CXios::usingServer2) delete [] srvGlobalRanks ;
+        oasis_enddef() ;
       }
-*/
-//      int rank;
-      MPI_Comm_rank(intraComm,&rank) ;
+
+
+      MPI_Comm_rank(intraComm, &rank) ;
       if (rank==0) isRoot=true;
       else isRoot=false;
@@ -154 +419 @@
       for (std::list<MPI_Comm>::iterator it = contextInterComms.begin(); it != contextInterComms.end(); it++)
         MPI_Comm_free(&(*it));
-      for (std::list<MPI_Comm>::iterator it = interComm.begin(); it != interComm.end(); it++)
+
+      for (std::list<MPI_Comm>::iterator it = contextIntraComms.begin(); it != contextIntraComms.end(); it++)
         MPI_Comm_free(&(*it));
+
+//      for (std::list<MPI_Comm>::iterator it = interComm.begin(); it != interComm.end(); it++)
+//        MPI_Comm_free(&(*it));
+
+//        for (std::list<MPI_Comm>::iterator it = interCommLeft.begin(); it != interCommLeft.end(); it++)
+//          MPI_Comm_free(&(*it));
+
+        for (std::list<MPI_Comm>::iterator it = interCommRight.begin(); it != interCommRight.end(); it++)
+          MPI_Comm_free(&(*it));
+
       MPI_Comm_free(&intraComm);
 
@@ -179 +455 @@
          {
            listenContext();
+           listenRootContext();
            if (!finished) listenFinalize() ;
          }
@@ -196 +473 @@
      void CServer::listenFinalize(void)
      {
-        list<MPI_Comm>::iterator it;
+        list<MPI_Comm>::iterator it, itr;
         int msg ;
         int flag ;
 
-        for(it=interComm.begin();it!=interComm.end();it++)
+        for(it=interCommLeft.begin();it!=interCommLeft.end();it++)
         {
            MPI_Status status ;
@@ -210 +487 @@
               MPI_Recv(&msg,1,MPI_INT,0,0,*it,&status) ;
               info(20)<<" CServer : Receive client finalize"<<endl ;
+              // Sending server finalize message to secondary servers (if any)
+              for(itr=interCommRight.begin();itr!=interCommRight.end();itr++)
+              {
+                MPI_Send(&msg,1,MPI_INT,0,0,*itr) ;
+              }
               MPI_Comm_free(&(*it));
-              interComm.erase(it) ;
+              interCommLeft.erase(it) ;
               break ;
             }
          }
 
-         if (interComm.empty())
+         if (interCommLeft.empty())
          {
            int i,size ;
@@ -300 +582 @@
      void CServer::recvContextMessage(void* buff,int count)
      {
-       static map<string,contextMessage> recvContextId ;
+       static map<string,contextMessage> recvContextId;
        map<string,contextMessage>::iterator it ;
-
        CBufferIn buffer(buff,count) ;
        string id ;
@@ -325 +606 @@
          int size ;
          MPI_Comm_size(intraComm,&size) ;
-         MPI_Request* requests= new MPI_Request[size-1] ;
-         MPI_Status* status= new MPI_Status[size-1] ;
-
-         for(int i=1;i<size;i++)
-         {
-            MPI_Isend(buff,count,MPI_CHAR,i,2,intraComm,&requests[i-1]) ;
-         }
-         MPI_Waitall(size-1,requests,status) ;
-         registerContext(buff,count,it->second.leaderRank) ;
+//         MPI_Request* requests= new MPI_Request[size-1] ;
+//         MPI_Status* status= new MPI_Status[size-1] ;
+         MPI_Request* requests= new MPI_Request[size] ;
+         MPI_Status* status= new MPI_Status[size] ;
+
+         CMessage msg ;
+         msg<<id<<it->second.leaderRank;
+         int messageSize=msg.size() ;
+         void * sendBuff = new char[messageSize] ;
+         CBufferOut sendBuffer(sendBuff,messageSize) ;
+         sendBuffer<<msg ;
+
+         // Include root itself in order not to have a divergence
+         for(int i=0; i<size; i++)
+         {
+           MPI_Isend(sendBuff,sendBuffer.count(),MPI_CHAR,i,2,intraComm,&requests[i]) ;
+         }
 
          recvContextId.erase(it) ;
@@ -344 +633 @@
      void CServer::listenRootContext(void)
      {
-
        MPI_Status status ;
        int flag ;
-       static char* buffer ;
-       static MPI_Request request ;
-       static bool recept=false ;
+       static std::vector<void*> buffers;
+       static std::vector<MPI_Request> requests ;
+       static std::vector<int> counts ;
+       static std::vector<bool> isEventRegistered ;
+       static std::vector<bool> isEventQueued ;
+       MPI_Request request;
+
        int rank ;
-       int count ;
        const int root=0 ;
-
-       if (recept==false)
-       {
-         traceOff() ;
-         MPI_Iprobe(root,2,intraComm, &flag, &status) ;
-         traceOn() ;
-         if (flag==true)
-         {
-           MPI_Get_count(&status,MPI_CHAR,&count) ;
-           buffer=new char[count] ;
-           MPI_Irecv((void*)buffer,count,MPI_CHAR,root,2,intraComm,&request) ;
-           recept=true ;
-         }
-       }
-       else
-       {
-         MPI_Test(&request,&flag,&status) ;
-         if (flag==true)
-         {
-           MPI_Get_count(&status,MPI_CHAR,&count) ;
-           registerContext((void*)buffer,count) ;
-           delete [] buffer ;
-           recept=false ;
-         }
-       }
+       boost::hash<string> hashString;
+       size_t hashId = hashString("RegisterContext");
+
+       // (1) Receive context id from the root, save it into a buffer
+       traceOff() ;
+       MPI_Iprobe(root,2,intraComm, &flag, &status) ;
+       traceOn() ;
+       if (flag==true)
+       {
+         counts.push_back(0);
+         MPI_Get_count(&status,MPI_CHAR,&(counts.back())) ;
+         buffers.push_back(new char[counts.back()]) ;
+         requests.push_back(request);
+         MPI_Irecv((void*)(buffers.back()),counts.back(),MPI_CHAR,root,2,intraComm,&(requests.back())) ;
+         isEventRegistered.push_back(false);
+         isEventQueued.push_back(false);
+         nbContexts++;
+       }
+
+       for (int ctxNb = 0; ctxNb < nbContexts; ctxNb++ )
+       {
+         // (2) If context id is received, register an event
+         MPI_Test(&requests[ctxNb],&flag,&status) ;
+         if (flag==true && !isEventRegistered[ctxNb])
+         {
+           eventScheduler->registerEvent(ctxNb,hashId);
+           isEventRegistered[ctxNb] = true;
+         }
+         // (3) If event has been scheduled, call register context
+         if (eventScheduler->queryEvent(ctxNb,hashId) && !isEventQueued[ctxNb])
+         {
+           registerContext(buffers[ctxNb],counts[ctxNb]) ;
+           isEventQueued[ctxNb] = true;
+           delete [] buffers[ctxNb] ;
+         }
+       }
+
      }
 
@@ -384 +687 @@
        string contextId;
        CBufferIn buffer(buff, count);
-       buffer >> contextId;
+//       buffer >> contextId;
+       buffer >> contextId>>leaderRank;
+       CContext* context;
 
        info(20) << "CServer : Register new Context : " << contextId << endl;
@@ -392 +697 @@
                << "Context '" << contextId << "' has already been registred");
 
-       MPI_Comm contextIntercomm;
-       //MPI_Barrier(CXios::globalComm);
-       
-       MPI_Intercomm_create(intraComm,0,CXios::globalComm,leaderRank,10+leaderRank,&contextIntercomm);
-
+       context=CContext::create(contextId);
+       contextList[contextId]=context;
+
+       // Primary or classical server: create communication channel with a client
+       // (1) create interComm (with a client)
+       // (2) initialize client and server (contextClient and contextServer)
        MPI_Comm inter;
-       MPI_Intercomm_merge(contextIntercomm,1,&inter);
-       MPI_Barrier(inter);
-
-       CContext* context=CContext::create(contextId);
-       contextList[contextId]=context;
-       context->initServer(intraComm,contextIntercomm);
-
-       contextInterComms.push_back(contextIntercomm);
-       MPI_Comm_free(&inter);
+       if (serverLevel < 2)
+       {
+         MPI_Comm contextInterComm;
+         MPI_Intercomm_create(intraComm, 0, CXios::globalComm, leaderRank, 10+leaderRank, &contextInterComm);
+         MPI_Intercomm_merge(contextInterComm,1,&inter);
+         MPI_Barrier(inter);
+         MPI_Comm_free(&inter);
+         context->initServer(intraComm,contextInterComm);
+         contextInterComms.push_back(contextInterComm);
+
+       }
+       // Secondary server: create communication channel with a primary server
+       // (1) duplicate interComm with a primary server
+       // (2) initialize client and server (contextClient and contextServer)
+       // Remark: in the case of the secondary server there is no need to create an interComm calling MPI_Intercomm_create,
+       //         because interComm of CContext is defined on the same processes as the interComm of CServer.
+       //         So just duplicate it.
+       else if (serverLevel == 2)
+       {
+         MPI_Comm_dup(interCommLeft.front(), &inter);
+         contextInterComms.push_back(inter);
+         context->initServer(intraComm, contextInterComms.back());
+       }
+
+       // Primary server:
+       // (1) send create context message to secondary servers
+       // (2) initialize communication channels with secondary servers (create contextClient and contextServer)
+       if (serverLevel == 1)
+       {
+         int i = 0, size;
+         MPI_Comm_size(intraComm, &size) ;
+         for (std::list<MPI_Comm>::iterator it = interCommRight.begin(); it != interCommRight.end(); it++, ++i)
+         {
+           StdString str = contextId +"_server_" + boost::lexical_cast<string>(i);
+           CMessage msg;
+           int messageSize;
+           msg<<str<<size<<rank_ ;
+           messageSize = msg.size() ;
+           buff = new char[messageSize] ;
+           CBufferOut buffer(buff,messageSize) ;
+           buffer<<msg ;
+           MPI_Send(buff, buffer.count(), MPI_CHAR, sndServerGlobalRanks[i], 1, CXios::globalComm) ;
+           MPI_Comm_dup(*it, &inter);
+           contextInterComms.push_back(inter);
+           MPI_Comm_dup(intraComm, &inter);
+           contextIntraComms.push_back(inter);
+           context->initClient(contextIntraComms.back(), contextInterComms.back()) ;
+           delete [] buff ;
+         }
+       }
      }
 
-     void CServer::contextEventLoop(void)
-     {
-       bool finished ;
+     void CServer::contextEventLoop(bool enableEventsProcessing /*= true*/)
+     {
+       bool isFinalized ;
        map<string,CContext*>::iterator it ;
+
        for(it=contextList.begin();it!=contextList.end();it++)
        {
-         finished=it->second->checkBuffersAndListen();
-         if (finished)
+         isFinalized=it->second->isFinalized();
+         if (isFinalized)
          {
            contextList.erase(it) ;
            break ;
          }
-       }
-
+         else
+           it->second->checkBuffersAndListen(enableEventsProcessing);
+       }
      }
 
-     //! Get rank of the current process
+     //! Get rank of the current process in the intraComm
      int CServer::getRank()
      {
+       int rank;
+       MPI_Comm_rank(intraComm,&rank);
        return rank;
+     }
+
+     vector<int>& CServer::getSecondaryServerGlobalRanks()
+     {
+       return sndServerGlobalRanks;
      }
 
@@ -444 +800 @@
       int numDigit = 0;
       int size = 0;
+      int id;
       MPI_Comm_size(CXios::globalComm, &size);
       while (size)
@@ -450 +807 @@
         ++numDigit;
       }
-
-      fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << getRank() << ext;
+      id = rank_; //getRank();
+
+      fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << id << ext;
       fb->open(fileNameClient.str().c_str(), std::ios::out);
       if (!fb->is_open())