Changeset 5861 for CONFIG_DEVT

Timestamp:

06/24/21 11:14:28 (3 years ago)

Author:

ymipsl

Message:

• Adapt XIOS sources to manage ensemble runs, using new version of oasis

YM

Location:

CONFIG_DEVT/IPSLCM6.5_work_ENSEMBLES/modeles/XIOS/src

Files:

• client.cpp (modified) (3 diffs)
• interface/c/oasis_cinterface.cpp (modified) (2 diffs)
• interface/c/oasis_cinterface.hpp (modified) (2 diffs)
• io/nc4_data_output.cpp (modified) (1 diff)
• server.cpp (modified) (1 diff)

CONFIG_DEVT/IPSLCM6.5_work_ENSEMBLES/modeles/XIOS/src/client.cpp

@@ -42 +42 @@
       else is_MPI_Initialized=false ;
       int rank ;
+      bool ensembleManagement ;
+      int ensembleSize ;
 
 // don't use OASIS
@@ -134 +136 @@
       else
       {
+
+        // manage ensemble with oasis
+        // code id is splitted follow the base code Id and the num number using the ":" separator
+        // codeId == baseCodeId:member
+        // member must integer convertible
+        vector<string> splittedCodeId=splitRegex(codeId,"\\s*:\\s*") ;
+        if (splittedCodeId.size()>2) ERROR("void CClient::initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)",
+                                            <<"CodeId is badly formed "<<codeId) ;
+        string baseCodeId=splittedCodeId[0] ;
+        int member ;
+        if (splittedCodeId.size()==2)
+        {
+          try
+          {
+            member = stoi(splittedCodeId[1]) ;
+          } 
+          catch(...)
+          {
+            ERROR("void CClient::initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)",
+                                            <<"CodeId is badly formed "<<codeId) ;
+          }
+          ensembleManagement=true ;
+        }
+        else ensembleManagement=false ;
+
+        string codesId=CXios::getin<string>("oasis_codes_id") ;
+        vector<string> oasisCodesId=splitRegex(codesId,"\\s*,\\s*") ;
+        for(auto& oasisCodeId : oasisCodesId)
+        {
+          info(10)<<"oasisCodeId "<<oasisCodeId<<endl ;
+          vector<string> splittedOasisCodeId = splitRegex(oasisCodeId,"\\s*:\\s*") ;
+          string baseOasisCodeId=splittedOasisCodeId[0] ;
+          info(10)<<"baseOasisCodeId "<<baseOasisCodeId<<"  baseCodeId "<<baseCodeId<<endl ;
+          if (baseOasisCodeId==baseCodeId)
+          {
+            if (splittedOasisCodeId.size()==2)
+            {
+              try
+              {
+                ensembleSize = stoi(splittedOasisCodeId[1]) ;
+                info(10)<<"ensembleSize "<<ensembleSize<<endl ;
+              } 
+              catch(...)
+              {
+                ERROR("void CClient::initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)",
+                                                <<"CodeId is badly formed "<<codeId) ;
+              }
+              if (ensembleManagement==false)  ERROR("void CClient::initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)",
+                                                <<"inconsistancy for ensemble management between local codeId and OASIS codes Id  : "
+                                                <<codeId<<"  "<< codesId); 
+            }
+            else if (ensembleManagement==true)  ERROR("void CClient::initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)",
+                                                <<"inconsistancy for ensemble management between local codeId and OASIS codes Id "
+                                                <<codeId<<"  "<< codesId); 
+          }
+        }
+        
+        
         // localComm isn't given
         if (localComm == MPI_COMM_NULL)
         {
           if (!is_MPI_Initialized) oasis_init(codeId) ;
-          oasis_get_localcomm(localComm) ;
+          if(ensembleManagement) 
+          {
+            vector<string> intraCommStr(ensembleSize) ;
+            vector<int>    rootRanks ;
+            for(int i=0;i<ensembleSize;i++) intraCommStr[i]=baseCodeId+":"+std::to_string(i) ;
+            oasis_get_multi_intracomm(localComm, intraCommStr, rootRanks) ;
+          }
+          else oasis_get_localcomm(localComm) ;
         }
         MPI_Comm_dup(localComm,&intraComm) ;
@@ -149 +216 @@
           MPI_Status status ;
           MPI_Comm_rank(intraComm,&rank_) ;
-
-          oasis_get_intercomm(interComm,CXios::xiosCodeId) ;
+          
+          if(ensembleManagement) 
+          {
+            vector<string> intraCommStr(ensembleSize+1) ;
+            vector<int>    rootRanks ;
+            for(int i=0;i<ensembleSize;i++) intraCommStr[i]=baseCodeId+":"+std::to_string(i) ;
+            intraCommStr[ensembleSize] = CXios::xiosCodeId ;
+            
+            oasis_get_multi_intracomm(localComm, intraCommStr, rootRanks) ;
+   
+            int myColor= 0 ; // 0:client 1:server
+            int myrank ;
+            MPI_Comm splittedComm ;
+            MPI_Comm_rank(localComm,&myrank) ;
+            MPI_Comm_split(localComm,myColor,myrank, &splittedComm) ;
+            MPI_Intercomm_create(splittedComm,0,localComm,rootRanks[ensembleSize],10,&interComm) ;
+          
+          }
+          else oasis_get_intercomm(interComm,CXios::xiosCodeId) ;
           if (rank_==0) MPI_Recv(&serverLeader,1, MPI_INT, 0, 0, interComm, &status) ;
           MPI_Bcast(&serverLeader,1,MPI_INT,0,intraComm) ;

CONFIG_DEVT/IPSLCM6.5_work_ENSEMBLES/modeles/XIOS/src/interface/c/oasis_cinterface.cpp

@@ -1 +1 @@
 #include "oasis_cinterface.hpp"
+#include <oasis_c.h>
 #include <string>
 #include "mpi.hpp"
+#include <cstring>
 
 namespace xios
@@ -44 +46 @@
     comm_client_server=MPI_Comm_f2c(f_comm) ;
   }
+  
+  void oasis_get_multi_intracomm(MPI_Comm& comm_clients_server,const std::vector<std::string>& codesId, std::vector<int> &rootRanks)
+  {
+    int size = codesId.size() ;
+    char** cdnam = new char*[size] ;
+    for(int i=0;i<size;i++)
+    {
+      cdnam[i]=new char[codesId[i].length()+1] ;
+      std::strcpy(cdnam[i], codesId[i].c_str());
+    }
+    rootRanks.resize(size) ;
+    oasis_c_get_multi_intracomm(&comm_clients_server, size, cdnam, rootRanks.data()) ;
+  }
+
 }

CONFIG_DEVT/IPSLCM6.5_work_ENSEMBLES/modeles/XIOS/src/interface/c/oasis_cinterface.hpp

@@ -2 +2 @@
 #define __OASIS_CINTERFACE__
 #include <string>
+#include <vector>
 #include "mpi.hpp"
 
@@ -23 +24 @@
   void oasis_get_intracomm(MPI_Comm& comm_client_server,const std::string& server_id) ;
   void oasis_get_intercomm(MPI_Comm& comm_client_server,const std::string& server_id) ;
+  void oasis_get_multi_intracomm(MPI_Comm& comm_clients_server,const std::vector<std::string>& codesId, std::vector<int>& rootRanks) ;
 }
 #endif

CONFIG_DEVT/IPSLCM6.5_work_ENSEMBLES/modeles/XIOS/src/io/nc4_data_output.cpp

@@ -2634 +2634 @@
                 }
 
-
+                // suppose first dim is dedicated for ensemble
+                double defaultValue=field->default_value ;
+                size_t slice=1 ;
+                for(int i=1 ; i<count.size() ; i++) slice=slice*count[i] ;
+                
+                size_t nEnsemble=count[0] ;
+                size_t pos=0 ;
+                start[0] = 0 ;
+                count[0] = 0 ;
+                for(size_t i=0 ; i<nEnsemble ; i++)
+                {
+                  bool allMissing=true ;
+                  for (size_t j=0 ; j<slice ; j++,pos++)
+                    if (fieldData(pos)!=defaultValue) { allMissing=false ; break;}
+                  if (allMissing) 
+                  {
+                    if (count[0]==0) start[0]++ ;
+                    else break ;
+                  }
+                  else count[0]++ ;  
+                }  
+
+                CArray<double,1> newData(fieldData.dataFirst()+start[0]*slice,shape(count[0]*slice),neverDeleteData) ;
+                info(0)<<"write data parameter  "<<fieldid<<"  "<<field->getNStep()<<"  start : " ;
+                for(auto it : start) info(0)<<" "<<it ;
+                info(0)<<"  count : " ; 
+                for(auto it : count) info(0)<<"  "<<it ;
+                info(0)<<endl ; 
+                 
                 CTimer::get("Files : writing data").resume();
-                SuperClassWriter::writeData(fieldData, fieldid, isCollective, field->getNStep() - 1, &start, &count);
+                SuperClassWriter::writeData(newData, fieldid, isCollective, field->getNStep() - 1, &start, &count);
                 CTimer::get("Files : writing data").suspend();
 

CONFIG_DEVT/IPSLCM6.5_work_ENSEMBLES/modeles/XIOS/src/server.cpp

@@ -352 +352 @@
         MPI_Comm_rank(CXios::globalComm,&globalRank);
 
+// (1.5) check ensemble management
+        vector<string> baseOasisCodeId(oasisCodeId.size()) ;
+        vector<int> ensembleSize(oasisCodeId.size()) ;
+        vector<bool> ensembleManagement(oasisCodeId.size()) ;
+        
+        for(int i=0 ; i<oasisCodeId.size() ;i++)
+        {
+          vector<string> splittedOasisCodeId = splitRegex(oasisCodeId[i],"\\s*:\\s*") ;
+          baseOasisCodeId[i]=splittedOasisCodeId[0] ;
+          if (splittedOasisCodeId.size()==1) 
+          {
+            ensembleManagement[i]=false ;
+            ensembleSize[i]=1 ;
+          }
+          else if (splittedOasisCodeId.size()==2)
+          {
+            
+            try 
+            { 
+              ensembleSize[i] = stoi(splittedOasisCodeId[1]) ;
+            }
+            catch(...) 
+            { 
+              ERROR("void CServer::initialize(void)",<<"OasisCodeId is badly formed "<<oasisCodeId[i]) ;
+            }
+            
+            ensembleManagement[i] = true ;
+          }
+          else ERROR("void CServer::initialize(void)",<<"OasisCodeId is badly formed "<<oasisCodeId[i]) ;
+        }
+
 //      (2) Create interComms with models
-        for(it=oasisCodeId.begin();it!=oasisCodeId.end();it++)
-        {
-          oasis_get_intercomm(newComm,*it) ;
+        for(int i=0; i<baseOasisCodeId.size();i++)
+        {
+          if (ensembleManagement[i]==false) oasis_get_intercomm(newComm,baseOasisCodeId[i]) ;
+          else
+          {
+            vector<string> intraCommStr(ensembleSize[i]+1) ;
+            vector<int>    rootRanks ;
+            for(int j=0;j<ensembleSize[i];j++) intraCommStr[j]=baseOasisCodeId[i]+":"+std::to_string(j) ;
+            intraCommStr[ensembleSize[i]] = CXios::xiosCodeId ;
+            oasis_get_multi_intracomm(localComm, intraCommStr, rootRanks) ;
+   
+            int myColor= 1 ; // 0:client 1:server
+            int myrank ;
+            MPI_Comm splittedComm ;
+            MPI_Comm_rank(localComm,&myrank) ;
+            MPI_Comm_split(localComm,myColor,myrank, &splittedComm) ;
+            MPI_Intercomm_create(splittedComm,0,localComm,rootRanks[0],10,&newComm) ;
+          }
+          
           if ( serverLevel == 0 || serverLevel == 1)
           {