Changeset 1938 for XIOS

Timestamp:

09/22/20 16:28:47 (4 years ago)

Author:

ymipsl

Message:

XIOS Coupling branch : Solve spurious situation :

• when client have no data on their local grid or local grid is 0 sized
• when sever have no data on their local grid or local grid is 0 sized
• holes in grid (missing global point) that cover a full client or a full server

YM

Location:

XIOS/dev/dev_ym/XIOS_COUPLING/src

Files:

• distribution/grid_remote_connector.cpp (modified) (5 diffs)
• distribution/grid_remote_connector.hpp (modified) (2 diffs)
• node/axis.cpp (modified) (1 diff)
• node/domain.cpp (modified) (6 diffs)
• node/grid.cpp (modified) (2 diffs)

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/grid_remote_connector.cpp

@@ -1 +1 @@
 #include "grid_remote_connector.hpp"
 #include "client_client_dht_template.hpp"
+#include "mpi.hpp"
 
 
@@ -7 +8 @@
 {
   
-  CGridRemoteConnector::CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, MPI_Comm localComm) 
-                       : srcView_(srcView), dstView_(dstView), localComm_(localComm) 
+  CGridRemoteConnector::CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, MPI_Comm localComm, int remoteSize) 
+                       : srcView_(srcView), dstView_(dstView), localComm_(localComm), remoteSize_(remoteSize) 
   {}
 
@@ -74 +75 @@
 
     CClientClientDHTTemplate<int> dataRanks(dataInfo, localComm_) ;
-/*
-    CClientClientDHTTemplate<int>::Index2VectorInfoTypeMap info ; // info map
-        
-// generate list of global index for dst view, and insert it into DHT map
-    int nDst = dstView_.size() ;
-    vector<size_t> dstGlobalIndex ;
-    vector<size_t> dstSliceSize(nDst) ;
-    dstSliceSize[nDst-1] = 1 ;  
-    for(int i=nDst-2; i>=0; i--)  dstSliceSize[i] = dstView_[i+1]->getGlobalSize()*dstSliceSize[i+1] ;
-    for(auto& ranks : dstView_[0]->getLocalSize())
-    {
-      dstGlobalIndex.clear() ;
-      int rank=ranks.first ;
-      size_t sliceIndex=0 ;
-      dstView_[nDst-1]->getGlobalIndex(rank, dstGlobalIndex, sliceIndex, dstSliceSize.data(), dstView_.data(), nDst-1) ;
-      for(auto globalIndex : dstGlobalIndex) info[globalIndex].push_back(rank) ; // insert into DHT
-    }
-    
-    CClientClientDHTTemplate<int> dataRanks(info, localComm_) ;
-*/    
+
     // generate list of global index for src view
     int nSrc = srcView_.size() ;
     vector<size_t> srcSliceSize(nSrc) ;
-//    srcSliceSize[nSrc-1] = 1 ;
-//    for(int i=nSrc-2; i>=0; i--)  srcSliceSize[i] = srcView_[i+1]->getGlobalSize()*srcSliceSize[i+1] ;
    
     srcSliceSize[0] = 1 ;  
@@ -112 +92 @@
 
     vector<map<int, set<size_t>>> elements(nSrc) ; // internal representation of elements composing the grid
-
-//    srcSliceSize[nSrc-1] = srcView_[nSrc-1]->getGlobalSize() ;
-//    for(int i=nSrc-2 ; i>=0 ; i--) srcSliceSize[i] = srcView_[i]->getGlobalSize()*srcSliceSize[i+1] ;
 
     for(auto& indRanks : returnInfo)
@@ -143 +120 @@
       }
     }
+    
+    // So what about when there is some server that have no data to receive
+    // they must be inform they receive an event with no data.
+    // So find remote servers with no data, and one client will take in charge 
+    // that it receive global index with no data (0-size) 
+    vector<int> ranks(remoteSize_,0) ;
+    for(auto& it : elements_[0]) ranks[it.first] = 1 ;
+    MPI_Allreduce(MPI_IN_PLACE, ranks.data(), remoteSize_, MPI_INT, MPI_SUM, localComm_) ;
+    int commRank, commSize ;
+    MPI_Comm_rank(localComm_,&commRank) ;
+    MPI_Comm_size(localComm_,&commSize) ;
+    int pos=0 ;
+    for(int i=0; i<remoteSize_ ; i++)
+      if (ranks[i]==0)
+      {
+        if (pos%commSize==commRank) for(auto& element : elements_) element[i] = CArray<size_t,1>(0) ;
+        pos++ ;
+      }
   }
 
+
 }

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/grid_remote_connector.hpp

@@ -18 +18 @@
     public:
 
-      CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, MPI_Comm localComm) ;
+      CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, MPI_Comm localComm, int remoteSize) ;
       void computeConnector(void) ;
       void computeGenericMethod(void) ;
@@ -28 +28 @@
       vector<CDistributedView*> dstView_ ;
       MPI_Comm localComm_ ;
+      int remoteSize_ ;
 
   } ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/node/axis.cpp

@@ -1840 +1840 @@
       gathererConnector_->transfer(event, 2, value, 0.); 
       bounds.resize(2,n) ;
-      bounds=CArray<double,2>(bounds.dataFirst(),shape(2,n),neverDeleteData) ; 
+      if (bounds.numElements() > 0 ) bounds=CArray<double,2>(bounds.dataFirst(),shape(2,n),neverDeleteData) ; 
     }
     else if (type=="label")

XIOS/dev/dev_ym/XIOS_COUPLING/src/node/domain.cpp

@@ -2275 +2275 @@
         for(int rank : ranks) vectGlobalIndex[rank].push_back(index) ;
       }
-      for(auto& vect : vectGlobalIndex ) globalIndex.emplace(vect.first, CArray<size_t,1>(vect.second.data(), shape(vect.second.size()),duplicateData)) ; 
+      for(auto& vect : vectGlobalIndex ) globalIndex.emplace(vect.first, CArray<size_t,1>(vect.second.data(), shape(vect.second.size()),duplicateData)) ;
+    // some servers receves no index (zeroIndex array) => root process take them into account.
+      if (context->getIntraCommRank()==0) 
+        for(auto& rank : zeroIndex) globalIndex[rank] = CArray<size_t,1>() ; 
     }
     else if (type==EDistributionType::NONE) // domain is not distributed ie all servers get the same local domain
@@ -2500 +2503 @@
       gathererConnector_->transfer(event, value, 0.); 
       lonvalue_2d.resize(ni,nj) ;
-      lonvalue_2d=CArray<double,2>(value.dataFirst(),shape(ni,nj),neverDeleteData) ; 
+      if (lonvalue_2d.numElements()>0) lonvalue_2d=CArray<double,2>(value.dataFirst(),shape(ni,nj),neverDeleteData) ; 
     }
     else if (type=="lat")
@@ -2507 +2510 @@
       gathererConnector_->transfer(event, value, 0.); 
       latvalue_2d.resize(ni,nj) ;
-      latvalue_2d=CArray<double,2>(value.dataFirst(),shape(ni,nj),neverDeleteData) ; 
+      if (latvalue_2d.numElements()>0) latvalue_2d=CArray<double,2>(value.dataFirst(),shape(ni,nj),neverDeleteData) ; 
     }
     else if (type=="boundslon")
@@ -2514 +2517 @@
       gathererConnector_->transfer(event, nvertex, value, 0.); 
       bounds_lon_2d.resize(nvertex,ni,nj) ;
-      bounds_lon_2d=CArray<double,3>(value.dataFirst(),shape(nvertex,ni,nj),neverDeleteData) ; 
+      if (bounds_lon_2d.numElements()>0) bounds_lon_2d=CArray<double,3>(value.dataFirst(),shape(nvertex,ni,nj),neverDeleteData) ; 
     }
     else if (type=="boundslat")
@@ -2521 +2524 @@
       gathererConnector_->transfer(event, nvertex, value, 0.); 
       bounds_lat_2d.resize(nvertex,ni,nj) ;
-      bounds_lat_2d=CArray<double,3>(value.dataFirst(),shape(nvertex,ni,nj),neverDeleteData) ; 
+      if (bounds_lat_2d.numElements()>0) bounds_lat_2d=CArray<double,3>(value.dataFirst(),shape(nvertex,ni,nj),neverDeleteData) ; 
     }
     else if (type=="area") 
@@ -2528 +2531 @@
       gathererConnector_->transfer(event, value, 0.); 
       area.resize(ni,nj) ;
-      area=CArray<double,2>(value.dataFirst(),shape(ni,nj),neverDeleteData) ; 
+      if (area.numElements()>0) area=CArray<double,2>(value.dataFirst(),shape(ni,nj),neverDeleteData) ; 
     }
   }

XIOS/dev/dev_ym/XIOS_COUPLING/src/node/grid.cpp

@@ -2668 +2668 @@
       }
     }
-    CGridRemoteConnector gridRemoteConnector(localViews, remoteViews, context->getIntraComm()) ;
+    CGridRemoteConnector gridRemoteConnector(localViews, remoteViews, context->getIntraComm(), client->getRemoteSize()) ;
     gridRemoteConnector.computeConnector() ;
     
@@ -2754 +2754 @@
       }
     }
-    CGridRemoteConnector gridRemoteConnector(localViews, remoteViews, context->getIntraComm()) ;
+    CGridRemoteConnector gridRemoteConnector(localViews, remoteViews, context->getIntraComm(), client->getRemoteSize()) ;
     gridRemoteConnector.computeConnector() ;