Changeset 2179 for XIOS

Timestamp:

07/12/21 20:02:30 (3 years ago)

Author:

ymipsl

Message:

Improve performance of CGridRemoteConnector when grid source and/or destination are non distributed.

YM

Location:

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution

Files:

• grid_remote_connector.cpp (modified) (9 diffs)
• grid_remote_connector.hpp (modified) (1 diff)

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/grid_remote_connector.cpp

@@ -1 +1 @@
 #include "grid_remote_connector.hpp"
 #include "client_client_dht_template.hpp"
+#include "leader_process.hpp"
 #include "mpi.hpp"
 
@@ -7 +8 @@
 namespace xios
 {
-  
+  /** 
+   * \brief class constructor. 
+   * \param srcView List of sources views.
+   * \param dstView List of remotes views.
+   * \param localComm Local MPI communicator
+   * \param remoteSize Size of the remote communicator
+   */ 
   CGridRemoteConnector::CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, MPI_Comm localComm, int remoteSize) 
                        : srcView_(srcView), dstView_(dstView), localComm_(localComm), remoteSize_(remoteSize) 
   {}
 
+  /** 
+   * \brief class constructor. 
+   * \param srcView List of sources views.
+   * \param dstView List of remotes views.
+   * \param localComm Local MPI communicator
+   * \param remoteSize Size of the remote communicator
+   */ 
   CGridRemoteConnector::CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CLocalView*>& dstView, MPI_Comm localComm, int remoteSize) 
                        : srcView_(srcView), localComm_(localComm), remoteSize_(remoteSize) 
@@ -18 +32 @@
   }
 
+
+  /**
+   * \brief Compute if each view composing the source grid and the remote grid is distributed or not. 
+   *         Result is stored on internal attributes \b isSrcViewDistributed_ and \b isDstViewDistributed_.
+   * \detail To compute this, a hash is computed for each array on indices. The hash must permutable, i.e. 
+   *         the order of the list of global indices doesn't influence the value of the hash. So simply a sum of 
+   *         hash of each indices is used for the whole array. After, the computed hash are compared with each other
+   *         ranks of \b localComm_ MPI communicator using an MPI_ALLReduce. If, for each ranks, the hash is the same
+   *         then the view is not distributed
+   */
+  void CGridRemoteConnector::computeViewDistribution(void)
+  {
+    HashXIOS<size_t> hashGlobalIndex; // hash function-object
+
+    int nDst = dstView_.size() ;
+    vector<size_t> hashRank(remoteSize_) ;
+    isDstViewDistributed_.resize(nDst) ;
+
+    for(int i=0; i<nDst; i++)
+    {
+      map<int,CArray<size_t,1>> globalIndexView ;
+      dstView_[i]->getGlobalIndexView(globalIndexView) ;
+      hashRank.assign(remoteSize_,0) ; // everybody ranks to 0 except rank of the remote view I have 
+                                       // that would be assign to my local hash 
+      for(auto& it : globalIndexView)
+      {
+        int rank=it.first ;
+        CArray<size_t,1>& globalIndex = it.second ;
+        size_t globalIndexSize = globalIndex.numElements();
+        size_t hashValue=0 ;
+        for(size_t ind=0;ind<globalIndexSize;ind++) hashValue += hashGlobalIndex(globalIndex(ind)) ;
+        hashRank[rank] += hashValue ;
+      }
+      // sum all the hash for every process of the local comm. The reduce is on the size of remote view (remoteSize_)
+      // after that for each rank of the remote view, we get the hash
+      MPI_Allreduce(MPI_IN_PLACE, hashRank.data(), remoteSize_, MPI_SIZE_T, MPI_SUM, localComm_) ;
+      size_t value = hashRank[0] ;
+      isDstViewDistributed_[i]=false ;
+      for(int j=0 ; j<remoteSize_ ; j++) 
+        if (value != hashRank[j]) 
+        { 
+          isDstViewDistributed_[i]=true ;
+          break ;
+        }
+    }
+
+    int nSrc = srcView_.size() ;
+    int commSize,commRank ;
+    MPI_Comm_size(localComm_,&commSize) ;
+    MPI_Comm_rank(localComm_,&commRank) ;
+    hashRank.resize(commSize,0) ;
+    isSrcViewDistributed_.resize(nSrc) ;
+
+    for(int i=0; i<nSrc; i++)
+    {
+      CArray<size_t,1> globalIndex ;
+      srcView_[i]->getGlobalIndexView(globalIndex) ;
+      hashRank.assign(commSize,0) ; // 0 for everybody except my rank
+      size_t globalIndexSize = globalIndex.numElements() ;
+      size_t hashValue=0 ;
+      for(size_t ind=0;ind<globalIndexSize;ind++) hashValue += hashGlobalIndex(globalIndex(ind)) ;
+        hashRank[commRank] += hashValue ;
+    
+      // Same method than for remote view 
+      MPI_Allreduce(MPI_IN_PLACE, hashRank.data(), commSize, MPI_SIZE_T, MPI_SUM, localComm_) ;
+      size_t value = hashRank[0] ;
+      isSrcViewDistributed_[i]=false ;
+      for(int j=0 ; j<commSize ; j++) 
+        if (value != hashRank[j]) 
+        { 
+          isSrcViewDistributed_[i]=true ;
+          break ;
+        }
+    }
+
+  }
+
+/**
+  * \brief Compute the connector, i.e. compute the \b elements_ attribute. 
+  * \detail In order to achive better optimisation,
+  *         we distingute the case when the grid is not distributed on source grid (\bcomputeSrcNonDistributed), 
+  *         or the remote grid (\b computeDstNonDistributed), or the both (\b computeSrcDstNonDistributed). 
+  *         Otherwise the generic method is called computeGenericMethod. Note that in the case, if one element view
+  *         is not distributed on the source and on the remote grid, then we can used the tensorial product
+  *         property to computing it independently using \b computeSrcDstNonDistributed method.
+  *         After that, we call the \b removeRedondantRanks method to supress blocks of data that can be sent 
+  *         redondantly the the remote servers
+  */
   void CGridRemoteConnector::computeConnector(void)
   {
-    computeGenericMethod() ;
-  }
-
-  void CGridRemoteConnector::computeGenericMethod(void)
+    computeViewDistribution() ;
+    vector<CLocalView*> srcView ;
+    vector<CDistributedView*> dstView ;
+    vector<int> indElements ;
+    elements_.resize(srcView_.size()) ;
+    
+    bool srcViewsNonDistributed=true ;
+    for(int i=0;i<srcView_.size();i++) srcViewsNonDistributed &= !isSrcViewDistributed_[i]  ;
+    
+    bool dstViewsNonDistributed=true ;
+    for(int i=0;i<dstView_.size();i++) dstViewsNonDistributed &= !isDstViewDistributed_[i] ;
+    
+    if (srcViewsNonDistributed) 
+    {
+      int commRank, commSize ;
+      MPI_Comm_rank(localComm_,&commRank) ;
+      MPI_Comm_size(localComm_,&commSize) ;
+      list<int> remoteRanks;
+      list<int> notUsed ;
+      map<int,bool> ranks ;  
+      computeLeaderProcess(commRank, commSize, remoteSize_, remoteRanks, notUsed) ;
+      for(int rank : remoteRanks) ranks[rank]=true ;
+      
+      for(int i=0; i<srcView_.size(); i++)  
+      {
+        if (isDstViewDistributed_[i]) computeSrcNonDistributed(i) ;
+        else computeSrcDstNonDistributed(i, ranks) ;
+      }
+    } 
+    else if (dstViewsNonDistributed)
+    {
+      map<int,bool> ranks ;
+      for(int i=0;i<remoteSize_;i++) ranks[i]=true ;
+      for(int i=0; i<srcView_.size(); i++)  
+      {
+        if (isSrcViewDistributed_[i]) computeDstNonDistributed(i,ranks) ;
+        else computeSrcDstNonDistributed(i,ranks) ;
+      }
+    } 
+    else
+    {
+      for(int i=0;i<srcView_.size();i++) 
+        if (isSrcViewDistributed_[i] || isDstViewDistributed_[i])
+        {
+          srcView.push_back(srcView_[i]) ;
+          dstView.push_back(dstView_[i]) ;
+          indElements.push_back(i) ;
+        }
+
+      computeGenericMethod(srcView, dstView, indElements) ;
+    
+      map<int,bool> ranks ;  
+      for(auto& it : elements_[indElements[0]]) 
+      {
+        if (it.second.numElements()==0) ranks[it.first] = false ;
+        else  ranks[it.first] = true ;
+      }
+    
+      for(int i=0;i<srcView_.size();i++) 
+        if (!isSrcViewDistributed_[i] && !isDstViewDistributed_[i]) computeSrcDstNonDistributed(i, ranks) ;
+    }
+
+    removeRedondantRanks() ;
+  }
+
+  
+/**
+  * \brief Compute the connector for the element \b i when the source view is not distributed. 
+  *        After the call element_[i] is defined.
+  *  \param i Indice of the element composing the source grid. 
+  */
+
+  void CGridRemoteConnector::computeSrcNonDistributed(int i)
+  {
+    auto& element = elements_[i] ;
+    map<int,CArray<size_t,1>> globalIndexView ;
+    dstView_[i]->getGlobalIndexView(globalIndexView) ;
+    
+    CClientClientDHTTemplate<int>::Index2InfoTypeMap dataInfo;
+    
+    for(auto& it : globalIndexView)
+    {
+      auto& globalIndex=it.second ;
+      for(size_t ind : globalIndex) dataInfo[ind]=it.first ;
+    }
+    
+    // First we feed the distributed hash map  with key (remote global index) 
+    // associated with the value of the remote rank
+    CClientClientDHTTemplate<int> DHT(dataInfo, localComm_) ;
+    // after we feed the DHT with the local global indices of the source view
+
+    int commRank, commSize ;
+    MPI_Comm_rank(localComm_,&commRank) ;
+    MPI_Comm_size(localComm_,&commSize) ;
+    CArray<size_t,1> srcIndex ;
+    // like the source view is not distributed, then only the rank 0 need to feed the DHT
+    if (commRank==0) srcView_[i]->getGlobalIndexView(srcIndex) ;
+    
+    // compute the mapping
+    DHT.computeIndexInfoMapping(srcIndex) ;
+    auto& returnInfo = DHT.getInfoIndexMap() ;
+    
+    // returnInfo contains now the map for each global indices to send to a list of remote rank
+    // only for the rank=0 because it is the one to feed the DHT
+    // so it need to send the list to each server leader i.e. the local process that handle specifically one or more 
+    // servers
+    
+    // rankIndGlo : rankIndGlo[rank][indGlo] : list of indice to send the the remote server of rank "rank"
+    vector<vector<size_t>> rankIndGlo(remoteSize_) ;
+    if (commRank==0) 
+      for(auto& it1 : returnInfo)
+        for(auto& it2 : it1.second) rankIndGlo[it2].push_back(it1.first) ;
+    
+    
+    vector<MPI_Request> requests ;
+    
+    if (commRank==0)
+    {
+      requests.resize(remoteSize_) ;
+      for(int i=0 ; i<remoteSize_;i++) 
+      {
+        // ok send only the global indices for a server to the "server leader"
+        int rank = getLeaderRank(commSize, remoteSize_, i) ;
+        MPI_Isend(rankIndGlo[i].data(), rankIndGlo[i].size(), MPI_SIZE_T, rank, i ,localComm_, &requests[i]) ;
+      }
+    }  
+   
+    list<int> remoteRanks;
+    list<int> notUsed ;
+    // I am a server leader of which remote ranks ?
+    computeLeaderProcess(commRank, commSize, remoteSize_, remoteRanks, notUsed) ;
+
+    for(auto remoteRank : remoteRanks)
+    {
+      MPI_Status status ;
+      int size ;
+      MPI_Probe(0,remoteRank,localComm_, &status);
+      MPI_Get_count(&status, MPI_SIZE_T, &size) ;
+      elements_[i][remoteRank].resize(size) ;
+      // for each remote ranks receive the global indices from proc 0
+      MPI_Recv(elements_[i][remoteRank].dataFirst(),size, MPI_SIZE_T,0,remoteRank, localComm_,&status) ;
+    }
+      
+    if (commRank==0)
+    {
+      vector<MPI_Status> status(remoteSize_) ;
+      // asynchronous for sender, wait for completion
+      MPI_Waitall(remoteSize_, requests.data(), status.data()) ;
+    }
+  }
+
+  /**
+   * \brief Compute the remote connector for the element \b i when the remote view is not distributed. 
+   *        After the call,  element_[i] is defined.
+   * \param i Indice of the element composing the remote grid. 
+   * \param ranks The list of rank for which the local proc is in charge to compute the connector 
+   *              (if leader server for exemple). if ranks[rank] == false the corresponding elements_ 
+   *              is set to void array (no data to sent) just in order to notify corresponding remote server
+   *              that the call is collective with each other one  
+   */
+  void CGridRemoteConnector::computeDstNonDistributed(int i, map<int,bool>& ranks)
+  {
+    auto& element = elements_[i] ;
+    map<int,CArray<size_t,1>> globalIndexView ;
+    dstView_[i]->getGlobalIndexView(globalIndexView) ;
+    
+    
+    CClientClientDHTTemplate<int>::Index2InfoTypeMap dataInfo;
+ 
+    // First we feed the distributed hash map  with key (remote global index) 
+    // associated with the value of the remote rank
+    for(auto& it : globalIndexView)
+      if (it.first==0) // since the remote view is not distributed, insert only the remote rank 0
+      {
+        auto& globalIndex=it.second ;
+        for(size_t ind : globalIndex) dataInfo[ind]=0 ; // associated the the rank 0
+      }
+    
+    CClientClientDHTTemplate<int> DHT(dataInfo, localComm_) ;
+    // after we feed the DHT with the local global indices of the source view
+
+    CArray<size_t,1> srcIndex ;
+    srcView_[i]->getGlobalIndexView(srcIndex) ;
+    DHT.computeIndexInfoMapping(srcIndex) ;
+    auto& returnInfo = DHT.getInfoIndexMap() ;
+    
+    // returnInfo contains now the map for each global indices to send to a list of remote rank
+    // now construct the element_ list of global indices for each rank in my list except if the erray must be empty
+    for (auto& rank : ranks)
+    {
+      if (rank.second) // non empty array => for rank that have not any data to be received
+      {
+        int size=0 ;
+        for(auto& it : returnInfo) if (!it.second.empty()) size++ ;
+        auto& array = element[rank.first] ;
+       array.resize(size) ;
+       size=0 ;
+       for(auto& it : returnInfo) 
+         if (!it.second.empty()) 
+         {
+           array(size)=it.first ;
+           size++ ;
+         }
+      }
+      else element[rank.first] = CArray<size_t,1>(0) ;  // empty array => for rank that have not any data to be received
+    }
+  }
+
+ /**
+  * \brief Compute the remote connector for the element \b i when the source and the remote view are not distributed. 
+  *        After the call, element_[i] is defined.
+  * \param i Indice of the element composing the remote grid. 
+  * \param ranks The list of rank for which the local proc is in charge to compute the connector 
+  *              (if leader server for exemple). if ranks[rank] == false the corresponding elements_ 
+  *              is set to void array (no data to sent) just in order to notify corresponding remote server
+  *              that the call is collective with each other one  
+  */
+
+  void CGridRemoteConnector::computeSrcDstNonDistributed(int i, map<int,bool>& ranks)
+  {
+    auto& element = elements_[i] ;
+    map<int,CArray<size_t,1>> globalIndexView ;
+    dstView_[i]->getGlobalIndexView(globalIndexView) ;
+    
+    
+    CClientClientDHTTemplate<int>::Index2InfoTypeMap dataInfo;
+    // First we feed the distributed hash map  with key (remote global index) 
+    // associated with the value of the remote rank
+
+    for(auto& it : globalIndexView)
+      if (it.first==0) // insert only the remote rank 0 since the remote view is not distributed
+      {
+        auto& globalIndex=it.second ;
+        for(size_t ind : globalIndex) dataInfo[ind]=0 ; // associated the the rank 0
+      }
+    
+    CClientClientDHTTemplate<int> DHT(dataInfo, localComm_) ;
+    // after we feed the DHT with the local global indices of the source view
+
+    int commRank, commSize ;
+    MPI_Comm_rank(localComm_,&commRank) ;
+    MPI_Comm_size(localComm_,&commSize) ;
+    CArray<size_t,1> srcIndex ;
+  
+    // like the source view is not distributed, then only the rank 0 need to feed the DHT
+    if (commRank==0) srcView_[i]->getGlobalIndexView(srcIndex) ;
+    DHT.computeIndexInfoMapping(srcIndex) ;
+    auto& returnInfo = DHT.getInfoIndexMap() ;
+    
+    vector<size_t> indGlo ;
+    if (commRank==0) 
+      for(auto& it1 : returnInfo) 
+        for(auto& it2 : it1.second) indGlo.push_back(it1.first) ;
+
+    // now local rank 0 know which indices to seed to remote rank 0, but all the server
+    // must receive the same information. So only the leader rank will sent this.
+    // So local rank 0 must broadcast the information to all leader.
+    // for this we create a new communicator composed of local process that must send data
+    // to a remote rank, data are broadcasted, and element_[i] is construction for each remote 
+    // rank in charge
+    int color=0 ;
+    if (ranks.empty()) color=0 ;
+    else color=1 ;
+    if (commRank==0) color=1 ;
+    MPI_Comm newComm ;
+    MPI_Comm_split(localComm_, color, commRank, &newComm) ;
+    if (color==1)
+    {
+      // ok, I am part of the process that must send something to one or more remote server
+      // so I get the list of global indices from rank 0
+      int dataSize ;
+      if (commRank==0) dataSize=indGlo.size() ;
+      MPI_Bcast(&dataSize,1,MPI_INT, 0, newComm) ;
+      indGlo.resize(dataSize) ;
+      MPI_Bcast(indGlo.data(),dataSize,MPI_SIZE_T,0,newComm) ;
+    }
+    MPI_Comm_free(&newComm) ;
+
+    // construct element_[i] from indGlo
+    for(auto& rank : ranks)
+    {
+      if (rank.second)
+      {
+        int dataSize=indGlo.size();
+        auto& element = elements_[i][rank.first] ;
+        element.resize(dataSize) ;
+        for(int i=0;i<dataSize; i++) element(i)=indGlo[i] ;
+      }
+      else element[rank.first] = CArray<size_t,1>(0) ;
+    }    
+
+  }
+
+  
+ /**
+  * \brief Generic method the compute the grid remote connector. Only distributed elements are specifed in the source view and remote view. 
+  *        Connector for non distributed elements are computed separatly to improve performance and memory consumption. After the call,
+  *        \b elements_  is defined.
+  *  \param srcView List of the source views composing the grid, without non distributed views
+  *  \param dstView List of the remote views composing the grid, without non distributed views
+  *  \param indElements Index of the view making the correspondance between all views and views distributed (that are in input)
+  */
+  void CGridRemoteConnector::computeGenericMethod(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, vector<int>& indElements)
   {
     // generic method, every element can be distributed
-    int nDst = dstView_.size() ;
+    int nDst = dstView.size() ;
     vector<size_t> dstSliceSize(nDst) ;
     dstSliceSize[0] = 1 ;  
-    for(int i=1; i<nDst; i++)  dstSliceSize[i] = dstView_[i-1]->getGlobalSize()*dstSliceSize[i-1] ;
+    for(int i=1; i<nDst; i++)  dstSliceSize[i] = dstView[i-1]->getGlobalSize()*dstSliceSize[i-1] ;
   
     CClientClientDHTTemplate<int>::Index2VectorInfoTypeMap dataInfo ;
-
     CClientClientDHTTemplate<size_t>::Index2VectorInfoTypeMap info ; // info map
+
+    // first, we need to feed the DHT with the global index of the remote server
+    // for that :
+    // First the first element insert the in a DHT with key as the rank and value the list of global index associated
+    // Then get the previously stored index associate with the remote rank I am in charge and reinsert the global index
+    // corresponding to the position of the element in the remote view suing tensorial product
+    // finaly we get only the list of remote global index I am in charge for the whole remote grid   
+
     for(int pos=0; pos<nDst; pos++)
     {
       size_t sliceSize=dstSliceSize[pos] ;
       map<int,CArray<size_t,1>> globalIndexView ;
-      dstView_[pos]->getGlobalIndexView(globalIndexView) ;
+      dstView[pos]->getGlobalIndexView(globalIndexView) ;
       
       CClientClientDHTTemplate<size_t>::Index2VectorInfoTypeMap lastInfo(info) ;
@@ -80 +488 @@
     }
 
+    // we feed the DHT with the remote global index
     CClientClientDHTTemplate<int> dataRanks(dataInfo, localComm_) ;
 
     // generate list of global index for src view
-    int nSrc = srcView_.size() ;
+    int nSrc = srcView.size() ;
     vector<size_t> srcSliceSize(nSrc) ;
    
     srcSliceSize[0] = 1 ;  
-    for(int i=1; i<nSrc; i++)  srcSliceSize[i] = srcView_[i-1]->getGlobalSize()*srcSliceSize[i-1] ;
+    for(int i=1; i<nSrc; i++)  srcSliceSize[i] = srcView[i-1]->getGlobalSize()*srcSliceSize[i-1] ;
 
     vector<size_t> srcGlobalIndex ;
     size_t sliceIndex=0 ;
-    srcView_[nSrc-1]->getGlobalIndex(srcGlobalIndex, sliceIndex, srcSliceSize.data(), srcView_.data(), nSrc-1) ;
-    
+    srcView[nSrc-1]->getGlobalIndex(srcGlobalIndex, sliceIndex, srcSliceSize.data(), srcView.data(), nSrc-1) ;
+    // now we have the global index of the source grid in srcGlobalIndex 
+    // we feed the DHT with the src global index (if we have)
     if (srcGlobalIndex.size()>0)
     {
@@ -104 +514 @@
     }
     const auto& returnInfo = dataRanks.getInfoIndexMap() ;
+    // returnInfo contains now the map for each global indices to send to a list of remote rank
+    // but we want to use the tensorial product property to get the same information using only global
+    // index of element view. So the idea is to reverse the information : for a global index of the grid 
+    // to send to the remote server, what is the global index of each element composing the grid ?
 
     vector<map<int, set<size_t>>> elements(nSrc) ; // internal representation of elements composing the grid
@@ -120 +534 @@
     }
 
-    elements_.resize(nSrc) ;
+//    elements_.resize(nSrc) ;
     for(int i=0 ; i<nSrc; i++)
     {
@@ -128 +542 @@
         int rank=rankInd.first ;
         set<size_t>& indGlo = rankInd.second ;
-        CArray<size_t,1>& indGloArray = elements_[i][rank] ;
+        CArray<size_t,1>& indGloArray = elements_[indElements[i]][rank] ;
         indGloArray.resize(indGlo.size()) ;
         int j=0 ;
@@ -140 +554 @@
     // that it receive global index with no data (0-size) 
     vector<int> ranks(remoteSize_,0) ;
-    for(auto& it : elements_[0]) ranks[it.first] = 1 ;
+    for(auto& it : elements_[indElements[0]]) ranks[it.first] = 1 ;
     MPI_Allreduce(MPI_IN_PLACE, ranks.data(), remoteSize_, MPI_INT, MPI_SUM, localComm_) ;
     int commRank, commSize ;
@@ -149 +563 @@
       if (ranks[i]==0)
       {
-        if (pos%commSize==commRank) for(auto& element : elements_) element[i] = CArray<size_t,1>(0) ;
+        if (pos%commSize==commRank) 
+          for(int j=0 ; j<nSrc; j++) elements_[indElements[j]][i] = CArray<size_t,1>(0) ;
         pos++ ;
       }
   }
 
-
+ /**
+  * \brief Once the connector is computed (compute \b elements_), redondant data can be send to the server. 
+  *        The goal of this method is to make a hash of each block of indice that determine wich data to send to a 
+  *        of a specific server rank using a hash method. So data to send to a rank is associated to a hash.
+  *        After we compare hash between local rank and remove redondant data corresponding to the same hash.
+  */
+  void CGridRemoteConnector::removeRedondantRanks(void)
+  {
+    int commRank ;
+    MPI_Comm_rank(localComm_,&commRank) ;
+
+    set<int> ranks;
+    for(auto& element : elements_) 
+      for(auto& it : element) ranks.insert(it.first) ;
+
+    for(auto& element : elements_)
+      for(auto& it : element) 
+        if (ranks.count(it.first)==0) ERROR("void CGridRemoteConnector::removeRedondantRanks(void)",<<"number of ranks in elements is not coherent between each element") ;
+    
+    HashXIOS<size_t> hashGlobalIndex;
+    
+    map<int,size_t> hashRanks ;
+    for(auto& element : elements_) 
+      for(auto& it : element)
+      {
+        auto& globalIndex=it.second ;
+        int rank=it.first ;
+        size_t hash ;
+        hash=0 ;
+        for(int i=0; i<globalIndex.numElements(); i++) hash+=hashGlobalIndex(globalIndex(i)) ;
+        if (hashRanks.count(rank)==0) hashRanks[rank]=hash ;
+        else hashRanks[rank]=hashGlobalIndex.hashCombine(hashRanks[rank],hash) ;
+      }
+    // a hash is now computed for data block I will sent to the server.
+
+    CClientClientDHTTemplate<int>::Index2InfoTypeMap info ;
+
+    map<size_t,int> hashRank ;
+    HashXIOS<int> hashGlobalIndexRank;
+    for(auto& it : hashRanks) 
+    {
+      it.second = hashGlobalIndexRank.hashCombine(it.first,it.second) ; 
+      info[it.second]=commRank ;
+      hashRank[it.second]=it.first ;
+    }
+
+    // we feed a DHT map with key : hash, value : myrank
+    CClientClientDHTTemplate<int> dataHash(info, localComm_) ;
+    CArray<size_t,1> hashList(hashRank.size()) ;
+    
+    int i=0 ;
+    for(auto& it : hashRank) { hashList(i)=it.first ; i++; }
+
+    // now who are the ranks that have the same hash : feed the DHT with my list of hash
+    dataHash.computeIndexInfoMapping(hashList) ;
+    auto& hashRankList = dataHash.getInfoIndexMap() ;
+    
+
+    for(auto& it : hashRankList)
+    {
+      size_t hash = it.first ;
+      auto& ranks = it.second ;
+      
+      bool first=true ;
+      // only the process with the lowest rank get in charge of sendinf data to remote server
+      for(int rank : ranks) if (commRank>rank) first=false ;
+      if (!first)
+      {
+        int rankToRemove=hashRank[hash] ;
+        for(auto& element : elements_) element.erase(rankToRemove) ;
+      }
+    }
+
+
+  }
 }

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/grid_remote_connector.hpp

@@ -20 +20 @@
       CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, MPI_Comm localComm, int remoteSize) ;
       CGridRemoteConnector(vector<CLocalView*>& srcView, vector<CLocalView*>& dstView, MPI_Comm localComm, int remoteSize) ;
+      void computeViewDistribution(void) ;
       void computeConnector(void) ;
-      void computeGenericMethod(void) ;
+      void computeGenericMethod(vector<CLocalView*>& srcView, vector<CDistributedView*>& dstView, vector<int>& indElements) ;
+      void computeSrcDstNonDistributed(int i, map<int,bool>& ranks) ;
+      void computeDstNonDistributed(int i, map<int,bool>& ranks) ;
+      void computeSrcNonDistributed(int i) ;
+      void removeRedondantRanks(void) ;
       std::map<int, CArray<size_t,1>>& getDistributedGlobalIndex(int pos) { return elements_[pos] ;} 
 
     private:
+  
+    /** 
+     * Source views composing the source grid. The vector store an internal copy of pointer elements composing the grid. 
+     * It is feed at construction time
+     */
+      vector<CLocalView*> srcView_ ;
+
+    /** 
+     * Destination views composing the source grid. The vector store an internal copy of pointer elements composing the grid
+     * It is feed at construction time
+     */
+      vector<CDistributedView*> dstView_ ;
+
+    /**  
+     * The list of global indices to send to each rank of the remote view (servers). The vector store the information for each element, and the map 
+     * specify a list of global indices to send to each rank of the remote view.
+     * size of element_[] -> number of elements composing the grids (source/destination)
+     * element[i][rank] == CArray<size_t,1> -> list of global indices  to send to the remote process of rank \b rank for the view \b i
+     * The is computed when calling computeConnector internal methods
+     * map can be returned trough the public accessor : getDistributedGlobalIndex(int)
+     */
       vector<map<int, CArray<size_t,1>>> elements_ ;
-      vector<CLocalView*> srcView_ ;
-      vector<CDistributedView*> dstView_ ;
+    
+      /**
+      /* internal copy of the local communicator (client workflow). Feed at construction time.
+       */
       MPI_Comm localComm_ ;
+
+      /**
+      /* size of the remote communicator (== nb of servers). Feed at consctruction time
+       */
       int remoteSize_ ;
 
+      /**
+      /* for each view composing the source grid, the vector store the information about the distribution of the element, i.e. 
+       * if each ranks of the local view has exactly the same global indices than each other. This is computed when calling  
+       * \b computeViewDistribution method.  
+       */
+      vector<bool> isSrcViewDistributed_ ;
+
+     /**
+      /* for each view composing the destination grid, the vector store the information about the distribution of the element, i.e. 
+       * if each ranks of the remote view has exactly the same global indices than each other. This is computed when calling  
+       * \b computeViewDistribution method. 
+       */
+      vector<bool> isDstViewDistributed_ ;
+      
+    
   } ;