Changeset 1930 for XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution

Timestamp:

09/10/20 13:51:02 (4 years ago)

Author:

ymipsl

Message:

Big update on on going work related to data distribution and transfer between clients and servers.
Revisite of the source and store filter using "connectors".

YM

Location:

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution

Files:

• distributed_view.cpp (modified) (2 diffs)
• distributed_view.hpp (modified) (2 diffs)
• distribution_type.hpp (added)
• element.cpp (modified) (2 diffs)
• element.hpp (modified) (2 diffs)
• gatherer_connector.cpp (modified) (2 diffs)
• gatherer_connector.hpp (modified) (7 diffs)
• grid_local_view.cpp (modified) (1 diff)
• grid_local_view.hpp (modified) (1 diff)
• grid_remote_connector.cpp (modified) (4 diffs)
• local_view.cpp (modified) (1 diff)
• local_view.hpp (modified) (1 diff)
• scatterer_connector.hpp (modified) (4 diffs)

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/distributed_view.cpp

@@ -9 +9 @@
                                     : globalIndex_(parent->globalIndex_), globalSize_(parent->globalSize_), localSize_(parent->localSize_) 
   {
-    for(auto index : indexView) 
+    for(auto& index : indexView) 
     {
       index_[index.first].reference(index.second.copy()) ;
@@ -16 +16 @@
   }
 
-
-} 
+  CDistributedView::CDistributedView(CDistributedElement* parent, CElementView::type, const std::map<int,CArray<bool,1>>& maskView) 
+                                    : globalIndex_(parent->globalIndex_), globalSize_(parent->globalSize_), localSize_(parent->localSize_) 
+  {
+    for(auto& it : maskView) 
+    {
+      int rank = it.first ;
+      auto& mask = it.second ;
+      int size = mask.numElements() ;
+      auto& index = index_[rank] ; 
+      index.resize(size) ;
+      int pos=0 ;
+      for(int i=0 ; i < size ; i++)
+        if (mask(i)) { index(pos) = i ; pos++ ; }
+      index.resizeAndPreserve(pos) ;
+      size_[rank] = pos ;
+    }
+  } 
+}

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/distributed_view.hpp

@@ -15 +15 @@
 
       CDistributedView(CDistributedElement* parent, CElementView::type type, const std::map<int,CArray<int,1>>& indexView) ;
+      CDistributedView(CDistributedElement* parent, CElementView::type type, const std::map<int,CArray<bool,1>>& maskView) ;
       
       std::map<int,CArray<int,1>>&  getIndex(void) { return index_ ;}
@@ -21 +22 @@
       std::map<int, int>& getLocalSize(void) { return localSize_ ;}
       size_t getGlobalSize(void) { return globalSize_ ;}
+
+      void getGlobalIndexView(map<int,CArray<size_t,1>>& globalIndexView)
+      {
+        for(auto& it : globalIndex_)
+        {
+          int rank=it.first ;
+          auto& globalIndex = it.second;
+          auto& globalIndView = globalIndexView[rank] ;
+          auto& index = index_[rank] ;
+          auto& size = size_[rank] ;
+          auto& localSize = localSize_[rank] ;
+          globalIndView.resize(size) ;
+          int pos=0 ;
+          for(int i=0 ; i<size ; i++)
+          {
+            if (index(i)>=0 && index(i)<localSize) 
+            {
+              globalIndView(i) = globalIndex(index(i)) ;
+              pos++ ;
+            }
+          }
+          globalIndView.resizeAndPreserve(pos) ;
+      }
+    }   
 
       void getGlobalIndex(int rank, vector<size_t>& globalIndex, size_t sliceIndex, size_t* sliceSize, CDistributedView** view, int pos)

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/element.cpp

@@ -29 +29 @@
   {
     views_[type] = new CDistributedView(this, type, indexView) ;
+  } 
+
+  void CDistributedElement::addView(CElementView::type type, std::map<int, CArray<bool,1>>& maskView)
+  {
+    views_[type] = new CDistributedView(this, type, maskView) ;
   } 
 
@@ -118 +123 @@
   } 
 
+  void CLocalElement::addView(CElementView::type type, CArray<bool,1>& maskView)
+  {
+    views_[type] = new CLocalView(this, type, maskView) ;
+  } 
+
   void CLocalElement::addFullView(void)
   {

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/element.hpp

@@ -40 +40 @@
 
     void addView(CElementView::type type, std::map<int, CArray<int,1>>& indexView) ;
+    void addView(CElementView::type type, std::map<int, CArray<bool,1>>& maskView) ;
     void sendToServer(CEventClient& event, const CMessage& messageHeader) ;
 
@@ -55 +56 @@
       CLocalElement(int localRank, CEventServer& event) ;
       void recvFromClient(int localRank, CEventServer& event) ;
-      
+      const CArray<size_t,1>& getGlobalIndex(void) { return globalIndex_ ;}
       void addView(CElementView::type type, CArray<int,1>& indexView) ;
+      void addView(CElementView::type type, CArray<bool,1>& maskView) ;
       void addFullView(void) ;
       CLocalView* getView(CElementView::type type) 

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/gatherer_connector.cpp

@@ -15 +15 @@
     unordered_map<size_t,int> mapGlobalLocalIndex ;
     int globalIndexSize=dstGlobalIndex.size() ;
-    for(auto& ind : dstIndex) mapGlobalLocalIndex[dstGlobalIndex(ind)] = ind ;
+    //for(auto& ind : dstIndex) mapGlobalLocalIndex[dstGlobalIndex(ind)] = ind ;
+    for(int i=0; i<dstSize_ ; i++) if (dstIndex(i)>=0 && dstIndex(i)<globalIndexSize) mapGlobalLocalIndex[dstGlobalIndex(dstIndex(i))] = i ;
+
     for(auto& rankIndex : srcIndex)
     {
@@ -27 +29 @@
       for(int ind=0; ind<indexSize ; ind++)
       {
-        if (ind>=0 && ind<globalIndexSize)
+        if (index(ind)>=0 && index(ind)<globalIndexSize)
         {
-           auto it=mapGlobalLocalIndex.find(globalIndex(ind)) ;
+           auto it=mapGlobalLocalIndex.find(globalIndex(index(ind))) ;
            if (it != mapGlobalLocalIndex.end()) 
            {

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/gatherer_connector.hpp

@@ -29 +29 @@
       
       template<typename T>
-      void transfer(map<int, CArray<T,1>>& dataIn, CArray<T,1>& dataOut)
+      void transfer(int sizeT, map<int, CArray<T,1>>& dataIn, CArray<T,1>& dataOut)
       {
-        dataOut.resize(dstSize_) ;
+        // for future, make a specific transfer function for sizeT=1 to avoid multiplication (increasing performance)
+        dataOut.resize(dstSize_*sizeT) ;  
         T* output = dataOut.dataFirst() ;
         for(auto& data : dataIn)
@@ -39 +40 @@
           auto& connector=connector_[rank] ;
           auto& mask=mask_[rank] ;
-          int size=data.second.numElements() ;
+          int size=mask.size() ;
 
           for(int i=0, j=0 ;i<size;i++)
@@ -45 +46 @@
             if (mask[i]) 
             {
-              output[connector[j]] = input[i] ;
+              int cj = connector[j]*sizeT ;
+              int ci = i*sizeT ;
+              for (int k=0;k<sizeT;k++) output[cj+k] = input[ci+k] ;
               j++ ;
             }
@@ -52 +55 @@
       }
     
+      template<typename T>
+      void transfer(map<int, CArray<T,1>>& dataIn, CArray<T,1>& dataOut)
+      {
+        transfer(1,dataIn,dataOut) ;
+      }
 
       template<typename T>
@@ -93 +101 @@
       void transfer(map<int, CArray<T,1>>& dataIn, CArray<T,1>& dataOut, T missingValue)
       {
-        dataOut.resize(dstSize_) ;
-        dataOut=missingValue ;
-        transfer(map<int, CArray<T,1>>& dataIn, CArray<T,1>& dataOut) ;
+        transfer(1, dataIn, dataout, missingValue)
       }
       
       template<typename T>
-      void transfer(CEventServer& event, CArray<T,1>& dataOut)
+      void transfer(int sizeT, map<int, CArray<T,1>>& dataIn, CArray<T,1>& dataOut, T missingValue)
+      {
+        dataOut.resize(dstSize_*sizeT) ;
+        dataOut=missingValue ;
+        transfer(sizeT, dataIn, dataOut) ;
+      }
+
+      template<typename T>
+      void transfer(CEventServer& event, int sizeT, CArray<T,1>& dataOut)
       {
         map<int, CArray<T,1>> dataIn ;
@@ -107 +121 @@
           (*subEvent.buffer) >> data ;
         }
-        transfer(dataIn, dataOut) ;
+        transfer(sizeT, dataIn, dataOut) ;
+      }
+      
+      template<typename T>
+      void transfer(CEventServer& event, CArray<T,1>& dataOut)
+      {
+        transfer(event, 1, dataOut) ;
+      }
+
+      template<typename T>
+      void transfer(CEventServer& event, int sizeT, CArray<T,1>& dataOut, T missingValue)
+      {
+        map<int, CArray<T,1>> dataIn ;
+        for (auto& subEvent : event.subEvents) 
+        {
+          auto& data = dataIn[subEvent.rank]; 
+          (*subEvent.buffer) >> data ;
+        }
+        transfer(sizeT, dataIn, dataOut, missingValue) ;
       }
 
@@ -119 +151 @@
           (*subEvent.buffer) >> data ;
         }
-        transfer(dataIn, dataOut, missingValue) ;
+        transfer(1, dataIn, dataOut, missingValue) ;
       }
 

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/grid_local_view.cpp

@@ -6 +6 @@
   CGridLocalView::CGridLocalView(CGridLocalElements* parent, CElementView::type type) : localMask_(parent->getLocalMask())
   {
-    for(auto element : parent->getElements()) views_.push_back(element->getView(type)) ;
+    size_ = 1 ;
+    for(auto element : parent->getElements()) 
+    {
+      views_.push_back(element->getView(type)) ;
+      size_ *= element->getView(type)->getSize() ;
+    }
+
   }
 

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/grid_local_view.hpp

@@ -14 +14 @@
       std::vector<CLocalView*> views_ ;
       std::vector<bool>& localMask_ ;
+      int size_ ;
     
     public:
       CGridLocalView(CGridLocalElements* parent, CElementView::type type) ;
       std::vector<CLocalView*>& getViews(void) {return views_ ;}
+      int getSize() { return size_ ;}
   } ;
 }

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/grid_remote_connector.cpp

@@ -19 +19 @@
   {
     // generic method, every element can be distributed
+    int nDst = dstView_.size() ;
+    vector<size_t> dstSliceSize(nDst) ;
+    dstSliceSize[0] = 1 ;  
+    for(int i=1; i<nDst; i++)  dstSliceSize[i] = dstView_[i-1]->getGlobalSize()*dstSliceSize[i-1] ;
+  
+    CClientClientDHTTemplate<int>::Index2VectorInfoTypeMap dataInfo ;
+
+    CClientClientDHTTemplate<size_t>::Index2VectorInfoTypeMap info ; // info map
+    for(int pos=0; pos<nDst; pos++)
+    {
+      size_t sliceSize=dstSliceSize[pos] ;
+      map<int,CArray<size_t,1>> globalIndexView ;
+      dstView_[pos]->getGlobalIndexView(globalIndexView) ;
+      
+      CClientClientDHTTemplate<size_t>::Index2VectorInfoTypeMap lastInfo(info) ;
+
+      if (pos>0)
+      {
+        CArray<size_t,1> ranks(globalIndexView.size()) ;
+        auto it=globalIndexView.begin() ;
+        for(int i=0 ; i<ranks.numElements();i++,it++) ranks(i)=it->first ;
+        CClientClientDHTTemplate<size_t> dataRanks(info, localComm_) ;
+        dataRanks.computeIndexInfoMapping(ranks) ;
+        lastInfo = dataRanks.getInfoIndexMap() ;
+      }
+      
+      info.clear() ;
+      for(auto& it : globalIndexView)
+      {
+        int rank = it.first ;
+        auto& globalIndex = it.second ;
+        auto& inf = info[rank] ;
+        if (pos==0) for(int i=0;i<globalIndex.numElements();i++) inf.push_back(globalIndex(i)) ;
+        else
+        {
+          auto& lastGlobalIndex = lastInfo[rank] ;
+          for(size_t lastGlobalInd : lastGlobalIndex)
+          {
+            for(int i=0;i<globalIndex.numElements();i++) inf.push_back(globalIndex(i)*sliceSize+lastGlobalInd) ;
+          }
+        } 
+      }
+
+      if (pos==nDst-1)
+      {
+         for(auto& it : info)
+         {
+           int rank=it.first ;
+           auto& globalIndex = it.second ;
+           for(auto globalInd : globalIndex) dataInfo[globalInd].push_back(rank) ;
+         }
+      } 
+    }
+
+    CClientClientDHTTemplate<int> dataRanks(dataInfo, localComm_) ;
+/*
     CClientClientDHTTemplate<int>::Index2VectorInfoTypeMap info ; // info map
         
@@ -37 +93 @@
     
     CClientClientDHTTemplate<int> dataRanks(info, localComm_) ;
-    
+*/    
     // generate list of global index for src view
     int nSrc = srcView_.size() ;
     vector<size_t> srcSliceSize(nSrc) ;
-    srcSliceSize[nSrc-1] = 1 ;
-    for(int i=nSrc-2; i>=0; i--)  srcSliceSize[i] = srcView_[i+1]->getGlobalSize()*srcSliceSize[i+1] ;
-    
+//    srcSliceSize[nSrc-1] = 1 ;
+//    for(int i=nSrc-2; i>=0; i--)  srcSliceSize[i] = srcView_[i+1]->getGlobalSize()*srcSliceSize[i+1] ;
+   
+    srcSliceSize[0] = 1 ;  
+    for(int i=1; i<nSrc; i++)  srcSliceSize[i] = srcView_[i-1]->getGlobalSize()*srcSliceSize[i-1] ;
+
     vector<size_t> srcGlobalIndex ;
     size_t sliceIndex=0 ;
@@ -54 +113 @@
     vector<map<int, set<size_t>>> elements(nSrc) ; // internal representation of elements composing the grid
 
-    srcSliceSize[nSrc-1] = srcView_[nSrc-1]->getGlobalSize() ;
-    for(int i=nSrc-2 ; i>=0 ; i--) srcSliceSize[i] = srcView_[i]->getGlobalSize()*srcSliceSize[i+1] ;
+//    srcSliceSize[nSrc-1] = srcView_[nSrc-1]->getGlobalSize() ;
+//    for(int i=nSrc-2 ; i>=0 ; i--) srcSliceSize[i] = srcView_[i]->getGlobalSize()*srcSliceSize[i+1] ;
 
     for(auto& indRanks : returnInfo)
@@ -61 +120 @@
       size_t gridIndexGlo=indRanks.first ;
       auto& ranks = indRanks.second ;
-      for(int i=0;i<nSrc;i++)
+      for(int i=nSrc-1; i>=0; i--)
       {
         auto& element = elements[i] ;
-        size_t localIndGlo = gridIndexGlo % srcSliceSize[i] ;
+        size_t localIndGlo = gridIndexGlo / srcSliceSize[i] ;
+        gridIndexGlo = gridIndexGlo % srcSliceSize[i] ;
         for(int rank : ranks) element[rank].insert(localIndGlo) ;
       }

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/local_view.cpp

@@ -15 +15 @@
   }
 
+  CLocalView::CLocalView(CLocalElement* parent, CElementView::type type, const CArray<bool,1>& maskView) 
+                        : CDistributedView( parent, type, {{  parent->localRank_,  maskView }} ),
+                          localRank_(parent->localRank_),
+                          globalIndex_(parent->globalIndex_), size_(CDistributedView::size_[parent->localRank_]),
+                          index_(CDistributedView::index_[parent->localRank_]), localSize_(CDistributedView::localSize_[parent->localRank_]) 
+  {
+
+  }
 
   void CLocalView::sendRemoteElement(CRemoteConnector& connector, CContextClient* client, CEventClient& event, const CMessage& messageHeader)

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/local_view.hpp

@@ -17 +17 @@
     public:
     CLocalView(CLocalElement* parent, CElementView::type type, const CArray<int,1>& indexView) ;
+    CLocalView(CLocalElement* parent, CElementView::type type, const CArray<bool,1>& maskView) ;
+
     const CArray<int,1>& getIndex(void) { return index_ ;}
     const CArray<size_t,1>& getGlobalIndex(void) { return globalIndex_ ;}
+    
+    void getGlobalIndexView(CArray<size_t,1>& globalIndexView)
+    {
+      globalIndexView.resize(size_) ;
+      int pos=0 ;
+      for(int i=0 ; i<size_ ; i++)
+      {
+        if (index_(i)>=0 && index_(i)<localSize_) 
+        {
+          globalIndexView(i) = globalIndex_(index_(i)) ;
+          pos++ ;
+        }
+      }
+      globalIndexView.resizeAndPreserve(pos) ;
+    }    
 
     void getGlobalIndex(vector<size_t>& globalIndex, size_t sliceIndex, size_t* sliceSize, CLocalView** localView, int pos)

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/scatterer_connector.hpp

@@ -34 +34 @@
     void computeConnector(void) ;
     
-    template<typename T>
-    void transfer(const CArray<T,1>& dataIn, map<int, CArray<T,1>>& dataOut)
+    template<typename T, int n>
+    void transfer(const CArray<T,n>& dataIn, map<int, CArray<T,1>>& dataOut)
     {
+      transfer(1, dataIn, dataOut) ;
+    }
+
+    template<typename T, int n>
+    void transfer(const CArray<T,n>& dataIn, map<int, CArray<T,1>>& dataOut, T missingValue)
+    {
+      transfer(1, dataIn, dataOut, missingValue) ;
+    }
+
+    template<typename T, int n>
+    void transfer(int sizeT, const CArray<T,n>& dataIn, map<int, CArray<T,1>>& dataOut)
+    {
+      // for future, make a specific transfer function for sizeT=1 to avoid multiplication (increasing performance)
       for(auto& rankConnector : connector_)
       {
@@ -44 +57 @@
         int dstSize = mask.size() ;
         auto& data = dataOut[rank] ;
-        data.resize(dstSize) ;
+        data.resize(dstSize * sizeT) ;
         T* dstData = data.dataFirst() ;
         const T* srcData = dataIn.dataFirst() ;
@@ -50 +63 @@
           if (mask[i]) 
           {
-            dstData[i] = srcData[connector[j]] ;
+            for(int k=0;k<sizeT;k++) dstData[i*sizeT+k] = srcData[connector[j]*sizeT+k] ;
+            j++ ;
+          }
+      }
+    }
+
+    template<typename T, int n>
+    void transfer(int sizeT, const CArray<T,n>& dataIn, map<int, CArray<T,1>>& dataOut, T missingValue)
+    {
+      // for future, make a specific transfer function for sizeT=1 to avoid multiplication (increasing performance)
+      for(auto& rankConnector : connector_)
+      {
+        int rank = rankConnector.first ;
+        auto& connector = rankConnector.second ;
+        auto& mask = mask_[rank] ;
+        int dstSize = mask.size() ;
+        auto& data = dataOut[rank] ;
+        data.resize(dstSize * sizeT) ;
+        T* dstData = data.dataFirst() ;
+        const T* srcData = dataIn.dataFirst() ;
+        for(int i=0, j=0; i<dstSize; i++)
+          if (mask[i]) 
+          {
+            for(int k=0;k<sizeT;k++) dstData[i*sizeT+k] = srcData[connector[j]*sizeT+k] ;
+            j++ ;
+          }
+          else 
+          {
+            for(int k=0;k<sizeT;k++) dstData[i*sizeT+k] = missingValue ;
             j++ ;
           }
@@ -56 +97 @@
     }
     
-    template<typename T>
-    void transfer(const CArray<T,1>& dataIn, CContextClient* client, CEventClient& event, const CMessage& messageHeader)
+    template<typename T,int n>
+    void transfer(const CArray<T,n>& dataIn, CContextClient* client, CEventClient& event, const CMessage& messageHeader)
+    {
+      transfer(1, dataIn, client, event, messageHeader) ;
+    }
+
+    template<typename T,int n>
+    void transfer(const CArray<T,n>& dataIn, T missingValue, CContextClient* client, CEventClient& event, const CMessage& messageHeader)
+    {
+      transfer(1, dataIn, missingValue, client, event, messageHeader) ;
+    }
+
+    template<typename T, int n>
+    void transfer(int sizeT, const CArray<T,n>& dataIn, CContextClient* client, CEventClient& event, const CMessage& messageHeader)
     {
       map<int, CArray<T,1>> dataOut ;
-      transfer(dataIn, dataOut) ;
+      transfer(sizeT, dataIn, dataOut) ;
+      sendToServer(dataOut, client, event, messageHeader) ;
+    }
+
+    template<typename T, int n>
+    void transfer(int sizeT, const CArray<T,n>& dataIn, T missingValue, CContextClient* client, CEventClient& event, const CMessage& messageHeader)
+    {
+      map<int, CArray<T,1>> dataOut ;
+      transfer(sizeT, dataIn, dataOut, missingValue) ;
       sendToServer(dataOut, client, event, messageHeader) ;
     }