Changeset 552

Timestamp:

02/11/15 16:23:07 (9 years ago)

Author:

mhnguyen

Message:

Doing some cleans and improving a little bit performance of creating local index on server

+) Add some comments, add some initialization
+) Change the way to calculate local index on server

Test
+) On Curie

Location:

XIOS/trunk/src

Files:

• distribution.cpp (modified) (1 diff)
• distribution.hpp (modified) (1 diff)
• node/distribution_client.cpp (modified) (4 diffs)
• node/distribution_client.hpp (modified) (2 diffs)
• node/distribution_server.cpp (modified) (2 diffs)
• node/distribution_server.hpp (modified) (1 diff)
• node/grid.cpp (modified) (5 diffs)
• node/grid.hpp (modified) (3 diffs)

XIOS/trunk/src/distribution.cpp

@@ -15 +15 @@
 CDistribution::~CDistribution()
 {
+  if (0 != globalIndex_) delete globalIndex_;
 }
 
-const CArray<size_t,1>* CDistribution::getGlobalIndex() const
-{
-  return globalIndex_;
-}
+//const CArray<size_t,1>* CDistribution::getGlobalIndex() const
+//{
+//  return globalIndex_;
+//}
 
 int CDistribution::getDims() const

XIOS/trunk/src/distribution.hpp

@@ -14 +14 @@
     /** Default destructor */
     virtual ~CDistribution();
-    const CArray<size_t,1>* getGlobalIndex() const;
 
     int getDims() const;
     int getRank() const;
   protected:
-    virtual void createGlobalIndex() = 0; // Maybe change to pure ? = 0;
+    virtual void createGlobalIndex() = 0;
   protected:
     CArray<size_t,1>* globalIndex_;

XIOS/trunk/src/node/distribution_client.cpp

@@ -5 +5 @@
 CDistributionClient::CDistributionClient(int rank, int dims, CArray<size_t,1>* globalIndex)
    : CDistribution(rank, dims, globalIndex),
-   axisDomainOrder_(), indexGlobalOnServer_(), isConnectedServerComputed_(false)
+   localDataIndex_(0), indexGlobalOnServer_(), localIndexSend2Server_(), axisDomainOrder_(),
+   nLocal_(), nGlob_(), nBeginLocal_(), nBeginGlobal_(),nZoomBegin_(), nZoomEnd_(),
+   dataNIndex_(), dataDims_(), dataBegin_(), dataIndex_(), domainMasks_(), axisMasks_(),
+   gridMask_(), localDomainIndex_(), localAxisIndex_(), indexMap_(), connectedClients_(),
+   isConnectedServerComputed_(false), indexDomainData_()
 {
 }
 
 CDistributionClient::CDistributionClient(int rank, CGrid* grid)
-   : CDistribution(rank, 0, 0), isConnectedServerComputed_(false)
+   : CDistribution(rank, 0, 0),
+   localDataIndex_(0), indexGlobalOnServer_(), localIndexSend2Server_(), axisDomainOrder_(),
+   nLocal_(), nGlob_(), nBeginLocal_(), nBeginGlobal_(),nZoomBegin_(), nZoomEnd_(),
+   dataNIndex_(), dataDims_(), dataBegin_(), dataIndex_(), domainMasks_(), axisMasks_(),
+   gridMask_(), localDomainIndex_(), localAxisIndex_(), indexMap_(), connectedClients_(),
+   isConnectedServerComputed_(false), indexDomainData_()
 {
   readDistributionInfo(grid);
@@ -18 +27 @@
 CDistributionClient::~CDistributionClient()
 {
-  if (0 != this->globalIndex_) delete globalIndex_;
   if (0 != localDataIndex_) delete localDataIndex_;
 }
@@ -625 +633 @@
 
   std::vector<CArray<size_t,1>::const_iterator> itBegin(nServer), itEnd(nServer), it(nServer);
-
   for (int i = 0; i < nServer; ++i)
   {
-    itBegin[i] = globalIndexServer[i]->begin();
+    itBegin[i] = it[i] = globalIndexServer[i]->begin();
     itEnd[i]   = globalIndexServer[i]->end();
   }
@@ -637 +644 @@
     for (int j = 0; j < nServer; ++j)
     {
-      // Just temporarily, it's so so bad.
-//      if (std::binary_search(itBegin[j], itEnd[j], (*this->globalIndex_)(i)))
-//      if (itEnd[j] != std::find(itBegin[j], itEnd[j], (*this->globalIndex_)(i)))
-      it[j] = std::find(itBegin[j], itEnd[j], (*this->globalIndex_)(i));
-      if (itEnd[j] != it[j])
-      {
-//        (indexGlobalOnServer_[j]).push_back((*this->globalIndex_)(i));
+      // Just temporarily, it's bad.
+
+      if (std::binary_search(itBegin[j], itEnd[j], (*this->globalIndex_)(i)))
+      {
         // Just try to calculate local index server on client side
-        (indexGlobalOnServer_[j]).push_back(std::distance(itBegin[j], it[j]));
+        (indexGlobalOnServer_[j]).push_back((*this->globalIndex_)(i));
         (localIndexSend2Server_[j]).push_back(i);
         continue;
       }
     }
-  }
-
-
-  for (int i = 0; i < nServer; ++i)
-  {
-    if (indexGlobalOnServer_[i].empty()) indexGlobalOnServer_.erase(i);
-    if (localIndexSend2Server_[i].empty()) localIndexSend2Server_.erase(i);
   }
 

XIOS/trunk/src/node/distribution_client.hpp

@@ -65 +65 @@
     std::vector<CArray<size_t,1>* > computeServerBandDistribution(int nServer);
   private:
-    CArray<int,1>* localDataIndex_; //!< LocalData index on client
-    CArray<size_t,1>* globalIndex_; //!< Global index on client
+    //!< LocalData index on client
+    CArray<int,1>* localDataIndex_;
 
+    //! Index of the local data which will be sent to the corresponding server(s)
+    std::map<int, std::vector<int> >  localIndexSend2Server_;
+
+    //! Global index of data on SERVER, which are calculated by client(s)
     std::map<int, std::vector<size_t> > indexGlobalOnServer_;
-    std::map<int, std::vector<int> >  localIndexSend2Server_;
   private:
     /*! Domains and axis are considered elements.
@@ -75 +78 @@
     int numElement_;
     CArray<bool,1> axisDomainOrder_; //!<
-
 
     std::vector<int> nLocal_; //!< Local size of each dimension (ni, nj, etc, ...)

XIOS/trunk/src/node/distribution_server.cpp

@@ -2 +2 @@
 
 namespace xios {
-CDistributionServer::CDistributionServer(int rank, int dims, int nServer, CArray<size_t,1>* globalIndex)
-  : CDistribution(rank, dims, globalIndex), nServer_(nServer), nGlobal_()
+CDistributionServer::CDistributionServer(int rank, int dims, CArray<size_t,1>* globalIndex)
+  : CDistribution(rank, dims, globalIndex), nGlobal_(), nZoomSize_(), nZoomBegin_()
 {
-
 }
 
-CDistributionServer::CDistributionServer(int rank, int nServer, const std::vector<int>& nGlobal)
-  : CDistribution(rank, nGlobal.size()), nServer_(nServer)
+CDistributionServer::CDistributionServer(int rank, const std::vector<int>& nZoomBegin,
+                                         const std::vector<int>& nZoomSize, const std::vector<int>& nGlobal)
+  : CDistribution(rank, nGlobal.size()), nGlobal_(nGlobal), nZoomSize_(nZoomSize), nZoomBegin_(nZoomBegin)
 {
-  readDistributionInfo(nGlobal);
   createGlobalIndex();
 }
@@ -17 +16 @@
 CDistributionServer::~CDistributionServer()
 {
-  if (0 != this->globalIndex_) delete globalIndex_;
 }
 
-void CDistributionServer::readDistributionInfo(const std::vector<int>& nGlobal)
+/*!
+  Create global index on server side
+  Like the similar function on client side, this function serves on creating global index
+for data written by the server. The global index is used to calculating local index of data
+written on each server
+*/
+void CDistributionServer::createGlobalIndex()
 {
-  if (nGlobal.empty())
+  size_t idx = 0, ssize = 1;
+  for (int i = 0; i < nZoomSize_.size(); ++i) ssize *= nZoomSize_[i];
+
+  this->globalIndex_ = new CArray<size_t,1>(ssize);
+  std::vector<int> idxLoop(this->getDims(),0);
+  std::vector<int> currentIndex(this->getDims());
+  int innerLoopSize = nZoomSize_[0];
+
+  while (idx<ssize)
   {
-    //! TODO: This error must be replaced a call to function processing scalar value
-    ERROR("CDistributionServer::readDistributionInfo(const std::vector<int>& nGlobal)",
-       << "At least one dimension must be defined for this field.");
+    for (int i = 0; i < this->dims_-1; ++i)
+    {
+      if (idxLoop[i] == nZoomSize_[i])
+      {
+        idxLoop[i] = 0;
+        ++idxLoop[i+1];
+      }
+    }
+
+    for (int i = 1; i < this->dims_; ++i)  currentIndex[i] = idxLoop[i] + nZoomBegin_[i];
+
+    size_t mulDim, globalIndex;
+    for (int i = 0; i < innerLoopSize; ++i)
+    {
+      mulDim = 1;
+      globalIndex = i + nZoomBegin_[0];
+
+      for (int k = 1; k < this->dims_; ++k)
+      {
+        mulDim *= nGlobal_[k-1];
+        globalIndex += (currentIndex[k])*mulDim;
+      }
+      (*this->globalIndex_)(idx) = globalIndex;
+      ++idx;
+    }
+    idxLoop[0] += innerLoopSize;
   }
-  nGlobal_ = nGlobal;
-  this->dims_ = nGlobal.size();
 }
 
-void CDistributionServer::createGlobalIndex()
+/*!
+  Compute local index for writing data on server
+  \param [in] globalIndex global index received from client
+  \return local index of written data
+*/
+CArray<size_t,1> CDistributionServer::computeLocalIndex(const CArray<size_t,1>& globalIndex)
 {
-  size_t globalIndexSize = 1;
-  for (int i = 0; i < nGlobal_.size(); ++i)
-    globalIndexSize *= nGlobal_[i];
-  size_t rangeSize    = globalIndexSize / nServer_;
-  size_t modulusSize = globalIndexSize % nServer_;
+  CArray<size_t,1>::const_iterator itBegin = (this->globalIndex_)->begin(),
+                                   itEnd   = (this->globalIndex_)->end(), it;
 
-  if ((this->rank_ == (nServer_-1)) && (0 != modulusSize))
+  int ssize = globalIndex.numElements(), idx = 0;
+  CArray<size_t,1> localIndex(ssize);
+  it = itBegin;
+  for (int i = 0; i < ssize; ++i)
   {
-    this->globalIndex_ = new CArray<size_t,1>(modulusSize);
-    globalIndexSize = modulusSize;
-  }
-  else
-  {
-    this->globalIndex_ = new CArray<size_t,1>(rangeSize);
-    globalIndexSize = rangeSize;
+    it = std::lower_bound(it, itEnd, globalIndex(i));
+    if (itEnd != it)
+    {
+      localIndex(idx) = std::distance(itBegin, it);
+      ++idx;
+    }
   }
 
-  size_t idxBegin = this->rank_ * rangeSize;
-  for (size_t i = 0; i < globalIndexSize;++i)
-    (*this->globalIndex_)(i) = i+idxBegin;
+  return localIndex;
 }
 
+/*!
+  Compute local index for writing data on server
+  \param [in] globalIndex Global index received from client
+*/
+void CDistributionServer::computeLocalIndex(CArray<size_t,1>& globalIndex)
+{
+  CArray<size_t,1>::const_iterator itBegin = (this->globalIndex_)->begin(),
+                                   itEnd   = (this->globalIndex_)->end(), it;
+
+  int ssize = globalIndex.numElements(), idx = 0;
+  CArray<size_t,1> localIndex(ssize);
+  it = itBegin;
+  for (int i = 0; i < ssize; ++i)
+  {
+    it = std::lower_bound(it, itEnd, globalIndex(i));
+    if (itEnd != it)
+    {
+      localIndex(idx) = std::distance(itBegin, it);
+      ++idx;
+    }
+  }
+
+  globalIndex = localIndex;
 }
+
+} // namespace xios

XIOS/trunk/src/node/distribution_server.hpp

@@ -5 +5 @@
 
 namespace xios {
+/*!
+\class CDistributionServer
+ The class, for now, plays a role of computing local index for writing data on server
+*/
 class CDistributionServer : public CDistribution
 {
   public:
     /** Default constructor */
-    CDistributionServer(int rank, int dims, int nServer, CArray<size_t,1>* globalIndex = 0);
-    CDistributionServer(int rank, int nServer, const std::vector<int>& nGlobal);
+    CDistributionServer(int rank, int dims, CArray<size_t,1>* globalIndex = 0);
+    CDistributionServer(int rank, const std::vector<int>& nZoomBegin,
+                        const std::vector<int>& nZoomSize, const std::vector<int>& nGlobal);
 
     /** Default destructor */
     virtual ~CDistributionServer();
 
+    CArray<size_t,1> computeLocalIndex(const CArray<size_t,1>& globalIndex);
+    void computeLocalIndex(CArray<size_t,1>& globalIndex);
+
   protected:
     virtual void createGlobalIndex();
-
-    //! Read info
-    void readDistributionInfo(const std::vector<int>& nGlobal);
   private:
-    int nServer_;
     std::vector<int> nGlobal_;
+    std::vector<int> nZoomSize_;
+    std::vector<int> nZoomBegin_;
 };
 
-}
+} // namespace xios
 #endif // __XIOS_DISTRIBUTION_SERVER_HPP__

XIOS/trunk/src/node/grid.cpp

@@ -11 +11 @@
 #include "context.hpp"
 #include "context_client.hpp"
+#include "context_server.hpp"
 #include "array_new.hpp"
 
@@ -22 +23 @@
       , storeIndex(1), out_i_index(1), out_j_index(1), out_l_index(1), isDomConServerComputed_(false)
       , vDomainGroup_(), vAxisGroup_(), axisList_(), isAxisListSet(false), isDomListSet(false), clientDistribution_(0), isIndexSent(false)
+      , serverDistribution_(0)
    {
      setVirtualDomainGroup();
@@ -32 +34 @@
       , storeIndex(1), out_i_index(1), out_j_index(1), out_l_index(1), isDomConServerComputed_(false)
       , vDomainGroup_(), vAxisGroup_(), axisList_(), isAxisListSet(false), isDomListSet(false), clientDistribution_(0), isIndexSent(false)
+      , serverDistribution_(0)
    {
      setVirtualDomainGroup();
@@ -55 +58 @@
 
     if (0 != clientDistribution_) delete clientDistribution_;
+    if (0 != serverDistribution_) delete serverDistribution_;
 
    }
@@ -807 +811 @@
   void CGrid::recvIndex(int rank, CBufferIn& buffer)
   {
+     if (0 == serverDistribution_)
+     {
+       CContext* context = CContext::getCurrent() ;
+       CContextServer* server=context->server ;
+       int idx = 0, numElement = axisDomainOrder.numElements();
+       int ssize = numElement;
+       std::vector<int> indexMap(numElement);
+       for (int i = 0; i < numElement; ++i)
+       {
+         indexMap[i] = idx;
+         if (true == axisDomainOrder(i))
+        {
+          ++ssize;
+          idx += 2;
+        }
+       }
+
+       int axisId = 0, domainId = 0;
+       std::vector<CDomain*> domainList = getDomains();
+       std::vector<CAxis*> axisList = getAxis();
+       std::vector<int> nZoomBegin(ssize), nZoomSize(ssize), nGlob(ssize);
+       for (int i = 0; i < numElement; ++i)
+       {
+         if (axisDomainOrder(i))
+         {
+            nZoomBegin[indexMap[i]]   = domainList[domainId]->zoom_ibegin_srv;
+            nZoomSize[indexMap[i]]    = domainList[domainId]->zoom_ni_srv;
+            nGlob[indexMap[i]]    = domainList[domainId]->ni_glo;
+
+            nZoomBegin[indexMap[i]+1] = domainList[domainId]->zoom_jbegin_srv;
+            nZoomSize[indexMap[i]+1]  = domainList[domainId]->zoom_nj_srv;
+            nGlob[indexMap[i]+1]    = domainList[domainId]->nj_glo;
+            ++domainId;
+         }
+         else
+         {
+            nZoomBegin[indexMap[i]] = axisList[axisId]->zoom_begin;
+            nZoomSize[indexMap[i]]  = axisList[axisId]->zoom_size;
+            nGlob[indexMap[i]]      = axisList[axisId]->size;
+            ++axisId;
+         }
+       }
+       serverDistribution_ = new CDistributionServer(server->intraCommRank, nZoomBegin, nZoomSize, nGlob);
+     }
+
      CArray<size_t,1> outIndex;
      buffer>>outIndex;
+     serverDistribution_->computeLocalIndex(outIndex);
      outIndexFromClient.insert(std::pair<int, CArray<size_t,1>* >(rank, new CArray<size_t,1>(outIndex)));
 

XIOS/trunk/src/node/grid.hpp

@@ -12 +12 @@
 #include "attribute_array.hpp"
 #include "distribution_client.hpp"
+#include "distribution_server.hpp"
 
 namespace xios {
@@ -23 +24 @@
    class CGrid;
    class CDistributionClient;
+   class CDistributionServer;
 
    ///--------------------------------------------------------------
@@ -217 +219 @@
         bool isAxisListSet, isDomListSet;
         CDistributionClient* clientDistribution_;
+        CDistributionServer* serverDistribution_;
    }; // class CGrid