Changeset 967

Timestamp:

10/12/16 11:17:49 (8 years ago)

Author:

mhnguyen

Message:

Ticket 106: Reading a scalar from file

+) Add method to read scalar value
+) Modify the way to send/receive scalar (and non-distributed) data

Test
+) On Curie
+) OK

Location:

XIOS/trunk/src

Files:

• io/nc4_data_input.cpp (modified) (8 diffs)
• io/nc4_data_input.hpp (modified) (3 diffs)
• node/field.cpp (modified) (2 diffs)
• node/grid.cpp (modified) (12 diffs)

XIOS/trunk/src/io/nc4_data_input.cpp

@@ -6 +6 @@
 #include "domain.hpp"
 #include "axis.hpp"
+#include "scalar.hpp"
 
 namespace xios
@@ -17 +18 @@
     , readMetaDataDomains_(), readValueDomains_()
     , readMetaDataAxis_(), readValueAxis_()
+    , readMetaDataScalar_(), readValueScalar_()
   {
     SuperClass::type = multifile ? MULTI_FILE : ONE_FILE;
@@ -113 +115 @@
             count.push_back(nZoomSizeServer[idx]);
             --idx;
-           }
+          }
+          else
+          {
+            if (1 == axisDomainOrder.numElements())
+            {
+              start.push_back(0);
+              count.push_back(1);
+            }
+            --idx;
+          }
         }
 
@@ -140 +151 @@
     std::vector<CDomain*> domainP = grid->getDomains();
     std::vector<CAxis*> axisP = grid->getAxis();
+    std::vector<CScalar*> scalarP = grid->getScalars();
     int gridDim = domainP.size() * 2 + axisP.size();
+
+    // Nothing to do with scalar without timestep
+    if ((0 == gridDim) && (!SuperClassWriter::isTemporal(fieldId))) 
+      return;
 
     // Verify the compatibility of dimension of declared grid and real grid in file
@@ -163 +179 @@
     }
 
-    int mapSize = dimSizeMap.size() - 1;
     std::list<std::pair<StdString, StdSize> > listDimSize;
 /*
@@ -174 +189 @@
     // Now process domain and axis
     CArray<int,1> axisDomainOrder = grid->axis_domain_order;
-    int numElement = domainP.size() + axisP.size();
+    int numElement = domainP.size() + axisP.size() + scalarP.size();
     int elementPosition = 0;
-    int idxDomain = 0, idxAxis = 0;
+    int idxDomain = 0, idxAxis = 0, idxScalar = 0;
 
     std::pair<std::set<StdString>::iterator,bool> it;
@@ -210 +225 @@
         }
         ++idxAxis;
+        ++elementPosition;
+      }
+      else
+      {
+        if (readAttributeValues)
+        {
+          it = readValueScalar_.insert(scalarP[idxScalar]->getId());
+          if (it.second) readScalarAttributeValueFromFile(scalarP[idxScalar], listDimSize, elementPosition, fieldId);
+        }
+        else
+        {
+          it = readMetaDataScalar_.insert(scalarP[idxScalar]->getId());
+          if (it.second) readScalarAttributesFromFile(scalarP[idxScalar], listDimSize, elementPosition, fieldId);
+        }
+        ++idxScalar;
         ++elementPosition;
       }
@@ -443 +473 @@
   }
 
+  /*!
+    Read attributes of a scalar from a file
+    \param [in] scalar scalar whose attributes are read from the file
+    \param [in] dimSizeMap Dimensions and and their corresponding names and size read from file
+    \param [in] emelentPosition position of scalar in grid
+    \param [in] fieldId id (or name) associated with the grid
+  */
+  void CNc4DataInput::readScalarAttributesFromFile(CScalar* scalar, std::list<std::pair<StdString, StdSize> >& dimSizeMap,
+                                                  int elementPosition, const StdString& fieldId)
+  {
+//    std::list<std::pair<StdString, StdSize> >::const_iterator itMapN = dimSizeMap.begin(),
+//                                                              iteMap = dimSizeMap.end();
+//    for (int i = 0; i < elementPosition; ++i, ++itMapN) {}
+//    axis->n_glo.setValue(itMapN->second);
+  }
+
+  /*!
+    Read attribute value of an axis from a file
+    \param [in] axis axis whose attributes are read from the file
+    \param [in] dimSizeMap Dimensions and and their corresponding names and size read from file
+    \param [in] emelentPosition position of axis in grid
+    \param [in] fieldId id (or name) associated with the grid
+  */
+  void CNc4DataInput::readScalarAttributeValueFromFile(CScalar* scalar, std::list<std::pair<StdString, StdSize> >& dimSizeMap,
+                                                      int elementPosition, const StdString& fieldId)
+  {
+//    std::list<std::pair<StdString, StdSize> >::const_iterator itMapN = dimSizeMap.begin(),
+//                                                              iteMap = dimSizeMap.end();
+//    for (int i = 0; i < elementPosition; ++i, ++itMapN) {}
+//
+//    { // Read axis value
+//      std::vector<StdSize> nBegin(1, 0), nSize(1, itMapN->second);
+//      CArray<double,1> readAxisValue(itMapN->second);
+//      readFieldVariableValue(readAxisValue, itMapN->first, nBegin, nSize, true);
+//      int begin = 0, n = itMapN->second;
+//      if (!axis->begin.isEmpty()) begin = axis->begin.getValue();
+//      if (!axis->n.isEmpty()) n = axis->n.getValue();
+//      axis->value.resize(n);
+//      for (int i = 0; i < n; ++i) axis->value(i) = readAxisValue(begin + i);
+//    }
+  }
+
   void CNc4DataInput::closeFile_(void)
   {

XIOS/trunk/src/io/nc4_data_input.hpp

@@ -11 +11 @@
   class CDomain;
   class CAxis;
+  class CScalar;
 
   class CNc4DataInput
@@ -51 +52 @@
                                         int elementPosition, const StdString& fieldId);
 
+    void readScalarAttributesFromFile(CScalar* scalar, std::list<std::pair<StdString, StdSize> >& dimSizeMap,
+                                      int elementPosition, const StdString& fieldId);
+    void readScalarAttributeValueFromFile(CScalar* scalar, std::list<std::pair<StdString, StdSize> >& dimSizeMap,
+                                          int elementPosition, const StdString& fieldId);
+
     template <typename T, int Ndim>
     void readFieldVariableValue(CArray<T, Ndim>& var, const StdString& varId,
@@ -60 +66 @@
     std::set<StdString> readMetaDataDomains_, readValueDomains_;
     std::set<StdString> readMetaDataAxis_, readValueAxis_;
+    std::set<StdString> readMetaDataScalar_, readValueScalar_;
 
   private:

XIOS/trunk/src/node/field.cpp

@@ -132 +132 @@
     if (!grid->doGridHaveDataDistributed())
     {
-       if (0 == client->clientRank)
+       if (client->isServerLeader())
        {
           for (it = grid->storeIndex_toSrv.begin(); it != grid->storeIndex_toSrv.end(); it++)
@@ -149 +149 @@
           }
           client->sendEvent(event);
-       } else client->sendEvent(event);
+       } 
+       else client->sendEvent(event);
     }
     else

XIOS/trunk/src/node/grid.cpp

@@ -499 +499 @@
      if (!doGridHaveDataDistributed())
      {
-        if (0 == client->clientRank)
+        if (client->isServerLeader())
         {
           size_t ssize = clientDistribution_->getLocalDataIndexOnClient().size();
-          for (int rank = 0; rank < client->serverSize; ++rank)
+          const std::list<int>& ranks = client->getRanksServerLeader();
+          for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
           {
-            connectedServerRank_.push_back(rank);
-            connectedDataSize_[rank] = ssize;
+            connectedServerRank_.push_back(*itRank);
+            connectedDataSize_[*itRank] = ssize;
           }
         }
@@ -975 +976 @@
         connectedServerRank_.push_back(rank);
         connectedDataSize_[rank] = 1;
+        nbSenders[rank] = 1;
       }
     }
@@ -1023 +1025 @@
     list<CArray<size_t,1> > listOutIndex;
 
-    if (0 == client->clientRank)
-    {
-      for (int rank = 0; rank < client->serverSize; ++rank)
-      {
+    if (client->isServerLeader())
+    {
+      const std::list<int>& ranks = client->getRanksServerLeader();
+      for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+      {
+        int rank = *itRank;
         int nb = 1;
         storeIndex_toSrv.insert(std::make_pair(rank, CArray<int,1>(nb)));
@@ -1045 +1049 @@
         event.push(rank, 1, listMsg.back());
       }
-
       client->sendEvent(event);
     }
@@ -1067 +1070 @@
     if (!doGridHaveDataDistributed())
     {
-      if (0 == client->clientRank)
+      if (client->isServerLeader())
       {
         int indexSize = globalLocalIndexSendToServer.size();
@@ -1078 +1081 @@
         }
 
-        for (rank = 0; rank < client->serverSize; ++rank)
-        {
-          storeIndex_toSrv.insert(std::make_pair(rank, CArray<int,1>(outLocalIndexToServer)));
+        //int nbClient = client->clientSize; // This stupid variable signals the servers the number of client connect to them
+        const std::list<int>& ranks = client->getRanksServerLeader();
+        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
+        {
+          storeIndex_toSrv.insert(std::make_pair(*itRank, CArray<int,1>(outLocalIndexToServer)));
           listOutIndex.push_back(CArray<size_t,1>(outGlobalIndexOnServer));
 
@@ -1086 +1091 @@
           listMsg.back() << getId() << isDataDistributed_ << isCompressible_ << listOutIndex.back();
 
-          event.push(rank, 1, listMsg.back());
-        }
-
+          event.push(*itRank, 1, listMsg.back());
+        }
         client->sendEvent(event);
       }
@@ -1164 +1168 @@
   }
 
-  void CGrid::computeGridGlobalDimension(const std::vector<CDomain*>& domains,
-                                         const std::vector<CAxis*>& axis,
-                                         const std::vector<CScalar*>& scalars,
-                                         const CArray<int,1>& axisDomainOrder)
-  {
-    globalDim_.resize(domains.size()*2+axis.size()+scalars.size());
-    int idx = 0, idxDomain = 0, idxAxis = 0, idxScalar = 0;
-    for (int i = 0; i < axisDomainOrder.numElements(); ++i)
-    {
-      if (2 == axisDomainOrder(i))
-      {
-        if (!(domains[idxDomain]->type.isEmpty()) && (domains[idxDomain]->type==CDomain::type_attr::unstructured))
-        {
-          positionDimensionDistributed_ = idx;
-        }
-        else
-        {
-          positionDimensionDistributed_ = idx +1;
-        }
-
-        globalDim_[idx]   = domains[idxDomain]->ni_glo.getValue();
-        globalDim_[idx+1] = domains[idxDomain]->nj_glo.getValue();
-
-        ++idxDomain;
-        idx += 2;
-      }
-      else if (1 == axisDomainOrder(i))
-      {
-        globalDim_[idx] = axis[idxAxis]->n_glo.getValue();
-        ++idxAxis;
-        ++idx;
-      }
-      else
-      {
-        globalDim_[idx] = 1;
-        ++idxScalar;
-        ++idx;
-      }
-    }
-  }
-
-  std::vector<int> CGrid::getGlobalDimension()
-  {
-    return globalDim_;
-  }
-
-  bool CGrid::isScalarGrid() const
-  {
-    return (axisList_.empty() && domList_.empty());
-  }
-
-  /*!
-    Verify whether one server need to write data
-    There are some cases on which one server has nodata to write. For example, when we
-    just only want to zoom on a domain.
-  */
-  bool CGrid::doGridHaveDataToWrite()
-  {
-     return (0 != writtenDataSize_);
-  }
-
-  /*!
-    Return size of data which is written on each server
-    Whatever dimension of a grid, data which are written on server must be presented as
-    an one dimension array.
-    \return size of data written on server
-  */
-  size_t CGrid::getWrittenDataSize() const
-  {
-    return writtenDataSize_;
-  }
-
-  /*!
-    Returns the number of indexes written by each server.
-    \return the number of indexes written by each server
-  */
-  int CGrid::getNumberWrittenIndexes() const
-  {
-    return numberWrittenIndexes_;
-  }
-
-  /*!
-    Returns the total number of indexes written by the servers.
-    \return the total number of indexes written by the servers
-  */
-  int CGrid::getTotalNumberWrittenIndexes() const
-  {
-    return totalNumberWrittenIndexes_;
-  }
-
-  /*!
-    Returns the offset of indexes written by each server.
-    \return the offset of indexes written by each server
-  */
-  int CGrid::getOffsetWrittenIndexes() const
-  {
-    return offsetWrittenIndexes_;
-  }
-
-  CDistributionServer* CGrid::getDistributionServer()
-  {
-    return serverDistribution_;
-  }
-
-  CDistributionClient* CGrid::getDistributionClient()
-  {
-    return clientDistribution_;
-  }
-
-  bool CGrid::doGridHaveDataDistributed()
-  {
-    if (isScalarGrid()) return false;
-    else
-      return isDataDistributed_;
-  }
-
   void CGrid::recvIndex(vector<int> ranks, vector<CBufferIn*> buffers)
   {
@@ -1294 +1182 @@
       buffer >> isDataDistributed_ >> isCompressible_;
       size_t dataSize = 0;
-
-      if (isScalarGrid())
-      {
-        writtenDataSize_ = numberWrittenIndexes_ = totalNumberWrittenIndexes_ = 1;
-        CArray<size_t,1> outIndex;
-        buffer >> outIndex;
-        outIndexFromClient.insert(std::make_pair(rank, outIndex));
-        std::vector<int> nZoomBegin(1,0), nZoomSize(1,1), nGlob(1,1), nZoomBeginGlobal(1,0);
-        serverDistribution_ = new CDistributionServer(server->intraCommRank, nZoomBegin, nZoomSize,
-                                                      nZoomBeginGlobal, nGlob);
-        return;
-      }
 
       if (0 == serverDistribution_)
@@ -1378 +1254 @@
       }
       writtenDataSize_ += dataSize;
-
+      
       outIndexFromClient.insert(std::make_pair(rank, outIndex));
       connectedDataSize_[rank] = outIndex.numElements();
@@ -1384 +1260 @@
     }
 
+    // if (isScalarGrid()) return;
+
     if (isDataDistributed_)
     {
@@ -1394 +1272 @@
 
     nbSenders = CClientServerMappingDistributed::computeConnectedClients(context->client->serverSize, context->client->clientSize, context->client->intraComm, ranks);
+  }
+
+  void CGrid::computeGridGlobalDimension(const std::vector<CDomain*>& domains,
+                                         const std::vector<CAxis*>& axis,
+                                         const std::vector<CScalar*>& scalars,
+                                         const CArray<int,1>& axisDomainOrder)
+  {
+    globalDim_.resize(domains.size()*2+axis.size()+scalars.size());
+    int idx = 0, idxDomain = 0, idxAxis = 0, idxScalar = 0;
+    for (int i = 0; i < axisDomainOrder.numElements(); ++i)
+    {
+      if (2 == axisDomainOrder(i))
+      {
+        if (!(domains[idxDomain]->type.isEmpty()) && (domains[idxDomain]->type==CDomain::type_attr::unstructured))
+        {
+          positionDimensionDistributed_ = idx;
+        }
+        else
+        {
+          positionDimensionDistributed_ = idx +1;
+        }
+
+        globalDim_[idx]   = domains[idxDomain]->ni_glo.getValue();
+        globalDim_[idx+1] = domains[idxDomain]->nj_glo.getValue();
+
+        ++idxDomain;
+        idx += 2;
+      }
+      else if (1 == axisDomainOrder(i))
+      {
+        globalDim_[idx] = axis[idxAxis]->n_glo.getValue();
+        ++idxAxis;
+        ++idx;
+      }
+      else
+      {
+        globalDim_[idx] = 1;
+        ++idxScalar;
+        ++idx;
+      }
+    }
+  }
+
+  std::vector<int> CGrid::getGlobalDimension()
+  {
+    return globalDim_;
+  }
+
+  bool CGrid::isScalarGrid() const
+  {
+    return (axisList_.empty() && domList_.empty());
+  }
+
+  /*!
+    Verify whether one server need to write data
+    There are some cases on which one server has nodata to write. For example, when we
+    just only want to zoom on a domain.
+  */
+  bool CGrid::doGridHaveDataToWrite()
+  {
+     return (0 != writtenDataSize_);
+  }
+
+  /*!
+    Return size of data which is written on each server
+    Whatever dimension of a grid, data which are written on server must be presented as
+    an one dimension array.
+    \return size of data written on server
+  */
+  size_t CGrid::getWrittenDataSize() const
+  {
+    return writtenDataSize_;
+  }
+
+  /*!
+    Returns the number of indexes written by each server.
+    \return the number of indexes written by each server
+  */
+  int CGrid::getNumberWrittenIndexes() const
+  {
+    return numberWrittenIndexes_;
+  }
+
+  /*!
+    Returns the total number of indexes written by the servers.
+    \return the total number of indexes written by the servers
+  */
+  int CGrid::getTotalNumberWrittenIndexes() const
+  {
+    return totalNumberWrittenIndexes_;
+  }
+
+  /*!
+    Returns the offset of indexes written by each server.
+    \return the offset of indexes written by each server
+  */
+  int CGrid::getOffsetWrittenIndexes() const
+  {
+    return offsetWrittenIndexes_;
+  }
+
+  CDistributionServer* CGrid::getDistributionServer()
+  {
+    return serverDistribution_;
+  }
+
+  CDistributionClient* CGrid::getDistributionClient()
+  {
+    return clientDistribution_;
+  }
+
+  bool CGrid::doGridHaveDataDistributed()
+  {
+    if (isScalarGrid()) return false;
+    else
+      return isDataDistributed_;
   }