Changeset 1018 for XIOS/trunk/src/filter

Timestamp:

01/10/17 13:52:53 (7 years ago)

Author:

mhnguyen

Message:

Improving missing-value processing
If detect_missing_value is activated, then all missing value will be converted to
NaN (Not-a-number) in input of data flow then they will be reconverted to missing value on output

+) Update SourceFilter?, TemporalFilter? and SpatialTransformFilter? with new processing
+) Update all transformations with new processing

Test
+) On Curie
+) Work

Location:

XIOS/trunk/src/filter

Files:

• file_writer_filter.cpp (modified) (2 diffs)
• source_filter.cpp (modified) (2 diffs)
• source_filter.hpp (modified) (2 diffs)
• spatial_transform_filter.cpp (modified) (3 diffs)
• spatial_transform_filter.hpp (modified) (1 diff)
• temporal_filter.cpp (modified) (2 diffs)

XIOS/trunk/src/filter/file_writer_filter.cpp

@@ -2 +2 @@
 #include "exception.hpp"
 #include "field.hpp"
+#include "utils.hpp"
 
 namespace xios
@@ -16 +17 @@
   void CFileWriterFilter::onInputReady(std::vector<CDataPacketPtr> data)
   {
+    bool ignoreMissingValue = (!field->detect_missing_value.isEmpty() && 
+                               !field->default_value.isEmpty() && 
+                               field->detect_missing_value == true);
+    if (ignoreMissingValue)
+    {
+      double missingValue = field->default_value;
+      size_t nbData = data[0]->data.numElements();
+      for (size_t idx = 0; idx < nbData; ++idx)
+      {
+        if (NumTraits<double>::isnan(data[0]->data(idx)))
+          data[0]->data(idx) = missingValue;
+      }
+    }    
+
     field->sendUpdateData(data[0]->data);
   }

XIOS/trunk/src/filter/source_filter.cpp

@@ -3 +3 @@
 #include "exception.hpp"
 #include "calendar_util.hpp"
+#include <limits> 
 
 namespace xios
 {
   CSourceFilter::CSourceFilter(CGarbageCollector& gc, CGrid* grid,
-                               const CDuration offset /*= NoneDu*/, bool manualTrigger /*= false*/)
+                               const CDuration offset /*= NoneDu*/, bool manualTrigger /*= false*/,
+                               bool hasMissingValue /*= false*/,
+                               double defaultValue /*= 0.0*/)
     : COutputPin(gc, manualTrigger)
     , grid(grid)
     , offset(offset)
+    , hasMissingValue(hasMissingValue), defaultValue(defaultValue)
   {
     if (!grid)
@@ -29 +33 @@
     packet->data.resize(grid->storeIndex_client.numElements());
     grid->inputField(data, packet->data);
+
+    // Convert missing values to NaN
+    if (hasMissingValue)
+    {
+      double nanValue = std::numeric_limits<double>::quiet_NaN();
+      size_t nbData = packet->data.numElements();
+      for (size_t idx = 0; idx < nbData; ++idx)
+      {
+        if (defaultValue == packet->data(idx))
+          packet->data(idx) = nanValue;
+      }
+    }
 
     onOutputReady(packet);

XIOS/trunk/src/filter/source_filter.hpp

@@ -23 +23 @@
        * \param offset the offset applied to the timestamp of all packets
        * \param manualTrigger whether the output should be triggered manually
+       * \param hasMissingValue whether data has missing value
+       * \param defaultValue missing value to detect
        */
       CSourceFilter(CGarbageCollector& gc, CGrid* grid,
-                    const CDuration offset = NoneDu, bool manualTrigger = false);
+                    const CDuration offset = NoneDu, bool manualTrigger = false,
+                    bool hasMissingValue = false,
+                    double defaultValue = 0.0);
 
       /*!
@@ -58 +62 @@
       CGrid* grid; //!< The grid attached to the data the filter can accept
       const CDuration offset; //!< The offset applied to the timestamp of all packets
+      bool hasMissingValue;
+      double defaultValue;
   }; // class CSourceFilter
 } // namespace xios

XIOS/trunk/src/filter/spatial_transform_filter.cpp

@@ -11 +11 @@
 
   std::pair<boost::shared_ptr<CSpatialTransformFilter>, boost::shared_ptr<CSpatialTransformFilter> >
-  CSpatialTransformFilter::buildFilterGraph(CGarbageCollector& gc, CGrid* srcGrid, CGrid* destGrid, double defaultValue)
+  CSpatialTransformFilter::buildFilterGraph(CGarbageCollector& gc, CGrid* srcGrid, CGrid* destGrid, bool hasMissingValue, double missingValue)
   {
     if (!srcGrid || !destGrid)
@@ -26 +26 @@
       const std::vector<StdString>& auxInputs = gridTransformation->getAuxInputs();
       size_t inputCount = 1 + (auxInputs.empty() ? 0 : auxInputs.size());
+      double defaultValue  = (hasMissingValue) ? std::numeric_limits<double>::quiet_NaN() : missingValue;
       boost::shared_ptr<CSpatialTransformFilter> filter(new CSpatialTransformFilter(gc, engine, defaultValue, inputCount));
 
@@ -103 +104 @@
       }
       packet->data.resize(gridTransformation->getGridDestination()->storeIndex_client.numElements());
-      packet->data = defaultValue;
+      if (0 != packet->data.numElements())
+        (packet->data)(0) = defaultValue;
       apply(data[0]->data, packet->data);
     }

XIOS/trunk/src/filter/spatial_transform_filter.hpp

@@ -32 +32 @@
        * \param srcGrid the source grid
        * \param destGrid the destination grid
+       * \param hasMissingValue whether field source has missing value
+       * \param defaultValue default value
        * \return the first and the last filters of the filter graph
        */
       static std::pair<boost::shared_ptr<CSpatialTransformFilter>, boost::shared_ptr<CSpatialTransformFilter> >
-      buildFilterGraph(CGarbageCollector& gc, CGrid* srcGrid, CGrid* destGrid, double defaultValue);
+      buildFilterGraph(CGarbageCollector& gc, CGrid* srcGrid, CGrid* destGrid, bool hasMissingValue, double defaultValue);
 
     protected:

XIOS/trunk/src/filter/temporal_filter.cpp

@@ -15 +15 @@
     , isFirstOperation(true)
   {
+    double defaultValue = ignoreMissingValue ? std::numeric_limits<double>::quiet_NaN() : missingValue;
+    
 #define DECLARE_FUNCTOR(MType, mtype) \
     if (opId.compare(#mtype) == 0) \
@@ -20 +22 @@
       if (ignoreMissingValue) \
       { \
-        functor.reset(new func::C##MType(tmpData, missingValue)); \
+        functor.reset(new func::C##MType(tmpData, defaultValue)); \
       } \
       else \