Changeset 1358 for XIOS/dev/XIOS_DEV_CMIP6/src

Timestamp:

12/06/17 08:34:41 (6 years ago)

Author:

rlacroix

Message:

Support reentrant workflows and workflows with temporal integration for fields read from files.

Location:

XIOS/dev/XIOS_DEV_CMIP6/src

Files:

• filter/file_server_writer_filter.cpp (modified) (1 diff)
• filter/file_server_writer_filter.hpp (modified) (1 diff)
• filter/file_writer_filter.cpp (modified) (1 diff)
• filter/file_writer_filter.hpp (modified) (1 diff)
• filter/filter.cpp (modified) (1 diff)
• filter/filter.hpp (modified) (1 diff)
• filter/input_pin.hpp (modified) (1 diff)
• filter/output_pin.cpp (modified) (1 diff)
• filter/output_pin.hpp (modified) (1 diff)
• filter/store_filter.cpp (modified) (1 diff)
• filter/store_filter.hpp (modified) (1 diff)
• filter/temporal_filter.cpp (modified) (1 diff)
• filter/temporal_filter.hpp (modified) (1 diff)
• node/context.cpp (modified) (4 diffs)
• node/context.hpp (modified) (1 diff)
• node/field.cpp (modified) (5 diffs)
• node/field.hpp (modified) (2 diffs)
• node/file.cpp (modified) (3 diffs)
• node/file.hpp (modified) (1 diff)

XIOS/dev/XIOS_DEV_CMIP6/src/filter/file_server_writer_filter.cpp

@@ -19 +19 @@
   }
 
+  bool CFileServerWriterFilter::mustAutoTrigger() const
+  {
+    return true;
+  }
+
   bool CFileServerWriterFilter::isDataExpected(const CDate& date) const
   {

XIOS/dev/XIOS_DEV_CMIP6/src/filter/file_server_writer_filter.hpp

@@ -23 +23 @@
       CFileServerWriterFilter(CGarbageCollector& gc, CField* field);
 
+      /*!
+       * Tests if the filter must auto-trigger.
+       *
+       * \return true if the filter must auto-trigger
+       */
+      bool virtual mustAutoTrigger() const;
+
+      /*!
+       * Tests whether data is expected for the specified date.
+       *
+       * \param date the date associated to the data
+       */
       bool virtual isDataExpected(const CDate& date) const;
+
     protected:
       /*!

XIOS/dev/XIOS_DEV_CMIP6/src/filter/file_writer_filter.cpp

@@ -37 +37 @@
   }
 
+  bool CFileWriterFilter::mustAutoTrigger() const
+  {
+    return true;
+  }
+
   bool CFileWriterFilter::isDataExpected(const CDate& date) const
   {

XIOS/dev/XIOS_DEV_CMIP6/src/filter/file_writer_filter.hpp

@@ -24 +24 @@
 
       /*!
+       * Tests if the filter must auto-trigger.
+       *
+       * \return true if the filter must auto-trigger
+       */
+      bool virtual mustAutoTrigger() const;
+
+      /*!
        * Tests whether data is expected for the specified date.
        *

XIOS/dev/XIOS_DEV_CMIP6/src/filter/filter.cpp

@@ -41 +41 @@
   }
 
+  bool CFilter::mustAutoTrigger() const
+  {
+    return COutputPin::mustAutoTrigger();
+  }
+
   bool CFilter::isDataExpected(const CDate& date) const
   {

XIOS/dev/XIOS_DEV_CMIP6/src/filter/filter.hpp

@@ -49 +49 @@
 
       /*!
+       * Tests if the filter must auto-trigger.
+       *
+       * \return true if the filter must auto-trigger
+       */
+      bool virtual mustAutoTrigger() const;
+
+      /*!
        * Tests whether data is expected for the specified date.
        *

XIOS/dev/XIOS_DEV_CMIP6/src/filter/input_pin.hpp

@@ -60 +60 @@
 
       /*!
+       * Tests if the pin must auto-trigger.
+       *
+       * \return true if the pin must auto-trigger
+       */
+      bool virtual mustAutoTrigger() const = 0;
+
+      /*!
        * Tests whether data is expected for the specified date.
        *

XIOS/dev/XIOS_DEV_CMIP6/src/filter/output_pin.cpp

@@ -66 +66 @@
   }
 
+  bool COutputPin::mustAutoTrigger() const
+  {
+    std::vector<std::pair<boost::shared_ptr<CInputPin>, size_t> >::const_iterator it, itEnd;
+    for (it = outputs.begin(), itEnd = outputs.end(); it != itEnd; ++it)
+    {
+      if (it->first->mustAutoTrigger())
+        return true;
+    }
+
+    return false;
+  }
+
   void COutputPin::setOutputTriggers()
   {

XIOS/dev/XIOS_DEV_CMIP6/src/filter/output_pin.hpp

@@ -44 +44 @@
        */
       bool virtual canBeTriggered() const;
+
+      /*!
+       * Tests if the pin must auto-trigger.
+       *
+       * \return true if the pin must auto-trigger
+       */
+      bool virtual mustAutoTrigger() const;
 
       /*!

XIOS/dev/XIOS_DEV_CMIP6/src/filter/store_filter.cpp

@@ -111 +111 @@
   }
 
+  bool CStoreFilter::mustAutoTrigger() const
+  {
+    return false;
+  }
+
   bool CStoreFilter::isDataExpected(const CDate& date) const
   {

XIOS/dev/XIOS_DEV_CMIP6/src/filter/store_filter.hpp

@@ -53 +53 @@
 
       /*!
+       * Tests if the filter must auto-trigger.
+       *
+       * \return true if the filter must auto-trigger
+       */
+      bool virtual mustAutoTrigger() const;
+
+      /*!
        * Tests whether data is expected for the specified date.
        *

XIOS/dev/XIOS_DEV_CMIP6/src/filter/temporal_filter.cpp

@@ -84 +84 @@
   }
 
+  bool CTemporalFilter::mustAutoTrigger() const
+  {
+    return true;
+  }
+
   bool CTemporalFilter::isDataExpected(const CDate& date) const
   {

XIOS/dev/XIOS_DEV_CMIP6/src/filter/temporal_filter.hpp

@@ -41 +41 @@
 
       /*!
+       * Tests if the filter must auto-trigger.
+       *
+       * \return true if the filter must auto-trigger
+       */
+      bool virtual mustAutoTrigger() const;
+
+      /*!
        * Tests whether data is expected for the specified date.
        *

XIOS/dev/XIOS_DEV_CMIP6/src/node/context.cpp

@@ -665 +665 @@
     {
       buildFilterGraphOfFieldsWithReadAccess();
+      postProcessFilterGraph();
     }
     
@@ -782 +783 @@
    }
 
+   void CContext::postProcessFilterGraph()
+   {
+     int size = enabledFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+        enabledFiles[i]->postProcessFilterGraph();
+     }
+   }
+
    void CContext::startPrefetchingOfEnabledReadModeFiles()
    {
@@ -788 +798 @@
      {
         enabledReadModeFiles[i]->prefetchEnabledReadModeFields();
+     }
+   }
+
+   void CContext::doPreTimestepOperationsForEnabledReadModeFiles()
+   {
+     int size = enabledReadModeFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+        enabledReadModeFiles[i]->doPreTimestepOperationsForEnabledReadModeFields();
      }
    }
@@ -1837 +1856 @@
       if (prevStep < step)
       {
+        if (hasClient && !hasServer) // For now we only use server level 1 to read data
+        {
+          doPreTimestepOperationsForEnabledReadModeFiles();
+        }
+
         info(50) << "updateCalendar : before : " << calendar->getCurrentDate() << endl;
         calendar->update(step);

XIOS/dev/XIOS_DEV_CMIP6/src/node/context.hpp

@@ -120 +120 @@
          void solveOnlyRefOfEnabledFields(bool sendToServer);         
          void buildFilterGraphOfEnabledFields();
+         void postProcessFilterGraph();
          void startPrefetchingOfEnabledReadModeFiles();
+         void doPreTimestepOperationsForEnabledReadModeFiles();
          void doPostTimestepOperationsForEnabledReadModeFiles();
          void findFieldsWithReadAccess(void);

XIOS/dev/XIOS_DEV_CMIP6/src/node/field.cpp

@@ -42 +42 @@
       , wasDataRequestedFromServer(false)
       , wasDataAlreadyReceivedFromServer(false)
+      , mustAutoTrigger(false)
       , isEOF(false), nstepMaxRead(false)
    { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
@@ -58 +59 @@
       , wasDataRequestedFromServer(false)
       , wasDataAlreadyReceivedFromServer(false)
+      , mustAutoTrigger(false)
       , isEOF(false), nstepMaxRead(false)
    { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
@@ -569 +571 @@
 
     // Check if data previously requested has been received as expected
-    if (wasDataRequestedFromServer && (!isEOF || currentDate <= dateEOF))
+    if (wasDataRequestedFromServer && !isEOF)
     {
       CTimer timer("CField::checkForLateDataFromServer");
@@ -577 +579 @@
       {
         const CDate nextDataDue = wasDataAlreadyReceivedFromServer ? (lastDataReceivedFromServer + file->output_freq) : context->getCalendar()->getInitDate();
-        isDataLate = nextDataDue < currentDate;
+        isDataLate = (nextDataDue <= currentDate);
 
         if (isDataLate)
@@ -594 +596 @@
               << "Late data at timestep = " << currentDate);
     }
+  }
+
+  void CField::checkIfMustAutoTrigger(void)
+  {
+    mustAutoTrigger = serverSourceFilter ? serverSourceFilter->mustAutoTrigger() : false;
+  }
+
+  void CField::autoTriggerIfNeeded(void)
+  {
+    if (mustAutoTrigger)
+      serverSourceFilter->trigger(CContext::getCurrent()->getCalendar()->getCurrentDate());
   }
 

XIOS/dev/XIOS_DEV_CMIP6/src/node/field.hpp

@@ -174 +174 @@
         void recvReadDataReady(vector<int> ranks, vector<CBufferIn*> buffers);
         void checkForLateDataFromServer(void);
+        void checkIfMustAutoTrigger(void);
+        void autoTriggerIfNeeded(void);
         void outputField(CArray<double,3>& fieldOut);
         void outputField(CArray<double,2>& fieldOut);
@@ -221 +223 @@
          CDate lastDataRequestedFromServer, lastDataReceivedFromServer, dateEOF;
          bool wasDataRequestedFromServer, wasDataAlreadyReceivedFromServer;
+         bool mustAutoTrigger;
 
          map<int,boost::shared_ptr<func::CFunctor> > foperation_srv;

XIOS/dev/XIOS_DEV_CMIP6/src/node/file.cpp

@@ -854 +854 @@
 
    /*!
+    * Post-process the filter graph for each active field.
+    */
+   void CFile::postProcessFilterGraph()
+   {
+     int size = this->enabledFields.size();
+     for (int i = 0; i < size; ++i)
+     {
+       this->enabledFields[i]->checkIfMustAutoTrigger();
+     }
+   }
+
+   /*!
      Prefetching the data for enabled fields read from file.
    */
@@ -867 +879 @@
 
    /*!
+     Do all pre timestep operations for enabled fields in read mode:
+      - Check that the data excepted from server has been received
+      - Check if some filters must auto-trigger
+   */
+   void CFile::doPreTimestepOperationsForEnabledReadModeFields(void)
+   {
+     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
+       return;
+
+     int size = this->enabledFields.size();
+     for (int i = 0; i < size; ++i)
+     {
+       this->enabledFields[i]->checkForLateDataFromServer();
+       this->enabledFields[i]->autoTriggerIfNeeded();
+     }
+   }
+
+   /*!
      Do all post timestep operations for enabled fields in read mode:
       - Prefetch the data read from file when needed
-      - Check that the data excepted from server has been received
    */
    void CFile::doPostTimestepOperationsForEnabledReadModeFields(void)
@@ -879 +908 @@
      for (int i = 0; i < size; ++i)
      {
-       this->enabledFields[i]->checkForLateDataFromServer();
        this->enabledFields[i]->sendReadDataRequestIfNeeded();
      }

XIOS/dev/XIOS_DEV_CMIP6/src/node/file.hpp

@@ -110 +110 @@
          
          void buildFilterGraphOfEnabledFields(CGarbageCollector& gc);
+         void postProcessFilterGraph();
          void prefetchEnabledReadModeFields();
+         void doPreTimestepOperationsForEnabledReadModeFields();
          void doPostTimestepOperationsForEnabledReadModeFields();