Changeset 2144 for XIOS/dev/dev_ym/XIOS_COUPLING/src/node

Timestamp:

06/04/21 11:55:52 (3 years ago)

Author:

yushan

Message:

Big commit on graph functionality. Enable the workflow graph when going through field dependency graph

Location:

XIOS/dev/dev_ym/XIOS_COUPLING/src/node

Files:

• field.cpp (modified) (19 diffs)
• grid.cpp (modified) (3 diffs)
• grid.hpp (modified) (1 diff)

XIOS/dev/dev_ym/XIOS_COUPLING/src/node/field.cpp

@@ -22 +22 @@
 #include "file_reader_source_filter.hpp"
 #include "tracer.hpp"
+#include "graph_package.hpp"
 
 namespace xios
@@ -585 +586 @@
     
     const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
-    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
-
+    const double defaultValue = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
+    bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    info(100)<<"=== Workflow Graph === field id="<<this->getId()<<" build_workflow_graph="<<buildGraph_<<std::endl;
     if (!inputFilter) inputFilter = std::shared_ptr<CPassThroughFilter>(new CPassThroughFilter(gc)); 
      
@@ -592 +594 @@
     {
       CField* fieldRef = getDirectFieldReference();
+      info(100)<<"=== Workflow Graph === fieldRef id="<<fieldRef->getId()<<std::endl;
+
+      //------ build_workflow_graph start
+      if(buildGraph_)
+      {
+        (*fieldRef).build_workflow_graph.set(build_workflow_graph); 
+      }    
+      else
+      {
+        this->build_workflow_graph.set(fieldRef->build_workflow_graph);
+        buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+        info(100)<<"=== Workflow Graph === field id="<<this->getId()<<" updated build_workflow_graph="<<buildGraph_<<std::endl;
+      }
+
+      
+      if(buildGraph_) this->build_workflow_graph.set(build_workflow_graph);
+      //------ build_workflow_graph end
+
       bool ret=fieldRef->buildWorkflowGraph(gc); 
       if (!ret) return false ; // workflow graph cannot be built at this stage
+      
+      this->build_workflow_graph.set(fieldRef->build_workflow_graph);
+      buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
     }
 
@@ -624 +647 @@
       {
         grid->solveElementsRefInheritance() ;
-        std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = grid->buildTransformationGraph(gc, false,  gridSrc, detectMissingValues, defaultValue, newGrid) ;
+        std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = grid->buildTransformationGraph(gc, false,  gridSrc, detectMissingValues, defaultValue, newGrid, buildGraph_, this) ;
         lastFilter->connectOutput(filters.first, 0);
         lastFilter = filters.second;
@@ -636 +659 @@
       // connect the input Filter to the reference
       getDirectFieldReference()->getInstantDataFilter()->connectOutput(inputFilter,0);
+      if(buildGraph_) 
+      {
+        info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a transformation filter 2 ============== "<<getDirectFieldReference()->getInstantDataFilter()<<" _ "<<inputFilter<<std::endl;
+        inputFilter->graphEnabled=true;
+        inputFilter->graphPackage = new CGraphPackage;
+        inputFilter->graphPackage->inFields.push_back(this);
+        inputFilter->label_field_id = getDirectFieldReference()->getId(); 
+      }
     }
     else 
@@ -647 +678 @@
          if (fileIn_->isClientSide()) fileIn_->readFieldAttributesMetaData(this);
          CGrid* newGrid ;
-         std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = grid_->buildTransformationGraph(gc, true, nullptr, detectMissingValues, defaultValue, newGrid) ;
+         std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = grid_->buildTransformationGraph(gc, true, nullptr, detectMissingValues, defaultValue, newGrid, buildGraph_, this) ;
          grid_ = newGrid ;
          grid_ref=grid_->getId() ; // for server 
@@ -668 +699 @@
         grid_->solveElementsRefInheritance() ;
         CGrid* newGrid ;
-        std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = grid_->buildTransformationGraph(gc, true, nullptr, detectMissingValues, defaultValue, newGrid) ;
+        std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = grid_->buildTransformationGraph(gc, true, nullptr, detectMissingValues, defaultValue, newGrid, buildGraph_, this) ;
         newGrid->duplicateAttributes(grid_) ; // for grid attributes (mask)
         grid_ = newGrid ;
@@ -701 +732 @@
     // insert temporal filter before sending to files
     getTemporalDataFilter(gc, fileOut_->output_freq)->connectOutput(clientToServerStoreFilter_, 0);
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_) 
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToFileServer  ============== "<<getTemporalDataFilter(gc, fileOut_->output_freq)<<" _ "<<clientToServerStoreFilter_<<std::endl;
+      clientToServerStoreFilter_->graphPackage = new CGraphPackage;
+      clientToServerStoreFilter_->graphEnabled = true;
+      clientToServerStoreFilter_->graphPackage->inFields.push_back(this);
+    }
   } 
 
@@ -708 +747 @@
     clientToServerStoreFilter_ = std::shared_ptr<CClientToServerStoreFilter>(new CClientToServerStoreFilter(gc, this, client));
     instantDataFilter->connectOutput(clientToServerStoreFilter_, 0);
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_) 
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToCouplerOut  ============== "<<instantDataFilter<<" _ "<<clientToServerStoreFilter_<<std::endl;
+      clientToServerStoreFilter_->graphPackage = new CGraphPackage;
+      clientToServerStoreFilter_->graphEnabled = true;
+      clientToServerStoreFilter_->graphPackage->inFields.push_back(this);
+    }
   } 
 
@@ -722 +769 @@
     modelToClientSourceFilter_ = std::shared_ptr<CModelToClientSourceFilter>(new CModelToClientSourceFilter(gc, grid_, detectMissingValues, defaultValue));
     modelToClientSourceFilter_ -> connectOutput(inputFilter,0) ;
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_ ) 
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToModelInput  ============== "<<modelToClientSourceFilter_<<" _ "<<inputFilter<<" ***** "<<std::endl;
+      modelToClientSourceFilter_->graphPackage = new CGraphPackage;
+      modelToClientSourceFilter_->graphEnabled = true;
+      modelToClientSourceFilter_->graphPackage->inFields.push_back(this);
+    }
   } 
  
@@ -731 +786 @@
     serverFromClientSourceFilter_ = std::shared_ptr<CServerFromClientSourceFilter>(new CServerFromClientSourceFilter(gc,  grid_));
     serverFromClientSourceFilter_ -> connectOutput(inputFilter,0) ;
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_)
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToClientInput  ============== "<<serverFromClientSourceFilter_ << " _ "<<inputFilter<<" ***** "<<std::endl;
+      serverFromClientSourceFilter_->graphPackage = new CGraphPackage;
+      serverFromClientSourceFilter_->graphEnabled = true;
+      serverFromClientSourceFilter_->graphPackage->inFields.push_back(this);
+    }
   } 
 
@@ -742 +805 @@
     clientFromServerSourceFilter_ = std::shared_ptr<CClientFromServerSourceFilter>(new CClientFromServerSourceFilter(gc,this)) ;
     clientFromServerSourceFilter_ -> connectOutput(inputFilter,0) ;
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_)
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToServerInput  ============== "<<clientFromServerSourceFilter_ << " _ "<<inputFilter<<std::endl;
+      clientFromServerSourceFilter_->graphPackage = new CGraphPackage;
+      clientFromServerSourceFilter_->graphEnabled = true;
+      clientFromServerSourceFilter_->graphPackage->inFields.push_back(this);
+    }
   } 
 
@@ -755 +826 @@
     clientFromClientSourceFilter_ = std::shared_ptr<CClientFromClientSourceFilter>(new CClientFromClientSourceFilter(gc, this)) ;
     clientFromClientSourceFilter_ -> connectOutput(inputFilter,0) ;
-   
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_)
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToCouplerIn  ============== "<<clientFromClientSourceFilter_ << " _ "<<inputFilter<<std::endl;
+      clientFromClientSourceFilter_->graphPackage = new CGraphPackage;
+      clientFromClientSourceFilter_->graphEnabled = true;
+      clientFromClientSourceFilter_->graphPackage->inFields.push_back(this);
+    }
   } 
 
@@ -765 +843 @@
     fileWriterStoreFilter_ = std::shared_ptr<CFileWriterStoreFilter>(new CFileWriterStoreFilter(gc, this));
     instantDataFilter->connectOutput(fileWriterStoreFilter_, 0);
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_)
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToFileWriter  ============== "<<instantDataFilter << " _ "<<fileWriterStoreFilter_<<std::endl;
+      fileWriterStoreFilter_->graphPackage = new CGraphPackage;
+      fileWriterStoreFilter_->graphEnabled = true;
+      fileWriterStoreFilter_->graphPackage->inFields.push_back(this);
+    }
   } 
 
@@ -774 +860 @@
     fileReaderSourceFilter_ = std::shared_ptr<CFileReaderSourceFilter>(new CFileReaderSourceFilter(gc, this));
     fileReaderSourceFilter_->connectOutput(inputFilter, 0);
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_)
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToFileReader  ============== "<<fileReaderSourceFilter_ << " _ "<<inputFilter<<std::endl;
+      fileReaderSourceFilter_->graphPackage = new CGraphPackage;
+      fileReaderSourceFilter_->graphEnabled = true;
+      fileReaderSourceFilter_->graphPackage->inFields.push_back(this);
+    }
   }
 
@@ -784 +878 @@
     clientToModelStoreFilter_ = std::shared_ptr<CClientToModelStoreFilter>(new CClientToModelStoreFilter(gc, this));
     instantDataFilter->connectOutput(clientToModelStoreFilter_, 0);
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_)
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToModelOutput  ============== "<<instantDataFilter << " _ "<<clientToModelStoreFilter_<<std::endl;
+      clientToModelStoreFilter_->graphPackage = new CGraphPackage;
+      clientToModelStoreFilter_->graphEnabled = true;
+      clientToModelStoreFilter_->graphPackage->inFields.push_back(this);
+    }
   }
 
@@ -792 +894 @@
     serverToClientStoreFilter_ = std::shared_ptr<CServerToClientStoreFilter>(new CServerToClientStoreFilter(gc, this, client));
     instantDataFilter->connectOutput(serverToClientStoreFilter_, 0);
+    const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+    if(buildGraph_)
+    {
+      info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a connectToServerToClient  ============== "<<instantDataFilter << " _ "<<serverToClientStoreFilter_<<std::endl;
+      serverToClientStoreFilter_->graphPackage = new CGraphPackage;
+      serverToClientStoreFilter_->graphEnabled = true;
+      serverToClientStoreFilter_->graphPackage->inFields.push_back(this);
+    }
   }
 
@@ -925 +1035 @@
 
       instantDataFilter->connectOutput(temporalFilter, 0);
+      const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+      if(buildGraph_) 
+      {
+        info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a getTemporalDataFilter  ============== "<<instantDataFilter << " _ "<<temporalFilter<<std::endl;
+        temporalFilter->graphPackage = new CGraphPackage;
+        temporalFilter->graphEnabled = true;
+        temporalFilter->graphPackage->inFields.push_back(this);
+      }
 
       it = temporalDataFilters.insert(std::make_pair(outFreq, temporalFilter)).first;
@@ -972 +1090 @@
 
       inputFilter->connectOutput(selfTemporalDataFilter, 0);
+      const bool buildGraph_ = !build_workflow_graph.isEmpty() && build_workflow_graph == true ;
+      if(buildGraph_)
+      {
+        info(100)<<"=== Workflow Graph === field "<<this->getId()<<" calls a getSelfTemporalDataFilter  ============== "<<inputFilter << " _ "<<selfTemporalDataFilter<<std::endl;
+        selfTemporalDataFilter->graphPackage = new CGraphPackage;
+        selfTemporalDataFilter->graphEnabled = true;
+        selfTemporalDataFilter->graphPackage->inFields.push_back(this);
+      }
       return selfTemporalDataFilter ;
     }

XIOS/dev/dev_ym/XIOS_COUPLING/src/node/grid.cpp

@@ -1640 +1640 @@
 
   std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > 
-  CGrid::buildTransformationGraph(CGarbageCollector& gc, bool isSource, CGrid* gridSrc, double detectMissingValues, double defaultValue, CGrid*& newGrid)
-  TRY
-  {
+  CGrid::buildTransformationGraph(CGarbageCollector& gc, bool isSource, CGrid* gridSrc, double detectMissingValues, double defaultValue, CGrid*& newGrid, bool graphEnabled, CField* field)
+  TRY
+  {
+    static bool transformationGoing = false;
     registerAlgorithmTransformation() ; // needed to enable self-registration of the transformations
                                         // big mystery why it doesn't work witout that...
@@ -1936 +1937 @@
         shared_ptr<CTransformFilter> transformFilter = shared_ptr<CTransformFilter>(gridAlgorithm->createTransformFilter(gc, detectMissingValues, defaultValue)) ;
         outputFilter->connectOutput(transformFilter,0) ;
+        if(graphEnabled)
+        {
+          transformFilter->graphEnabled=true;
+          transformFilter->graphPackage = new CGraphPackage;
+          transformFilter->graphPackage->inFields.push_back(field);
+          transformFilter->graphPackage->show = !transformationGoing;
+        }
+        
         vector<string> auxFieldId = algo->getAuxFieldId() ; // better to do that at transformation not algo ??
         int i=1; 
@@ -1950 +1959 @@
       if (hasRemainTransform)
       {
+        transformationGoing = true;
         gridSrc=newGrid ;
-        pair<shared_ptr<CFilter>, shared_ptr<CFilter> > filters = this->buildTransformationGraph(gc, isSource, gridSrc, detectMissingValues, defaultValue, newGrid) ;
+        CField *field_bis = field;
+        pair<shared_ptr<CFilter>, shared_ptr<CFilter> > filters = this->buildTransformationGraph(gc, isSource, gridSrc, detectMissingValues, defaultValue, newGrid, graphEnabled, field_bis) ;
         outputFilter->connectOutput(filters.first,0) ;
         outputFilter=filters.second ;
       }
+      transformationGoing = false;
     }
      

XIOS/dev/dev_ym/XIOS_COUPLING/src/node/grid.hpp

@@ -216 +216 @@
       public:
          pair<shared_ptr<CFilter>, shared_ptr<CFilter> > buildTransformationGraph(CGarbageCollector& gc, bool isSource, CGrid* gridSrc, double detectMissingValues,
-                                                                                  double defaultValue, CGrid*& newGrid) ;
+                                                                                  double defaultValue, CGrid*& newGrid, bool graphEnabled=false, CField* field=0) ;
       private:
         CGridAlgorithm* gridAlgorithm_ = nullptr ;