source: XIOS/dev/dev_ym/XIOS_COUPLING/src/node/field.hpp @ 1930

Last change on this file since 1930 was 1930, checked in by ymipsl, 14 months ago

Big update on on going work related to data distribution and transfer between clients and servers.
Revisite of the source and store filter using "connectors".

YM

  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
  • Property svn:executable set to *
File size: 17.7 KB
Line 
1#ifndef __XIOS_CField__
2#define __XIOS_CField__
3
4/// XIOS headers ///
5#include "xios_spl.hpp"
6#include "group_factory.hpp"
7#include "functor.hpp"
8#include "functor_type.hpp"
9#include "duration.hpp"
10#include "date.hpp"
11#include "declare_group.hpp"
12#include "calendar_util.hpp"
13#include "array_new.hpp"
14#include "attribute_array.hpp"
15#include "declare_ref_func.hpp"
16#include "transformation_enum.hpp"
17#include "variable.hpp"
18#include "context_client.hpp"
19#include "pass_through_filter.hpp"
20#include "temporal_filter.hpp"
21#include "model_to_client_source_filter.hpp"
22#include "client_from_client_source_filter.hpp"
23#include "client_from_server_source_filter.hpp"
24#include "client_to_model_store_filter.hpp"
25
26
27
28
29namespace xios
30{
31
32   /// ////////////////////// Déclarations ////////////////////// ///
33
34   class CFieldGroup;
35   class CFieldAttributes;
36   class CField;
37
38   class CFile;
39   class CCouplerIn ;
40   class CCouplerOut ;
41   class CGrid;
42   class CContext;
43   class CGenericFilter;
44
45   class CGarbageCollector;
46   class COutputPin;
47   class CSourceFilter;
48   class CFileWriterFilter;
49   class CFileServerWriterFilter;
50   class CFileReaderSourceFilter;
51   class CServerToClientFilter;
52   class CModelToClientSourceFilter;
53   class CServerFromClientSourceFilter;
54   ///--------------------------------------------------------------
55
56   // Declare/Define CFieldAttribute
57   BEGIN_DECLARE_ATTRIBUTE_MAP(CField)
58#  include "field_attribute.conf"
59   END_DECLARE_ATTRIBUTE_MAP(CField)
60
61   class CContextClient ;
62
63   ///--------------------------------------------------------------
64   class CField
65      : public CObjectTemplate<CField>
66      , public CFieldAttributes
67   {
68         /// friend ///
69         friend class CFile;
70
71         /// typedef ///
72         typedef CObjectTemplate<CField>   SuperClass;
73         typedef CFieldAttributes SuperClassAttribute;
74
75      public :   
76         enum EReadField
77         {
78           RF_NODATA, RF_EOF, RF_DATA
79         };
80         
81      public:
82
83         typedef CFieldAttributes RelAttributes;
84         typedef CFieldGroup      RelGroup;
85
86         enum EEventId
87         {
88           EVENT_ID_UPDATE_DATA, EVENT_ID_READ_DATA, EVENT_ID_READ_DATA_READY,
89           EVENT_ID_ADD_VARIABLE, EVENT_ID_ADD_VARIABLE_GROUP, EVENT_ID_GRID_COMPLETED
90         };
91
92         /// Constructeurs ///
93         CField(void);
94         explicit CField(const StdString& id);
95         CField(const CField& field);       // Not implemented yet.
96         CField(const CField* const field); // Not implemented yet.
97
98         /// Accesseurs ///
99
100         CGrid* getRelGrid(void) const;
101         CFile* getRelFile(void) const;
102
103         func::CFunctor::ETimeType getOperationTimeType() const;
104
105      public:
106         int getNStep(void) const;
107
108         template <int N> void getData(CArray<double, N>& _data) const;
109
110         std::shared_ptr<COutputPin> getInstantDataFilter();
111
112         /// Mutateur ///
113         void setRelFile(CFile* _file);
114         void incrementNStep(void);
115         void resetNStep(int nstep = 0);
116         void resetNStepMax();
117
118         std::map<int, StdSize> getGridAttributesBufferSize(CContextClient* client, bool bufferForWriting = false);
119         // Grid data buffer size for each connection of contextclient
120         std::map<int, StdSize> getGridDataBufferSize(CContextClient* client, bool bufferForWriting = false);
121
122       public:
123          void makeGridAliasForCoupling(void) ;
124       public:
125         bool isActive(bool atCurrentTimestep = false) const;
126         bool hasOutputFile;
127
128         bool wasWritten() const;
129         void setWritten();
130
131         bool getUseCompressedOutput() const;
132         void setUseCompressedOutput();
133
134         /// Traitements ///
135         void solveGridReference(void);
136         void solveServerOperation(void);
137         void solveCheckMaskIndex(bool doSendingIndex);
138         void solveAllReferenceEnabledField(bool doSending2Server);
139         void solveOnlyReferenceEnabledField(void);
140         void generateNewTransformationGridDest();
141         void updateRef(CGrid* grid);
142         void buildGridTransformationGraph();
143         void solveGridDomainAxisRef(bool checkAtt);
144         void solveTransformedGrid();
145         void solveGenerateGrid();
146         void solveGridDomainAxisBaseRef();
147
148         void solveAllEnabledFieldsAndTransform();
149         void checkGridOfEnabledFields();
150         void sendGridOfEnabledFields();
151         void sendGridComponentOfEnabledFields();
152
153         void sendFieldToFileServer(void) ;
154         void sendCloseDefinition(void) ;
155     
156      public:
157         void sendFieldToCouplerOut(void) ;
158      private:
159         bool sendFieldToCouplerOut_done_=false;
160      public:
161
162         void sendFieldToInputFileServer(void) ;
163
164         /// Vérifications ///
165         void checkTimeAttributes(CDuration* freqOp=NULL);
166
167         void buildFilterGraph(CGarbageCollector& gc, bool enableOutput);
168
169         bool buildWorkflowGraph(CGarbageCollector& gc) ;
170         bool buildWorkflowGraphDone_ = false ;
171
172         size_t getGlobalWrittenSize(void) ;
173         
174         
175         std::shared_ptr<COutputPin> getFieldReference(CGarbageCollector& gc);
176         std::shared_ptr<COutputPin> getSelfReference(CGarbageCollector& gc);
177         std::shared_ptr<COutputPin> getTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq);
178         std::shared_ptr<COutputPin> getSelfTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq);
179
180//         virtual void fromBinary(StdIStream& is);
181
182         /// Destructeur ///
183         virtual ~CField(void);
184
185         /// Accesseurs statiques ///
186         static StdString GetName(void);
187         static StdString GetDefName(void);
188
189         static ENodeType GetType(void);
190
191        template <int N> void setData(const CArray<double, N>& _data);
192        static bool dispatchEvent(CEventServer& event);
193        void sendAllAttributesToServer(CContextClient* client) ; 
194        void sendUpdateData(Time timestamp, const CArray<double,1>& data, CContextClient* client);
195        void sendUpdateDataServerToClient(bool isEOF, const CArray<double,1>& data, CContextClient* client) ;
196
197        static void recvUpdateData(CEventServer& event);
198        void receiveUpdateData(CEventServer& event); 
199       
200        void recvUpdateData(std::map<int,CBufferIn*>& rankBuffers); // old interface to be removed
201        void recvUpdateDataFromClient(std::map<int,CBufferIn*>& rankBuffers); // old interface to be removed
202        void recvUpdateDataFromCoupler(std::map<int,CBufferIn*>& rankBuffers); // old interface to be removed
203       
204        void writeField(const CArray<double,1>& data);
205        bool sendReadDataRequest(const CDate& tsDataRequested, CContextClient* client);
206        bool sendReadDataRequestIfNeeded(void);
207        static void recvReadDataRequest(CEventServer& event);
208        void recvReadDataRequest(void);
209        EReadField readField(CArray<double,1>& data);
210        static void recvReadDataReady(CEventServer& event);
211        void receiveReadDataReady(CEventServer& event);
212        void recvReadDataReady(vector<int> ranks, vector<CBufferIn*> buffers); // old interface to remove
213        void recvDataFromCoupler(vector<int> ranks, vector<CBufferIn*> buffers) ; // old interface to remove
214        void checkForLateDataFromServer(void);
215        void checkForLateDataFromCoupler(void) ;
216
217        void checkIfMustAutoTrigger(void); // ym obsolete
218        void autoTriggerIfNeeded(void); //ym obsolete
219        void triggerLateField(void) ;
220
221//        void outputField(CArray<double,3>& fieldOut);
222//        void outputField(CArray<double,2>& fieldOut);
223        void outputField(const CArray<double,1>& dataIn, CArray<double,1>& dataOut);
224
225//        void inputField(CArray<double,3>& fieldOut);
226//        void inputField(CArray<double,2>& fieldOut);
227        void inputField(const CArray<double,1>& dataIn, CArray<double,1>& dataOut);
228        void outputCompressedField(const CArray<double,1>& dataIn, CArray<double, 1>& dataOut);
229        void scaleFactorAddOffset(CArray<double,1>& data, double scaleFactor, double addOffset);
230        void invertScaleFactorAddOffset(CArray<double,1>& data, double scaleFactor, double addOffset);
231        void parse(xml::CXMLNode& node);
232
233        void setVirtualVariableGroup(CVariableGroup* newVVariableGroup);
234        CVariableGroup* getVirtualVariableGroup(void) const;
235        vector<CVariable*> getAllVariables(void) const;
236        virtual void solveDescInheritance(bool apply, const CAttributeMap* const parent = 0);
237
238        CVariable* addVariable(const string& id = "");
239        CVariableGroup* addVariableGroup(const string& id = "");       
240        void sendAddVariable(const string& id, CContextClient* client);
241        void sendAddVariableGroup(const string& id, CContextClient* client);
242        static void recvAddVariable(CEventServer& event);
243        void recvAddVariable(CBufferIn& buffer);
244        static void recvAddVariableGroup(CEventServer& event);
245        void recvAddVariableGroup(CBufferIn& buffer);       
246        void sendAddAllVariables(CContextClient* client);
247        void writeUpdateData(const CArray<double,1>& data);
248
249        const std::vector<StdString>& getRefDomainAxisIds();
250
251        const string& getExpression(void);
252        bool hasExpression(void) const;
253
254        bool hasGridMask(void) const;
255        CGrid* getGrid(void) { return grid_; } 
256
257        void connectToFileServer(CGarbageCollector& gc) ;
258        void connectToCouplerOut(CGarbageCollector& gc) ;
259        void connectToCouplerIn(CGarbageCollector& gc) ;
260        void connectToModelInput(CGarbageCollector& gc) ;
261        void connectToFileWriter(CGarbageCollector& gc) ;
262        void connectToClientInput(CGarbageCollector& gc) ;
263        void connectToServerInput(CGarbageCollector& gc) ;
264        void connectToModelOutput(CGarbageCollector& gc);
265        void connectToFileReader(CGarbageCollector& gc) ;
266        void connectToServerToClient(CGarbageCollector& gc) ;
267
268        void computeGridIndexToFileServer(void) ;
269
270        void setContextClientDataBufferSize(map<CContextClient*,map<int,size_t>>& bufferSize, 
271                                        map<CContextClient*,map<int,size_t>>& maxEventSize, 
272                                        bool bufferForWriting) ;
273        void setContextClientAttributesBufferSize(map<CContextClient*,map<int,size_t>>& bufferSize, 
274                                                 map<CContextClient*,map<int,size_t>>& maxEventSize, 
275                                                 bool bufferForWriting) ;
276      private:
277          bool isGridCompleted_ = true ;
278      public:
279          bool isGridCompleted() { return isGridCompleted_ ;} 
280          void setGridCompleted(void) { isGridCompleted_= true; }
281          void unsetGridCompleted(void) { isGridCompleted_ = false ;}
282     
283      public:     
284          void sendGridCompleted(void) ;
285      private:   
286          static void recvGridCompleted(CEventServer& event);
287          void recvGridCompleted(CBufferIn& buffer);
288
289
290      private:
291        std::vector<CGrid*> getGridPath(void) ;
292
293      public:
294         /// Propriétés privées ///
295         CVariableGroup* vVariableGroup;
296
297         CGrid*  grid_=nullptr;
298//         CFile*  file;
299         
300         CFile* fileIn_ = nullptr ; //<! pointer to input related file
301         bool hasFileIn(void) const { return fileIn_==nullptr ? false : true ;} 
302         CFile* getFileIn(void) {return fileIn_;}
303         void setFileIn(CFile* fileIn) { fileIn_ = fileIn ;}
304         void unsetFileIn(void) { fileIn_ = nullptr ;}
305
306         CFile* fileOut_ = nullptr ; //<! pointer to output related file
307         bool hasFileOut(void) const { return fileOut_==nullptr ? false : true ;} 
308         CFile* getFileOut(void) {return fileOut_;}
309         void setFileOut(CFile* fileOut) { fileOut_ = fileOut ;}
310         void unsetFileOut(void) { fileOut_ = nullptr ;}
311
312         CCouplerIn* couplerIn_ = nullptr ; //<!pointer to input related coupler
313         bool hasCouplerIn(void) const { return couplerIn_==nullptr ? false : true ;}
314         CCouplerIn* getCouplerIn(void) {return couplerIn_;}
315         void setCouplerIn(CCouplerIn* couplerIn) { couplerIn_ = couplerIn ;}
316         void unsetCouplerIn(void) { couplerIn_ = nullptr ;}
317
318         CCouplerOut* couplerOut_ = nullptr ; //<!pointer to output related coupler
319         bool hasCouplerOut(void) const { return couplerOut_==nullptr ? false : true ;}
320         CCouplerOut* getCouplerOut(void) {return couplerOut_;}
321         void setCouplerOut(CCouplerOut* couplerOut) { couplerOut_ = couplerOut ;}
322         void unsetCouplerOut(void) { couplerOut_ = nullptr ;}
323
324         bool modelIn_ = false ; //<! field can be received from model == true
325         bool getModelIn(void) { return modelIn_ ;}
326         void setModelIn(void) { modelIn_ = true ;}
327         void unsetModelIn(void) { modelIn_ = false ;}
328         
329         bool modelOut_ = false ; //<! field can be retrieve to model == true
330         bool getModelOut(void) { return modelOut_ ;}
331         void setModelOut(void) { modelOut_ = true ;}
332         void unsetModelOut(void) { modelOut_ = false ;}
333
334         CDuration freq_operation_srv, freq_write_srv;
335
336         bool written; //<! Was the field written at least once
337         int nstep, nstepMax;
338         bool isEOF;
339         CDate lastlast_Write_srv, last_Write_srv, last_operation_srv;
340         CDate lastDataRequestedFromServer, lastDataReceivedFromServer, dateEOF;
341         bool wasDataRequestedFromServer, wasDataAlreadyReceivedFromServer;
342         bool mustAutoTrigger;
343
344         map<int,std::shared_ptr<func::CFunctor> > foperation_srv;
345
346         // map<int, CArray<double,1> > data_srv;
347//         CArray<double,1> recvDataSrv; // not usefull anymore
348         
349         std::shared_ptr<func::CFunctor> recvFoperationSrv;
350         string content;
351
352         std::vector<StdString> domAxisScalarIds_;
353         bool useCompressedOutput;
354
355         // Two variables to identify the time_counter meta data written in file, which has no time_counter
356         bool hasTimeInstant;
357         bool hasTimeCentered;
358
359
360         DECLARE_REF_FUNC(Field,field)
361       
362      private:
363         CContextClient* client;
364      public:
365         void setContextClient(CContextClient* newContextClient);
366         CContextClient* getContextClient(void) {return client;}
367
368      private:
369
370         bool areAllReferenceSolved;
371         bool isReferenceSolved;
372         bool isReferenceSolvedAndTransformed;
373         bool isGridChecked;
374         bool nstepMaxRead;
375
376       private: 
377         //! define if the field is part of the active workflow. It will be tagged to true when CField::buildWorkflowGraph is successfull
378         bool workflowEnabled_ = false ;
379       public: 
380         /*! workflowEnabled_ public accessor
381          * \return Value of workflowEnabled_ */
382         bool getWorkflowEnabled(void) { return  workflowEnabled_; }
383 
384
385      private:
386     
387         //! The type of operation attached to the field
388         func::CFunctor::ETimeType operationTimeType;
389
390         //! The output pin of the input filter of the field
391         std::shared_ptr<CPassThroughFilter> inputFilter;
392
393         //! The self temporal data filter
394         std::shared_ptr<CTemporalFilter> selfTemporalDataFilter ;
395         
396         //! The output pin of the filter providing the instant data for the field
397         std::shared_ptr<COutputPin> instantDataFilter;
398         
399         //! The output pin of the filters providing the result of the field's temporal operation
400         std::map<CDuration, std::shared_ptr<COutputPin>, DurationFakeLessComparator> temporalDataFilters;
401         
402         //! The output pin of the filter providing the instant data for self references
403         std::shared_ptr<COutputPin> selfReferenceFilter; // probably redondant with inputFilter
404
405         //! The source filter for data provided by the client
406         std::shared_ptr<CSourceFilter> clientSourceFilter; // obsolete to remove
407 
408         //! The source filter for data provided by the model to enter the client workflow
409         std::shared_ptr<CModelToClientSourceFilter> modelToClientSourceFilter_;
410
411         //! The source filter for data provided by the model to enter the client workflow
412         std::shared_ptr<CClientToModelStoreFilter> clientToModelStoreFilter_;
413
414         //! The source filter for data provided by the client that send data to server workflow
415         std::shared_ptr<CServerFromClientSourceFilter> serverFromClientSourceFilter_;
416
417         //! The source filter for data provided by an other to enter the current client workflow (coupling mode)
418         std::shared_ptr<CClientFromClientSourceFilter> clientFromClientSourceFilter_;
419
420         //! The source filter for data provided by server to enter the current client workflow (reading mode)
421         std::shared_ptr<CClientFromServerSourceFilter> clientFromServerSourceFilter_;
422         
423         //! The source filter for data read from file on server side
424         std::shared_ptr<CFileReaderSourceFilter> fileReaderSourceFilter_;
425
426         //! The source filter for data provided by the server
427         std::shared_ptr<CSourceFilter> serverSourceFilter; // obsolete to remove
428       
429         //! The terminal filter which writes the data to file
430         std::shared_ptr<CFileWriterFilter> fileWriterFilter;
431       
432         //! The terminal filter which writes data to file
433         std::shared_ptr<CFileServerWriterFilter> fileServerWriterFilter;
434
435         //! The terminal filter which send data from server to client
436         std::shared_ptr<CServerToClientFilter> serverToClientFilter_;
437
438
439   }; // class CField
440
441   ///--------------------------------------------------------------
442
443   // Declare/Define CFieldGroup and CFieldDefinition
444   DECLARE_GROUP(CField);
445
446   ///-----------------------------------------------------------------
447
448   template <>
449      void CGroupTemplate<CField, CFieldGroup, CFieldAttributes>::solveRefInheritance(void);
450
451   ///-----------------------------------------------------------------
452} // namespace xios
453
454
455#endif // __XIOS_CField__
Note: See TracBrowser for help on using the repository browser.