source: XIOS/dev/branch_openmp/src/node/file.cpp @ 1491

Last change on this file since 1491 was 1491, checked in by yushan, 3 years ago

Branch EP merged with Dev_cmip6 @r1490

  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
File size: 40.2 KB
Line 
1#include "file.hpp"
2
3#include "attribute_template.hpp"
4#include "object_template.hpp"
5#include "group_template.hpp"
6#include "object_factory.hpp"
7#include "context.hpp"
8#include "context_server.hpp"
9#include "nc4_data_output.hpp"
10#include "nc4_data_input.hpp"
11#include "calendar_util.hpp"
12#include "date.hpp"
13#include "message.hpp"
14#include "type.hpp"
15#include "xios_spl.hpp"
16#include "context_client.hpp"
17#include "mpi.hpp"
18#include "timer.hpp"
19#include "server.hpp"
20
21namespace xios {
22
23   /// ////////////////////// Dfinitions ////////////////////// ///
24
25   CFile::CFile(void)
26      : CObjectTemplate<CFile>(), CFileAttributes()
27      , vFieldGroup(), data_out(), enabledFields()
28      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
29   {
30     setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
31     setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group"));
32   }
33
34   CFile::CFile(const StdString & id)
35      : CObjectTemplate<CFile>(id), CFileAttributes()
36      , vFieldGroup(), data_out(), enabledFields()
37      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
38    {
39      setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
40      setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group"));
41    }
42
43   CFile::~CFile(void)
44   { /* Ne rien faire de plus */ }
45
46   ///---------------------------------------------------------------
47  //! Get name of file
48   StdString CFile::GetName(void)   { return (StdString("file")); }
49   StdString CFile::GetDefName(void){ return (CFile::GetName()); }
50   ENodeType CFile::GetType(void)   { return (eFile); }
51
52   //----------------------------------------------------------------
53
54   const StdString CFile::getFileOutputName(void) const
55   {
56     return (name.isEmpty() ? getId() : name) + (name_suffix.isEmpty() ? StdString("") :  name_suffix.getValue());
57   }
58
59   //----------------------------------------------------------------
60   /*!
61   \brief Get data writer object.
62   Each enabled file in xml represents a physical netcdf file.
63   This function allows to access the data writer object.
64   \return data writer object.
65   */
66   boost::shared_ptr<CDataOutput> CFile::getDataOutput(void) const
67   {
68      return data_out;
69   }
70
71   /*!
72   \brief Get data reader object.
73   Each enabled file in xml represents a physical netcdf file.
74   This function allows to access the data reader object.
75   \return data reader object.
76   */
77   boost::shared_ptr<CDataInput> CFile::getDataInput(void) const
78   {
79      return data_in;
80   }
81
82   /*!
83   \brief Get virtual field group
84      In each file, there always exists a field group which is the ancestor of all
85   fields in the file. This is considered be virtual because it is created automatically during
86   file initialization and it normally doesn't appear on xml file
87   \return Pointer to field group
88   */
89   CFieldGroup* CFile::getVirtualFieldGroup(void) const
90   {
91      return (this->vFieldGroup);
92   }
93
94   /*!
95   \brief Get virtual variable group
96      In each file, there always exists a variable group which is the ancestor of all
97   variable in the file. This is considered be virtual because it is created automatically during
98   file initialization and it normally doesn't appear on xml file
99   \return Pointer to variable group
100   */
101   CVariableGroup* CFile::getVirtualVariableGroup(void) const
102   {
103      return (this->vVariableGroup);
104   }
105
106   //! Get all fields of a file
107   std::vector<CField*> CFile::getAllFields(void) const
108   {
109      return (this->vFieldGroup->getAllChildren());
110   }
111
112   //! Get all variables of a file
113   std::vector<CVariable*> CFile::getAllVariables(void) const
114   {
115      return (this->vVariableGroup->getAllChildren());
116   }
117
118   //----------------------------------------------------------------
119   /*!
120   \brief Get all enabled fields of file
121      A field is considered to be enabled if it fullfil these conditions: it is enabled, inside a enabled file
122   and its own level is not larger than file output level.
123   \param [in] default_outputlevel default value output level of file
124   \param [in] default_level default value level of field
125   \param [in] default_enabled flag determine by default if field is enabled
126   \return Vector of pointers of enabled fields
127   */
128   std::vector<CField*> CFile::getEnabledFields(int default_outputlevel,
129                                                int default_level,
130                                                bool default_enabled)
131   {
132      if (!this->enabledFields.empty())
133         return (this->enabledFields);
134
135      const int _outputlevel =
136         (!output_level.isEmpty()) ? output_level.getValue() : default_outputlevel;
137      std::vector<CField*>::iterator it;
138      this->enabledFields = this->getAllFields();
139
140      std::vector<CField*> newEnabledFields;
141
142      for ( it = this->enabledFields.begin(); it != this->enabledFields.end(); it++ )
143      {
144         if (!(*it)->enabled.isEmpty()) // Si l'attribut 'enabled' est dfini ...
145         {
146            if (! (*it)->enabled.getValue()) continue;
147//            { it--; this->enabledFields.erase(it+1); continue; }
148         }
149         else // Si l'attribut 'enabled' n'est pas dfini ...
150         {
151            if (!default_enabled) continue;
152//            { it--; this->enabledFields.erase(it+1); continue; }
153         }
154
155         if (!(*it)->level.isEmpty()) // Si l'attribut 'level' est dfini ...
156         {
157            if ((*it)->level.getValue() > _outputlevel) continue;
158//            { it--; this->enabledFields.erase(it+1); continue; }
159         }
160         else // Si l'attribut 'level' n'est pas dfini ...
161         {
162            if (default_level > _outputlevel) continue;
163//            { it--; this->enabledFields.erase(it+1); continue; }
164         }
165
166//         CField* field_tmp=(*it).get();
167//         shared_ptr<CField> sptfield=*it;
168//         field_tmp->refObject.push_back(sptfield);
169         newEnabledFields.push_back(*it);
170         // Le champ est finalement actif, on y ajoute sa propre reference.
171//         (*it)->refObject.push_back(*it);
172         // Le champ est finalement actif, on y ajoute la rfrence au champ de base.
173         (*it)->setRelFile(CFile::get(this));
174      }
175      enabledFields = newEnabledFields;
176
177      return (this->enabledFields);
178   }
179
180   //----------------------------------------------------------------
181   //! Change virtual field group to a new one
182   void CFile::setVirtualFieldGroup(CFieldGroup* newVFieldGroup)
183   {
184      this->vFieldGroup = newVFieldGroup;
185   }
186
187   //! Change virtual variable group to new one
188   void CFile::setVirtualVariableGroup(CVariableGroup* newVVariableGroup)
189   {
190      this->vVariableGroup = newVVariableGroup;
191   }
192
193   //----------------------------------------------------------------
194   bool CFile::isSyncTime(void)
195   {
196     CContext* context = CContext::getCurrent();
197     const CDate& currentDate = context->calendar->getCurrentDate();
198     if (!sync_freq.isEmpty())
199     {
200       if (lastSync + sync_freq.getValue() < currentDate)
201       {
202         lastSync = currentDate;
203         return true;
204        }
205      }
206      return false;
207    }
208
209   //! Initialize a file in order to write into it
210   void CFile::initWrite(void)
211   {
212      CContext* context = CContext::getCurrent();
213      const CDate& currentDate = context->calendar->getCurrentDate();
214      CContextServer* server = context->server;
215
216      lastSync  = currentDate;
217      lastSplit = currentDate;
218      if (!split_freq.isEmpty())
219      {
220        StdString keySuffix("CFile::"+getFileOutputName()+"::") ; 
221        if (context->registryIn->foundKey(keySuffix+"splitStart") && context->registryIn->foundKey(keySuffix+"splitEnd"))
222        {
223          CDate savedSplitStart(*context->getCalendar()), savedSplitEnd(*context->getCalendar());
224          context->registryIn->getKey(keySuffix+"splitStart", savedSplitStart);
225          context->registryIn->getKey(keySuffix+"splitEnd",   savedSplitEnd);
226
227          if (savedSplitStart <= lastSplit && lastSplit <= savedSplitEnd)
228            lastSplit = savedSplitStart;
229        }
230      }
231      isOpen = false;     
232
233//      if (!record_offset.isEmpty() && record_offset < 0)
234//        ERROR("void CFile::initFile(void)",
235//              "Invalid 'record_offset', this attribute cannot be negative.");
236      const int recordOffset = record_offset.isEmpty() ? 0 : record_offset;
237
238      set<StdString> setAxis;
239      set<StdString> setDomains;
240
241      std::vector<CField*>::iterator it, end = this->enabledFields.end();
242      for (it = this->enabledFields.begin(); it != end; it++)
243      {
244         CField* field = *it;         
245         std::vector<CAxis*> vecAxis = field->grid->getAxis();
246         for (size_t i = 0; i < vecAxis.size(); ++i)
247           setAxis.insert(vecAxis[i]->getAxisOutputName());
248         std::vector<CDomain*> vecDomains = field->grid->getDomains();
249         for (size_t i = 0; i < vecDomains.size(); ++i)
250           setDomains.insert(vecDomains[i]->getDomainOutputName());
251
252         field->resetNStep(recordOffset);
253      }
254      nbAxis = setAxis.size();
255      nbDomains = setDomains.size();
256
257      // create sub communicator for file
258      createSubComFile();
259
260      // if (time_counter.isEmpty()) time_counter.setValue(time_counter_attr::centered);
261      if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
262    }
263
264    //! Initialize a file in order to write into it
265    void CFile::initRead(void)
266    {
267      if (checkRead) return;
268      createSubComFile();
269      checkRead = true;
270    }
271
272    /*!
273      Create a sub communicator in which processes participate in reading/opening file
274    */
275    void CFile::createSubComFile()
276    {
277      CContext* context = CContext::getCurrent();
278      CContextServer* server = context->server;
279
280      // create sub communicator for file
281      allZoneEmpty = true;     
282      std::vector<CField*>::iterator it, end = this->enabledFields.end();
283      for (it = this->enabledFields.begin(); it != end; it++)
284      {
285         CField* field = *it;
286         bool nullGrid = (0 == field->grid);
287         allZoneEmpty &= nullGrid ? false : !field->grid->doGridHaveDataToWrite();
288      }
289
290      int color = allZoneEmpty ? 0 : 1;
291      ep_lib::MPI_Comm_split(server->intraComm, color, server->intraCommRank, &fileComm);
292      if (allZoneEmpty) ep_lib::MPI_Comm_free(&fileComm);
293    }
294
295    /*
296       Check condition to write into a file
297       For now, we only use the level-2 server to write files (if this mode is activated)
298       or classical server to do this job.
299    */
300    void CFile::checkWriteFile(void)
301    {
302      CContext* context = CContext::getCurrent();
303      // Done by classical server or secondary server
304      // This condition should be changed soon
305      if (CServer::serverLevel == 0 || CServer::serverLevel == 2)
306      {
307        if (mode.isEmpty() || mode.getValue() == mode_attr::write)
308        {
309          CTimer::get("Files : create headers").resume();
310          if (!isOpen) createHeader();
311          CTimer::get("Files : create headers").suspend();
312          checkSync();
313        }       
314        checkSplit(); // REally need this?
315      }
316    }
317
318    /*
319       Check condition to read from a file
320       For now, we only use the level-1 server to write files (if this mode is activated)
321       or classical server to do this job.
322       This function can be used by client for reading metadata
323    */
324    void CFile::checkReadFile(void)
325    {
326      CContext* context = CContext::getCurrent();
327      // Done by classical server or secondary server
328      // TODO: This condition should be changed soon. It only works with maximum number of level as 2
329      if (CServer::serverLevel == 0 || CServer::serverLevel == 1)
330      {
331        if (!mode.isEmpty() && mode.getValue() == mode_attr::read)
332        {
333          CTimer::get("Files : open headers").resume();
334         
335          if (!isOpen) openInReadMode();
336
337          CTimer::get("Files : open headers").suspend();
338        }
339        //checkSplit(); // Really need for reading?
340      }
341    }
342
343    /*!
344      Verify if a process participates in an opening-file communicator
345      \return true if the process doesn't participate in opening file
346    */
347    bool CFile::isEmptyZone()
348    {
349      return allZoneEmpty;
350    }
351
352    /*!
353    \brief Verify if synchronisation should be done
354        If syn option is enabled, syn frequence and current time will be used to
355    calculate the moment to syn file(s)
356    \return True if it is the moment to synchronize file, otherwise false
357    */
358   bool CFile::checkSync(void)
359   {
360     CContext* context = CContext::getCurrent();
361     const CDate& currentDate = context->calendar->getCurrentDate();
362     if (!sync_freq.isEmpty())
363     {
364       if (lastSync + sync_freq.getValue() <= currentDate)
365       {
366         lastSync = currentDate;
367         data_out->syncFile();
368         return true;
369        }
370      }
371      return false;
372    }
373
374    /*!
375    \brief Verify if splitting should be done
376        If split option is enabled, split frequence and current time will be used to
377    calculate the moment to split file
378    \return True if it is the moment to split file, otherwise false
379    */
380    bool CFile::checkSplit(void)
381    {
382      CContext* context = CContext::getCurrent();
383      const CDate& currentDate = context->calendar->getCurrentDate();
384      if (!split_freq.isEmpty())
385      {
386        if (currentDate > lastSplit + split_freq.getValue())
387        {
388          lastSplit = lastSplit + split_freq.getValue();
389          std::vector<CField*>::iterator it, end = this->enabledFields.end();
390          for (it = this->enabledFields.begin(); it != end; it++)
391          {
392            (*it)->resetNStep();
393            (*it)->resetNStepMax();
394          }
395          if (mode.isEmpty() || mode.getValue() == mode_attr::write)
396            createHeader();
397          else
398            openInReadMode();
399          return true;
400        }
401      }
402      return false;
403    }
404
405   /*!
406   \brief Create header of netcdf file
407   There are some information to fill in header of each netcdf.
408   */
409   void CFile::createHeader(void)
410   {
411      CContext* context = CContext::getCurrent();
412      CContextServer* server = context->server;
413
414      if (!allZoneEmpty)
415      {
416         StdString filename = getFileOutputName();
417
418// determine splitting format in the file name  : firstPart%start_date%middlePart%end_date%lastPart
419
420         std::string strStartDate="%start_date%" ;
421         std::string strEndDate="%end_date%" ;
422
423         std::string firstPart ;
424         std::string middlePart ;
425         std::string lastPart ;
426         size_t pos1, pos2 ;
427         bool hasStartDate=false ;
428         bool hasEndDate=false ;
429         bool hasSplit = (!split_freq.isEmpty());
430                 
431         pos1=filename.find(strStartDate) ;
432         if (pos1!=std::string::npos)
433         {
434           firstPart=filename.substr(0,pos1) ;
435           pos1+=strStartDate.size() ;
436           hasStartDate=true ;
437         }
438         else pos1=0 ;
439
440         pos2=filename.find(strEndDate,pos1) ;
441         if (pos2!=std::string::npos)
442         {
443           middlePart=filename.substr(pos1,pos2-pos1) ;
444           pos2+=strEndDate.size() ;
445           lastPart=filename.substr(pos2,filename.size()-pos2) ;
446           hasEndDate=true ;
447         }
448         else middlePart=filename.substr(pos1,filename.size()) ;
449
450         if (!hasStartDate && !hasEndDate)
451         {
452           hasStartDate=true ;
453           hasEndDate=true;
454           firstPart=middlePart ;
455           if (hasSplit) firstPart +="_";
456           middlePart="-" ;
457         }
458   
459         StdOStringStream oss;
460
461         if (!split_freq.isEmpty())
462         {
463           CDate split_start ;
464           CDate splitEnd ;
465           if (!split_start_offset.isEmpty()) split_start=lastSplit + split_start_offset ;
466           else split_start=lastSplit ;
467
468           splitEnd = lastSplit + split_freq ;
469           if (!split_last_date.isEmpty())
470           {
471             CDate splitLastDate=CDate::FromString(split_last_date,*CContext::getCurrent()->getCalendar()) ;
472             if( splitLastDate < splitEnd)  splitEnd=splitLastDate ;
473           }
474           
475           if (!split_end_offset.isEmpty()) splitEnd = splitEnd + split_end_offset;
476           else splitEnd = splitEnd - 1 * Second;
477
478           string splitFormat;
479           if (split_freq_format.isEmpty())
480           {
481             if (split_freq.getValue().second != 0) splitFormat = "%y%mo%d%h%mi%s";
482             else if (split_freq.getValue().minute != 0) splitFormat = "%y%mo%d%h%mi";
483             else if (split_freq.getValue().hour != 0) splitFormat = "%y%mo%d%h";
484             else if (split_freq.getValue().day != 0) splitFormat = "%y%mo%d";
485             else if (split_freq.getValue().month != 0) splitFormat = "%y%mo";
486             else splitFormat = "%y";
487           }
488           else splitFormat = split_freq_format;
489
490           oss << firstPart ;
491           if (hasStartDate) oss << split_start.getStr(splitFormat) ;
492           oss << middlePart ;
493           if (hasEndDate) oss << splitEnd.getStr(splitFormat);
494           oss << lastPart ;
495
496           StdString keySuffix("CFile::"+getFileOutputName()+"::") ; 
497           context->registryOut->setKey(keySuffix+"splitStart", lastSplit);
498           context->registryOut->setKey(keySuffix+"splitEnd",   splitEnd);
499         }
500         else oss<<firstPart<<lastPart ;
501
502        bool append = !this->append.isEmpty() && this->append.getValue();
503
504         bool useClassicFormat = !format.isEmpty() && format == format_attr::netcdf4_classic;
505         bool useCFConvention = convention.isEmpty() || convention == convention_attr::CF;
506
507         bool multifile = true;
508         if (!type.isEmpty())
509         {
510           if (type == type_attr::one_file) multifile = false;
511           else if (type == type_attr::multiple_file) multifile = true;
512
513         }
514#ifndef USING_NETCDF_PAR
515         if (!multifile)
516         {
517            info(0) << "!!! Warning -> Using non parallel version of netcdf, switching in multiple_file mode for file : " << filename << " ..." << endl;
518            multifile = true;
519          }
520#endif
521         if (multifile)
522         {
523            int commSize, commRank;
524            ep_lib::MPI_Comm_size(fileComm, &commSize);
525            ep_lib::MPI_Comm_rank(fileComm, &commRank);
526
527            if (server->intraCommSize > 1)
528            {
529              oss << "_" ;
530              int width=0; int n = commSize-1;
531              while (n != 0) { n = n / 10; width++;}
532              if (!min_digits.isEmpty())
533                if (width < min_digits) width = min_digits;
534              oss.width(width);
535              oss.fill('0');
536              oss << right << commRank;
537            }
538         }
539         oss << ".nc";
540
541         bool isCollective = par_access.isEmpty() ||  par_access == par_access_attr::collective;
542
543         if (isOpen) data_out->closeFile();
544
545        data_out = boost::shared_ptr<CDataOutput>(new CNc4DataOutput(this, oss.str(), append, useClassicFormat, useCFConvention,
546                                                              fileComm, multifile, isCollective, time_counter_name));
547        isOpen = true;
548
549        data_out->writeFile(CFile::get(this));
550
551        // Do not recreate the file structure if opening an existing file
552        if (!data_out->IsInAppendMode())
553        {
554          std::vector<CField*>::iterator it, end = this->enabledFields.end();
555          for (it = this->enabledFields.begin(); it != end; it++)
556          {
557            CField* field = *it;
558            this->data_out->writeFieldGrid(field);
559          }
560          this->data_out->writeTimeDimension();
561
562          for (it = this->enabledFields.begin(); it != end; it++)
563          {
564            CField* field = *it;
565            this->data_out->writeFieldTimeAxis(field);
566          }
567         
568          for (it = this->enabledFields.begin(); it != end; it++)
569          {
570            CField* field = *it;
571            this->data_out->writeField(field);
572          }
573
574          vector<CVariable*> listVars = getAllVariables();
575          for (vector<CVariable*>::iterator it = listVars.begin(); it != listVars.end(); it++)
576            this->data_out->writeAttribute(*it);
577
578          this->data_out->definition_end();
579        }
580        else
581        {
582          // check time axis even in append mode
583          std::vector<CField*>::iterator it, end = this->enabledFields.end();
584          for (it = this->enabledFields.begin(); it != end; it++)
585          {
586            CField* field = *it;
587            this->data_out->writeFieldTimeAxis(field);
588          }
589        }
590      }
591   }
592
593  /*!
594  \brief Open an existing NetCDF file in read-only mode
595  */
596  void CFile::openInReadMode()
597  {
598    CContext* context = CContext::getCurrent();
599    CContextServer* server = context->server;
600    ep_lib::MPI_Comm readComm = this->fileComm;
601
602    if (!allZoneEmpty)
603    {
604      StdString filename = getFileOutputName();
605      StdOStringStream oss;
606      oss << filename;
607
608      if (!split_freq.isEmpty())
609      {
610        string splitFormat;
611        if (split_freq_format.isEmpty())
612        {
613          if (split_freq.getValue().second != 0) splitFormat = "%y%mo%d%h%mi%s";
614          else if (split_freq.getValue().minute != 0) splitFormat = "%y%mo%d%h%mi";
615          else if (split_freq.getValue().hour != 0) splitFormat = "%y%mo%d%h";
616          else if (split_freq.getValue().day != 0) splitFormat = "%y%mo%d";
617          else if (split_freq.getValue().month != 0) splitFormat = "%y%mo";
618          else splitFormat = "%y";
619        }
620        else splitFormat = split_freq_format;
621        oss << "_" << lastSplit.getStr(splitFormat)
622        << "-" << (lastSplit + split_freq.getValue() - 1 * Second).getStr(splitFormat);
623      }
624
625      bool multifile = true;
626      if (!type.isEmpty())
627      {
628        if (type == type_attr::one_file) multifile = false;
629        else if (type == type_attr::multiple_file) multifile = true;
630      }
631  #ifndef USING_NETCDF_PAR
632      if (!multifile)
633      {
634        info(0) << "!!! Warning -> Using non parallel version of netcdf, switching in multiple_file mode for file : " << filename << " ..." << endl;
635        multifile = true;
636      }
637  #endif
638      if (multifile)
639      {
640        int commSize, commRank;
641        ep_lib::MPI_Comm_size(readComm, &commSize);
642        ep_lib::MPI_Comm_rank(readComm, &commRank);
643
644        if (server->intraCommSize > 1)
645        {
646          oss << "_";
647          int width = 0, n = commSize - 1;
648          while (n != 0) { n = n / 10; width++; }
649          if (!min_digits.isEmpty() && width < min_digits)
650            width = min_digits;
651          oss.width(width);
652          oss.fill('0');
653          oss << right << commRank;
654        }
655      }
656      oss << ".nc";
657
658      bool isCollective = par_access.isEmpty() || par_access == par_access_attr::collective;
659      bool  readMetaDataPar = true;
660      if (!context->hasServer) readMetaDataPar = (read_metadata_par.isEmpty()) ? false : read_metadata_par;
661     
662      if (isOpen) data_out->closeFile();
663      //if (time_counter_name.isEmpty()) data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective));
664      //else data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, time_counter_name));
665      bool ugridConvention = !convention.isEmpty() ? (convention == convention_attr::UGRID) : false;
666      if (time_counter_name.isEmpty())
667        data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, readMetaDataPar, ugridConvention));
668      else
669        data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, readMetaDataPar, ugridConvention, time_counter_name));
670
671      isOpen = true;
672    }
673  }
674
675   //! Close file
676   void CFile::close(void)
677   {
678     if (!allZoneEmpty)
679       if (isOpen)
680       {
681         if (mode.isEmpty() || mode.getValue() == mode_attr::write)
682          this->data_out->closeFile();
683         else
684          this->data_in->closeFile();
685        isOpen = false;
686       }
687      //if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
688   }
689   //----------------------------------------------------------------
690
691   void CFile::readAttributesOfEnabledFieldsInReadMode()
692   {
693     if (enabledFields.empty()) return;
694
695     // Just check file and try to open it
696     if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
697
698     checkReadFile();
699
700     for (int idx = 0; idx < enabledFields.size(); ++idx)
701     {
702        // First of all, find out which domain and axis associated with this field
703        enabledFields[idx]->solveGridReference();
704
705        // Read attributes of domain and axis from this file
706        this->data_in->readFieldAttributesMetaData(enabledFields[idx]);
707
708        // Now complete domain and axis associated with this field
709        enabledFields[idx]->solveGenerateGrid();
710
711        // Read necessary value from file
712        #pragma omp critical (_func)
713        this->data_in->readFieldAttributesValues(enabledFields[idx]);
714
715        // Fill attributes for base reference
716        enabledFields[idx]->solveGridDomainAxisBaseRef();
717     }
718
719     // Now everything is ok, close it
720     //close();
721   }
722
723
724   /*!
725   \brief Parse xml file and write information into file object
726   \param [in] node xmld node corresponding in xml file
727   */
728   void CFile::parse(xml::CXMLNode & node)
729   {
730      SuperClass::parse(node);
731
732      if (node.goToChildElement())
733      {
734        do
735        {
736           if (node.getElementName()=="field" || node.getElementName()=="field_group") this->getVirtualFieldGroup()->parseChild(node);
737           else if (node.getElementName()=="variable" || node.getElementName()=="variable_group") this->getVirtualVariableGroup()->parseChild(node);
738        } while (node.goToNextElement());
739        node.goToParentElement();
740      }
741
742   }
743   //----------------------------------------------------------------
744
745   /*!
746   \brief Represent a file in form of string with all its info
747   \return String
748   */
749   StdString CFile::toString(void) const
750   {
751      StdOStringStream oss;
752
753      oss << "<" << CFile::GetName() << " ";
754      if (this->hasId())
755         oss << " id=\"" << this->getId() << "\" ";
756      oss << SuperClassAttribute::toString() << ">" << std::endl;
757      if (this->getVirtualFieldGroup() != NULL)
758         oss << *this->getVirtualFieldGroup() << std::endl;
759      oss << "</" << CFile::GetName() << " >";
760      return (oss.str());
761   }
762
763   //----------------------------------------------------------------
764
765   /*!
766   \brief Find all inheritace among objects in a file.
767   \param [in] apply (true) write attributes of parent into ones of child if they are empty
768                     (false) write attributes of parent into a new container of child
769   \param [in] parent
770   */
771   void CFile::solveDescInheritance(bool apply, const CAttributeMap * const parent)
772   {
773      SuperClassAttribute::setAttributes(parent,apply);
774      this->getVirtualFieldGroup()->solveDescInheritance(apply, NULL);
775      this->getVirtualVariableGroup()->solveDescInheritance(apply, NULL);
776   }
777
778   //----------------------------------------------------------------
779
780   /*!
781   \brief Resolve all reference of active fields.
782      In order to know exactly which data each active field has, a search for all its
783   reference to find its parents or/and its base reference object must be done. Moreover
784   during this search, there are some information that can only be sent to server AFTER
785   all information of active fields are created on server side, e.g: checking mask or index
786   \param [in] sendToServer: Send all info to server (true) or only a part of it (false)
787   */
788   void CFile::solveOnlyRefOfEnabledFields(bool sendToServer)
789   {
790     int size = this->enabledFields.size();
791     for (int i = 0; i < size; ++i)
792     {
793       this->enabledFields[i]->solveOnlyReferenceEnabledField(sendToServer);
794     }
795   }
796
797   void CFile::checkGridOfEnabledFields()
798   { 
799     int size = this->enabledFields.size();
800     for (int i = 0; i < size; ++i)
801     {
802       this->enabledFields[i]->checkGridOfEnabledFields();
803     }
804   }
805
806   void CFile::sendGridComponentOfEnabledFields()
807   { 
808     int size = this->enabledFields.size();
809     for (int i = 0; i < size; ++i)
810     {
811       this->enabledFields[i]->sendGridComponentOfEnabledFields();
812     }
813   }
814
815   void CFile::sendGridOfEnabledFields()
816   { 
817     int size = this->enabledFields.size();
818     for (int i = 0; i < size; ++i)
819     {
820       this->enabledFields[i]->sendGridOfEnabledFields();
821     }
822   }
823
824   void CFile::generateNewTransformationGridDest()
825   {
826     int size = this->enabledFields.size();
827     for (int i = 0; i < size; ++i)
828     {
829       this->enabledFields[i]->generateNewTransformationGridDest();
830     }
831   }
832
833   /*!
834   \brief Resolve all reference of active fields.
835      In order to know exactly which data each active field has, a search for all its
836   reference to find its parents or/and its base reference object must be done. Moreover
837   during this search, there are some information that can only be sent to server AFTER
838   all information of active fields are created on server side, e.g: checking mask or index
839   \param [in] sendToServer: Send all info to server (true) or only a part of it (false)
840   */
841   void CFile::solveAllRefOfEnabledFieldsAndTransform(bool sendToServer)
842   {
843     int size = this->enabledFields.size();
844     for (int i = 0; i < size; ++i)
845     {
846       this->enabledFields[i]->solveAllEnabledFieldsAndTransform();
847     }
848   }
849
850   /*!
851    * Constructs the filter graph for each active field.
852    *
853    * \param gc the garbage collector to use when building the filter graph
854    */
855   void CFile::buildFilterGraphOfEnabledFields(CGarbageCollector& gc)
856   {
857     int size = this->enabledFields.size();
858     for (int i = 0; i < size; ++i)
859     {
860       this->enabledFields[i]->buildFilterGraph(gc, true);
861     }
862   }
863
864   /*!
865    * Post-process the filter graph for each active field.
866    */
867   void CFile::postProcessFilterGraph()
868   {
869     int size = this->enabledFields.size();
870     for (int i = 0; i < size; ++i)
871     {
872       this->enabledFields[i]->checkIfMustAutoTrigger();
873     }
874   }
875
876   /*!
877     Prefetching the data for enabled fields read from file.
878   */
879   void CFile::prefetchEnabledReadModeFields(void)
880   {
881     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
882       return;
883
884     int size = this->enabledFields.size();
885     for (int i = 0; i < size; ++i)
886       this->enabledFields[i]->sendReadDataRequest(CContext::getCurrent()->getCalendar()->getCurrentDate());
887   }
888
889   /*!
890     Do all pre timestep operations for enabled fields in read mode:
891      - Check that the data excepted from server has been received
892      - Check if some filters must auto-trigger
893   */
894   void CFile::doPreTimestepOperationsForEnabledReadModeFields(void)
895   {
896     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
897       return;
898
899     int size = this->enabledFields.size();
900     for (int i = 0; i < size; ++i)
901     {
902       this->enabledFields[i]->checkForLateDataFromServer();
903       this->enabledFields[i]->autoTriggerIfNeeded();
904     }
905   }
906
907   /*!
908     Do all post timestep operations for enabled fields in read mode:
909      - Prefetch the data read from file when needed
910   */
911   void CFile::doPostTimestepOperationsForEnabledReadModeFields(void)
912   {
913     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
914       return;
915
916     int size = this->enabledFields.size();
917     for (int i = 0; i < size; ++i)
918     {
919       this->enabledFields[i]->sendReadDataRequestIfNeeded();
920     }
921   }
922
923   void CFile::solveFieldRefInheritance(bool apply)
924   {
925      // Rsolution des hritages par rfrence de chacun des champs contenus dans le fichier.
926      std::vector<CField*> allF = this->getAllFields();
927      for (unsigned int i = 0; i < allF.size(); i++)
928         allF[i]->solveRefInheritance(apply);
929   }
930
931   //----------------------------------------------------------------
932
933   /*!
934   \brief Add a field into file.
935      A field is added into file and it will be written out if the file is enabled and
936   level of this field is smaller than level_output. A new field won't be created if one
937   with id has already existed
938   \param [in] id String identity of new field
939   \return Pointer to added (or already existed) field
940   */
941   CField* CFile::addField(const string& id)
942   {
943     return vFieldGroup->createChild(id);
944   }
945
946   /*!
947   \brief Add a field group into file.
948      A field group is added into file and it will play a role as parents for fields.
949   A new field group won't be created if one with id has already existed
950   \param [in] id String identity of new field group
951   \return Pointer to added (or already existed) field group
952   */
953   CFieldGroup* CFile::addFieldGroup(const string& id)
954   {
955     return vFieldGroup->createChildGroup(id);
956   }
957
958   /*!
959   \brief Add a variable into file.
960      A variable is added into file and if one with id has already existed, pointer to
961   it will be returned.
962      Variable as long as attributes are information container of file.
963   However, whereas attributes are "fixed" information, variables provides a more flexible way to user
964   to fill in (extra) information for a file.
965   \param [in] id String identity of new variable
966   \return Pointer to added (or already existed) variable
967   */
968   CVariable* CFile::addVariable(const string& id)
969   {
970     return vVariableGroup->createChild(id);
971   }
972
973   /*!
974   \brief Add a variable group into file.
975      A variable group is added into file and it will play a role as parents for variables.
976   A new variable group won't be created if one with id has already existed
977   \param [in] id String identity of new variable group
978   \return Pointer to added (or already existed) variable group
979   */
980   CVariableGroup* CFile::addVariableGroup(const string& id)
981   {
982     return vVariableGroup->createChildGroup(id);
983   }
984
985   void CFile::setContextClient(CContextClient* newContextClient)
986   {
987     client = newContextClient;
988     size_t size = this->enabledFields.size();
989     for (size_t i = 0; i < size; ++i)
990     {
991       this->enabledFields[i]->setContextClient(newContextClient);
992     }
993   }
994
995   CContextClient* CFile::getContextClient()
996   {
997     return client;
998   }
999
1000   void CFile::setReadContextClient(CContextClient* readContextclient)
1001   {
1002     read_client = readContextclient;
1003   }
1004
1005   CContextClient* CFile::getReadContextClient()
1006   {
1007     return read_client;
1008   }
1009
1010   /*!
1011   \brief Send a message to create a field on server side
1012   \param[in] id String identity of field that will be created on server
1013   */
1014   void CFile::sendAddField(const string& id, CContextClient* client)
1015   {
1016      sendAddItem(id, EVENT_ID_ADD_FIELD, client);
1017   }
1018
1019   /*!
1020   \brief Send a message to create a field group on server side
1021   \param[in] id String identity of field group that will be created on server
1022   */
1023   void CFile::sendAddFieldGroup(const string& id, CContextClient* client)
1024   {
1025      sendAddItem(id, (int)EVENT_ID_ADD_FIELD_GROUP, client);
1026   }
1027
1028   /*!
1029   \brief Receive a message annoucing the creation of a field on server side
1030   \param[in] event Received event
1031   */
1032   void CFile::recvAddField(CEventServer& event)
1033   {
1034
1035      CBufferIn* buffer = event.subEvents.begin()->buffer;
1036      string id;
1037      *buffer>>id;
1038      get(id)->recvAddField(*buffer);
1039   }
1040
1041   /*!
1042   \brief Receive a message annoucing the creation of a field on server side
1043   \param[in] buffer Buffer containing message
1044   */
1045   void CFile::recvAddField(CBufferIn& buffer)
1046   {
1047      string id;
1048      buffer>>id;
1049      addField(id);
1050   }
1051
1052   /*!
1053   \brief Receive a message annoucing the creation of a field group on server side
1054   \param[in] event Received event
1055   */
1056   void CFile::recvAddFieldGroup(CEventServer& event)
1057   {
1058
1059      CBufferIn* buffer = event.subEvents.begin()->buffer;
1060      string id;
1061      *buffer>>id;
1062      get(id)->recvAddFieldGroup(*buffer);
1063   }
1064
1065   /*!
1066   \brief Receive a message annoucing the creation of a field group on server side
1067   \param[in] buffer Buffer containing message
1068   */
1069   void CFile::recvAddFieldGroup(CBufferIn& buffer)
1070   {
1071      string id;
1072      buffer>>id;
1073      addFieldGroup(id);
1074   }
1075
1076   /*!
1077   \brief Send messages to duplicate all variables on server side
1078      Because each variable has also its attributes. So first thing to do is replicate
1079   all these attributes on server side. Because variable can have a value, the second thing
1080   is to duplicate this value on server, too.
1081   */
1082   void CFile::sendAddAllVariables(CContextClient* client)
1083   {
1084     std::vector<CVariable*> allVar = getAllVariables();
1085     std::vector<CVariable*>::const_iterator it = allVar.begin();
1086     std::vector<CVariable*>::const_iterator itE = allVar.end();
1087
1088     for (; it != itE; ++it)
1089     {
1090       this->sendAddVariable((*it)->getId(), client);
1091       (*it)->sendAllAttributesToServer(client);
1092       (*it)->sendValue(client);
1093     }
1094   }
1095
1096   /*!
1097   \brief Send a message to create a variable group on server side
1098   \param[in] id String identity of variable group that will be created on server
1099   \param [in] client client to which we will send this adding action
1100   */
1101   void CFile::sendAddVariableGroup(const string& id, CContextClient* client)
1102   {
1103      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE_GROUP, client);
1104   }
1105
1106   /*
1107     Send message to add a variable into a file within a certain client
1108     \param [in] id String identity of a variable
1109     \param [in] client client to which we will send this adding action
1110   */
1111   void CFile::sendAddVariable(const string& id, CContextClient* client)
1112   {
1113      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE, client);
1114   }
1115
1116   /*!
1117   \brief Receive a message annoucing the creation of a variable on server side
1118   \param[in] event Received event
1119   */
1120   void CFile::recvAddVariable(CEventServer& event)
1121   {
1122
1123      CBufferIn* buffer = event.subEvents.begin()->buffer;
1124      string id;
1125      *buffer>>id;
1126      get(id)->recvAddVariable(*buffer);
1127   }
1128
1129   /*!
1130   \brief Receive a message annoucing the creation of a variable on server side
1131   \param[in] buffer Buffer containing message
1132   */
1133   void CFile::recvAddVariable(CBufferIn& buffer)
1134   {
1135      string id;
1136      buffer>>id;
1137      addVariable(id);
1138   }
1139
1140   /*!
1141   \brief Receive a message annoucing the creation of a variable group on server side
1142   \param[in] event Received event
1143   */
1144   void CFile::recvAddVariableGroup(CEventServer& event)
1145   {
1146
1147      CBufferIn* buffer = event.subEvents.begin()->buffer;
1148      string id;
1149      *buffer>>id;
1150      get(id)->recvAddVariableGroup(*buffer);
1151   }
1152
1153   /*!
1154   \brief Receive a message annoucing the creation of a variable group on server side
1155   \param[in] buffer Buffer containing message
1156   */
1157   void CFile::recvAddVariableGroup(CBufferIn& buffer)
1158   {
1159      string id;
1160      buffer>>id;
1161      addVariableGroup(id);
1162   }
1163
1164   /*!
1165     \brief Sending all active (enabled) fields from client to server.
1166   Each field is identified uniquely by its string identity. Not only should we
1167   send the id to server but also we need to send ids of reference domain and reference axis.
1168   With these two id, it's easier to make reference to grid where all data should be written.
1169   Remark: This function must be called AFTER all active (enabled) files have been created on the server side
1170   */
1171   void CFile::sendEnabledFields(CContextClient* client)
1172   {
1173     size_t size = this->enabledFields.size();
1174     for (size_t i = 0; i < size; ++i)
1175     {
1176       CField* field = this->enabledFields[i];
1177       this->sendAddField(field->getId(), client);
1178       field->checkTimeAttributes();
1179       field->sendAllAttributesToServer(client);
1180       field->sendAddAllVariables(client);
1181     }
1182   }
1183
1184
1185   /*!
1186   \brief Dispatch event received from client
1187      Whenever a message is received in buffer of server, it will be processed depending on
1188   its event type. A new event type should be added in the switch list to make sure
1189   it processed on server side.
1190   \param [in] event: Received message
1191   */
1192   bool CFile::dispatchEvent(CEventServer& event)
1193   {
1194      if (SuperClass::dispatchEvent(event)) return true;
1195      else
1196      {
1197        switch(event.type)
1198        {
1199           case EVENT_ID_ADD_FIELD :
1200             recvAddField(event);
1201             return true;
1202             break;
1203
1204           case EVENT_ID_ADD_FIELD_GROUP :
1205             recvAddFieldGroup(event);
1206             return true;
1207             break;
1208
1209            case EVENT_ID_ADD_VARIABLE :
1210             recvAddVariable(event);
1211             return true;
1212             break;
1213
1214           case EVENT_ID_ADD_VARIABLE_GROUP :
1215             recvAddVariableGroup(event);
1216             return true;
1217             break;
1218           default :
1219              ERROR("bool CFile::dispatchEvent(CEventServer& event)", << "Unknown Event");
1220           return false;
1221        }
1222      }
1223   }
1224
1225
1226
1227
1228   ///---------------------------------------------------------------
1229
1230} // namespace xios
Note: See TracBrowser for help on using the repository browser.