source: XIOS/dev/XIOS_DEV_CMIP6/src/node/file.cpp @ 1516

Last change on this file since 1516 was 1516, checked in by oabramkina, 4 years ago

Bugfix for cases of split_freq defined in ts.

  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
File size: 42.0 KB
Line 
1#include "file.hpp"
2
3#include "attribute_template.hpp"
4#include "object_template.hpp"
5#include "group_template.hpp"
6#include "object_factory.hpp"
7#include "context.hpp"
8#include "context_server.hpp"
9#include "nc4_data_output.hpp"
10#include "nc4_data_input.hpp"
11#include "calendar_util.hpp"
12#include "date.hpp"
13#include "message.hpp"
14#include "type.hpp"
15#include "xios_spl.hpp"
16#include "context_client.hpp"
17#include "mpi.hpp"
18#include "timer.hpp"
19#include "server.hpp"
20
21namespace xios {
22
23   /// ////////////////////// Dfinitions ////////////////////// ///
24
25   CFile::CFile(void)
26      : CObjectTemplate<CFile>(), CFileAttributes()
27      , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
28      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
29   {
30     setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
31     setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group"));
32   }
33
34   CFile::CFile(const StdString & id)
35      : CObjectTemplate<CFile>(id), CFileAttributes()
36      , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
37      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
38    {
39      setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
40      setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group"));
41    }
42
43   CFile::~CFile(void)
44   { /* Ne rien faire de plus */ }
45
46   ///---------------------------------------------------------------
47  //! Get name of file
48   StdString CFile::GetName(void)   { return (StdString("file")); }
49   StdString CFile::GetDefName(void){ return (CFile::GetName()); }
50   ENodeType CFile::GetType(void)   { return (eFile); }
51
52   //----------------------------------------------------------------
53
54   const StdString CFile::getFileOutputName(void) const
55   {
56     return (name.isEmpty() ? getId() : name) + (name_suffix.isEmpty() ? StdString("") :  name_suffix.getValue());
57   }
58
59   //----------------------------------------------------------------
60   /*!
61   \brief Get data writer object.
62   Each enabled file in xml represents a physical netcdf file.
63   This function allows to access the data writer object.
64   \return data writer object.
65   */
66   boost::shared_ptr<CDataOutput> CFile::getDataOutput(void) const
67   {
68      return data_out;
69   }
70
71   /*!
72   \brief Get data reader object.
73   Each enabled file in xml represents a physical netcdf file.
74   This function allows to access the data reader object.
75   \return data reader object.
76   */
77   boost::shared_ptr<CDataInput> CFile::getDataInput(void) const
78   {
79      return data_in;
80   }
81
82   /*!
83   \brief Get virtual field group
84      In each file, there always exists a field group which is the ancestor of all
85   fields in the file. This is considered be virtual because it is created automatically during
86   file initialization and it normally doesn't appear on xml file
87   \return Pointer to field group
88   */
89   CFieldGroup* CFile::getVirtualFieldGroup(void) const
90   {
91      return (this->vFieldGroup);
92   }
93
94   /*!
95   \brief Get virtual variable group
96      In each file, there always exists a variable group which is the ancestor of all
97   variable in the file. This is considered be virtual because it is created automatically during
98   file initialization and it normally doesn't appear on xml file
99   \return Pointer to variable group
100   */
101   CVariableGroup* CFile::getVirtualVariableGroup(void) const
102   {
103      return (this->vVariableGroup);
104   }
105
106   //! Get all fields of a file
107   std::vector<CField*> CFile::getAllFields(void) const
108   {
109      return (this->vFieldGroup->getAllChildren());
110   }
111
112   //! Get all variables of a file
113   std::vector<CVariable*> CFile::getAllVariables(void) const
114   {
115      return (this->vVariableGroup->getAllChildren());
116   }
117
118   //----------------------------------------------------------------
119   /*!
120   \brief Get all enabled fields of file
121      A field is considered to be enabled if it fullfil these conditions: it is enabled, inside a enabled file
122   and its own level is not larger than file output level.
123   \param [in] default_outputlevel default value output level of file
124   \param [in] default_level default value level of field
125   \param [in] default_enabled flag determine by default if field is enabled
126   \return Vector of pointers of enabled fields
127   */
128   std::vector<CField*> CFile::getEnabledFields(int default_outputlevel,
129                                                int default_level,
130                                                bool default_enabled)
131   {
132      if (!this->enabledFields.empty())
133         return (this->enabledFields);
134
135      const int _outputlevel =
136         (!output_level.isEmpty()) ? output_level.getValue() : default_outputlevel;
137      std::vector<CField*>::iterator it;
138      this->enabledFields = this->getAllFields();
139
140      std::vector<CField*> newEnabledFields;
141
142      for ( it = this->enabledFields.begin(); it != this->enabledFields.end(); it++ )
143      {
144         if (!(*it)->enabled.isEmpty()) // Si l'attribut 'enabled' est dfini ...
145         {
146            if (! (*it)->enabled.getValue()) continue;
147//            { it--; this->enabledFields.erase(it+1); continue; }
148         }
149         else // Si l'attribut 'enabled' n'est pas dfini ...
150         {
151            if (!default_enabled) continue;
152//            { it--; this->enabledFields.erase(it+1); continue; }
153         }
154
155         if (!(*it)->level.isEmpty()) // Si l'attribut 'level' est dfini ...
156         {
157            if ((*it)->level.getValue() > _outputlevel) continue;
158//            { it--; this->enabledFields.erase(it+1); continue; }
159         }
160         else // Si l'attribut 'level' n'est pas dfini ...
161         {
162            if (default_level > _outputlevel) continue;
163//            { it--; this->enabledFields.erase(it+1); continue; }
164         }
165
166//         CField* field_tmp=(*it).get();
167//         shared_ptr<CField> sptfield=*it;
168//         field_tmp->refObject.push_back(sptfield);
169         newEnabledFields.push_back(*it);
170         // Le champ est finalement actif, on y ajoute sa propre reference.
171//         (*it)->refObject.push_back(*it);
172         // Le champ est finalement actif, on y ajoute la rfrence au champ de base.
173         (*it)->setRelFile(CFile::get(this));
174      }
175      enabledFields = newEnabledFields;
176
177      return (this->enabledFields);
178   }
179
180   //----------------------------------------------------------------
181   //! Change virtual field group to a new one
182   void CFile::setVirtualFieldGroup(CFieldGroup* newVFieldGroup)
183   {
184      this->vFieldGroup = newVFieldGroup;
185   }
186
187   //! Change virtual variable group to new one
188   void CFile::setVirtualVariableGroup(CVariableGroup* newVVariableGroup)
189   {
190      this->vVariableGroup = newVVariableGroup;
191   }
192
193   //----------------------------------------------------------------
194   bool CFile::isSyncTime(void)
195   {
196     CContext* context = CContext::getCurrent();
197     const CDate& currentDate = context->calendar->getCurrentDate();
198     if (!sync_freq.isEmpty())
199     {
200       if (lastSync + sync_freq.getValue() < currentDate)
201       {
202         lastSync = currentDate;
203         return true;
204        }
205      }
206      return false;
207    }
208
209   //! Initialize a file in order to write into it
210   void CFile::initWrite(void)
211   {
212      CContext* context = CContext::getCurrent();
213      const CDate& currentDate = context->calendar->getCurrentDate();
214      CContextServer* server = context->server;
215
216      lastSync  = currentDate;
217      lastSplit = currentDate;
218      if (!split_freq.isEmpty())
219      {
220        StdString keySuffix("CFile::"+getFileOutputName()+"::") ; 
221        if (context->registryIn->foundKey(keySuffix+"splitStart") && context->registryIn->foundKey(keySuffix+"splitEnd"))
222        {
223          CDate savedSplitStart(*context->getCalendar()), savedSplitEnd(*context->getCalendar());
224          context->registryIn->getKey(keySuffix+"splitStart", savedSplitStart);
225          context->registryIn->getKey(keySuffix+"splitEnd",   savedSplitEnd);
226
227          if (savedSplitStart <= lastSplit && lastSplit <= savedSplitEnd)
228            lastSplit = savedSplitStart;
229        }
230      }
231      isOpen = false;     
232
233//      if (!record_offset.isEmpty() && record_offset < 0)
234//        ERROR("void CFile::initFile(void)",
235//              "Invalid 'record_offset', this attribute cannot be negative.");
236      const int recordOffset = record_offset.isEmpty() ? 0 : record_offset;
237
238      set<StdString> setAxis;
239      set<StdString> setDomains;
240
241      std::vector<CField*>::iterator it, end = this->enabledFields.end();
242      for (it = this->enabledFields.begin(); it != end; it++)
243      {
244         CField* field = *it;         
245         std::vector<CAxis*> vecAxis = field->grid->getAxis();
246         for (size_t i = 0; i < vecAxis.size(); ++i)
247           setAxis.insert(vecAxis[i]->getAxisOutputName());
248         std::vector<CDomain*> vecDomains = field->grid->getDomains();
249         for (size_t i = 0; i < vecDomains.size(); ++i)
250           setDomains.insert(vecDomains[i]->getDomainOutputName());
251
252         field->resetNStep(recordOffset);
253      }
254      nbAxis = setAxis.size();
255      nbDomains = setDomains.size();
256
257      // create sub communicator for file
258      createSubComFile();
259
260      // if (time_counter.isEmpty()) time_counter.setValue(time_counter_attr::centered);
261      if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
262    }
263
264    //! Initialize a file in order to write into it
265    void CFile::initRead(void)
266    {
267      if (checkRead) return;
268      createSubComFile();
269      checkRead = true;
270    }
271
272    /*!
273      Create a sub communicator in which processes participate in reading/opening file
274    */
275    void CFile::createSubComFile()
276    {
277      CContext* context = CContext::getCurrent();
278      CContextServer* server = context->server;
279
280      // create sub communicator for file
281      allZoneEmpty = true;     
282      std::vector<CField*>::iterator it, end = this->enabledFields.end();
283      for (it = this->enabledFields.begin(); it != end; it++)
284      {
285         CField* field = *it;
286         bool nullGrid = (0 == field->grid);
287         allZoneEmpty &= nullGrid ? false : !field->grid->doGridHaveDataToWrite();
288      }
289
290      int color = allZoneEmpty ? 0 : 1;
291      MPI_Comm_split(server->intraComm, color, server->intraCommRank, &fileComm);
292      if (allZoneEmpty) MPI_Comm_free(&fileComm);
293    }
294
295    /*
296       Check condition to write into a file
297       For now, we only use the level-2 server to write files (if this mode is activated)
298       or classical server to do this job.
299    */
300    void CFile::checkWriteFile(void)
301    {
302      CContext* context = CContext::getCurrent();
303      // Done by classical server or secondary server
304      // This condition should be changed soon
305      if (CServer::serverLevel == 0 || CServer::serverLevel == 2)
306      {
307        if (mode.isEmpty() || mode.getValue() == mode_attr::write)
308        {
309          CTimer::get("Files : create headers").resume();
310          if (!isOpen) createHeader();
311          CTimer::get("Files : create headers").suspend();
312          checkSync();
313        }       
314        checkSplit(); // REally need this?
315      }
316    }
317
318    /*
319       Check condition to read from a file
320       For now, we only use the level-1 server to write files (if this mode is activated)
321       or classical server to do this job.
322       This function can be used by client for reading metadata
323    */
324    void CFile::checkReadFile(void)
325    {
326      CContext* context = CContext::getCurrent();
327      // Done by classical server or secondary server
328      // TODO: This condition should be changed soon. It only works with maximum number of level as 2
329      if (CServer::serverLevel == 0 || CServer::serverLevel == 1)
330      {
331        if (!mode.isEmpty() && mode.getValue() == mode_attr::read)
332        {
333          CTimer::get("Files : open headers").resume();
334         
335          if (!isOpen) openInReadMode();
336
337          CTimer::get("Files : open headers").suspend();
338        }
339        //checkSplit(); // Really need for reading?
340      }
341    }
342
343    /*!
344      Verify if a process participates in an opening-file communicator
345      \return true if the process doesn't participate in opening file
346    */
347    bool CFile::isEmptyZone()
348    {
349      return allZoneEmpty;
350    }
351
352    /*!
353    \brief Verify if synchronisation should be done
354        If syn option is enabled, syn frequence and current time will be used to
355    calculate the moment to syn file(s)
356    \return True if it is the moment to synchronize file, otherwise false
357    */
358   bool CFile::checkSync(void)
359   {
360     CContext* context = CContext::getCurrent();
361     const CDate& currentDate = context->calendar->getCurrentDate();
362     if (!sync_freq.isEmpty())
363     {
364       if (lastSync + sync_freq.getValue() <= currentDate)
365       {
366         lastSync = currentDate;
367         data_out->syncFile();
368         return true;
369        }
370      }
371      return false;
372    }
373
374    /*!
375    \brief Verify if splitting should be done
376        If split option is enabled, split frequence and current time will be used to
377    calculate the moment to split file
378    \return True if it is the moment to split file, otherwise false
379    */
380    bool CFile::checkSplit(void)
381    {
382      CContext* context = CContext::getCurrent();
383      const CDate& currentDate = context->calendar->getCurrentDate();
384      if (!split_freq.isEmpty())
385      {
386        if (currentDate > lastSplit + split_freq.getValue())
387        {
388          lastSplit = lastSplit + split_freq.getValue();
389          std::vector<CField*>::iterator it, end = this->enabledFields.end();
390          for (it = this->enabledFields.begin(); it != end; it++)
391          {
392            (*it)->resetNStep();
393            (*it)->resetNStepMax();
394          }
395          if (mode.isEmpty() || mode.getValue() == mode_attr::write)
396            createHeader();
397          else
398            openInReadMode();
399          return true;
400        }
401      }
402      return false;
403    }
404
405   /*!
406   \brief Create header of netcdf file
407   There are some information to fill in header of each netcdf.
408   */
409   void CFile::createHeader(void)
410   {
411      CContext* context = CContext::getCurrent();
412      CContextServer* server = context->server;
413
414      if (!allZoneEmpty)
415      {
416         StdString filename = getFileOutputName();
417
418// determine splitting format in the file name  : firstPart%start_date%middlePart%end_date%lastPart
419
420         std::string strStartDate="%start_date%" ;
421         std::string strEndDate="%end_date%" ;
422
423         std::string firstPart ;
424         std::string middlePart ;
425         std::string lastPart ;
426         size_t pos1, pos2 ;
427         bool hasStartDate=false ;
428         bool hasEndDate=false ;
429         bool hasSplit = (!split_freq.isEmpty());
430                 
431         pos1=filename.find(strStartDate) ;
432         if (pos1!=std::string::npos)
433         {
434           firstPart=filename.substr(0,pos1) ;
435           pos1+=strStartDate.size() ;
436           hasStartDate=true ;
437         }
438         else pos1=0 ;
439
440         pos2=filename.find(strEndDate,pos1) ;
441         if (pos2!=std::string::npos)
442         {
443           middlePart=filename.substr(pos1,pos2-pos1) ;
444           pos2+=strEndDate.size() ;
445           lastPart=filename.substr(pos2,filename.size()-pos2) ;
446           hasEndDate=true ;
447         }
448         else middlePart=filename.substr(pos1,filename.size()) ;
449
450         if (!hasStartDate && !hasEndDate)
451         {
452           hasStartDate=true ;
453           hasEndDate=true;
454           firstPart=middlePart ;
455           if (hasSplit) firstPart +="_";
456           middlePart="-" ;
457         }
458   
459         StdOStringStream oss;
460
461         if (!split_freq.isEmpty())
462         {
463           CDate split_start ;
464           CDate splitEnd ;
465           if (!split_start_offset.isEmpty()) split_start=lastSplit + split_start_offset ;
466           else split_start=lastSplit ;
467
468           splitEnd = lastSplit + split_freq ;
469           if (!split_last_date.isEmpty())
470           {
471             CDate splitLastDate=CDate::FromString(split_last_date,*CContext::getCurrent()->getCalendar()) ;
472             if( splitLastDate < splitEnd)  splitEnd=splitLastDate ;
473           }
474           
475           if (!split_end_offset.isEmpty()) splitEnd = splitEnd + split_end_offset;
476           else splitEnd = splitEnd - 1 * Second;
477
478           string splitFormat;
479           if (split_freq_format.isEmpty())
480           {
481             CDuration splitFreq = split_freq.getValue();
482             splitFreq.solveTimeStep(*CContext::getCurrent()->getCalendar());
483             if (splitFreq.second != 0) splitFormat = "%y%mo%d%h%mi%s";
484             else if (splitFreq.minute != 0) splitFormat = "%y%mo%d%h%mi";
485             else if (splitFreq.hour != 0) splitFormat = "%y%mo%d%h";
486             else if (splitFreq.day != 0) splitFormat = "%y%mo%d";
487             else if (splitFreq.month != 0) splitFormat = "%y%mo";
488             else splitFormat = "%y";
489           }
490           else splitFormat = split_freq_format;
491
492           oss << firstPart ;
493           if (hasStartDate) oss << split_start.getStr(splitFormat) ;
494           oss << middlePart ;
495           if (hasEndDate) oss << splitEnd.getStr(splitFormat);
496           oss << lastPart ;
497
498           StdString keySuffix("CFile::"+getFileOutputName()+"::") ; 
499           context->registryOut->setKey(keySuffix+"splitStart", lastSplit);
500           context->registryOut->setKey(keySuffix+"splitEnd",   splitEnd);
501         }
502         else oss<<firstPart<<lastPart ;
503
504        bool append = !this->append.isEmpty() && this->append.getValue();
505
506         bool useClassicFormat = !format.isEmpty() && format == format_attr::netcdf4_classic;
507         bool useCFConvention = convention.isEmpty() || convention == convention_attr::CF;
508
509         bool multifile = true;
510         if (!type.isEmpty())
511         {
512           if (type == type_attr::one_file) multifile = false;
513           else if (type == type_attr::multiple_file) multifile = true;
514
515         }
516#ifndef USING_NETCDF_PAR
517         if (!multifile)
518         {
519            info(0) << "!!! Warning -> Using non parallel version of netcdf, switching in multiple_file mode for file : " << filename << " ..." << endl;
520            multifile = true;
521          }
522#endif
523         if (multifile)
524         {
525            int commSize, commRank;
526            MPI_Comm_size(fileComm, &commSize);
527            MPI_Comm_rank(fileComm, &commRank);
528
529            if (server->intraCommSize > 1)
530            {
531              oss << "_" ;
532              int width=0; int n = commSize-1;
533              while (n != 0) { n = n / 10; width++;}
534              if (!min_digits.isEmpty())
535                if (width < min_digits) width = min_digits;
536              oss.width(width);
537              oss.fill('0');
538              oss << right << commRank;
539            }
540         }
541         oss << ".nc";
542
543         bool isCollective = par_access.isEmpty() ||  par_access == par_access_attr::collective;
544
545         if (isOpen) data_out->closeFile();
546
547        data_out = shared_ptr<CDataOutput>(new CNc4DataOutput(this, oss.str(), append, useClassicFormat, useCFConvention,
548                                                              fileComm, multifile, isCollective, time_counter_name));
549        isOpen = true;
550
551        data_out->writeFile(CFile::get(this));
552
553        if (!useCFConvention) sortEnabledFieldsForUgrid();
554
555        // Do not recreate the file structure if opening an existing file
556        if (!data_out->IsInAppendMode())
557        {
558          std::vector<CField*>::iterator it, end = this->enabledFields.end();
559          for (it = this->enabledFields.begin(); it != end; it++)
560          {
561            CField* field = *it;
562            this->data_out->writeFieldGrid(field);
563          }
564          this->data_out->writeTimeDimension();
565
566          for (it = this->enabledFields.begin(); it != end; it++)
567          {
568            CField* field = *it;
569            this->data_out->writeFieldTimeAxis(field);
570          }
571         
572          for (it = this->enabledFields.begin(); it != end; it++)
573          {
574            CField* field = *it;
575            this->data_out->writeField(field);
576          }
577
578          vector<CVariable*> listVars = getAllVariables();
579          for (vector<CVariable*>::iterator it = listVars.begin(); it != listVars.end(); it++)
580            this->data_out->writeAttribute(*it);
581
582          this->data_out->definition_end();
583        }
584        else
585        {
586          // check time axis even in append mode
587          std::vector<CField*>::iterator it, end = this->enabledFields.end();
588          for (it = this->enabledFields.begin(); it != end; it++)
589          {
590            CField* field = *it;
591            this->data_out->writeFieldTimeAxis(field);
592          }
593        }
594      }
595   }
596
597  /*!
598  \brief Open an existing NetCDF file in read-only mode
599  */
600  void CFile::openInReadMode()
601  {
602    CContext* context = CContext::getCurrent();
603    CContextServer* server = context->server;
604    MPI_Comm readComm = this->fileComm;
605
606    if (!allZoneEmpty)
607    {
608      StdString filename = getFileOutputName();
609      StdOStringStream oss;
610      oss << filename;
611
612      if (!split_freq.isEmpty())
613      {
614        string splitFormat;
615        if (split_freq_format.isEmpty())
616        {
617          CDuration splitFreq = split_freq.getValue();
618          splitFreq.solveTimeStep(*CContext::getCurrent()->getCalendar());
619          if (splitFreq.second != 0) splitFormat = "%y%mo%d%h%mi%s";
620          else if (splitFreq.minute != 0) splitFormat = "%y%mo%d%h%mi";
621          else if (splitFreq.hour != 0) splitFormat = "%y%mo%d%h";
622          else if (splitFreq.day != 0) splitFormat = "%y%mo%d";
623          else if (splitFreq.month != 0) splitFormat = "%y%mo";
624          else splitFormat = "%y";
625        }
626        else splitFormat = split_freq_format;
627        oss << "_" << lastSplit.getStr(splitFormat)
628        << "-" << (lastSplit + split_freq.getValue() - 1 * Second).getStr(splitFormat);
629      }
630
631      bool multifile = true;
632      if (!type.isEmpty())
633      {
634        if (type == type_attr::one_file) multifile = false;
635        else if (type == type_attr::multiple_file) multifile = true;
636      }
637  #ifndef USING_NETCDF_PAR
638      if (!multifile)
639      {
640        info(0) << "!!! Warning -> Using non parallel version of netcdf, switching in multiple_file mode for file : " << filename << " ..." << endl;
641        multifile = true;
642      }
643  #endif
644      if (multifile)
645      {
646        int commSize, commRank;
647        MPI_Comm_size(readComm, &commSize);
648        MPI_Comm_rank(readComm, &commRank);
649
650        if (server->intraCommSize > 1)
651        {
652          oss << "_";
653          int width = 0, n = commSize - 1;
654          while (n != 0) { n = n / 10; width++; }
655          if (!min_digits.isEmpty() && width < min_digits)
656            width = min_digits;
657          oss.width(width);
658          oss.fill('0');
659          oss << right << commRank;
660        }
661      }
662      oss << ".nc";
663
664      bool isCollective = par_access.isEmpty() || par_access == par_access_attr::collective;
665      bool readMetaDataPar = true;
666      if (!context->hasServer) readMetaDataPar = (read_metadata_par.isEmpty()) ? false : read_metadata_par;
667
668      if (isOpen) data_out->closeFile();
669      bool ugridConvention = !convention.isEmpty() ? (convention == convention_attr::UGRID) : false;
670      if (time_counter_name.isEmpty())
671        data_in = shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, readMetaDataPar, ugridConvention));
672      else
673        data_in = shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, readMetaDataPar, ugridConvention, time_counter_name));
674      isOpen = true;
675    }
676  }
677
678   //! Close file
679   void CFile::close(void)
680   {
681     if (!allZoneEmpty)
682       if (isOpen)
683       {
684         if (mode.isEmpty() || mode.getValue() == mode_attr::write)
685          this->data_out->closeFile();
686         else
687          this->data_in->closeFile();
688        isOpen = false;
689       }
690      if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
691   }
692   //----------------------------------------------------------------
693
694   void CFile::readAttributesOfEnabledFieldsInReadMode()
695   {
696     if (enabledFields.empty()) return;
697
698     // Just check file and try to open it
699     if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
700
701     checkReadFile();
702
703     for (int idx = 0; idx < enabledFields.size(); ++idx)
704     {
705        // First of all, find out which domain and axis associated with this field
706        enabledFields[idx]->solveGridReference();
707
708        // Read attributes of domain and axis from this file
709        this->data_in->readFieldAttributesMetaData(enabledFields[idx]);
710
711        // Now complete domain and axis associated with this field
712        enabledFields[idx]->solveGenerateGrid();
713
714        // Read necessary value from file
715        this->data_in->readFieldAttributesValues(enabledFields[idx]);
716
717        // Fill attributes for base reference
718        enabledFields[idx]->solveGridDomainAxisBaseRef();
719     }
720
721     // Now everything is ok, close it
722     close();
723   }
724
725
726   /*!
727   \brief Parse xml file and write information into file object
728   \param [in] node xmld node corresponding in xml file
729   */
730   void CFile::parse(xml::CXMLNode & node)
731   {
732      SuperClass::parse(node);
733
734      if (node.goToChildElement())
735      {
736        do
737        {
738           if (node.getElementName()=="field" || node.getElementName()=="field_group") this->getVirtualFieldGroup()->parseChild(node);
739           else if (node.getElementName()=="variable" || node.getElementName()=="variable_group") this->getVirtualVariableGroup()->parseChild(node);
740        } while (node.goToNextElement());
741        node.goToParentElement();
742      }
743
744   }
745   //----------------------------------------------------------------
746
747   /*!
748   \brief Represent a file in form of string with all its info
749   \return String
750   */
751   StdString CFile::toString(void) const
752   {
753      StdOStringStream oss;
754
755      oss << "<" << CFile::GetName() << " ";
756      if (this->hasId())
757         oss << " id=\"" << this->getId() << "\" ";
758      oss << SuperClassAttribute::toString() << ">" << std::endl;
759      if (this->getVirtualFieldGroup() != NULL)
760         oss << *this->getVirtualFieldGroup() << std::endl;
761      oss << "</" << CFile::GetName() << " >";
762      return (oss.str());
763   }
764
765   //----------------------------------------------------------------
766
767   /*!
768   \brief Find all inheritace among objects in a file.
769   \param [in] apply (true) write attributes of parent into ones of child if they are empty
770                     (false) write attributes of parent into a new container of child
771   \param [in] parent
772   */
773   void CFile::solveDescInheritance(bool apply, const CAttributeMap * const parent)
774   {
775      SuperClassAttribute::setAttributes(parent,apply);
776      this->getVirtualFieldGroup()->solveDescInheritance(apply, NULL);
777      this->getVirtualVariableGroup()->solveDescInheritance(apply, NULL);
778   }
779
780   //----------------------------------------------------------------
781
782   /*!
783   \brief Resolve all reference of active fields.
784      In order to know exactly which data each active field has, a search for all its
785   reference to find its parents or/and its base reference object must be done. Moreover
786   during this search, there are some information that can only be sent to server AFTER
787   all information of active fields are created on server side, e.g: checking mask or index
788   \param [in] sendToServer: Send all info to server (true) or only a part of it (false)
789   */
790   void CFile::solveOnlyRefOfEnabledFields(bool sendToServer)
791   {
792     int size = this->enabledFields.size();
793     for (int i = 0; i < size; ++i)
794     {
795       this->enabledFields[i]->solveOnlyReferenceEnabledField(sendToServer);
796     }
797   }
798
799   void CFile::checkGridOfEnabledFields()
800   { 
801     int size = this->enabledFields.size();
802     for (int i = 0; i < size; ++i)
803     {
804       this->enabledFields[i]->checkGridOfEnabledFields();
805     }
806   }
807
808   void CFile::sendGridComponentOfEnabledFields()
809   { 
810     int size = this->enabledFields.size();
811     for (int i = 0; i < size; ++i)
812     {
813       this->enabledFields[i]->sendGridComponentOfEnabledFields();
814     }
815   }
816
817   /*!
818   \brief Sorting domains with the same name (= describing the same mesh) in the decreasing order of nvertex for UGRID files.
819   This insures that the domain with the highest nvertex is written first and thus all known mesh connectivity is generated at once by this domain.
820   */
821   void CFile::sortEnabledFieldsForUgrid()
822   {
823     int size = this->enabledFields.size();
824     std::vector<int> domainNvertices;
825     std::vector<StdString> domainNames;
826
827     for (int i = 0; i < size; ++i)
828     {
829       std::vector<CDomain*> domain = this->enabledFields[i]->getRelGrid()->getDomains();
830       if (domain.size() != 1)
831       {
832         ERROR("void CFile::sortEnabledFieldsForUgrid()",
833               "A domain, and only one, should be defined for grid "<< this->enabledFields[i]->getRelGrid()->getId() << ".");
834       }
835       StdString domainName = domain[0]->getDomainOutputName();
836       int nvertex;
837       if (domain[0]->nvertex.isEmpty())
838       {
839         ERROR("void CFile::sortEnabledFieldsForUgrid()",
840               "Attributes nvertex must be defined for domain "<< domain[0]->getDomainOutputName() << ".");
841       }
842       else
843         nvertex = domain[0]->nvertex;
844
845       for (int j = 0; j < i; ++j)
846       {
847         if (domainName == domainNames[j] && nvertex > domainNvertices[j])
848         {
849           CField* tmpSwap = this->enabledFields[j];
850           this->enabledFields[j] = this->enabledFields[i];
851           this->enabledFields[i] = tmpSwap;
852           domainNames.push_back(domainNames[j]);
853           domainNames[j] = domainName;
854           domainNvertices.push_back(domainNvertices[j]);
855           domainNvertices[j] = nvertex;
856         }
857         else
858         {
859           domainNames.push_back(domainName);
860           domainNvertices.push_back(nvertex);
861         }
862       }
863       if (i==0)
864       {
865         domainNames.push_back(domainName);
866         domainNvertices.push_back(nvertex);
867       }
868     }
869   }
870
871   void CFile::sendGridOfEnabledFields()
872   { 
873     int size = this->enabledFields.size();
874     for (int i = 0; i < size; ++i)
875     {
876       this->enabledFields[i]->sendGridOfEnabledFields();
877     }
878   }
879
880   void CFile::generateNewTransformationGridDest()
881   {
882     int size = this->enabledFields.size();
883     for (int i = 0; i < size; ++i)
884     {
885       this->enabledFields[i]->generateNewTransformationGridDest();
886     }
887   }
888
889   /*!
890   \brief Resolve all reference of active fields.
891      In order to know exactly which data each active field has, a search for all its
892   reference to find its parents or/and its base reference object must be done. Moreover
893   during this search, there are some information that can only be sent to server AFTER
894   all information of active fields are created on server side, e.g: checking mask or index
895   \param [in] sendToServer: Send all info to server (true) or only a part of it (false)
896   */
897   void CFile::solveAllRefOfEnabledFieldsAndTransform(bool sendToServer)
898   {
899     int size = this->enabledFields.size();
900     for (int i = 0; i < size; ++i)
901     {       
902      this->enabledFields[i]->solveAllEnabledFieldsAndTransform();
903     }
904   }
905
906   /*!
907    * Constructs the filter graph for each active field.
908    *
909    * \param gc the garbage collector to use when building the filter graph
910    */
911   void CFile::buildFilterGraphOfEnabledFields(CGarbageCollector& gc)
912   {
913     int size = this->enabledFields.size();
914     for (int i = 0; i < size; ++i)
915     {
916       this->enabledFields[i]->buildFilterGraph(gc, true);
917     }
918   }
919
920   /*!
921    * Post-process the filter graph for each active field.
922    */
923   void CFile::postProcessFilterGraph()
924   {
925     int size = this->enabledFields.size();
926     for (int i = 0; i < size; ++i)
927     {
928       this->enabledFields[i]->checkIfMustAutoTrigger();
929     }
930   }
931
932   /*!
933     Prefetching the data for enabled fields read from file.
934   */
935   void CFile::prefetchEnabledReadModeFields(void)
936   {
937     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
938       return;
939
940     int size = this->enabledFields.size();
941     for (int i = 0; i < size; ++i)
942       this->enabledFields[i]->sendReadDataRequest(CContext::getCurrent()->getCalendar()->getCurrentDate());
943   }
944
945   /*!
946     Do all pre timestep operations for enabled fields in read mode:
947      - Check that the data excepted from server has been received
948      - Check if some filters must auto-trigger
949   */
950   void CFile::doPreTimestepOperationsForEnabledReadModeFields(void)
951   {
952     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
953       return;
954
955     int size = this->enabledFields.size();
956     for (int i = 0; i < size; ++i)
957     {
958       this->enabledFields[i]->checkForLateDataFromServer();
959       this->enabledFields[i]->autoTriggerIfNeeded();
960     }
961   }
962
963   /*!
964     Do all post timestep operations for enabled fields in read mode:
965      - Prefetch the data read from file when needed
966   */
967   void CFile::doPostTimestepOperationsForEnabledReadModeFields(void)
968   {
969     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
970       return;
971
972     int size = this->enabledFields.size();
973     for (int i = 0; i < size; ++i)
974     {
975       this->enabledFields[i]->sendReadDataRequestIfNeeded();
976     }
977   }
978
979   void CFile::solveFieldRefInheritance(bool apply)
980   {
981      // Rsolution des hritages par rfrence de chacun des champs contenus dans le fichier.
982      std::vector<CField*> allF = this->getAllFields();
983      for (unsigned int i = 0; i < allF.size(); i++)
984         allF[i]->solveRefInheritance(apply);
985   }
986
987   //----------------------------------------------------------------
988
989   /*!
990   \brief Add a field into file.
991      A field is added into file and it will be written out if the file is enabled and
992   level of this field is smaller than level_output. A new field won't be created if one
993   with id has already existed
994   \param [in] id String identity of new field
995   \return Pointer to added (or already existed) field
996   */
997   CField* CFile::addField(const string& id)
998   {
999     return vFieldGroup->createChild(id);
1000   }
1001
1002   /*!
1003   \brief Add a field group into file.
1004      A field group is added into file and it will play a role as parents for fields.
1005   A new field group won't be created if one with id has already existed
1006   \param [in] id String identity of new field group
1007   \return Pointer to added (or already existed) field group
1008   */
1009   CFieldGroup* CFile::addFieldGroup(const string& id)
1010   {
1011     return vFieldGroup->createChildGroup(id);
1012   }
1013
1014   /*!
1015   \brief Add a variable into file.
1016      A variable is added into file and if one with id has already existed, pointer to
1017   it will be returned.
1018      Variable as long as attributes are information container of file.
1019   However, whereas attributes are "fixed" information, variables provides a more flexible way to user
1020   to fill in (extra) information for a file.
1021   \param [in] id String identity of new variable
1022   \return Pointer to added (or already existed) variable
1023   */
1024   CVariable* CFile::addVariable(const string& id)
1025   {
1026     return vVariableGroup->createChild(id);
1027   }
1028
1029   /*!
1030   \brief Add a variable group into file.
1031      A variable group is added into file and it will play a role as parents for variables.
1032   A new variable group won't be created if one with id has already existed
1033   \param [in] id String identity of new variable group
1034   \return Pointer to added (or already existed) variable group
1035   */
1036   CVariableGroup* CFile::addVariableGroup(const string& id)
1037   {
1038     return vVariableGroup->createChildGroup(id);
1039   }
1040
1041   void CFile::setContextClient(CContextClient* newContextClient)
1042   {
1043     client = newContextClient;
1044     size_t size = this->enabledFields.size();
1045     for (size_t i = 0; i < size; ++i)
1046     {
1047       this->enabledFields[i]->setContextClient(newContextClient);
1048     }
1049   }
1050
1051   CContextClient* CFile::getContextClient()
1052   {
1053     return client;
1054   }
1055
1056   void CFile::setReadContextClient(CContextClient* readContextclient)
1057   {
1058     read_client = readContextclient;
1059   }
1060
1061   CContextClient* CFile::getReadContextClient()
1062   {
1063     return read_client;
1064   }
1065
1066   /*!
1067   \brief Send a message to create a field on server side
1068   \param[in] id String identity of field that will be created on server
1069   */
1070   void CFile::sendAddField(const string& id, CContextClient* client)
1071   {
1072      sendAddItem(id, EVENT_ID_ADD_FIELD, client);
1073   }
1074
1075   /*!
1076   \brief Send a message to create a field group on server side
1077   \param[in] id String identity of field group that will be created on server
1078   */
1079   void CFile::sendAddFieldGroup(const string& id, CContextClient* client)
1080   {
1081      sendAddItem(id, (int)EVENT_ID_ADD_FIELD_GROUP, client);
1082   }
1083
1084   /*!
1085   \brief Receive a message annoucing the creation of a field on server side
1086   \param[in] event Received event
1087   */
1088   void CFile::recvAddField(CEventServer& event)
1089   {
1090
1091      CBufferIn* buffer = event.subEvents.begin()->buffer;
1092      string id;
1093      *buffer>>id;
1094      get(id)->recvAddField(*buffer);
1095   }
1096
1097   /*!
1098   \brief Receive a message annoucing the creation of a field on server side
1099   \param[in] buffer Buffer containing message
1100   */
1101   void CFile::recvAddField(CBufferIn& buffer)
1102   {
1103      string id;
1104      buffer>>id;
1105      addField(id);
1106   }
1107
1108   /*!
1109   \brief Receive a message annoucing the creation of a field group on server side
1110   \param[in] event Received event
1111   */
1112   void CFile::recvAddFieldGroup(CEventServer& event)
1113   {
1114
1115      CBufferIn* buffer = event.subEvents.begin()->buffer;
1116      string id;
1117      *buffer>>id;
1118      get(id)->recvAddFieldGroup(*buffer);
1119   }
1120
1121   /*!
1122   \brief Receive a message annoucing the creation of a field group on server side
1123   \param[in] buffer Buffer containing message
1124   */
1125   void CFile::recvAddFieldGroup(CBufferIn& buffer)
1126   {
1127      string id;
1128      buffer>>id;
1129      addFieldGroup(id);
1130   }
1131
1132   /*!
1133   \brief Send messages to duplicate all variables on server side
1134      Because each variable has also its attributes. So first thing to do is replicate
1135   all these attributes on server side. Because variable can have a value, the second thing
1136   is to duplicate this value on server, too.
1137   */
1138   void CFile::sendAddAllVariables(CContextClient* client)
1139   {
1140     std::vector<CVariable*> allVar = getAllVariables();
1141     std::vector<CVariable*>::const_iterator it = allVar.begin();
1142     std::vector<CVariable*>::const_iterator itE = allVar.end();
1143
1144     for (; it != itE; ++it)
1145     {
1146       this->sendAddVariable((*it)->getId(), client);
1147       (*it)->sendAllAttributesToServer(client);
1148       (*it)->sendValue(client);
1149     }
1150   }
1151
1152   /*!
1153   \brief Send a message to create a variable group on server side
1154   \param[in] id String identity of variable group that will be created on server
1155   \param [in] client client to which we will send this adding action
1156   */
1157   void CFile::sendAddVariableGroup(const string& id, CContextClient* client)
1158   {
1159      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE_GROUP, client);
1160   }
1161
1162   /*
1163     Send message to add a variable into a file within a certain client
1164     \param [in] id String identity of a variable
1165     \param [in] client client to which we will send this adding action
1166   */
1167   void CFile::sendAddVariable(const string& id, CContextClient* client)
1168   {
1169      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE, client);
1170   }
1171
1172   /*!
1173   \brief Receive a message annoucing the creation of a variable on server side
1174   \param[in] event Received event
1175   */
1176   void CFile::recvAddVariable(CEventServer& event)
1177   {
1178
1179      CBufferIn* buffer = event.subEvents.begin()->buffer;
1180      string id;
1181      *buffer>>id;
1182      get(id)->recvAddVariable(*buffer);
1183   }
1184
1185   /*!
1186   \brief Receive a message annoucing the creation of a variable on server side
1187   \param[in] buffer Buffer containing message
1188   */
1189   void CFile::recvAddVariable(CBufferIn& buffer)
1190   {
1191      string id;
1192      buffer>>id;
1193      addVariable(id);
1194   }
1195
1196   /*!
1197   \brief Receive a message annoucing the creation of a variable group on server side
1198   \param[in] event Received event
1199   */
1200   void CFile::recvAddVariableGroup(CEventServer& event)
1201   {
1202
1203      CBufferIn* buffer = event.subEvents.begin()->buffer;
1204      string id;
1205      *buffer>>id;
1206      get(id)->recvAddVariableGroup(*buffer);
1207   }
1208
1209   /*!
1210   \brief Receive a message annoucing the creation of a variable group on server side
1211   \param[in] buffer Buffer containing message
1212   */
1213   void CFile::recvAddVariableGroup(CBufferIn& buffer)
1214   {
1215      string id;
1216      buffer>>id;
1217      addVariableGroup(id);
1218   }
1219
1220   /*!
1221     \brief Sending all active (enabled) fields from client to server.
1222   Each field is identified uniquely by its string identity. Not only should we
1223   send the id to server but also we need to send ids of reference domain and reference axis.
1224   With these two id, it's easier to make reference to grid where all data should be written.
1225   Remark: This function must be called AFTER all active (enabled) files have been created on the server side
1226   */
1227   void CFile::sendEnabledFields(CContextClient* client)
1228   {
1229     size_t size = this->enabledFields.size();
1230     for (size_t i = 0; i < size; ++i)
1231     {
1232       CField* field = this->enabledFields[i];
1233       this->sendAddField(field->getId(), client);
1234       field->checkTimeAttributes();
1235       field->sendAllAttributesToServer(client);
1236       field->sendAddAllVariables(client);
1237     }
1238   }
1239
1240
1241   /*!
1242   \brief Dispatch event received from client
1243      Whenever a message is received in buffer of server, it will be processed depending on
1244   its event type. A new event type should be added in the switch list to make sure
1245   it processed on server side.
1246   \param [in] event: Received message
1247   */
1248   bool CFile::dispatchEvent(CEventServer& event)
1249   {
1250      if (SuperClass::dispatchEvent(event)) return true;
1251      else
1252      {
1253        switch(event.type)
1254        {
1255           case EVENT_ID_ADD_FIELD :
1256             recvAddField(event);
1257             return true;
1258             break;
1259
1260           case EVENT_ID_ADD_FIELD_GROUP :
1261             recvAddFieldGroup(event);
1262             return true;
1263             break;
1264
1265            case EVENT_ID_ADD_VARIABLE :
1266             recvAddVariable(event);
1267             return true;
1268             break;
1269
1270           case EVENT_ID_ADD_VARIABLE_GROUP :
1271             recvAddVariableGroup(event);
1272             return true;
1273             break;
1274           default :
1275              ERROR("bool CFile::dispatchEvent(CEventServer& event)", << "Unknown Event");
1276           return false;
1277        }
1278      }
1279   }
1280
1281
1282
1283
1284   ///---------------------------------------------------------------
1285
1286} // namespace xios
Note: See TracBrowser for help on using the repository browser.