source: XIOS/dev/dev_ym/XIOS_COUPLING/src/node/file.cpp @ 1853

Last change on this file since 1853 was 1853, checked in by ymipsl, 21 months ago

Coupling branch : replace hasServer and hasClient combination by the name of correct service : CLIENT, GATHERER or OUT_SERVER.

YM

  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
File size: 43.6 KB
Line 
1#include "file.hpp"
2
3#include "attribute_template.hpp"
4#include "object_template.hpp"
5#include "group_template.hpp"
6#include "object_factory.hpp"
7#include "context.hpp"
8#include "context_server.hpp"
9#include "nc4_data_output.hpp"
10#include "nc4_data_input.hpp"
11#include "calendar_util.hpp"
12#include "date.hpp"
13#include "message.hpp"
14#include "type.hpp"
15#include "xios_spl.hpp"
16#include "context_client.hpp"
17#include "mpi.hpp"
18#include "timer.hpp"
19#include "server.hpp"
20
21namespace xios {
22
23   /// ////////////////////// Dfinitions ////////////////////// ///
24
25   CFile::CFile(void)
26      : CObjectTemplate<CFile>(), CFileAttributes()
27      , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
28      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
29   {
30     setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
31     setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group"));
32   }
33
34   CFile::CFile(const StdString & id)
35      : CObjectTemplate<CFile>(id), CFileAttributes()
36      , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
37      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
38    {
39      setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
40      setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group"));
41    }
42
43   CFile::~CFile(void)
44   { /* Ne rien faire de plus */ }
45
46   ///---------------------------------------------------------------
47  //! Get name of file
48   StdString CFile::GetName(void)   { return (StdString("file")); }
49   StdString CFile::GetDefName(void){ return (CFile::GetName()); }
50   ENodeType CFile::GetType(void)   { return (eFile); }
51
52   //----------------------------------------------------------------
53
54   const StdString CFile::getFileOutputName(void) const
55   TRY
56   {
57     return (name.isEmpty() ? getId() : name) + (name_suffix.isEmpty() ? StdString("") :  name_suffix.getValue());
58   }
59   CATCH
60
61   //----------------------------------------------------------------
62   /*!
63   \brief Get data writer object.
64   Each enabled file in xml represents a physical netcdf file.
65   This function allows to access the data writer object.
66   \return data writer object.
67   */
68   std::shared_ptr<CDataOutput> CFile::getDataOutput(void) const
69   TRY
70   {
71      return data_out;
72   }
73   CATCH
74
75   /*!
76   \brief Get data reader object.
77   Each enabled file in xml represents a physical netcdf file.
78   This function allows to access the data reader object.
79   \return data reader object.
80   */
81   std::shared_ptr<CDataInput> CFile::getDataInput(void) const
82   TRY
83   {
84      return data_in;
85   }
86   CATCH
87
88   /*!
89   \brief Get virtual field group
90      In each file, there always exists a field group which is the ancestor of all
91   fields in the file. This is considered be virtual because it is created automatically during
92   file initialization and it normally doesn't appear on xml file
93   \return Pointer to field group
94   */
95   CFieldGroup* CFile::getVirtualFieldGroup(void) const
96   TRY
97   {
98      return (this->vFieldGroup);
99   }
100   CATCH
101
102   /*!
103   \brief Get virtual variable group
104      In each file, there always exists a variable group which is the ancestor of all
105   variable in the file. This is considered be virtual because it is created automatically during
106   file initialization and it normally doesn't appear on xml file
107   \return Pointer to variable group
108   */
109   CVariableGroup* CFile::getVirtualVariableGroup(void) const
110   TRY
111   {
112      return (this->vVariableGroup);
113   }
114   CATCH
115
116   //! Get all fields of a file
117   std::vector<CField*> CFile::getAllFields(void) const
118   TRY
119   {
120      return (this->vFieldGroup->getAllChildren());
121   }
122   CATCH
123
124   //! Get all variables of a file
125   std::vector<CVariable*> CFile::getAllVariables(void) const
126   TRY
127   {
128      return (this->vVariableGroup->getAllChildren());
129   }
130   CATCH
131
132   //----------------------------------------------------------------
133   /*!
134   \brief Get all enabled fields of file
135      A field is considered to be enabled if it fullfil these conditions: it is enabled, inside a enabled file
136   and its own level is not larger than file output level.
137   \param [in] default_outputlevel default value output level of file
138   \param [in] default_level default value level of field
139   \param [in] default_enabled flag determine by default if field is enabled
140   \return Vector of pointers of enabled fields
141   */
142   std::vector<CField*> CFile::getEnabledFields(int default_outputlevel,
143                                                int default_level,
144                                                bool default_enabled)
145   TRY
146   {
147      if (!this->enabledFields.empty())
148         return (this->enabledFields);
149
150      const int _outputlevel =
151         (!output_level.isEmpty()) ? output_level.getValue() : default_outputlevel;
152      std::vector<CField*>::iterator it;
153      this->enabledFields = this->getAllFields();
154
155      std::vector<CField*> newEnabledFields;
156
157      for ( it = this->enabledFields.begin(); it != this->enabledFields.end(); it++ )
158      {
159         if (!(*it)->enabled.isEmpty()) // Si l'attribut 'enabled' est dfini ...
160         {
161            if (! (*it)->enabled.getValue()) continue;
162         }
163         else // Si l'attribut 'enabled' n'est pas dfini ...
164         {
165            if (!default_enabled) continue;
166         }
167
168         if (!(*it)->level.isEmpty()) // Si l'attribut 'level' est dfini ...
169         {
170            if ((*it)->level.getValue() > _outputlevel) continue;
171         }
172         else // Si l'attribut 'level' n'est pas dfini ...
173         {
174            if (default_level > _outputlevel) continue;
175         }
176
177         newEnabledFields.push_back(*it);
178         // Le champ est finalement actif, on y ajoute la rfrence au champ de base.
179         (*it)->setRelFile(CFile::get(this));
180      }
181      enabledFields = newEnabledFields;
182
183      return (this->enabledFields);
184   }
185   CATCH_DUMP_ATTR
186
187   //----------------------------------------------------------------
188   //! Change virtual field group to a new one
189   void CFile::setVirtualFieldGroup(CFieldGroup* newVFieldGroup)
190   TRY
191   {
192      this->vFieldGroup = newVFieldGroup;
193   }
194   CATCH_DUMP_ATTR
195
196   //! Change virtual variable group to new one
197   void CFile::setVirtualVariableGroup(CVariableGroup* newVVariableGroup)
198   TRY
199   {
200      this->vVariableGroup = newVVariableGroup;
201   }
202   CATCH_DUMP_ATTR
203
204   //----------------------------------------------------------------
205   bool CFile::isSyncTime(void)
206   TRY
207   {
208     CContext* context = CContext::getCurrent();
209     const CDate& currentDate = context->calendar->getCurrentDate();
210     if (!sync_freq.isEmpty())
211     {
212       if (lastSync + sync_freq.getValue() < currentDate)
213       {
214         lastSync = currentDate;
215         return true;
216        }
217      }
218      return false;
219    }
220    CATCH_DUMP_ATTR
221
222   //! Initialize a file in order to write into it
223   void CFile::initWrite(void)
224   TRY
225   {
226      CContext* context = CContext::getCurrent();
227      const CDate& currentDate = context->calendar->getCurrentDate();
228
229      lastSync  = currentDate;
230      lastSplit = currentDate;
231      if (!split_freq.isEmpty())
232      {
233        StdString keySuffix("CFile::"+getFileOutputName()+"::") ; 
234        if (context->registryIn->foundKey(keySuffix+"splitStart") && context->registryIn->foundKey(keySuffix+"splitEnd"))
235        {
236          CDate savedSplitStart(*context->getCalendar()), savedSplitEnd(*context->getCalendar());
237          context->registryIn->getKey(keySuffix+"splitStart", savedSplitStart);
238          context->registryIn->getKey(keySuffix+"splitEnd",   savedSplitEnd);
239
240          if (savedSplitStart <= lastSplit && lastSplit <= savedSplitEnd)
241            lastSplit = savedSplitStart;
242        }
243      }
244      isOpen = false;     
245
246//      if (!record_offset.isEmpty() && record_offset < 0)
247//        ERROR("void CFile::initFile(void)",
248//              "Invalid 'record_offset', this attribute cannot be negative.");
249      const int recordOffset = record_offset.isEmpty() ? 0 : record_offset;
250
251      set<StdString> setAxis;
252      set<StdString> setDomains;
253
254      std::vector<CField*>::iterator it, end = this->enabledFields.end();
255      for (it = this->enabledFields.begin(); it != end; it++)
256      {
257         CField* field = *it;         
258         std::vector<CAxis*> vecAxis = field->grid->getAxis();
259         for (size_t i = 0; i < vecAxis.size(); ++i)
260           setAxis.insert(vecAxis[i]->getAxisOutputName());
261         std::vector<CDomain*> vecDomains = field->grid->getDomains();
262         for (size_t i = 0; i < vecDomains.size(); ++i)
263           setDomains.insert(vecDomains[i]->getDomainOutputName());
264
265         field->resetNStep(recordOffset);
266      }
267      nbAxis = setAxis.size();
268      nbDomains = setDomains.size();
269
270      // create sub communicator for file
271      createSubComFile();
272
273      // if (time_counter.isEmpty()) time_counter.setValue(time_counter_attr::centered);
274      if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
275    }
276    CATCH_DUMP_ATTR
277
278    //! Initialize a file in order to write into it
279    void CFile::initRead(void)
280    TRY
281    {
282      if (checkRead) return;
283      createSubComFile();
284      checkRead = true;
285    }
286    CATCH_DUMP_ATTR
287
288    /*!
289      Create a sub communicator in which processes participate in reading/opening file
290    */
291    void CFile::createSubComFile()
292    TRY
293    {
294      CContext* context = CContext::getCurrent();
295
296      // create sub communicator for file
297      allZoneEmpty = true;     
298      std::vector<CField*>::iterator it, end = this->enabledFields.end();
299      for (it = this->enabledFields.begin(); it != end; it++)
300      {
301         CField* field = *it;
302         bool nullGrid = (0 == field->grid);
303         allZoneEmpty &= nullGrid ? false : !field->grid->doGridHaveDataToWrite();
304      }
305
306      int color = allZoneEmpty ? 0 : 1;
307      MPI_Comm_split(context->intraComm_, color, context->intraCommRank_, &fileComm);
308      if (allZoneEmpty) MPI_Comm_free(&fileComm);
309    }
310    CATCH_DUMP_ATTR
311
312    /*
313       Check condition to write into a file
314       For now, we only use the level-2 server to write files (if this mode is activated)
315       or classical server to do this job.
316    */
317    void CFile::checkWriteFile(void)
318    TRY
319    {
320      CContext* context = CContext::getCurrent();
321      // Done by classical server or secondary server
322      // This condition should be changed soon
323//ym      if (CServer::serverLevel == 0 || CServer::serverLevel == 2)
324       if (context->getServiceType()==CServicesManager::IO_SERVER || context->getServiceType()==CServicesManager::OUT_SERVER)
325       {
326        if (mode.isEmpty() || mode.getValue() == mode_attr::write)
327        {
328          CTimer::get("Files : create headers").resume();
329          if (!isOpen) createHeader();
330          CTimer::get("Files : create headers").suspend();
331          checkSync();
332        }       
333        checkSplit(); // REally need this?
334      }
335    }
336    CATCH_DUMP_ATTR
337
338    /*
339       Check condition to read from a file
340       For now, we only use the level-1 server to write files (if this mode is activated)
341       or classical server to do this job.
342       This function can be used by client for reading metadata
343    */
344    void CFile::checkReadFile(void)
345    TRY
346    {
347      CContext* context = CContext::getCurrent();
348      // Done by classical server or secondary server
349      // TODO: This condition should be changed soon. It only works with maximum number of level as 2
350
351//ym      if (CServer::serverLevel == 0 || CServer::serverLevel == 1)
352      if (context->getServiceType()==CServicesManager::IO_SERVER || context->getServiceType()==CServicesManager::GATHERER)
353      {
354        if (!mode.isEmpty() && mode.getValue() == mode_attr::read)
355        {
356          CTimer::get("Files : open headers").resume();
357         
358          if (!isOpen) openInReadMode();
359
360          CTimer::get("Files : open headers").suspend();
361        }
362        //checkSplit(); // Really need for reading?
363      }
364    }
365    CATCH_DUMP_ATTR
366
367    /*!
368      Verify if a process participates in an opening-file communicator
369      \return true if the process doesn't participate in opening file
370    */
371    bool CFile::isEmptyZone()
372    TRY
373    {
374      return allZoneEmpty;
375    }
376    CATCH_DUMP_ATTR
377
378    /*!
379    \brief Verify if synchronisation should be done
380        If syn option is enabled, syn frequence and current time will be used to
381    calculate the moment to syn file(s)
382    \return True if it is the moment to synchronize file, otherwise false
383    */
384   bool CFile::checkSync(void)
385   TRY
386   {
387     CContext* context = CContext::getCurrent();
388     const CDate& currentDate = context->calendar->getCurrentDate();
389     if (!sync_freq.isEmpty())
390     {
391       if (lastSync + sync_freq.getValue() <= currentDate)
392       {
393         lastSync = currentDate;
394         data_out->syncFile();
395         return true;
396        }
397      }
398      return false;
399    }
400   CATCH_DUMP_ATTR
401
402    /*!
403    \brief Verify if splitting should be done
404        If split option is enabled, split frequence and current time will be used to
405    calculate the moment to split file
406    \return True if it is the moment to split file, otherwise false
407    */
408    bool CFile::checkSplit(void)
409    TRY
410    {
411      CContext* context = CContext::getCurrent();
412      const CDate& currentDate = context->calendar->getCurrentDate();
413      if (!split_freq.isEmpty())
414      {
415        if (currentDate > lastSplit + split_freq.getValue())
416        {
417          lastSplit = lastSplit + split_freq.getValue();
418          std::vector<CField*>::iterator it, end = this->enabledFields.end();
419          for (it = this->enabledFields.begin(); it != end; it++)
420          {
421            (*it)->resetNStep();
422            (*it)->resetNStepMax();
423          }
424          if (mode.isEmpty() || mode.getValue() == mode_attr::write)
425            createHeader();
426          else
427            openInReadMode();
428          return true;
429        }
430      }
431      return false;
432    }
433    CATCH_DUMP_ATTR
434
435   /*!
436   \brief Create header of netcdf file
437   There are some information to fill in header of each netcdf.
438   */
439   void CFile::createHeader(void)
440   TRY
441   {
442      CContext* context = CContext::getCurrent();
443     
444      if (!allZoneEmpty)
445      {
446         StdString filename = getFileOutputName();
447
448// determine splitting format in the file name  : firstPart%start_date%middlePart%end_date%lastPart
449
450         std::string strStartDate="%start_date%" ;
451         std::string strEndDate="%end_date%" ;
452
453         std::string firstPart ;
454         std::string middlePart ;
455         std::string lastPart ;
456         size_t pos1, pos2 ;
457         bool hasStartDate=false ;
458         bool hasEndDate=false ;
459         bool hasSplit = (!split_freq.isEmpty());
460                 
461         pos1=filename.find(strStartDate) ;
462         if (pos1!=std::string::npos)
463         {
464           firstPart=filename.substr(0,pos1) ;
465           pos1+=strStartDate.size() ;
466           hasStartDate=true ;
467         }
468         else pos1=0 ;
469
470         pos2=filename.find(strEndDate,pos1) ;
471         if (pos2!=std::string::npos)
472         {
473           middlePart=filename.substr(pos1,pos2-pos1) ;
474           pos2+=strEndDate.size() ;
475           lastPart=filename.substr(pos2,filename.size()-pos2) ;
476           hasEndDate=true ;
477         }
478         else middlePart=filename.substr(pos1,filename.size()) ;
479
480         if (!hasStartDate && !hasEndDate)
481         {
482           hasStartDate=true ;
483           hasEndDate=true;
484           firstPart=middlePart ;
485           if (hasSplit) firstPart +="_";
486           middlePart="-" ;
487         }
488   
489         StdOStringStream oss;
490
491         if (!split_freq.isEmpty())
492         {
493           CDate split_start ;
494           CDate splitEnd ;
495           if (!split_start_offset.isEmpty()) split_start=lastSplit + split_start_offset ;
496           else split_start=lastSplit ;
497
498           splitEnd = lastSplit + split_freq ;
499           if (!split_last_date.isEmpty())
500           {
501             CDate splitLastDate=CDate::FromString(split_last_date,*CContext::getCurrent()->getCalendar()) ;
502             if( splitLastDate < splitEnd)  splitEnd=splitLastDate ;
503           }
504           
505           if (!split_end_offset.isEmpty()) splitEnd = splitEnd + split_end_offset;
506           else splitEnd = splitEnd - 1 * Second;
507
508           string splitFormat;
509           if (split_freq_format.isEmpty())
510           {
511             CDuration splitFreq = split_freq.getValue();
512             splitFreq.solveTimeStep(*CContext::getCurrent()->getCalendar());
513             if (splitFreq.second != 0) splitFormat = "%y%mo%d%h%mi%s";
514             else if (splitFreq.minute != 0) splitFormat = "%y%mo%d%h%mi";
515             else if (splitFreq.hour != 0) splitFormat = "%y%mo%d%h";
516             else if (splitFreq.day != 0) splitFormat = "%y%mo%d";
517             else if (splitFreq.month != 0) splitFormat = "%y%mo";
518             else splitFormat = "%y";
519           }
520           else splitFormat = split_freq_format;
521
522           oss << firstPart ;
523           if (hasStartDate) oss << split_start.getStr(splitFormat) ;
524           oss << middlePart ;
525           if (hasEndDate) oss << splitEnd.getStr(splitFormat);
526           oss << lastPart ;
527
528           StdString keySuffix("CFile::"+getFileOutputName()+"::") ; 
529           context->registryOut->setKey(keySuffix+"splitStart", lastSplit);
530           context->registryOut->setKey(keySuffix+"splitEnd",   splitEnd);
531         }
532         else oss<<firstPart<<lastPart ;
533
534        bool append = !this->append.isEmpty() && this->append.getValue();
535
536         bool useClassicFormat = !format.isEmpty() && format == format_attr::netcdf4_classic;
537         bool useCFConvention = convention.isEmpty() || convention == convention_attr::CF;
538
539         bool multifile = true;
540         if (!type.isEmpty())
541         {
542           if (type == type_attr::one_file) multifile = false;
543           else if (type == type_attr::multiple_file) multifile = true;
544
545         }
546#ifndef USING_NETCDF_PAR
547         if (!multifile)
548         {
549            info(0) << "!!! Warning -> Using non parallel version of netcdf, switching in multiple_file mode for file : " << filename << " ..." << endl;
550            multifile = true;
551          }
552#endif
553         if (multifile)
554         {
555            int commSize, commRank;
556            MPI_Comm_size(fileComm, &commSize);
557            MPI_Comm_rank(fileComm, &commRank);
558
559            if (context->intraCommSize_ > 1)
560            {
561              oss << "_" ;
562              int width=0; int n = commSize-1;
563              while (n != 0) { n = n / 10; width++;}
564              if (!min_digits.isEmpty())
565                if (width < min_digits) width = min_digits;
566              oss.width(width);
567              oss.fill('0');
568              oss << right << commRank;
569            }
570         }
571         oss << ".nc";
572
573         bool isCollective = par_access.isEmpty() ||  par_access == par_access_attr::collective;
574
575         if (isOpen) data_out->closeFile();
576
577        data_out = std::shared_ptr<CDataOutput>(new CNc4DataOutput(this, oss.str(), append, useClassicFormat, useCFConvention,
578                                                              fileComm, multifile, isCollective, time_counter_name));
579        isOpen = true;
580
581        data_out->writeFile(CFile::get(this));
582
583        if (!useCFConvention) sortEnabledFieldsForUgrid();
584
585        // Do not recreate the file structure if opening an existing file
586        if (!data_out->IsInAppendMode())
587        {
588          std::vector<CField*>::iterator it, end = this->enabledFields.end();
589          for (it = this->enabledFields.begin(); it != end; it++)
590          {
591            CField* field = *it;
592            this->data_out->writeFieldGrid(field);
593          }
594          this->data_out->writeTimeDimension();
595
596          for (it = this->enabledFields.begin(); it != end; it++)
597          {
598            CField* field = *it;
599            this->data_out->writeFieldTimeAxis(field);
600          }
601         
602          for (it = this->enabledFields.begin(); it != end; it++)
603          {
604            CField* field = *it;
605            this->data_out->writeField(field);
606          }
607
608          vector<CVariable*> listVars = getAllVariables();
609          for (vector<CVariable*>::iterator it = listVars.begin(); it != listVars.end(); it++)
610            this->data_out->writeAttribute(*it);
611
612          this->data_out->definition_end();
613        }
614        else
615        {
616          // check time axis even in append mode
617          std::vector<CField*>::iterator it, end = this->enabledFields.end();
618          for (it = this->enabledFields.begin(); it != end; it++)
619          {
620            CField* field = *it;
621            this->data_out->writeFieldTimeAxis(field);
622          }
623        }
624      }
625   }
626   CATCH_DUMP_ATTR
627
628  /*!
629  \brief Open an existing NetCDF file in read-only mode
630  */
631  void CFile::openInReadMode()
632  TRY
633  {
634    CContext* context = CContext::getCurrent();
635    MPI_Comm readComm = this->fileComm;
636
637    if (!allZoneEmpty)
638    {
639      StdString filename = getFileOutputName();
640      StdOStringStream oss;
641      oss << filename;
642
643      if (!split_freq.isEmpty())
644      {
645        string splitFormat;
646        if (split_freq_format.isEmpty())
647        {
648          CDuration splitFreq = split_freq.getValue();
649          splitFreq.solveTimeStep(*CContext::getCurrent()->getCalendar());
650          if (splitFreq.second != 0) splitFormat = "%y%mo%d%h%mi%s";
651          else if (splitFreq.minute != 0) splitFormat = "%y%mo%d%h%mi";
652          else if (splitFreq.hour != 0) splitFormat = "%y%mo%d%h";
653          else if (splitFreq.day != 0) splitFormat = "%y%mo%d";
654          else if (splitFreq.month != 0) splitFormat = "%y%mo";
655          else splitFormat = "%y";
656        }
657        else splitFormat = split_freq_format;
658        oss << "_" << lastSplit.getStr(splitFormat)
659        << "-" << (lastSplit + split_freq.getValue() - 1 * Second).getStr(splitFormat);
660      }
661
662      bool multifile = true;
663      if (!type.isEmpty())
664      {
665        if (type == type_attr::one_file) multifile = false;
666        else if (type == type_attr::multiple_file) multifile = true;
667      }
668  #ifndef USING_NETCDF_PAR
669      if (!multifile)
670      {
671        info(0) << "!!! Warning -> Using non parallel version of netcdf, switching in multiple_file mode for file : " << filename << " ..." << endl;
672        multifile = true;
673      }
674  #endif
675      if (multifile)
676      {
677        int commSize, commRank;
678        MPI_Comm_size(readComm, &commSize);
679        MPI_Comm_rank(readComm, &commRank);
680
681        if (context->intraCommSize_ > 1)
682        {
683          oss << "_";
684          int width = 0, n = commSize - 1;
685          while (n != 0) { n = n / 10; width++; }
686          if (!min_digits.isEmpty() && width < min_digits)
687            width = min_digits;
688          oss.width(width);
689          oss.fill('0');
690          oss << right << commRank;
691        }
692      }
693      oss << ".nc";
694
695      bool isCollective = par_access.isEmpty() || par_access == par_access_attr::collective;
696      bool readMetaDataPar = true;
697      if (context->getServiceType()==CServicesManager::CLIENT) readMetaDataPar = (read_metadata_par.isEmpty()) ? false : read_metadata_par;
698
699      if (isOpen) data_out->closeFile();
700      bool ugridConvention = !convention.isEmpty() ? (convention == convention_attr::UGRID) : false;
701      if (time_counter_name.isEmpty())
702        data_in = std::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, readMetaDataPar, ugridConvention));
703      else
704        data_in = std::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, readMetaDataPar, ugridConvention, time_counter_name));
705      isOpen = true;
706    }
707  }
708  CATCH_DUMP_ATTR
709
710   //! Close file
711   void CFile::close(void)
712   TRY
713   {
714     if (!allZoneEmpty)
715       if (isOpen)
716       {
717         if (mode.isEmpty() || mode.getValue() == mode_attr::write)
718          this->data_out->closeFile();
719         else
720          this->data_in->closeFile();
721        isOpen = false;
722       }
723      if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
724   }
725   CATCH_DUMP_ATTR
726
727   //----------------------------------------------------------------
728
729   void CFile::readAttributesOfEnabledFieldsInReadMode()
730   TRY
731   {
732     if (enabledFields.empty()) return;
733
734     // Just check file and try to open it
735     if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
736
737     checkReadFile();
738
739     for (int idx = 0; idx < enabledFields.size(); ++idx)
740     {
741        // First of all, find out which domain and axis associated with this field
742        enabledFields[idx]->solveGridReference();
743
744        // Read attributes of domain and axis from this file
745        this->data_in->readFieldAttributesMetaData(enabledFields[idx]);
746
747        // Now complete domain and axis associated with this field
748        enabledFields[idx]->solveGenerateGrid();
749
750        // Read necessary value from file
751        this->data_in->readFieldAttributesValues(enabledFields[idx]);
752
753        // Fill attributes for base reference
754        enabledFields[idx]->solveGridDomainAxisBaseRef();
755     }
756
757     // Now everything is ok, close it
758     close();
759   }
760   CATCH_DUMP_ATTR
761
762   /*!
763   \brief Parse xml file and write information into file object
764   \param [in] node xmld node corresponding in xml file
765   */
766   void CFile::parse(xml::CXMLNode & node)
767   TRY
768   {
769      SuperClass::parse(node);
770
771      if (node.goToChildElement())
772      {
773        do
774        {
775           if (node.getElementName()=="field" || node.getElementName()=="field_group") this->getVirtualFieldGroup()->parseChild(node);
776           else if (node.getElementName()=="variable" || node.getElementName()=="variable_group") this->getVirtualVariableGroup()->parseChild(node);
777        } while (node.goToNextElement());
778        node.goToParentElement();
779      }
780   }
781   CATCH_DUMP_ATTR
782
783   //----------------------------------------------------------------
784
785   /*!
786   \brief Represent a file in form of string with all its info
787   \return String
788   */
789   StdString CFile::toString(void) const
790   TRY
791   {
792      StdOStringStream oss;
793
794      oss << "<" << CFile::GetName() << " ";
795      if (this->hasId())
796         oss << " id=\"" << this->getId() << "\" ";
797      oss << SuperClassAttribute::toString() << ">" << std::endl;
798      if (this->getVirtualFieldGroup() != NULL)
799         oss << *this->getVirtualFieldGroup() << std::endl;
800      oss << "</" << CFile::GetName() << " >";
801      return (oss.str());
802   }
803   CATCH
804
805   //----------------------------------------------------------------
806
807   /*!
808   \brief Find all inheritace among objects in a file.
809   \param [in] apply (true) write attributes of parent into ones of child if they are empty
810                     (false) write attributes of parent into a new container of child
811   \param [in] parent
812   */
813   void CFile::solveDescInheritance(bool apply, const CAttributeMap * const parent)
814   TRY
815   {
816      SuperClassAttribute::setAttributes(parent,apply);
817      this->getVirtualFieldGroup()->solveDescInheritance(apply, NULL);
818      this->getVirtualVariableGroup()->solveDescInheritance(apply, NULL);
819   }
820   CATCH_DUMP_ATTR
821
822   //----------------------------------------------------------------
823
824   /*!
825   \brief Resolve all reference of active fields.
826      In order to know exactly which data each active field has, a search for all its
827   reference to find its parents or/and its base reference object must be done. Moreover
828   during this search, there are some information that can only be sent to server AFTER
829   all information of active fields are created on server side, e.g: checking mask or index
830   \param [in] sendToServer: Send all info to server (true) or only a part of it (false)
831   */
832   void CFile::solveOnlyRefOfEnabledFields(void)
833   TRY
834   {
835     int size = this->enabledFields.size();
836     for (int i = 0; i < size; ++i)
837     {
838       this->enabledFields[i]->solveOnlyReferenceEnabledField();
839     }
840   }
841   CATCH_DUMP_ATTR
842
843   void CFile::checkGridOfEnabledFields()
844   TRY
845   { 
846     int size = this->enabledFields.size();
847     for (int i = 0; i < size; ++i)
848     {
849       this->enabledFields[i]->checkGridOfEnabledFields();
850     }
851   }
852   CATCH_DUMP_ATTR
853
854   void CFile::sendGridComponentOfEnabledFields()
855   TRY
856   { 
857     int size = this->enabledFields.size();
858     for (int i = 0; i < size; ++i)
859     {
860       this->enabledFields[i]->sendGridComponentOfEnabledFields();
861     }
862   }
863   CATCH_DUMP_ATTR
864
865   /*!
866   \brief Sorting domains with the same name (= describing the same mesh) in the decreasing order of nvertex for UGRID files.
867   This insures that the domain with the highest nvertex is written first and thus all known mesh connectivity is generated at once by this domain.
868   */
869   void CFile::sortEnabledFieldsForUgrid()
870   TRY
871   {
872     int size = this->enabledFields.size();
873     std::vector<int> domainNvertices;
874     std::vector<StdString> domainNames;
875
876     for (int i = 0; i < size; ++i)
877     {
878       std::vector<CDomain*> domain = this->enabledFields[i]->getRelGrid()->getDomains();
879       if (domain.size() != 1)
880       {
881         ERROR("void CFile::sortEnabledFieldsForUgrid()",
882               "A domain, and only one, should be defined for grid "<< this->enabledFields[i]->getRelGrid()->getId() << ".");
883       }
884       StdString domainName = domain[0]->getDomainOutputName();
885       int nvertex;
886       if (domain[0]->nvertex.isEmpty())
887       {
888         ERROR("void CFile::sortEnabledFieldsForUgrid()",
889               "Attributes nvertex must be defined for domain "<< domain[0]->getDomainOutputName() << ".");
890       }
891       else
892         nvertex = domain[0]->nvertex;
893
894       for (int j = 0; j < i; ++j)
895       {
896         if (domainName == domainNames[j] && nvertex > domainNvertices[j])
897         {
898           CField* tmpSwap = this->enabledFields[j];
899           this->enabledFields[j] = this->enabledFields[i];
900           this->enabledFields[i] = tmpSwap;
901           domainNames.push_back(domainNames[j]);
902           domainNames[j] = domainName;
903           domainNvertices.push_back(domainNvertices[j]);
904           domainNvertices[j] = nvertex;
905         }
906         else
907         {
908           domainNames.push_back(domainName);
909           domainNvertices.push_back(nvertex);
910         }
911       }
912       if (i==0)
913       {
914         domainNames.push_back(domainName);
915         domainNvertices.push_back(nvertex);
916       }
917     }
918   }
919   CATCH_DUMP_ATTR
920
921   void CFile::sendGridOfEnabledFields()
922   TRY
923   { 
924     int size = this->enabledFields.size();
925     for (int i = 0; i < size; ++i)
926     {
927       this->enabledFields[i]->sendGridOfEnabledFields();
928     }
929   }
930   CATCH_DUMP_ATTR
931
932   void CFile::generateNewTransformationGridDest()
933   TRY
934   {
935     int size = this->enabledFields.size();
936     for (int i = 0; i < size; ++i)
937     {
938       this->enabledFields[i]->generateNewTransformationGridDest();
939     }
940   }
941   CATCH_DUMP_ATTR
942
943   /*!
944   \brief Resolve all reference of active fields.
945      In order to know exactly which data each active field has, a search for all its
946   reference to find its parents or/and its base reference object must be done. Moreover
947   during this search, there are some information that can only be sent to server AFTER
948   all information of active fields are created on server side, e.g: checking mask or index
949   \param [in] sendToServer: Send all info to server (true) or only a part of it (false)
950   */
951   void CFile::solveAllRefOfEnabledFieldsAndTransform(void)
952   TRY
953   {
954     int size = this->enabledFields.size();
955     for (int i = 0; i < size; ++i)
956     {       
957      this->enabledFields[i]->solveAllEnabledFieldsAndTransform();
958     }
959   }
960   CATCH_DUMP_ATTR
961
962   /*!
963    * Constructs the filter graph for each active field.
964    *
965    * \param gc the garbage collector to use when building the filter graph
966    */
967   void CFile::buildFilterGraphOfEnabledFields(CGarbageCollector& gc)
968   TRY
969   {
970     int size = this->enabledFields.size();
971     for (int i = 0; i < size; ++i)
972     {
973       this->enabledFields[i]->buildFilterGraph(gc, true);
974     }
975   }
976   CATCH_DUMP_ATTR
977
978   /*!
979    * Post-process the filter graph for each active field.
980    */
981   void CFile::postProcessFilterGraph()
982   TRY
983   {
984     int size = this->enabledFields.size();
985     for (int i = 0; i < size; ++i)
986     {
987       this->enabledFields[i]->checkIfMustAutoTrigger();
988     }
989   }
990   CATCH_DUMP_ATTR
991
992   /*!
993     Prefetching the data for enabled fields read from file.
994   */
995   void CFile::prefetchEnabledReadModeFields(void)
996   TRY
997   {
998     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
999       return;
1000
1001     int size = this->enabledFields.size();
1002     for (int i = 0; i < size; ++i)
1003       this->enabledFields[i]->sendReadDataRequest(CContext::getCurrent()->getCalendar()->getCurrentDate(), getContextClient());
1004   }
1005   CATCH_DUMP_ATTR
1006
1007   /*!
1008     Do all pre timestep operations for enabled fields in read mode:
1009      - Check that the data excepted from server has been received
1010      - Check if some filters must auto-trigger
1011   */
1012   void CFile::doPreTimestepOperationsForEnabledReadModeFields(void)
1013   TRY
1014   {
1015     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
1016       return;
1017
1018     int size = this->enabledFields.size();
1019     for (int i = 0; i < size; ++i)
1020     {
1021       this->enabledFields[i]->checkForLateDataFromServer();
1022       this->enabledFields[i]->autoTriggerIfNeeded();
1023     }
1024   }
1025   CATCH_DUMP_ATTR
1026
1027   /*!
1028     Do all post timestep operations for enabled fields in read mode:
1029      - Prefetch the data read from file when needed
1030   */
1031   void CFile::doPostTimestepOperationsForEnabledReadModeFields(void)
1032   TRY
1033   {
1034     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
1035       return;
1036
1037     int size = this->enabledFields.size();
1038     for (int i = 0; i < size; ++i)
1039     {
1040       this->enabledFields[i]->sendReadDataRequestIfNeeded();
1041     }
1042   }
1043   CATCH_DUMP_ATTR
1044
1045   void CFile::solveFieldRefInheritance(bool apply)
1046   TRY
1047   {
1048      // Rsolution des hritages par rfrence de chacun des champs contenus dans le fichier.
1049      std::vector<CField*> allF = this->getAllFields();
1050      for (unsigned int i = 0; i < allF.size(); i++)
1051         allF[i]->solveRefInheritance(apply);
1052   }
1053   CATCH_DUMP_ATTR
1054
1055   //----------------------------------------------------------------
1056
1057   /*!
1058   \brief Add a field into file.
1059      A field is added into file and it will be written out if the file is enabled and
1060   level of this field is smaller than level_output. A new field won't be created if one
1061   with id has already existed
1062   \param [in] id String identity of new field
1063   \return Pointer to added (or already existed) field
1064   */
1065   CField* CFile::addField(const string& id)
1066   TRY
1067   {
1068     return vFieldGroup->createChild(id);
1069   }
1070   CATCH_DUMP_ATTR
1071
1072   /*!
1073   \brief Add a field group into file.
1074      A field group is added into file and it will play a role as parents for fields.
1075   A new field group won't be created if one with id has already existed
1076   \param [in] id String identity of new field group
1077   \return Pointer to added (or already existed) field group
1078   */
1079   CFieldGroup* CFile::addFieldGroup(const string& id)
1080   TRY
1081   {
1082     return vFieldGroup->createChildGroup(id);
1083   }
1084   CATCH_DUMP_ATTR
1085
1086   /*!
1087   \brief Add a variable into file.
1088      A variable is added into file and if one with id has already existed, pointer to
1089   it will be returned.
1090      Variable as long as attributes are information container of file.
1091   However, whereas attributes are "fixed" information, variables provides a more flexible way to user
1092   to fill in (extra) information for a file.
1093   \param [in] id String identity of new variable
1094   \return Pointer to added (or already existed) variable
1095   */
1096   CVariable* CFile::addVariable(const string& id)
1097   TRY
1098   {
1099     return vVariableGroup->createChild(id);
1100   }
1101   CATCH_DUMP_ATTR
1102
1103   /*!
1104   \brief Add a variable group into file.
1105      A variable group is added into file and it will play a role as parents for variables.
1106   A new variable group won't be created if one with id has already existed
1107   \param [in] id String identity of new variable group
1108   \return Pointer to added (or already existed) variable group
1109   */
1110   CVariableGroup* CFile::addVariableGroup(const string& id)
1111   TRY
1112   {
1113     return vVariableGroup->createChildGroup(id);
1114   }
1115   CATCH_DUMP_ATTR
1116
1117   void CFile::setContextClient(CContextClient* newContextClient)
1118   TRY
1119   {
1120     client = newContextClient;
1121     size_t size = this->enabledFields.size();
1122     for (size_t i = 0; i < size; ++i)
1123     {
1124       this->enabledFields[i]->setContextClient(newContextClient);
1125     }
1126   }
1127   CATCH_DUMP_ATTR
1128
1129   CContextClient* CFile::getContextClient()
1130   TRY
1131   {
1132     return client;
1133   }
1134   CATCH_DUMP_ATTR
1135
1136   void CFile::setReadContextClient(CContextClient* readContextclient)
1137   TRY
1138   {
1139     read_client = readContextclient;
1140   }
1141   CATCH_DUMP_ATTR
1142
1143   CContextClient* CFile::getReadContextClient()
1144   TRY
1145   {
1146     return read_client;
1147   }
1148   CATCH_DUMP_ATTR
1149
1150   /*!
1151   \brief Send a message to create a field on server side
1152   \param[in] id String identity of field that will be created on server
1153   */
1154   void CFile::sendAddField(const string& id, CContextClient* client)
1155   TRY
1156   {
1157      sendAddItem(id, EVENT_ID_ADD_FIELD, client);
1158   }
1159   CATCH_DUMP_ATTR
1160
1161   /*!
1162   \brief Send a message to create a field group on server side
1163   \param[in] id String identity of field group that will be created on server
1164   */
1165   void CFile::sendAddFieldGroup(const string& id, CContextClient* client)
1166   TRY
1167   {
1168      sendAddItem(id, (int)EVENT_ID_ADD_FIELD_GROUP, client);
1169   }
1170   CATCH_DUMP_ATTR
1171
1172   /*!
1173   \brief Receive a message annoucing the creation of a field on server side
1174   \param[in] event Received event
1175   */
1176   void CFile::recvAddField(CEventServer& event)
1177   TRY
1178   {
1179
1180      CBufferIn* buffer = event.subEvents.begin()->buffer;
1181      string id;
1182      *buffer>>id;
1183      get(id)->recvAddField(*buffer);
1184   }
1185   CATCH
1186
1187   /*!
1188   \brief Receive a message annoucing the creation of a field on server side
1189   \param[in] buffer Buffer containing message
1190   */
1191   void CFile::recvAddField(CBufferIn& buffer)
1192   TRY
1193   {
1194      string id;
1195      buffer>>id;
1196      addField(id);
1197   }
1198   CATCH_DUMP_ATTR
1199
1200   /*!
1201   \brief Receive a message annoucing the creation of a field group on server side
1202   \param[in] event Received event
1203   */
1204   void CFile::recvAddFieldGroup(CEventServer& event)
1205   TRY
1206   {
1207
1208      CBufferIn* buffer = event.subEvents.begin()->buffer;
1209      string id;
1210      *buffer>>id;
1211      get(id)->recvAddFieldGroup(*buffer);
1212   }
1213   CATCH
1214
1215   /*!
1216   \brief Receive a message annoucing the creation of a field group on server side
1217   \param[in] buffer Buffer containing message
1218   */
1219   void CFile::recvAddFieldGroup(CBufferIn& buffer)
1220   TRY
1221   {
1222      string id;
1223      buffer>>id;
1224      addFieldGroup(id);
1225   }
1226   CATCH_DUMP_ATTR
1227
1228   /*!
1229   \brief Send messages to duplicate all variables on server side
1230      Because each variable has also its attributes. So first thing to do is replicate
1231   all these attributes on server side. Because variable can have a value, the second thing
1232   is to duplicate this value on server, too.
1233   */
1234   void CFile::sendAddAllVariables(CContextClient* client)
1235   TRY
1236   {
1237     std::vector<CVariable*> allVar = getAllVariables();
1238     std::vector<CVariable*>::const_iterator it = allVar.begin();
1239     std::vector<CVariable*>::const_iterator itE = allVar.end();
1240
1241     for (; it != itE; ++it)
1242     {
1243       this->sendAddVariable((*it)->getId(), client);
1244       (*it)->sendAllAttributesToServer(client);
1245       (*it)->sendValue(client);
1246     }
1247   }
1248   CATCH_DUMP_ATTR
1249
1250   /*!
1251   \brief Send a message to create a variable group on server side
1252   \param[in] id String identity of variable group that will be created on server
1253   \param [in] client client to which we will send this adding action
1254   */
1255   void CFile::sendAddVariableGroup(const string& id, CContextClient* client)
1256   TRY
1257   {
1258      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE_GROUP, client);
1259   }
1260   CATCH_DUMP_ATTR
1261
1262   /*
1263     Send message to add a variable into a file within a certain client
1264     \param [in] id String identity of a variable
1265     \param [in] client client to which we will send this adding action
1266   */
1267   void CFile::sendAddVariable(const string& id, CContextClient* client)
1268   TRY
1269   {
1270      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE, client);
1271   }
1272   CATCH_DUMP_ATTR
1273
1274   /*!
1275   \brief Receive a message annoucing the creation of a variable on server side
1276   \param[in] event Received event
1277   */
1278   void CFile::recvAddVariable(CEventServer& event)
1279   TRY
1280   {
1281      CBufferIn* buffer = event.subEvents.begin()->buffer;
1282      string id;
1283      *buffer>>id;
1284      get(id)->recvAddVariable(*buffer);
1285   }
1286   CATCH
1287
1288   /*!
1289   \brief Receive a message annoucing the creation of a variable on server side
1290   \param[in] buffer Buffer containing message
1291   */
1292   void CFile::recvAddVariable(CBufferIn& buffer)
1293   TRY
1294   {
1295      string id;
1296      buffer>>id;
1297      addVariable(id);
1298   }
1299   CATCH_DUMP_ATTR
1300
1301   /*!
1302   \brief Receive a message annoucing the creation of a variable group on server side
1303   \param[in] event Received event
1304   */
1305   void CFile::recvAddVariableGroup(CEventServer& event)
1306   TRY
1307   {
1308
1309      CBufferIn* buffer = event.subEvents.begin()->buffer;
1310      string id;
1311      *buffer>>id;
1312      get(id)->recvAddVariableGroup(*buffer);
1313   }
1314   CATCH
1315
1316   /*!
1317   \brief Receive a message annoucing the creation of a variable group on server side
1318   \param[in] buffer Buffer containing message
1319   */
1320   void CFile::recvAddVariableGroup(CBufferIn& buffer)
1321   TRY
1322   {
1323      string id;
1324      buffer>>id;
1325      addVariableGroup(id);
1326   }
1327   CATCH_DUMP_ATTR
1328
1329   /*!
1330     \brief Sending all active (enabled) fields from client to server.
1331   Each field is identified uniquely by its string identity. Not only should we
1332   send the id to server but also we need to send ids of reference domain and reference axis.
1333   With these two id, it's easier to make reference to grid where all data should be written.
1334   Remark: This function must be called AFTER all active (enabled) files have been created on the server side
1335   */
1336   void CFile::sendEnabledFields(CContextClient* client)
1337   TRY
1338   {
1339     size_t size = this->enabledFields.size();
1340     for (size_t i = 0; i < size; ++i)
1341     {
1342       CField* field = this->enabledFields[i];
1343       this->sendAddField(field->getId(), client);
1344       field->checkTimeAttributes();
1345       field->sendAllAttributesToServer(client);
1346       field->sendAddAllVariables(client);
1347     }
1348   }
1349   CATCH_DUMP_ATTR
1350
1351   /*!
1352   \brief Dispatch event received from client
1353      Whenever a message is received in buffer of server, it will be processed depending on
1354   its event type. A new event type should be added in the switch list to make sure
1355   it processed on server side.
1356   \param [in] event: Received message
1357   */
1358   bool CFile::dispatchEvent(CEventServer& event)
1359   TRY
1360   {
1361      if (SuperClass::dispatchEvent(event)) return true;
1362      else
1363      {
1364        switch(event.type)
1365        {
1366           case EVENT_ID_ADD_FIELD :
1367             recvAddField(event);
1368             return true;
1369             break;
1370
1371           case EVENT_ID_ADD_FIELD_GROUP :
1372             recvAddFieldGroup(event);
1373             return true;
1374             break;
1375
1376            case EVENT_ID_ADD_VARIABLE :
1377             recvAddVariable(event);
1378             return true;
1379             break;
1380
1381           case EVENT_ID_ADD_VARIABLE_GROUP :
1382             recvAddVariableGroup(event);
1383             return true;
1384             break;
1385           default :
1386              ERROR("bool CFile::dispatchEvent(CEventServer& event)", << "Unknown Event");
1387           return false;
1388        }
1389      }
1390   }
1391   CATCH
1392
1393   ///--------------------------------------------------------------
1394   /*!
1395   */
1396   StdString CFile::dumpClassAttributes(void)
1397   {
1398     StdString str;
1399     CContext* context = CContext::getCurrent();
1400     str.append("context=\"");
1401     str.append(context->getId());
1402     str.append("\"");
1403     str.append(" enabled fields=\"");
1404     int size = this->enabledFields.size();
1405     for (int i = 0; i < size; ++i)
1406     {
1407       str.append(this->enabledFields[i]->getId());
1408       str.append(" ");
1409     }
1410     str.append("\"");
1411     return str;
1412   }
1413
1414   ///---------------------------------------------------------------
1415
1416} // namespace xios
Note: See TracBrowser for help on using the repository browser.