source: XIOS/trunk/src/node/context.cpp @ 711

Last change on this file since 711 was 711, checked in by rlacroix, 9 years ago

Support creating the timeseries automatically.

Fields which should be outputted as timeseries must have the ts_enabled attribute set to true.

Files have a new attribute timeseries which must be one of:

  • none: no timeseries are outputted, only the regular file (default behavior when the attribute is ommited).
  • only: only the timeseries are outputted, the regular file is not created.
  • both: both the timeseries and the regular files are outputted.
  • exclusive: the timeseries are outputted and a regular file is created with only the fields which were not marked for output as a timeserie (if any).

The name of the files created for the timeseries is composed of a prefix followed by the name of the variable. The file attribute ts_prefix can be used to control the prefix used (by default, the name of the regular file is used).

All the attributes of the regular file are inherited by the files created for the timeseries. The field attribute ts_split_freq can be used to configure the splitting for each timeseries (by default, the splitting frequency of the regular file is used).

  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
File size: 36.7 KB
Line 
1#include "context.hpp"
2#include "attribute_template.hpp"
3#include "object_template.hpp"
4#include "group_template.hpp"
5
6#include "calendar_type.hpp"
7#include "duration.hpp"
8
9#include "context_client.hpp"
10#include "context_server.hpp"
11#include "nc4_data_output.hpp"
12#include "node_type.hpp"
13#include "message.hpp"
14#include "type.hpp"
15#include "xios_spl.hpp"
16
17
18namespace xios {
19
20  shared_ptr<CContextGroup> CContext::root;
21
22   /// ////////////////////// Définitions ////////////////////// ///
23
24   CContext::CContext(void)
25      : CObjectTemplate<CContext>(), CContextAttributes()
26      , calendar(), hasClient(false), hasServer(false), isPostProcessed(false), finalized(false)
27      , dataSize_(), idServer_(), client(0), server(0)
28   { /* Ne rien faire de plus */ }
29
30   CContext::CContext(const StdString & id)
31      : CObjectTemplate<CContext>(id), CContextAttributes()
32      , calendar(), hasClient(false), hasServer(false), isPostProcessed(false), finalized(false)
33      , dataSize_(), idServer_(), client(0), server(0)
34   { /* Ne rien faire de plus */ }
35
36   CContext::~CContext(void)
37   {
38     delete client;
39     delete server;
40   }
41
42   //----------------------------------------------------------------
43   //! Get name of context
44   StdString CContext::GetName(void)   { return (StdString("context")); }
45   StdString CContext::GetDefName(void){ return (CContext::GetName()); }
46   ENodeType CContext::GetType(void)   { return (eContext); }
47
48   //----------------------------------------------------------------
49
50   /*!
51   \brief Get context group (context root)
52   \return Context root
53   */
54   CContextGroup* CContext::getRoot(void)
55   {
56      if (root.get()==NULL) root=shared_ptr<CContextGroup>(new CContextGroup(xml::CXMLNode::GetRootName()));
57      return root.get();
58   }
59
60   //----------------------------------------------------------------
61
62   /*!
63   \brief Get calendar of a context
64   \return Calendar
65   */
66   boost::shared_ptr<CCalendar> CContext::getCalendar(void) const
67   {
68      return (this->calendar);
69   }
70
71   //----------------------------------------------------------------
72
73   /*!
74   \brief Set a context with a calendar
75   \param[in] newCalendar new calendar
76   */
77   void CContext::setCalendar(boost::shared_ptr<CCalendar> newCalendar)
78   {
79      this->calendar = newCalendar;
80   }
81
82   //----------------------------------------------------------------
83   /*!
84   \brief Parse xml file and write information into context object
85   \param [in] node xmld node corresponding in xml file
86   */
87   void CContext::parse(xml::CXMLNode & node)
88   {
89      CContext::SuperClass::parse(node);
90
91      // PARSING POUR GESTION DES ENFANTS
92      xml::THashAttributes attributes = node.getAttributes();
93
94      if (attributes.end() != attributes.find("src"))
95      {
96         StdIFStream ifs ( attributes["src"].c_str() , StdIFStream::in );
97         if ( (ifs.rdstate() & std::ifstream::failbit ) != 0 )
98            ERROR("void CContext::parse(xml::CXMLNode & node)",
99                  <<endl<< "Can not open <"<<attributes["src"].c_str()<<"> file" );
100         if (!ifs.good())
101            ERROR("CContext::parse(xml::CXMLNode & node)",
102                  << "[ filename = " << attributes["src"] << " ] Bad xml stream !");
103         xml::CXMLParser::ParseInclude(ifs, attributes["src"], *this);
104      }
105
106      if (node.getElementName().compare(CContext::GetName()))
107         DEBUG("Le noeud is wrong defined but will be considered as a context !");
108
109      if (!(node.goToChildElement()))
110      {
111         DEBUG("Le context ne contient pas d'enfant !");
112      }
113      else
114      {
115         do { // Parcours des contextes pour traitement.
116
117            StdString name = node.getElementName();
118            attributes.clear();
119            attributes = node.getAttributes();
120
121            if (attributes.end() != attributes.find("id"))
122            {
123              DEBUG(<< "Definition node has an id,"
124                    << "it will not be taking account !");
125            }
126
127#define DECLARE_NODE(Name_, name_)    \
128   if (name.compare(C##Name_##Definition::GetDefName()) == 0) \
129   { C##Name_##Definition::create(C##Name_##Definition::GetDefName()) -> parse(node); continue; }
130#define DECLARE_NODE_PAR(Name_, name_)
131#include "node_type.conf"
132
133            DEBUG(<< "The element \'"     << name
134                  << "\' in the context \'" << CContext::getCurrent()->getId()
135                  << "\' is not a definition !");
136
137         } while (node.goToNextElement());
138
139         node.goToParentElement(); // Retour au parent
140      }
141   }
142
143   //----------------------------------------------------------------
144   //! Show tree structure of context
145   void CContext::ShowTree(StdOStream & out)
146   {
147      StdString currentContextId = CContext::getCurrent() -> getId();
148      std::vector<CContext*> def_vector =
149         CContext::getRoot()->getChildList();
150      std::vector<CContext*>::iterator
151         it = def_vector.begin(), end = def_vector.end();
152
153      out << "<? xml version=\"1.0\" ?>" << std::endl;
154      out << "<"  << xml::CXMLNode::GetRootName() << " >" << std::endl;
155
156      for (; it != end; it++)
157      {
158         CContext* context = *it;
159         CContext::setCurrent(context->getId());
160         out << *context << std::endl;
161      }
162
163      out << "</" << xml::CXMLNode::GetRootName() << " >" << std::endl;
164      CContext::setCurrent(currentContextId);
165   }
166
167
168   //----------------------------------------------------------------
169
170   //! Convert context object into string (to print)
171   StdString CContext::toString(void) const
172   {
173      StdOStringStream oss;
174      oss << "<" << CContext::GetName()
175          << " id=\"" << this->getId() << "\" "
176          << SuperClassAttribute::toString() << ">" << std::endl;
177      if (!this->hasChild())
178      {
179         //oss << "<!-- No definition -->" << std::endl; // fait planter l'incrémentation
180      }
181      else
182      {
183
184#define DECLARE_NODE(Name_, name_)    \
185   if (C##Name_##Definition::has(C##Name_##Definition::GetDefName())) \
186   oss << * C##Name_##Definition::get(C##Name_##Definition::GetDefName()) << std::endl;
187#define DECLARE_NODE_PAR(Name_, name_)
188#include "node_type.conf"
189
190      }
191
192      oss << "</" << CContext::GetName() << " >";
193
194      return (oss.str());
195   }
196
197   //----------------------------------------------------------------
198
199   /*!
200   \brief Find all inheritace among objects in a context.
201   \param [in] apply (true) write attributes of parent into ones of child if they are empty
202                     (false) write attributes of parent into a new container of child
203   \param [in] parent unused
204   */
205   void CContext::solveDescInheritance(bool apply, const CAttributeMap * const UNUSED(parent))
206   {
207#define DECLARE_NODE(Name_, name_)    \
208   if (C##Name_##Definition::has(C##Name_##Definition::GetDefName())) \
209     C##Name_##Definition::get(C##Name_##Definition::GetDefName())->solveDescInheritance(apply);
210#define DECLARE_NODE_PAR(Name_, name_)
211#include "node_type.conf"
212   }
213
214   //----------------------------------------------------------------
215
216   //! Verify if all root definition in the context have child.
217   bool CContext::hasChild(void) const
218   {
219      return (
220#define DECLARE_NODE(Name_, name_)    \
221   C##Name_##Definition::has(C##Name_##Definition::GetDefName())   ||
222#define DECLARE_NODE_PAR(Name_, name_)
223#include "node_type.conf"
224      false);
225}
226
227   //----------------------------------------------------------------
228
229   void CContext::CleanTree(void)
230   {
231#define DECLARE_NODE(Name_, name_) C##Name_##Definition::ClearAllAttributes();
232#define DECLARE_NODE_PAR(Name_, name_)
233#include "node_type.conf"
234   }
235   ///---------------------------------------------------------------
236
237   //! Initialize client side
238   void CContext::initClient(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtServer /*= 0*/)
239   {
240     hasClient=true;
241     client = new CContextClient(this,intraComm, interComm, cxtServer);
242     registryIn=new CRegistry(intraComm);
243     registryIn->setPath(getId()) ;
244     if (client->clientRank==0) registryIn->fromFile("xios_registry.bin") ;
245     registryIn->bcastRegistry() ;
246
247     registryOut=new CRegistry(intraComm) ;
248     registryOut->setPath(getId()) ;
249
250     MPI_Comm intraCommServer, interCommServer;
251     if (cxtServer) // Attached mode
252     {
253       intraCommServer = intraComm;
254       interCommServer = interComm;
255     }
256     else
257     {
258       MPI_Comm_dup(intraComm, &intraCommServer);
259       comms.push_back(intraCommServer);
260       MPI_Comm_dup(interComm, &interCommServer);
261       comms.push_back(interCommServer);
262     }
263     server = new CContextServer(this,intraCommServer,interCommServer);
264   }
265
266   void CContext::setClientServerBuffer()
267   {
268     size_t bufferSizeMin = 10 * sizeof(size_t) * 1024;
269#define DECLARE_NODE(Name_, name_)    \
270     bufferSizeMin = (bufferSizeMin < sizeof(C##Name_##Definition)) ?  sizeof(C##Name_##Definition) : bufferSizeMin;
271#define DECLARE_NODE_PAR(Name_, name_)
272#include "node_type.conf"
273     std::map<int, StdSize> bufferSize = getDataSize();
274     if (bufferSize.empty())
275     {
276       if (client->isServerLeader())
277       {
278         const std::list<int>& ranks = client->getRanksServerLeader();
279         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
280           bufferSize[*itRank] = bufferSizeMin;
281       }
282       else
283        return;
284     }
285     else
286     {
287       std::map<int, StdSize>::iterator it  = bufferSize.begin(),
288                                        ite = bufferSize.end();
289       for (; it != ite; ++it)
290         it->second = (it->second < bufferSizeMin) ? bufferSizeMin : it->second;
291     }
292
293     client->setBufferSize(bufferSize);
294   }
295
296   //! Verify whether a context is initialized
297   bool CContext::isInitialized(void)
298   {
299     return hasClient;
300   }
301
302   //! Initialize server
303   void CContext::initServer(MPI_Comm intraComm,MPI_Comm interComm, CContext* cxtClient /*= 0*/)
304   {
305     hasServer=true;
306     server = new CContextServer(this,intraComm,interComm);
307
308     registryIn=new CRegistry(intraComm);
309     registryIn->setPath(getId()) ;
310     if (server->intraCommRank==0) registryIn->fromFile("xios_registry.bin") ;
311     registryIn->bcastRegistry() ;
312     registryOut=new CRegistry(intraComm) ;
313     registryOut->setPath(getId()) ;
314 
315     MPI_Comm intraCommClient, interCommClient;
316     if (cxtClient) // Attached mode
317     {
318       intraCommClient = intraComm;
319       interCommClient = interComm;
320     }
321     else
322     {
323       MPI_Comm_dup(intraComm, &intraCommClient);
324       comms.push_back(intraCommClient);
325       MPI_Comm_dup(interComm, &interCommClient);
326       comms.push_back(interCommClient);
327     }
328     client = new CContextClient(this,intraCommClient,interCommClient, cxtClient);
329   }
330
331   //! Server side: Put server into a loop in order to listen message from client
332   bool CContext::eventLoop(void)
333   {
334     return server->eventLoop();
335   }
336
337   //! Try to send the buffers and receive possible answers
338   bool CContext::checkBuffersAndListen(void)
339   {
340     client->checkBuffers();
341     return server->eventLoop();
342   }
343
344   //! Terminate a context
345   void CContext::finalize(void)
346   {
347      if (!finalized)
348      {
349        finalized = true;
350        if (hasClient) sendRegistry() ;
351        client->finalize();
352        while (!server->hasFinished())
353        {
354          server->eventLoop();
355        }
356
357        if (hasServer)
358        {
359          closeAllFile();
360          registryOut->hierarchicalGatherRegistry() ;
361          if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
362        }
363       
364        for (std::list<MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
365          MPI_Comm_free(&(*it));
366        comms.clear();
367      }
368   }
369
370   /*!
371   \brief Close all the context defintion and do processing data
372      After everything is well defined on client side, they will be processed and sent to server
373   From the version 2.0, sever and client work no more on the same database. Moreover, client(s) will send
374   all necessary information to server, from which each server can build its own database.
375   Because the role of server is to write out field data on a specific netcdf file,
376   the only information that it needs is the enabled files
377   and the active fields (fields will be written onto active files)
378   */
379   void CContext::closeDefinition(void)
380   {
381     // There is nothing client need to send to server
382     if (hasClient)
383     {
384       // After xml is parsed, there are some more works with post processing
385       postProcessing();
386     }
387     setClientServerBuffer();
388
389     if (hasClient && !hasServer)
390     {
391      // Send all attributes of current context to server
392      this->sendAllAttributesToServer();
393
394      // Send all attributes of current calendar
395      CCalendarWrapper::get(CCalendarWrapper::GetDefName())->sendAllAttributesToServer();
396
397      // We have enough information to send to server
398      // First of all, send all enabled files
399       sendEnabledFiles();
400
401      // Then, send all enabled fields
402       sendEnabledFields();
403
404      // At last, we have all info of domain and axis, then send them
405       sendRefDomainsAxis();
406
407      // After that, send all grid (if any)
408       sendRefGrid();
409    }
410
411    // We have a xml tree on the server side and now, it should be also processed
412    if (hasClient && !hasServer) sendPostProcessing();
413
414    // There are some processings that should be done after all of above. For example: check mask or index
415    if (hasClient)
416    {
417      this->buildFilterGraphOfEnabledFields();
418      buildFilterGraphOfFieldsWithReadAccess();
419      this->solveAllRefOfEnabledFields(true);
420    }
421
422    // Now tell server that it can process all messages from client
423    if (hasClient && !hasServer) this->sendCloseDefinition();
424
425    // Nettoyage de l'arborescence
426    if (hasClient && !hasServer) CleanTree(); // Only on client side??
427
428    if (hasClient)
429    {
430      sendCreateFileHeader();
431
432      startPrefetchingOfEnabledReadModeFiles();
433    }
434   }
435
436   void CContext::findAllEnabledFields(void)
437   {
438     for (unsigned int i = 0; i < this->enabledFiles.size(); i++)
439     (void)this->enabledFiles[i]->getEnabledFields();
440   }
441
442   void CContext::solveAllRefOfEnabledFields(bool sendToServer)
443   {
444     int size = this->enabledFiles.size();
445     for (int i = 0; i < size; ++i)
446     {
447       this->enabledFiles[i]->solveAllRefOfEnabledFields(sendToServer);
448     }
449   }
450
451   void CContext::buildFilterGraphOfEnabledFields()
452   {
453     int size = this->enabledFiles.size();
454     for (int i = 0; i < size; ++i)
455     {
456       this->enabledFiles[i]->buildFilterGraphOfEnabledFields(garbageCollector);
457     }
458   }
459
460   void CContext::startPrefetchingOfEnabledReadModeFiles()
461   {
462     int size = enabledReadModeFiles.size();
463     for (int i = 0; i < size; ++i)
464     {
465        enabledReadModeFiles[i]->prefetchEnabledReadModeFields();
466     }
467   }
468
469   void CContext::checkPrefetchingOfEnabledReadModeFiles()
470   {
471     int size = enabledReadModeFiles.size();
472     for (int i = 0; i < size; ++i)
473     {
474        enabledReadModeFiles[i]->prefetchEnabledReadModeFieldsIfNeeded();
475     }
476   }
477
478  void CContext::findFieldsWithReadAccess(void)
479  {
480    fieldsWithReadAccess.clear();
481    const vector<CField*> allFields = CField::getAll();
482    for (size_t i = 0; i < allFields.size(); ++i)
483    {
484      if (allFields[i]->file && !allFields[i]->file->mode.isEmpty() && allFields[i]->file->mode.getValue() == CFile::mode_attr::read)
485        allFields[i]->read_access = true;
486      if (!allFields[i]->read_access.isEmpty() && allFields[i]->read_access.getValue())
487        fieldsWithReadAccess.push_back(allFields[i]);
488    }
489  }
490
491  void CContext::solveAllRefOfFieldsWithReadAccess()
492  {
493    for (size_t i = 0; i < fieldsWithReadAccess.size(); ++i)
494      fieldsWithReadAccess[i]->solveAllReferenceEnabledField(false);
495  }
496
497  void CContext::buildFilterGraphOfFieldsWithReadAccess()
498  {
499    for (size_t i = 0; i < fieldsWithReadAccess.size(); ++i)
500      fieldsWithReadAccess[i]->buildFilterGraph(garbageCollector, true);
501  }
502
503   void CContext::solveAllInheritance(bool apply)
504   {
505     // Résolution des héritages descendants (càd des héritages de groupes)
506     // pour chacun des contextes.
507      solveDescInheritance(apply);
508
509     // Résolution des héritages par référence au niveau des fichiers.
510      const vector<CFile*> allFiles=CFile::getAll();
511      const vector<CGrid*> allGrids= CGrid::getAll();
512
513     //if (hasClient && !hasServer)
514      if (hasClient)
515      {
516        for (unsigned int i = 0; i < allFiles.size(); i++)
517          allFiles[i]->solveFieldRefInheritance(apply);
518      }
519
520      unsigned int vecSize = allGrids.size();
521      unsigned int i = 0;
522      for (i = 0; i < vecSize; ++i)
523        allGrids[i]->solveDomainAxisRefInheritance(apply);
524
525   }
526
527   void CContext::findEnabledFiles(void)
528   {
529      const std::vector<CFile*> allFiles = CFile::getAll();
530
531      for (unsigned int i = 0; i < allFiles.size(); i++)
532         if (!allFiles[i]->enabled.isEmpty()) // Si l'attribut 'enabled' est défini.
533         {
534            if (allFiles[i]->enabled.getValue()) // Si l'attribut 'enabled' est fixé à vrai.
535               enabledFiles.push_back(allFiles[i]);
536         }
537         else enabledFiles.push_back(allFiles[i]); // otherwise true by default
538
539
540      if (enabledFiles.size() == 0)
541         DEBUG(<<"Aucun fichier ne va être sorti dans le contexte nommé \""
542               << getId() << "\" !");
543   }
544
545   void CContext::findEnabledReadModeFiles(void)
546   {
547     int size = this->enabledFiles.size();
548     for (int i = 0; i < size; ++i)
549     {
550       if (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::read)
551        enabledReadModeFiles.push_back(enabledFiles[i]);
552     }
553   }
554
555   void CContext::closeAllFile(void)
556   {
557     std::vector<CFile*>::const_iterator
558            it = this->enabledFiles.begin(), end = this->enabledFiles.end();
559
560     for (; it != end; it++)
561     {
562       info(30)<<"Closing File : "<<(*it)->getId()<<endl;
563       (*it)->close();
564     }
565   }
566
567   /*!
568   \brief Dispatch event received from client
569      Whenever a message is received in buffer of server, it will be processed depending on
570   its event type. A new event type should be added in the switch list to make sure
571   it processed on server side.
572   \param [in] event: Received message
573   */
574   bool CContext::dispatchEvent(CEventServer& event)
575   {
576
577      if (SuperClass::dispatchEvent(event)) return true;
578      else
579      {
580        switch(event.type)
581        {
582           case EVENT_ID_CLOSE_DEFINITION :
583             recvCloseDefinition(event);
584             return true;
585             break;
586           case EVENT_ID_UPDATE_CALENDAR:
587             recvUpdateCalendar(event);
588             return true;
589             break;
590           case EVENT_ID_CREATE_FILE_HEADER :
591             recvCreateFileHeader(event);
592             return true;
593             break;
594           case EVENT_ID_POST_PROCESS:
595             recvPostProcessing(event);
596             return true;
597            case EVENT_ID_SEND_REGISTRY:
598             recvRegistry(event);
599             return true;
600            break;
601
602           default :
603             ERROR("bool CContext::dispatchEvent(CEventServer& event)",
604                    <<"Unknown Event");
605           return false;
606         }
607      }
608   }
609
610   //! Client side: Send a message to server to make it close
611   void CContext::sendCloseDefinition(void)
612   {
613     CEventClient event(getType(),EVENT_ID_CLOSE_DEFINITION);
614     if (client->isServerLeader())
615     {
616       CMessage msg;
617       msg<<this->getIdServer();
618       const std::list<int>& ranks = client->getRanksServerLeader();
619       for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
620         event.push(*itRank,1,msg);
621       client->sendEvent(event);
622     }
623     else client->sendEvent(event);
624   }
625
626   //! Server side: Receive a message of client announcing a context close
627   void CContext::recvCloseDefinition(CEventServer& event)
628   {
629
630      CBufferIn* buffer=event.subEvents.begin()->buffer;
631      string id;
632      *buffer>>id;
633      get(id)->closeDefinition();
634   }
635
636   //! Client side: Send a message to update calendar in each time step
637   void CContext::sendUpdateCalendar(int step)
638   {
639     if (!hasServer)
640     {
641       CEventClient event(getType(),EVENT_ID_UPDATE_CALENDAR);
642       if (client->isServerLeader())
643       {
644         CMessage msg;
645         msg<<this->getIdServer()<<step;
646         const std::list<int>& ranks = client->getRanksServerLeader();
647         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
648           event.push(*itRank,1,msg);
649         client->sendEvent(event);
650       }
651       else client->sendEvent(event);
652     }
653   }
654
655   //! Server side: Receive a message of client annoucing calendar update
656   void CContext::recvUpdateCalendar(CEventServer& event)
657   {
658      CBufferIn* buffer=event.subEvents.begin()->buffer;
659      string id;
660      *buffer>>id;
661      get(id)->recvUpdateCalendar(*buffer);
662   }
663
664   //! Server side: Receive a message of client annoucing calendar update
665   void CContext::recvUpdateCalendar(CBufferIn& buffer)
666   {
667      int step;
668      buffer>>step;
669      updateCalendar(step);
670   }
671
672   //! Client side: Send a message to create header part of netcdf file
673   void CContext::sendCreateFileHeader(void)
674   {
675     CEventClient event(getType(),EVENT_ID_CREATE_FILE_HEADER);
676     if (client->isServerLeader())
677     {
678       CMessage msg;
679       msg<<this->getIdServer();
680       const std::list<int>& ranks = client->getRanksServerLeader();
681       for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
682         event.push(*itRank,1,msg) ;
683       client->sendEvent(event);
684     }
685     else client->sendEvent(event);
686   }
687
688   //! Server side: Receive a message of client annoucing the creation of header part of netcdf file
689   void CContext::recvCreateFileHeader(CEventServer& event)
690   {
691      CBufferIn* buffer=event.subEvents.begin()->buffer;
692      string id;
693      *buffer>>id;
694      get(id)->recvCreateFileHeader(*buffer);
695   }
696
697   //! Server side: Receive a message of client annoucing the creation of header part of netcdf file
698   void CContext::recvCreateFileHeader(CBufferIn& buffer)
699   {
700      createFileHeader();
701   }
702
703   //! Client side: Send a message to do some post processing on server
704   void CContext::sendPostProcessing()
705   {
706     if (!hasServer)
707     {
708       CEventClient event(getType(),EVENT_ID_POST_PROCESS);
709       if (client->isServerLeader())
710       {
711         CMessage msg;
712         msg<<this->getIdServer();
713         const std::list<int>& ranks = client->getRanksServerLeader();
714         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
715           event.push(*itRank,1,msg);
716         client->sendEvent(event);
717       }
718       else client->sendEvent(event);
719     }
720   }
721
722   //! Server side: Receive a message to do some post processing
723   void CContext::recvPostProcessing(CEventServer& event)
724   {
725      CBufferIn* buffer=event.subEvents.begin()->buffer;
726      string id;
727      *buffer>>id;
728      get(id)->recvPostProcessing(*buffer);
729   }
730
731   //! Server side: Receive a message to do some post processing
732   void CContext::recvPostProcessing(CBufferIn& buffer)
733   {
734      CCalendarWrapper::get(CCalendarWrapper::GetDefName())->createCalendar();
735      postProcessing();
736   }
737
738   const StdString& CContext::getIdServer()
739   {
740      if (hasClient)
741      {
742        idServer_ = this->getId();
743        idServer_ += "_server";
744        return idServer_;
745      }
746      if (hasServer) return (this->getId());
747   }
748
749   /*!
750   \brief Do some simple post processings after parsing xml file
751      After the xml file (iodef.xml) is parsed, it is necessary to build all relations among
752   created object, e.g: inhertance among fields, domain, axis. After that, all fiels as well as their parents (reference fields),
753   which will be written out into netcdf files, are processed
754   */
755   void CContext::postProcessing()
756   {
757     if (isPostProcessed) return;
758
759      // Make sure the calendar was correctly created
760      if (!calendar)
761        ERROR("CContext::postProcessing()", << "A calendar must be defined for the context \"" << getId() << "!\"")
762      else if (calendar->getTimeStep() == NoneDu)
763        ERROR("CContext::postProcessing()", << "A timestep must be defined for the context \"" << getId() << "!\"")
764      // Calendar first update to set the current date equals to the start date
765      calendar->update(0);
766
767      // Find all inheritance in xml structure
768      this->solveAllInheritance();
769
770      // Check if some axis, domains or grids are eligible to for compressed indexed output.
771      // Warning: This must be done after solving the inheritance and before the rest of post-processing
772      checkAxisDomainsGridsEligibilityForCompressedOutput();
773
774      // Check if some automatic time series should be generated
775      // Warning: This must be done after solving the inheritance and before the rest of post-processing
776      prepareTimeseries();
777
778      //Initialisation du vecteur 'enabledFiles' contenant la liste des fichiers à sortir.
779      this->findEnabledFiles();
780      this->findEnabledReadModeFiles();
781
782      // Find all enabled fields of each file
783      this->findAllEnabledFields();
784
785      // Search and rebuild all reference object of enabled fields
786      this->solveAllRefOfEnabledFields(false);
787
788      // Find all fields with read access from the public API
789      findFieldsWithReadAccess();
790      // and solve the all reference for them
791      solveAllRefOfFieldsWithReadAccess();
792
793      isPostProcessed = true;
794   }
795
796   std::map<int, StdSize>& CContext::getDataSize()
797   {
798     CFile::mode_attr::t_enum mode = hasClient ? CFile::mode_attr::write : CFile::mode_attr::read;
799
800     // Set of grid used by enabled fields
801     std::set<StdString> usedGrid;
802
803     // Find all reference domain and axis of all active fields
804     int numEnabledFiles = this->enabledFiles.size();
805     for (int i = 0; i < numEnabledFiles; ++i)
806     {
807       CFile* file = this->enabledFiles[i];
808       CFile::mode_attr::t_enum fileMode = file->mode.isEmpty() ? CFile::mode_attr::write : file->mode.getValue();
809
810       if (fileMode == mode)
811       {
812         std::vector<CField*> enabledFields = file->getEnabledFields();
813         int numEnabledFields = enabledFields.size();
814         for (int j = 0; j < numEnabledFields; ++j)
815         {
816           StdString currentGrid = enabledFields[j]->grid->getId();
817           const std::map<int, StdSize> mapSize = enabledFields[j]->getGridDataSize();
818           if (dataSize_.empty())
819           {
820             dataSize_ = mapSize;
821             usedGrid.insert(currentGrid);
822           }
823           else
824           {
825             std::map<int, StdSize>::const_iterator it = mapSize.begin(), itE = mapSize.end();
826             if (usedGrid.find(currentGrid) == usedGrid.end())
827             {
828               for (; it != itE; ++it)
829               {
830                 if (0 < dataSize_.count(it->first)) dataSize_[it->first] += it->second;
831                 else dataSize_.insert(make_pair(it->first, it->second));
832               }
833             } else
834             {
835               for (; it != itE; ++it)
836               {
837                 if (0 < dataSize_.count(it->first))
838                  if (CXios::isOptPerformance) dataSize_[it->first] += it->second;
839                  else
840                  {
841                    if (dataSize_[it->first] < it->second) dataSize_[it->first] = it->second;
842                  }
843                 else dataSize_.insert(make_pair(it->first, it->second));
844               }
845             }
846           }
847         }
848       }
849     }
850
851     return dataSize_;
852   }
853
854   //! Client side: Send infomation of active files (files are enabled to write out)
855   void CContext::sendEnabledFiles()
856   {
857     int size = this->enabledFiles.size();
858
859     // In a context, each type has a root definition, e.g: axis, domain, field.
860     // Every object must be a child of one of these root definition. In this case
861     // all new file objects created on server must be children of the root "file_definition"
862     StdString fileDefRoot("file_definition");
863     CFileGroup* cfgrpPtr = CFileGroup::get(fileDefRoot);
864
865     for (int i = 0; i < size; ++i)
866     {
867       cfgrpPtr->sendCreateChild(this->enabledFiles[i]->getId());
868       this->enabledFiles[i]->sendAllAttributesToServer();
869       this->enabledFiles[i]->sendAddAllVariables();
870     }
871   }
872
873   //! Client side: Send information of active fields (ones are written onto files)
874   void CContext::sendEnabledFields()
875   {
876     int size = this->enabledFiles.size();
877     for (int i = 0; i < size; ++i)
878     {
879       this->enabledFiles[i]->sendEnabledFields();
880     }
881   }
882
883   //! Client side: Check if the defined axis, domains and grids are eligible for compressed indexed output
884   void CContext::checkAxisDomainsGridsEligibilityForCompressedOutput()
885   {
886     if (!hasClient) return;
887
888     const vector<CAxis*> allAxis = CAxis::getAll();
889     for (vector<CAxis*>::const_iterator it = allAxis.begin(); it != allAxis.end(); it++)
890       (*it)->checkEligibilityForCompressedOutput();
891
892     const vector<CDomain*> allDomains = CDomain::getAll();
893     for (vector<CDomain*>::const_iterator it = allDomains.begin(); it != allDomains.end(); it++)
894       (*it)->checkEligibilityForCompressedOutput();
895
896     const vector<CGrid*> allGrids = CGrid::getAll();
897     for (vector<CGrid*>::const_iterator it = allGrids.begin(); it != allGrids.end(); it++)
898       (*it)->checkEligibilityForCompressedOutput();
899   }
900
901   //! Client side: Prepare the timeseries by adding the necessary files
902   void CContext::prepareTimeseries()
903   {
904     if (!hasClient) return;
905
906     const std::vector<CFile*> allFiles = CFile::getAll();
907     for (size_t i = 0; i < allFiles.size(); i++)
908     {
909       CFile* file = allFiles[i];
910
911       if (!file->timeseries.isEmpty() && file->timeseries != CFile::timeseries_attr::none)
912       {
913         StdString tsPrefix = !file->ts_prefix.isEmpty() ? file->ts_prefix : (!file->name.isEmpty() ? file->name : file->getId());
914
915         const std::vector<CField*> allFields = file->getAllFields();
916         for (size_t j = 0; j < allFields.size(); j++)
917         {
918           CField* field = allFields[j];
919
920           if (!field->ts_enabled.isEmpty() && field->ts_enabled)
921           {
922             CFile* tsFile = CFile::create();
923             tsFile->duplicateAttributes(file);
924
925             tsFile->name = tsPrefix + "_";
926             if (!field->name.isEmpty())
927               tsFile->name.get() += field->name;
928             else if (field->hasDirectFieldReference()) // We cannot use getBaseFieldReference() just yet
929               tsFile->name.get() += field->field_ref;
930             else
931               tsFile->name.get() += field->getId();
932
933             if (!field->ts_split_freq.isEmpty())
934               tsFile->split_freq = field->ts_split_freq;
935
936             CField* tsField = tsFile->addField();
937             tsField->field_ref = field->getId();
938
939             tsFile->solveFieldRefInheritance(true);
940
941             if (file->timeseries == CFile::timeseries_attr::exclusive)
942               field->enabled = false;
943           }
944         }
945
946         // Finally disable the original file is need be
947         if (file->timeseries == CFile::timeseries_attr::only)
948          file->enabled = false;
949       }
950     }
951   }
952
953   //! Client side: Send information of reference grid of active fields
954   void CContext::sendRefGrid()
955   {
956     std::set<StdString> gridIds;
957     int sizeFile = this->enabledFiles.size();
958     CFile* filePtr(NULL);
959
960     // Firstly, find all reference grids of all active fields
961     for (int i = 0; i < sizeFile; ++i)
962     {
963       filePtr = this->enabledFiles[i];
964       std::vector<CField*> enabledFields = filePtr->getEnabledFields();
965       int sizeField = enabledFields.size();
966       for (int numField = 0; numField < sizeField; ++numField)
967       {
968         if (0 != enabledFields[numField]->getRelGrid())
969           gridIds.insert(CGrid::get(enabledFields[numField]->getRelGrid())->getId());
970       }
971     }
972
973     // Create all reference grids on server side
974     StdString gridDefRoot("grid_definition");
975     CGridGroup* gridPtr = CGridGroup::get(gridDefRoot);
976     std::set<StdString>::const_iterator it, itE = gridIds.end();
977     for (it = gridIds.begin(); it != itE; ++it)
978     {
979       gridPtr->sendCreateChild(*it);
980       CGrid::get(*it)->sendAllAttributesToServer();
981       CGrid::get(*it)->sendAllDomains();
982       CGrid::get(*it)->sendAllAxis();
983     }
984   }
985
986
987   //! Client side: Send information of reference domain and axis of active fields
988   void CContext::sendRefDomainsAxis()
989   {
990     std::set<StdString> domainIds;
991     std::set<StdString> axisIds;
992
993     // Find all reference domain and axis of all active fields
994     int numEnabledFiles = this->enabledFiles.size();
995     for (int i = 0; i < numEnabledFiles; ++i)
996     {
997       std::vector<CField*> enabledFields = this->enabledFiles[i]->getEnabledFields();
998       int numEnabledFields = enabledFields.size();
999       for (int j = 0; j < numEnabledFields; ++j)
1000       {
1001         const std::pair<StdString, StdString>& prDomAxisId = enabledFields[j]->getRefDomainAxisIds();
1002         if ("" != prDomAxisId.first) domainIds.insert(prDomAxisId.first);
1003         if ("" != prDomAxisId.second) axisIds.insert(prDomAxisId.second);
1004       }
1005     }
1006
1007     // Create all reference axis on server side
1008     std::set<StdString>::iterator itDom, itAxis;
1009     std::set<StdString>::const_iterator itE;
1010
1011     StdString axiDefRoot("axis_definition");
1012     CAxisGroup* axisPtr = CAxisGroup::get(axiDefRoot);
1013     itE = axisIds.end();
1014     for (itAxis = axisIds.begin(); itAxis != itE; ++itAxis)
1015     {
1016       if (!itAxis->empty())
1017       {
1018         axisPtr->sendCreateChild(*itAxis);
1019         CAxis::get(*itAxis)->sendAllAttributesToServer();
1020       }
1021     }
1022
1023     // Create all reference domains on server side
1024     StdString domDefRoot("domain_definition");
1025     CDomainGroup* domPtr = CDomainGroup::get(domDefRoot);
1026     itE = domainIds.end();
1027     for (itDom = domainIds.begin(); itDom != itE; ++itDom)
1028     {
1029       if (!itDom->empty()) {
1030          domPtr->sendCreateChild(*itDom);
1031          CDomain::get(*itDom)->sendAllAttributesToServer();
1032       }
1033     }
1034   }
1035
1036   //! Update calendar in each time step
1037   void CContext::updateCalendar(int step)
1038   {
1039      info(50) << "updateCalendar : before : " << calendar->getCurrentDate() << endl;
1040      calendar->update(step);
1041      info(50) << "updateCalendar : after : " << calendar->getCurrentDate() << endl;
1042
1043      if (hasClient)
1044      {
1045        checkPrefetchingOfEnabledReadModeFiles();
1046        garbageCollector.invalidate(calendar->getCurrentDate());
1047      }
1048   }
1049
1050   //! Server side: Create header of netcdf file
1051   void CContext::createFileHeader(void )
1052   {
1053      vector<CFile*>::const_iterator it;
1054
1055      for (it=enabledFiles.begin(); it != enabledFiles.end(); it++)
1056      {
1057         (*it)->initFile();
1058      }
1059   }
1060
1061   //! Get current context
1062   CContext* CContext::getCurrent(void)
1063   {
1064     return CObjectFactory::GetObject<CContext>(CObjectFactory::GetCurrentContextId()).get();
1065   }
1066
1067   /*!
1068   \brief Set context with an id be the current context
1069   \param [in] id identity of context to be set to current
1070   */
1071   void CContext::setCurrent(const string& id)
1072   {
1073     CObjectFactory::SetCurrentContextId(id);
1074     CGroupFactory::SetCurrentContextId(id);
1075   }
1076
1077  /*!
1078  \brief Create a context with specific id
1079  \param [in] id identity of new context
1080  \return pointer to the new context or already-existed one with identity id
1081  */
1082  CContext* CContext::create(const StdString& id)
1083  {
1084    CContext::setCurrent(id);
1085
1086    bool hasctxt = CContext::has(id);
1087    CContext* context = CObjectFactory::CreateObject<CContext>(id).get();
1088    getRoot();
1089    if (!hasctxt) CGroupFactory::AddChild(root, context->getShared());
1090
1091#define DECLARE_NODE(Name_, name_) \
1092    C##Name_##Definition::create(C##Name_##Definition::GetDefName());
1093#define DECLARE_NODE_PAR(Name_, name_)
1094#include "node_type.conf"
1095
1096    return (context);
1097  }
1098
1099
1100
1101     //! Server side: Receive a message to do some post processing
1102  void CContext::recvRegistry(CEventServer& event)
1103  {
1104    CBufferIn* buffer=event.subEvents.begin()->buffer;
1105    string id;
1106    *buffer>>id;
1107    get(id)->recvRegistry(*buffer);
1108  }
1109
1110  void CContext::recvRegistry(CBufferIn& buffer)
1111  {
1112    if (server->intraCommRank==0)
1113    {
1114      CRegistry registry(server->intraComm) ;
1115      registry.fromBuffer(buffer) ;
1116      registryOut->mergeRegistry(registry) ;
1117    }
1118  }
1119
1120  void CContext::sendRegistry(void)
1121  {
1122    registryOut->hierarchicalGatherRegistry() ;
1123
1124    CEventClient event(CContext::GetType(), CContext::EVENT_ID_SEND_REGISTRY);
1125    if (client->isServerLeader())
1126    {
1127       CMessage msg ;
1128       msg<<this->getIdServer();
1129       if (client->clientRank==0) msg<<*registryOut ;
1130       const std::list<int>& ranks = client->getRanksServerLeader();
1131       for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
1132         event.push(*itRank,1,msg);
1133       client->sendEvent(event);
1134     }
1135     else client->sendEvent(event);
1136  }
1137
1138} // namespace xios
Note: See TracBrowser for help on using the repository browser.