source: XIOS/dev/dev_ym/XIOS_COUPLING/src/node/field.cpp @ 1882

Last change on this file since 1882 was 1882, checked in by ymipsl, 4 years ago

Xios coupling branch
Refactor and simplify file writer filter management on server side.

YM
  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
  • Property svn:executable set to *
File size: 77.2 KB
Line 
1#include "field.hpp"
2
3#include "attribute_template.hpp"
4#include "object_template.hpp"
5#include "group_template.hpp"
6
7#include "node_type.hpp"
8#include "calendar_util.hpp"
9#include "message.hpp"
10#include "xios_spl.hpp"
11#include "type.hpp"
12#include "timer.hpp"
13#include "context_client.hpp"
14#include "context_server.hpp"
15#include <set>
16#include "garbage_collector.hpp"
17#include "source_filter.hpp"
18#include "store_filter.hpp"
19#include "file_writer_filter.hpp"
20#include "pass_through_filter.hpp"
21#include "filter_expr_node.hpp"
22#include "lex_parser.hpp"
23#include "temporal_filter.hpp"
24#include "spatial_transform_filter.hpp"
25#include "file_server_writer_filter.hpp"
26#include "tracer.hpp"
27
28namespace xios
29{
30
31  /// ////////////////////// Définitions ////////////////////// ///
32
33  CField::CField(void)
34    : CObjectTemplate<CField>(), CFieldAttributes()
35    , written(false)
36    , nstep(0), nstepMax(0)
37    , hasOutputFile(false)
38    , domAxisScalarIds_(vector<StdString>(3,""))
39    , areAllReferenceSolved(false), isReferenceSolved(false), isReferenceSolvedAndTransformed(false)
40    , isGridChecked(false)
41    , useCompressedOutput(false)
42    , hasTimeInstant(false)
43    , hasTimeCentered(false)
44    , wasDataRequestedFromServer(false)
45    , wasDataAlreadyReceivedFromServer(false)
46    , mustAutoTrigger(false)
47    , isEOF(false), nstepMaxRead(false)
48  { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
49
50  CField::CField(const StdString& id)
51    : CObjectTemplate<CField>(id), CFieldAttributes()
52    , written(false)
53    , nstep(0), nstepMax(0)
54    , hasOutputFile(false)
55    , domAxisScalarIds_(vector<StdString>(3,""))
56    , areAllReferenceSolved(false), isReferenceSolved(false), isReferenceSolvedAndTransformed(false)
57    , isGridChecked(false)
58    , useCompressedOutput(false)
59    , hasTimeInstant(false)
60    , hasTimeCentered(false)
61    , wasDataRequestedFromServer(false)
62    , wasDataAlreadyReceivedFromServer(false)
63    , mustAutoTrigger(false)
64    , isEOF(false), nstepMaxRead(false)
65  { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
66
67  CField::~CField(void)
68  {}
69
70  //----------------------------------------------------------------
71
72  void CField::setVirtualVariableGroup(CVariableGroup* newVVariableGroup)
73  TRY
74  {
75    this->vVariableGroup = newVVariableGroup;
76  }
77  CATCH
78
79  CVariableGroup* CField::getVirtualVariableGroup(void) const
80  TRY
81  {
82     return this->vVariableGroup;
83  }
84  CATCH
85
86  std::vector<CVariable*> CField::getAllVariables(void) const
87  TRY
88  {
89    return this->vVariableGroup->getAllChildren();
90  }
91  CATCH
92
93  void CField::solveDescInheritance(bool apply, const CAttributeMap* const parent)
94  TRY
95  {
96    SuperClassAttribute::setAttributes(parent, apply);
97    this->getVirtualVariableGroup()->solveDescInheritance(apply, NULL);
98  }
99  CATCH_DUMP_ATTR
100
101  //----------------------------------------------------------------
102
103  bool CField::dispatchEvent(CEventServer& event)
104  TRY
105  {
106    if (SuperClass::dispatchEvent(event)) return true;
107    else
108    {
109      switch(event.type)
110      {
111        case EVENT_ID_UPDATE_DATA :
112          recvUpdateData(event);
113          return true;
114          break;
115
116        case EVENT_ID_READ_DATA :
117          recvReadDataRequest(event);
118          return true;
119          break;
120
121        case EVENT_ID_READ_DATA_READY :
122          recvReadDataReady(event);
123          return true;
124          break;
125
126        case EVENT_ID_ADD_VARIABLE :
127          recvAddVariable(event);
128          return true;
129          break;
130
131        case EVENT_ID_ADD_VARIABLE_GROUP :
132          recvAddVariableGroup(event);
133          return true;
134          break;
135     
136        case EVENT_ID_GRID_COMPLETED :
137          recvGridCompleted(event);
138          return true;
139          break;
140        default :
141          ERROR("bool CField::dispatchEvent(CEventServer& event)", << "Unknown Event");
142          return false;
143      }
144    }
145  }
146  CATCH
147
148  void CField::sendUpdateData(Time timeStamp, const CArray<double,1>& data, CContextClient* client)
149  TRY
150  {
151    CTimer::get("Field : send data").resume();
152    int receiverSize = client->serverSize;
153
154    CEventClient event(getType(), EVENT_ID_UPDATE_DATA);
155
156    map<int, CArray<int,1> >::iterator it;
157    list<CMessage> list_msg;
158    list<CArray<double,1> > list_data;
159
160    if (!grid_->doGridHaveDataDistributed(client))
161    {
162       if (client->isServerLeader())
163       {
164          for (it = grid_->storeIndex_toSrv_[client].begin(); it != grid_->storeIndex_toSrv_[client].end(); it++)
165          {
166            int rank = it->first;
167            CArray<int,1>& index = it->second;
168
169            list_msg.push_back(CMessage());
170            list_data.push_back(CArray<double,1>(index.numElements()));
171
172            CArray<double,1>& data_tmp = list_data.back();
173            for (int n = 0; n < data_tmp.numElements(); n++) data_tmp(n) = data(index(n));
174
175            list_msg.back() << getId() << timeStamp << data_tmp;
176            event.push(rank, 1, list_msg.back());
177          }
178          client->sendEvent(event);
179        }
180      else client->sendEvent(event);
181    }
182    else
183    {
184      for (it = grid_->storeIndex_toSrv_[client].begin(); it != grid_->storeIndex_toSrv_[client].end(); it++)
185      {
186        int rank = it->first;
187        CArray<int,1>& index = it->second;
188
189        list_msg.push_back(CMessage());
190        list_data.push_back(CArray<double,1>(index.numElements()));
191
192        CArray<double,1>& data_tmp = list_data.back();
193        for (int n = 0; n < data_tmp.numElements(); n++) data_tmp(n) = data(index(n));
194
195        list_msg.back() << getId() << timeStamp << data_tmp;
196        event.push(rank, grid_->nbSenders_[receiverSize][rank], list_msg.back());
197      }
198      client->sendEvent(event);
199    }
200
201    CTimer::get("Field : send data").suspend();
202  }
203  CATCH_DUMP_ATTR
204
205  void CField::recvUpdateData(CEventServer& event)
206  TRY
207  {
208    std::map<int,CBufferIn*> rankBuffers;
209
210    list<CEventServer::SSubEvent>::iterator it;
211    string fieldId;
212    CTimer::get("Field : recv data").resume();
213    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
214    {
215      int rank = it->rank;
216      CBufferIn* buffer = it->buffer;
217      *buffer >> fieldId;
218      rankBuffers[rank] = buffer;
219    }
220    get(fieldId)->recvUpdateData(rankBuffers);
221    CTimer::get("Field : recv data").suspend();
222  }
223  CATCH
224
225
226  void  CField::recvUpdateData(std::map<int,CBufferIn*>& rankBuffers)
227  TRY
228  {
229    if (hasCouplerIn()) recvUpdateDataFromCoupler(rankBuffers) ;
230    else recvUpdateDataFromClient(rankBuffers) ;
231  }
232  CATCH
233
234  void  CField::recvUpdateDataFromClient(std::map<int,CBufferIn*>& rankBuffers)
235  TRY
236  {
237    // ym to remove latter
238    /*
239    CContext* context = CContext::getCurrent();
240    Time timeStamp ;
241    size_t sizeData = 0;
242    if (0 == recvDataSrv.numElements())
243    {           
244      CArray<int,1>& storeClient = grid_->getStoreIndex_client();
245
246      // Gather all data from different clients     
247      recvDataSrv.resize(storeClient.numElements());
248      recvFoperationSrv = std::shared_ptr<func::CFunctor>(new func::CInstant(recvDataSrv));
249    }
250
251    CArray<double,1> recv_data_tmp(recvDataSrv.numElements());   
252    const CDate& currDate = context->getCalendar()->getCurrentDate();
253    CDuration offsetAllButMonth (freq_offset.getValue().year, 0 , freq_offset.getValue().day,
254                                 freq_offset.getValue().hour, freq_offset.getValue().minute,
255                                 freq_offset.getValue().second, freq_offset.getValue().timestep);
256    const CDate opeDate   = (last_operation_srv - offsetAllButMonth + context->getCalendar()->getTimeStep())
257                             + freq_op + freq_operation_srv - freq_op - context->getCalendar()->getTimeStep() + offsetAllButMonth;
258
259    if (opeDate <= currDate)
260    {
261     
262      auto& outLocalIndexStoreOnClient = grid_-> getOutLocalIndexStoreOnClient() ;
263      for (auto it = outLocalIndexStoreOnClient.begin(); it != outLocalIndexStoreOnClient.end(); ++it)
264      {
265        CArray<double,1> tmp;
266        CArray<size_t,1>& indexTmp = it->second;
267        *(rankBuffers[it->first]) >> timeStamp >> tmp;
268        for (int idx = 0; idx < indexTmp.numElements(); ++idx) recv_data_tmp(indexTmp(idx)) = tmp(idx);
269      }
270    }
271*/
272    Time timeStamp ;
273    CArray<int,1>& storeClient = grid_->getStoreIndex_client(); // replace it with local size
274    CArray<double,1> recv_data_tmp(storeClient.numElements());
275    auto& outLocalIndexStoreOnClient = grid_-> getOutLocalIndexStoreOnClient() ;
276    for (auto it = outLocalIndexStoreOnClient.begin(); it != outLocalIndexStoreOnClient.end(); ++it)
277    {
278      CArray<double,1> tmp;
279      CArray<size_t,1>& indexTmp = it->second;
280      *(rankBuffers[it->first]) >> timeStamp >> tmp;
281      for (int idx = 0; idx < indexTmp.numElements(); ++idx) recv_data_tmp(indexTmp(idx)) = tmp(idx);
282    }
283
284    this->setData(recv_data_tmp);
285/*
286    // delete incomming flux for server only
287    recvFoperationSrv.reset() ;
288    recvDataSrv.reset() ;
289*/
290  }
291  CATCH_DUMP_ATTR
292
293  /* ym : old interface : to be removed...
294  void CField::writeUpdateData(const CArray<double,1>& data)
295  TRY
296  {
297    CContext* context = CContext::getCurrent();
298
299    const CDate& currDate = context->getCalendar()->getCurrentDate();
300    CDuration offsetAllButMonth (freq_offset.getValue().year, 0 , freq_offset.getValue().day,
301                                   freq_offset.getValue().hour, freq_offset.getValue().minute,
302                                   freq_offset.getValue().second, freq_offset.getValue().timestep);
303    const CDate opeDate   = (last_operation_srv - offsetAllButMonth + context->getCalendar()->getTimeStep())
304                              + freq_op + freq_operation_srv - freq_op - context->getCalendar()->getTimeStep() + offsetAllButMonth;
305    const CDate writeDate = last_Write_srv + freq_write_srv;
306
307    if (opeDate <= currDate)
308    {
309      (*recvFoperationSrv)(data);
310      last_operation_srv = currDate;
311    }
312
313    if (writeDate < (currDate + freq_operation_srv))
314    {
315      recvFoperationSrv->final();
316      last_Write_srv = writeDate;
317      grid_->computeWrittenIndex();
318      writeField();
319      lastlast_Write_srv = last_Write_srv;
320    }
321  }
322  CATCH_DUMP_ATTR
323  */
324
325  void CField::writeUpdateData(const CArray<double,1>& data)
326  TRY
327  {
328    const CDate writeDate = last_Write_srv + freq_write_srv;
329    last_Write_srv = writeDate;
330    grid_->computeWrittenIndex();
331    /*
332    recvDataSrv.resize(data.numElements()) ;
333    recvDataSrv = data ;
334    */
335    writeField(data);
336    lastlast_Write_srv = last_Write_srv;
337  }
338  CATCH_DUMP_ATTR
339
340
341  void CField::writeField(const CArray<double,1>& data)
342  TRY
343  {
344    if (!getRelFile()->isEmptyZone())
345    {
346      if (grid_->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)
347      {
348        getRelFile()->checkWriteFile();
349        this->incrementNStep();
350        getRelFile()->getDataOutput()->writeFieldData(CField::get(this), data);
351      }
352    }
353  }
354  CATCH_DUMP_ATTR
355
356  /*
357    Send a request for reading data.
358    Client sends a request to server for demanding server to read data and send back to it.
359    For now, this function is called only by client
360    In the future, it can be called by level-1 servers
361    \param [in] tsDataRequested timestamp when the call is made
362  */
363  bool CField::sendReadDataRequest(const CDate& tsDataRequested, CContextClient* client)
364  TRY
365  {
366    CContext* context = CContext::getCurrent();
367
368    lastDataRequestedFromServer = tsDataRequested;
369
370    // No need to send the request if we are sure that we are already at EOF
371    if (!isEOF || context->getCalendar()->getCurrentDate() <= dateEOF)
372    {
373      CEventClient event(getType(), EVENT_ID_READ_DATA);
374      if (client->isServerLeader())
375      {
376        CMessage msg;
377        msg << getId();
378        const std::list<int>& ranks = client->getRanksServerLeader();
379        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
380          event.push(*itRank, 1, msg);
381        client->sendEvent(event);
382      }
383      else client->sendEvent(event);
384    }
385    else serverSourceFilter->signalEndOfStream(tsDataRequested);
386
387    wasDataRequestedFromServer = true;
388
389    return !isEOF;
390  }
391  CATCH_DUMP_ATTR
392
393  /*!
394  Send request new data read from file if need be, that is the current data is out-of-date.
395  \return true if and only if some data was requested
396  */
397  bool CField::sendReadDataRequestIfNeeded(void)
398  TRY
399  {
400    const CDate& currentDate = CContext::getCurrent()->getCalendar()->getCurrentDate();
401
402    bool dataRequested = false;
403
404    while (currentDate >= lastDataRequestedFromServer)
405    {
406      info(20) << "currentDate : " << currentDate << endl ;
407      info(20) << "lastDataRequestedFromServer : " << lastDataRequestedFromServer << endl ;
408      info(20) << "fileIn_->output_freq.getValue() : " << fileIn_->output_freq.getValue() << endl ;
409      info(20) << "lastDataRequestedFromServer + fileIn_->output_freq.getValue() : " << lastDataRequestedFromServer + fileIn_->output_freq << endl ;
410
411      dataRequested |= sendReadDataRequest(lastDataRequestedFromServer + fileIn_->output_freq, fileIn_->getContextClient());
412    }
413
414    return dataRequested;
415  }
416  CATCH_DUMP_ATTR
417
418  void CField::recvReadDataRequest(CEventServer& event)
419  TRY
420  {
421    CBufferIn* buffer = event.subEvents.begin()->buffer;
422    StdString fieldId;
423    *buffer >> fieldId;
424    get(fieldId)->recvReadDataRequest(event.getContextServer());
425  }
426  CATCH
427
428  /*!
429    Receive data request sent from client and process it
430    Every time server receives this request, it will try to read data and sent read data back to client
431    At the moment, this function is called by server level 1
432    In the future, this should (only) be done by the last level servers.
433  */
434  void CField::recvReadDataRequest(CContextServer* server)
435  TRY
436  {
437    CContextClient* client = server->getAssociatedClient() ;
438    CEventClient event(getType(), EVENT_ID_READ_DATA_READY);
439    std::list<CMessage> msgs;
440   
441    CArray<double,1> data ;
442    EReadField hasData = readField(data);
443
444    map<int, CArray<double,1> >::iterator it;
445    if (!grid_->doGridHaveDataDistributed(client))
446    {
447       if (client->isServerLeader())
448       {
449          if (0 != data.numElements())
450          {           
451            const std::list<int>& ranks = client->getRanksServerLeader();
452            for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
453            {
454              msgs.push_back(CMessage());
455              CMessage& msg = msgs.back();
456              msg << getId();
457              switch (hasData)
458              {
459                case RF_DATA:
460                  msg << getNStep() - 1 << data;
461                  break;
462                case RF_NODATA:
463                  msg << int(-2) << data;
464                  break;
465                case RF_EOF:                 
466                default:
467                  msg << int(-1);
468                  break;
469              }
470
471              event.push(*itRank, 1, msg);
472            }
473          }
474          client->sendEvent(event);
475       }
476       else
477       {
478          client->sendEvent(event);
479       }
480    }
481    else
482    {
483      auto& outLocalIndexStoreOnClient = grid_-> getOutLocalIndexStoreOnClient() ;
484      for (auto it = outLocalIndexStoreOnClient.begin(); it != outLocalIndexStoreOnClient.end(); ++it)
485      {
486        CArray<size_t,1>& indexTmp = it->second;
487        CArray<double,1> tmp(indexTmp.numElements());
488        for (int idx = 0; idx < indexTmp.numElements(); ++idx)
489        {
490          tmp(idx) = data(indexTmp(idx));
491        } 
492
493        msgs.push_back(CMessage());
494        CMessage& msg = msgs.back();
495        msg << getId();
496        switch (hasData)
497        {
498          case RF_DATA:
499            msg << getNStep() - 1 << tmp;
500            break;
501          case RF_NODATA:
502            msg << int(-2) << tmp;
503            break;
504          case RF_EOF:                 
505          default:
506            msg << int(-1);
507            break;
508        }
509
510        event.push(it->first, grid_->getNbReadSenders(client)[it->first], msg);
511      }
512      client->sendEvent(event);
513    }
514  }
515  CATCH_DUMP_ATTR
516
517  /*!
518    Read field from a file.
519    A field is read with the distribution of data on the server side
520    \return State of field can be read from a file
521  */
522  CField::EReadField CField::readField(CArray<double,1>& data)
523  TRY
524  {
525    CContext* context = CContext::getCurrent();
526    grid_->computeWrittenIndex();
527    getRelFile()->initRead();
528    EReadField readState = RF_DATA;
529
530    if (!getRelFile()->isEmptyZone())
531    {     
532      if (grid_->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)     
533      {
534          CArray<int,1>& storeClient = grid_->getStoreIndex_client();         
535          data.resize(storeClient.numElements());         
536       
537        getRelFile()->checkReadFile();
538
539        if (!nstepMax)
540        {
541          nstepMax = getRelFile()->getDataInput()->getFieldNbRecords(CField::get(this));
542        }
543
544        this->incrementNStep();
545
546        if (getNStep() > nstepMax && (getRelFile()->cyclic.isEmpty() || !getRelFile()->cyclic) )
547          readState = RF_EOF;
548
549        if (RF_EOF != readState)
550          getRelFile()->getDataInput()->readFieldData(CField::get(this),data);
551      }
552    }
553    else
554    {
555      this->incrementNStep();
556      if (getNStep() > nstepMax && (getRelFile()->cyclic.isEmpty() || !getRelFile()->cyclic) )
557        readState = RF_EOF;
558      else
559        readState = RF_NODATA;
560
561      if (!nstepMaxRead) // This can be a bug if we try to read field from zero time record
562        readState = RF_NODATA;
563    }
564
565    if (!nstepMaxRead)
566    {
567       MPI_Allreduce(MPI_IN_PLACE, &nstepMax, 1, MPI_INT, MPI_MAX, context->intraComm_);
568       nstepMaxRead = true;
569    }
570
571    return readState;
572  }
573  CATCH_DUMP_ATTR
574
575  /*
576    Receive read data from server.
577    At the moment, this function is called in the client side.
578    In the future, this function can be called hiearachically (server n-1, server n -2, ..., client)
579    \param event event containing read data
580  */
581  void CField::recvReadDataReady(CEventServer& event)
582  TRY
583  {
584    string fieldId;
585    vector<int> ranks;
586    vector<CBufferIn*> buffers;
587
588    list<CEventServer::SSubEvent>::iterator it;
589    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
590    {
591      ranks.push_back(it->rank);
592      CBufferIn* buffer = it->buffer;
593      *buffer >> fieldId;
594      buffers.push_back(buffer);
595    }
596    get(fieldId)->recvReadDataReady(ranks, buffers);
597  }
598  CATCH
599
600 
601  void CField::recvUpdateDataFromCoupler(std::map<int,CBufferIn*>& rankBuffers)
602  TRY
603  {
604    CContext* context = CContext::getCurrent();
605    Time timeStamp ;
606    if (wasDataAlreadyReceivedFromServer)
607    { 
608      lastDataReceivedFromServer = lastDataReceivedFromServer + freq_op;
609    }
610    else
611    {
612      // unlikely to input from file server where data are received at ts=0
613      // for coupling, it would be after the first freq_op, because for now we don't have
614      // restart mecanism to send the value at ts=0. It mus be changed in future
615      lastDataReceivedFromServer = context->getCalendar()->getInitDate();
616      wasDataAlreadyReceivedFromServer = true;
617    }
618
619    CArray<int,1>& storeClient = grid_->getStoreIndex_client();
620    CArray<double,1> recv_data_tmp(storeClient.numElements()); 
621
622    auto& outLocalIndexStoreOnClient = grid_-> getOutLocalIndexStoreOnClient() ;
623    for (auto it = outLocalIndexStoreOnClient.begin(); it != outLocalIndexStoreOnClient.end(); ++it)
624    {
625      CArray<double,1> tmp;
626      CArray<size_t,1>& indexTmp = it->second;
627      *(rankBuffers[it->first]) >> timeStamp >> tmp;
628      for (int idx = 0; idx < indexTmp.numElements(); ++idx) recv_data_tmp(indexTmp(idx)) = tmp(idx);
629    }
630   
631    clientSourceFilter->streamData(lastDataReceivedFromServer, recv_data_tmp);
632   
633  }
634  CATCH_DUMP_ATTR
635  /*!
636    Receive read data from server
637    \param [in] ranks Ranks of sending processes
638    \param [in] buffers buffers containing read data
639  */
640  void CField::recvReadDataReady(vector<int> ranks, vector<CBufferIn*> buffers)
641  TRY
642  {
643    CContext* context = CContext::getCurrent();
644    std::map<int, CArray<double,1> > data;
645    const bool wasEOF = isEOF;
646
647    for (int i = 0; i < ranks.size(); i++)
648    {
649      int rank = ranks[i];
650      int record;
651      *buffers[i] >> record;
652      isEOF = (record == int(-1));
653
654      if (!isEOF)
655        *buffers[i] >> data[rank];
656      else
657        break;
658    }
659
660    if (wasDataAlreadyReceivedFromServer)
661      lastDataReceivedFromServer = lastDataReceivedFromServer + fileIn_->output_freq;
662    else
663    {
664      lastDataReceivedFromServer = context->getCalendar()->getInitDate();
665      wasDataAlreadyReceivedFromServer = true;
666    }
667
668    if (isEOF)
669    {
670      if (!wasEOF)
671        dateEOF = lastDataReceivedFromServer;
672
673      serverSourceFilter->signalEndOfStream(lastDataReceivedFromServer);
674    }
675    else
676      serverSourceFilter->streamDataFromServer(lastDataReceivedFromServer, data);
677  }
678  CATCH_DUMP_ATTR
679
680  void CField::checkForLateDataFromCoupler(void)
681  TRY
682  {
683    CContext* context = CContext::getCurrent();
684    const CDate& currentDate = context->getCalendar()->getCurrentDate();
685
686    CTimer timer("CField::checkForLateDataFromCoupler");
687    timer.resume();
688    traceOff() ;
689    timer.suspend();
690     
691    bool isDataLate;
692    do
693    {
694      if (wasDataAlreadyReceivedFromServer) isDataLate = lastDataReceivedFromServer + freq_offset + freq_op <= currentDate ;
695      else isDataLate = context->getCalendar()->getInitDate()+freq_offset <= currentDate ;
696
697      if (isDataLate)
698      {
699        timer.resume();
700//ym          context->checkBuffersAndListen();
701//ym            context->eventLoop();
702        context->globalEventLoop();
703
704        timer.suspend();
705      }
706    } while (isDataLate && timer.getCumulatedTime() < CXios::recvFieldTimeout);
707   
708    timer.resume();
709    traceOn() ;
710    timer.suspend() ;
711
712    if (isDataLate) ERROR("void CField::checkForLateDataFromCoupler(void)",
713                            << "Late data at timestep = " << currentDate);
714  }
715  CATCH_DUMP_ATTR
716
717
718  void CField::checkForLateDataFromServer(void)
719  TRY
720  {
721    CContext* context = CContext::getCurrent();
722    const CDate& currentDate = context->getCalendar()->getCurrentDate();
723
724    // Check if data previously requested has been received as expected
725    if (wasDataRequestedFromServer && !isEOF)
726    {
727      CTimer timer("CField::checkForLateDataFromServer");
728      timer.resume();
729      traceOff() ;
730      timer.suspend();
731     
732      bool isDataLate;
733      do
734      {
735        const CDate nextDataDue = wasDataAlreadyReceivedFromServer ? (lastDataReceivedFromServer + fileIn_->output_freq) : context->getCalendar()->getInitDate();
736        isDataLate = (nextDataDue <= currentDate);
737
738        if (isDataLate)
739        {
740          timer.resume();
741
742//ym          context->checkBuffersAndListen();
743//ym            context->eventLoop();
744          context->globalEventLoop();
745
746          timer.suspend();
747        }
748      }
749      while (isDataLate && timer.getCumulatedTime() < CXios::recvFieldTimeout);
750      timer.resume();
751      traceOn() ;
752      timer.suspend() ;
753
754
755      if (isDataLate)
756        ERROR("void CField::checkForLateDataFromServer(void)",
757              << "Late data at timestep = " << currentDate);
758    }
759  }
760  CATCH_DUMP_ATTR
761
762  void CField::triggerLateField(void)
763  TRY
764  {
765    if (hasFileIn()) 
766    {
767      checkForLateDataFromServer() ;
768      serverSourceFilter->trigger(CContext::getCurrent()->getCalendar()->getCurrentDate()) ;
769    } 
770    else if (hasCouplerIn())
771    {
772      checkForLateDataFromCoupler() ;
773      clientSourceFilter->trigger(CContext::getCurrent()->getCalendar()->getCurrentDate()) ;
774    }
775  }
776  CATCH_DUMP_ATTR
777
778
779  void CField::checkIfMustAutoTrigger(void)
780  TRY
781  {
782    mustAutoTrigger = serverSourceFilter ? serverSourceFilter->mustAutoTrigger() : false;
783  }
784  CATCH_DUMP_ATTR
785
786  void CField::autoTriggerIfNeeded(void)
787  TRY
788  {
789    if (mustAutoTrigger)
790      serverSourceFilter->trigger(CContext::getCurrent()->getCalendar()->getCurrentDate());
791  }
792  CATCH_DUMP_ATTR
793
794  //----------------------------------------------------------------
795/*
796  void CField::setRelFile(CFile* _file)
797  TRY
798  {
799    this->file = _file;
800    hasOutputFile = true;
801  }
802  CATCH_DUMP_ATTR
803*/
804  //----------------------------------------------------------------
805
806  StdString CField::GetName(void)    { return StdString("field"); }
807  StdString CField::GetDefName(void) { return CField::GetName(); }
808  ENodeType CField::GetType(void)    { return eField; }
809
810  //----------------------------------------------------------------
811
812  CGrid* CField::getRelGrid(void) const
813  TRY
814  {
815    return this->grid_;
816  }
817  CATCH
818
819  //----------------------------------------------------------------
820
821  CFile* CField::getRelFile(void) const
822  TRY
823  {
824    if (hasFileIn()) return this->fileIn_;
825    else if (hasFileOut()) return this->fileOut_ ;
826    else return nullptr ;
827  }
828  CATCH
829
830  int CField::getNStep(void) const
831  TRY
832  {
833    return this->nstep;
834  }
835  CATCH
836
837  func::CFunctor::ETimeType CField::getOperationTimeType() const
838  TRY
839  {
840    return operationTimeType;
841  }
842  CATCH
843
844   //----------------------------------------------------------------
845
846  void CField::incrementNStep(void)
847  TRY
848  {
849    this->nstep++;
850  }
851  CATCH_DUMP_ATTR
852
853  void CField::resetNStep(int nstep /*= 0*/)
854  TRY
855  {
856    this->nstep = nstep;
857  }
858  CATCH_DUMP_ATTR
859
860  void CField::resetNStepMax(void)
861  TRY
862  {
863    this->nstepMax = 0;
864    nstepMaxRead = false;
865  }
866  CATCH_DUMP_ATTR
867
868  //----------------------------------------------------------------
869
870  bool CField::isActive(bool atCurrentTimestep /*= false*/) const
871  TRY
872  {
873    if (clientSourceFilter)
874      return atCurrentTimestep ? clientSourceFilter->isDataExpected(CContext::getCurrent()->getCalendar()->getCurrentDate()) : true;
875    else if (storeFilter)
876      return true;
877    else if (instantDataFilter)
878      ERROR("bool CField::isActive(bool atCurrentTimestep)",
879            << "Impossible to check if field [ id = " << getId() << " ] is active as it cannot be used to receive nor send data.");
880
881    return false;
882  }
883  CATCH
884
885  //----------------------------------------------------------------
886
887  bool CField::wasWritten() const
888  TRY
889  {
890    return written;
891  }
892  CATCH
893
894  void CField::setWritten()
895  TRY
896  {
897    written = true;
898  }
899  CATCH_DUMP_ATTR
900
901  //----------------------------------------------------------------
902
903  bool CField::getUseCompressedOutput() const
904  TRY
905  {
906    return useCompressedOutput;
907  }
908  CATCH
909
910  void CField::setUseCompressedOutput()
911  TRY
912  {
913    useCompressedOutput = true;
914  }
915  CATCH_DUMP_ATTR
916
917  //----------------------------------------------------------------
918
919  std::shared_ptr<COutputPin> CField::getInstantDataFilter()
920  TRY
921  {
922    return instantDataFilter;
923  }
924  CATCH_DUMP_ATTR
925
926  //----------------------------------------------------------------
927
928  /*!
929    Build up graph of grids which plays role of destination and source in grid transformation
930    This function should be called before \func solveGridReference()
931  */
932  void CField::buildGridTransformationGraph()
933  TRY
934  {
935    CContext* context = CContext::getCurrent();
936    if (context->getServiceType()==CServicesManager::CLIENT)
937    {
938      if (grid_ && !grid_->isTransformed() && hasDirectFieldReference() && grid_ != getDirectFieldReference()->grid_)
939      {
940        grid_->addTransGridSource(getDirectFieldReference()->grid_);
941      }
942    }
943  }
944  CATCH_DUMP_ATTR
945
946  /*!
947    Generate a new grid destination if there are more than one grid source pointing to a same grid destination
948  */
949  void CField::generateNewTransformationGridDest()
950  TRY
951  {
952    CContext* context = CContext::getCurrent();
953    if (context->getServiceType()==CServicesManager::CLIENT)
954    {
955      std::map<CGrid*,std::pair<bool,StdString> >& gridSrcMap = grid_->getTransGridSource();
956      if (1 < gridSrcMap.size())
957      {
958        // Search for grid source
959        CGrid* gridSrc = grid_;
960        CField* currField = this;
961        std::vector<CField*> hieraField;
962        while (currField->hasDirectFieldReference() && (gridSrc == grid_))
963        {
964          hieraField.push_back(currField);
965          CField* tmp = currField->getDirectFieldReference();
966          currField = tmp;
967          gridSrc = currField->grid_;
968        }
969
970        if (gridSrcMap.end() != gridSrcMap.find(gridSrc))
971        {
972          CGrid* gridTmp;
973          std::pair<bool,StdString> newGridDest = gridSrcMap[gridSrc];
974          if (newGridDest.first)
975          {
976            StdString newIdGridDest = newGridDest.second;
977            if (!CGrid::has(newIdGridDest))
978            {
979              ERROR("CGrid* CGrid::generateNewTransformationGridDest()",
980                << " Something wrong happened! Grid whose id " << newIdGridDest
981                << "should exist ");
982            }
983            gridTmp = CGrid::get(newIdGridDest);
984          }
985          else
986          {
987            StdString newIdGridDest = CGrid::generateId(gridSrc, grid_);
988            gridTmp = CGrid::cloneGrid(newIdGridDest, grid_);
989
990            (gridSrcMap[gridSrc]).first = true;
991            (gridSrcMap[gridSrc]).second = newIdGridDest;
992          }
993
994          // Update all descendants
995          for (std::vector<CField*>::iterator it = hieraField.begin(); it != hieraField.end(); ++it)
996          {
997            (*it)->grid_ = gridTmp;
998            (*it)->updateRef((*it)->grid_);
999          }
1000        }
1001      }
1002    }
1003  }
1004  CATCH_DUMP_ATTR
1005
1006  void CField::updateRef(CGrid* grid)
1007  TRY
1008  {
1009    if (!grid_ref.isEmpty()) grid_ref.setValue(grid->getId());
1010    else
1011    {
1012      std::vector<CAxis*> axisTmp = grid->getAxis();
1013      std::vector<CDomain*> domainTmp = grid->getDomains();
1014      if ((1<axisTmp.size()) || (1<domainTmp.size()))
1015        ERROR("void CField::updateRef(CGrid* grid)",
1016          << "More than one domain or axis is available for domain_ref/axis_ref of field " << this->getId());
1017
1018      if ((!domain_ref.isEmpty()) && (domainTmp.empty()))
1019        ERROR("void CField::updateRef(CGrid* grid)",
1020          << "Incoherent between available domain and domain_ref of field " << this->getId());
1021      if ((!axis_ref.isEmpty()) && (axisTmp.empty()))
1022        ERROR("void CField::updateRef(CGrid* grid)",
1023          << "Incoherent between available axis and axis_ref of field " << this->getId());
1024
1025      if (!domain_ref.isEmpty()) domain_ref.setValue(domainTmp[0]->getId());
1026      if (!axis_ref.isEmpty()) axis_ref.setValue(axisTmp[0]->getId());
1027    }
1028  }
1029  CATCH_DUMP_ATTR
1030   
1031  /*!
1032    Solve reference of all enabled fields even the source fields .
1033    In this step, we do transformations.
1034  */
1035  void CField::solveAllEnabledFieldsAndTransform()
1036  TRY
1037  {
1038    CContext* context = CContext::getCurrent();
1039
1040    if (!isReferenceSolvedAndTransformed)
1041    {
1042      isReferenceSolvedAndTransformed = true;
1043
1044      if (context->getServiceType()==CServicesManager::CLIENT)
1045      {
1046        solveRefInheritance(true);
1047        if (hasDirectFieldReference()) getDirectFieldReference()->solveAllEnabledFieldsAndTransform();
1048      }
1049
1050      if (context->getServiceType()==CServicesManager::GATHERER || context->getServiceType()==CServicesManager::OUT_SERVER)
1051        solveServerOperation();
1052
1053      solveGridReference();
1054
1055      if (context->getServiceType()==CServicesManager::CLIENT)
1056      {
1057        solveGenerateGrid();
1058        buildGridTransformationGraph();
1059      }
1060
1061      solveGridDomainAxisRef(false);
1062
1063      if (context->getServiceType()==CServicesManager::CLIENT)
1064      {
1065        solveTransformedGrid();
1066      }
1067
1068      solveGridDomainAxisRef(false);
1069    }
1070  }
1071  CATCH_DUMP_ATTR
1072
1073  void CField::checkGridOfEnabledFields()
1074  TRY
1075  {
1076    if (!isGridChecked)
1077    {
1078      isGridChecked = true;
1079      solveCheckMaskIndex(false);
1080    }
1081  }
1082  CATCH_DUMP_ATTR
1083
1084  void CField::sendGridComponentOfEnabledFields()
1085  TRY
1086  {
1087    solveGridDomainAxisRef(true);
1088    // solveCheckMaskIndex(true);
1089  }
1090  CATCH_DUMP_ATTR
1091
1092  void CField::sendGridOfEnabledFields()
1093  TRY
1094  {
1095    // solveGridDomainAxisRef(true);
1096    solveCheckMaskIndex(true);
1097  }   
1098  CATCH_DUMP_ATTR
1099
1100  void CField::solveOnlyReferenceEnabledField(void)
1101  TRY
1102  {
1103    CContext* context = CContext::getCurrent();
1104    if (!isReferenceSolved)
1105    {
1106      isReferenceSolved = true;
1107
1108      if (context->getServiceType()==CServicesManager::CLIENT)
1109      {
1110        solveRefInheritance(true);
1111        if (hasDirectFieldReference()) getDirectFieldReference()->solveOnlyReferenceEnabledField();
1112      }
1113
1114      if (context->getServiceType()==CServicesManager::GATHERER || context->getServiceType()==CServicesManager::OUT_SERVER)
1115        solveServerOperation();
1116
1117      solveGridReference();
1118      grid_->solveElementsRefInheritance(true); // make it again to solve grid reading from file
1119
1120      if (context->getServiceType()==CServicesManager::CLIENT)
1121      {
1122        solveGenerateGrid();
1123        buildGridTransformationGraph();
1124      }
1125    }
1126  }
1127  CATCH_DUMP_ATTR
1128
1129  void CField::solveAllReferenceEnabledField(bool doSending2Server)
1130  TRY
1131  {
1132    CContext* context = CContext::getCurrent();
1133    solveOnlyReferenceEnabledField();
1134
1135    if (!areAllReferenceSolved)
1136    {
1137      areAllReferenceSolved = true;
1138     
1139      if (context->getServiceType()==CServicesManager::CLIENT)
1140      {
1141        solveRefInheritance(true);
1142        if (hasDirectFieldReference()) getDirectFieldReference()->solveAllReferenceEnabledField(false);
1143      }
1144      else if (context->getServiceType()==CServicesManager::GATHERER || context->getServiceType()==CServicesManager::OUT_SERVER)
1145        solveServerOperation();
1146
1147      solveGridReference();
1148    }
1149
1150    solveGridDomainAxisRef(doSending2Server);
1151
1152    if (context->getServiceType()==CServicesManager::CLIENT)
1153    {
1154      solveTransformedGrid();
1155    }
1156
1157    solveCheckMaskIndex(doSending2Server);
1158  }
1159  CATCH_DUMP_ATTR
1160
1161  /*!
1162   * Compute the required buffer size to send the fields data.
1163   * \param [in/out] bufferSize Modifying the bufferSize for the client context
1164   * \param [in/out] maxEventSize Modifying the maximum event size for the client context
1165   * \param [in] bufferForWriting True if buffers are used for sending data for writing
1166   */ 
1167  void CField::setContextClientDataBufferSize(map<CContextClient*,map<int,size_t>>& bufferSize, 
1168                                              map<CContextClient*,map<int,size_t>>& maxEventSize, 
1169                                              bool bufferForWriting)
1170  {
1171    auto& contextBufferSize = bufferSize[client] ;
1172    auto& contextMaxEventSize = maxEventSize[client] ;
1173    const std::map<int, size_t> mapSize = grid_->getDataBufferSize(client, getId(), bufferForWriting);
1174    for(auto& it : mapSize )
1175    {
1176      // If contextBufferSize[it.first] does not exist, it will be zero-initialized
1177      // so we can use it safely without checking for its existance
1178      if (CXios::isOptPerformance) contextBufferSize[it.first] += it.second;
1179      else if (contextBufferSize[it.first] < it.second) contextBufferSize[it.first] = it.second;
1180
1181      if (contextMaxEventSize[it.first] < it.second) contextMaxEventSize[it.first] = it.second;
1182    }
1183
1184  }
1185
1186  void CField::setContextClientAttributesBufferSize(map<CContextClient*,map<int,size_t>>& bufferSize, 
1187                                                   map<CContextClient*,map<int,size_t>>& maxEventSize, 
1188                                                   bool bufferForWriting)
1189  {
1190    auto& contextBufferSize = bufferSize[client] ;
1191    auto& contextMaxEventSize = maxEventSize[client] ;
1192    const std::map<int, size_t> mapSize = grid_->getAttributesBufferSize(client, bufferForWriting);
1193    for(auto& it : mapSize )
1194    {
1195      // If contextBufferSize[it.first] does not exist, it will be zero-initialized
1196      // so we can use it safely without checking for its existance
1197      if (contextBufferSize[it.first] < it.second) contextBufferSize[it.first] = it.second;
1198      if (contextMaxEventSize[it.first] < it.second) contextMaxEventSize[it.first] = it.second;
1199    }
1200
1201  }
1202
1203
1204// ym obsolete to be removed
1205  std::map<int, StdSize> CField::getGridAttributesBufferSize(CContextClient* client, bool bufferForWriting /*= "false"*/)
1206  TRY
1207  {
1208    return grid_->getAttributesBufferSize(client, bufferForWriting);
1209  }
1210  CATCH_DUMP_ATTR
1211
1212// ym obsolete to be removed
1213  std::map<int, StdSize> CField::getGridDataBufferSize(CContextClient* client, bool bufferForWriting /*= "false"*/)
1214  TRY
1215  {
1216    return grid_->getDataBufferSize(client, getId(), bufferForWriting);
1217  }
1218  CATCH_DUMP_ATTR
1219
1220
1221
1222  size_t CField::getGlobalWrittenSize()
1223  TRY
1224  {
1225    return grid_->getGlobalWrittenSize();
1226  }
1227  CATCH_DUMP_ATTR
1228
1229  //----------------------------------------------------------------
1230
1231  void CField::solveServerOperation(void)
1232  TRY
1233  {
1234    CContext* context = CContext::getCurrent();
1235
1236    if (freq_op.isEmpty()) freq_op.setValue(TimeStep);
1237
1238    if (freq_offset.isEmpty()) freq_offset.setValue(NoneDu);
1239
1240    freq_operation_srv = fileOut_->output_freq.getValue();
1241    freq_write_srv     = fileOut_->output_freq.getValue();
1242
1243    lastlast_Write_srv = context->getCalendar()->getInitDate();
1244    last_Write_srv     = context->getCalendar()->getInitDate();
1245    last_operation_srv = context->getCalendar()->getInitDate();
1246
1247    const CDuration toffset = freq_operation_srv - freq_offset.getValue() - context->getCalendar()->getTimeStep();
1248    last_operation_srv     = last_operation_srv - toffset;
1249
1250    if (operation.isEmpty())
1251      ERROR("void CField::solveServerOperation(void)",
1252            << "An operation must be defined for field \"" << getId() << "\".");
1253
1254    std::shared_ptr<func::CFunctor> functor;
1255    CArray<double, 1> dummyData;
1256
1257#define DECLARE_FUNCTOR(MType, mtype) \
1258    if (operation.getValue().compare(#mtype) == 0) \
1259    { \
1260      functor.reset(new func::C##MType(dummyData)); \
1261    }
1262
1263#include "functor_type.conf"
1264
1265    if (!functor)
1266      ERROR("void CField::solveServerOperation(void)",
1267            << "\"" << operation << "\" is not a valid operation.");
1268
1269    operationTimeType = functor->timeType();
1270  }
1271  CATCH_DUMP_ATTR
1272
1273 //----------------------------------------------------------------
1274
1275  bool CField::buildWorkflowGraph(CGarbageCollector& gc)
1276  {
1277    if (buildWorkflowGraphDone_) return true ;
1278   
1279    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1280    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1281
1282    if (!inputFilter) inputFilter = std::shared_ptr<CPassThroughFilter>(new CPassThroughFilter(gc)); 
1283     
1284    if (hasDirectFieldReference())
1285    {
1286      CField* fieldRef = getDirectFieldReference();
1287      bool ret=fieldRef->buildWorkflowGraph(gc); 
1288      if (!ret) return false ; // workflow graph cannot be built at this stage
1289    }
1290
1291    // now construct grid and check if element are enabled
1292    solveGridReference() ; // grid_ is now defined
1293    if (!isGridCompleted()) return false;
1294
1295    // Check if we have an expression to parse
1296    std::shared_ptr<COutputPin> filterExpr ;
1297    if (hasExpression())
1298    {
1299      boost::scoped_ptr<IFilterExprNode> expr(parseExpr(getExpression() + '\0'));
1300      filterExpr = expr->reduce(gc, *this);
1301      if (!filterExpr) return false ; // workflow graph cannot be built at this stage
1302    }
1303   
1304    // prepare transformation. Need to know before if workflow of auxillary field can be built
1305    if (hasDirectFieldReference())
1306    {
1307      auto gridPath=getGridPath() ;
1308      gridPath.push_back(grid_) ;
1309
1310      CGrid* gridSrc=getDirectFieldReference()->getGrid() ;
1311      for(auto grid : gridPath)
1312      {
1313        grid->solveElementsRefInheritance() ;
1314        grid->completeGrid(gridSrc); // grid generation, to be checked
1315        grid->checkElementsAttributes() ;
1316        grid->prepareTransformGrid(gridSrc) ; // prepare the grid tranformation
1317        for(auto fieldId : grid->getAuxInputTransformGrid()) // try to build workflow graph for auxillary field tranformation
1318          if (!CField::get(fieldId)->buildWorkflowGraph(gc)) return false ;
1319        gridSrc=grid ;
1320      }
1321     
1322      std::shared_ptr<COutputPin> lastFilter ;
1323      if (filterExpr) lastFilter=filterExpr ;
1324      else lastFilter = inputFilter ;
1325     
1326      gridSrc=getDirectFieldReference()->getGrid() ;
1327      for(auto grid : gridPath) 
1328      {
1329        grid->makeTransformGrid() ; // make the grid transformation
1330        if (grid->hasTransform()) 
1331        {
1332          std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = CSpatialTransformFilter::buildFilterGraph(gc, gridSrc, grid, detectMissingValues, defaultValue); 
1333          lastFilter->connectOutput(filters.first, 0);
1334          lastFilter = filters.second;
1335          gridSrc=grid ;
1336        }
1337      }
1338      instantDataFilter = lastFilter ;
1339     
1340      // connect the input Filter to the reference
1341      getDirectFieldReference()->getInstantDataFilter()->connectOutput(inputFilter,0);
1342    }
1343    else 
1344    {
1345      if (hasFileIn()) // input file, attemp to read the grid from file
1346      {
1347         // must be checked
1348         fileIn_->initRead() ;
1349         fileIn_->checkReadFile();
1350         grid_->solveElementsRefInheritance() ;
1351         if (fileIn_->isClientSide()) fileIn_->readFieldAttributesMetaData(this);
1352         grid_->completeGrid(); // grid generation, to be checked
1353         if (fileIn_->isClientSide()) fileIn_->readFieldAttributesValues(this);
1354         grid_->checkElementsAttributes() ;
1355         grid_->solveDomainAxisBaseRef();
1356         // probably in future tag grid incomplete if coming from a reading
1357         instantDataFilter=inputFilter ;
1358      } 
1359      else if (hasCouplerIn())
1360      {
1361        grid_->checkElementsAttributes() ;
1362        instantDataFilter=inputFilter ;
1363      }
1364      else
1365      {
1366        setModelIn() ; // no reference, the field is potentially a source field from model
1367
1368        grid_->solveElementsRefInheritance() ;
1369        grid_->completeGrid(); // grid generation, to be checked
1370        grid_->checkElementsAttributes() ;
1371        instantDataFilter=inputFilter ;
1372      }
1373    }
1374   
1375    if (hasFileOut())
1376    {
1377      if (fileOut_->isServerSide())
1378      {
1379        this->solveServerOperation() ;
1380      }
1381    }
1382
1383    buildWorkflowGraphDone_ = true ;
1384    workflowEnabled_ = true ;
1385    return true ;
1386  }
1387   
1388  /*!
1389   * Connect field to filter to send data to server. A temporal filter is inserted before accordingly to the
1390   * output frequency of the file
1391   * \param gc the garbage collector to use when building the filter graph
1392   */
1393  void CField::connectToFileServer(CGarbageCollector& gc)
1394  {
1395    // insert temporal filter before sending to files
1396    fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, client));
1397    // insert temporal filter before sending to files
1398    getTemporalDataFilter(gc, fileOut_->output_freq)->connectOutput(fileWriterFilter, 0);
1399  } 
1400
1401  void CField::connectToCouplerOut(CGarbageCollector& gc)
1402  {
1403    // insert temporal filter before sending to files
1404    fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, client));
1405    instantDataFilter->connectOutput(fileWriterFilter, 0);
1406  } 
1407
1408  /*!
1409   * Compute grid index needed to send grid and data to server
1410   */
1411  void CField::computeGridIndexToFileServer(void)
1412  {
1413    grid_->computeGridIndexToFileServer(client) ;
1414  }
1415
1416  /*!
1417   * Connect field to a source filter to receive data from model.
1418   */
1419  void CField::connectToModelInput(CGarbageCollector& gc)
1420  {
1421    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1422    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1423
1424    if (check_if_active.isEmpty()) check_if_active = false; 
1425    clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, false, true, NoneDu, false, detectMissingValues, defaultValue));
1426    clientSourceFilter -> connectOutput(inputFilter,0) ;
1427  } 
1428 
1429  /*!
1430   * Connect field to a source filter to receive data from a client (on server side).
1431   */
1432  void CField::connectToClientInput(CGarbageCollector& gc)
1433  {
1434    clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc,  grid_, false, false));
1435    clientSourceFilter -> connectOutput(inputFilter,0) ;
1436  } 
1437
1438
1439  /*!
1440   * Connect field to a source filter to receive data from a server (on client side).
1441   */
1442  void CField::connectToServerInput(CGarbageCollector& gc)
1443  {
1444    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1445    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1446
1447    checkTimeAttributes();
1448    serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, false, freq_offset, true,
1449                                                                          detectMissingValues, defaultValue));
1450    //serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc,  grid_, false, false));
1451    serverSourceFilter -> connectOutput(inputFilter,0) ;
1452  } 
1453
1454  /*!
1455   * Connect field to a source filter to receive data from coupler (on client side).
1456   */
1457   void CField::connectToCouplerIn(CGarbageCollector& gc)
1458  {
1459    CContext* context = CContext::getCurrent();
1460
1461    if (freq_op.isEmpty()) freq_op.setValue(TimeStep);
1462    if (freq_offset.isEmpty()) freq_offset.setValue(freq_op.getValue() - TimeStep);
1463
1464    freq_operation_srv = freq_op ;
1465    last_operation_srv = context->getCalendar()->getInitDate();
1466    const CDuration toffset = freq_operation_srv - freq_offset.getValue() - context->getCalendar()->getTimeStep();
1467    last_operation_srv     = last_operation_srv - toffset;
1468
1469    clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, false, false, freq_offset, true)) ;
1470    clientSourceFilter -> connectOutput(inputFilter,0) ;
1471  } 
1472
1473
1474  /*!
1475   * Connect field to a file writer filter to write data in file (on server side).
1476   */
1477  void CField::connectToFileWriter(CGarbageCollector& gc)
1478  {
1479    fileServerWriterFilter = std::shared_ptr<CFileServerWriterFilter>(new CFileServerWriterFilter(gc, this));
1480    instantDataFilter->connectOutput(fileServerWriterFilter, 0);
1481  } 
1482 
1483
1484  /*!
1485   * Connect field to a store filter to output data to model on client Side
1486   */
1487  void CField::connectToModelOutput(CGarbageCollector& gc)
1488  {
1489    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1490    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1491
1492    storeFilter = std::shared_ptr<CStoreFilter>(new CStoreFilter(gc, CContext::getCurrent(), grid_, detectMissingValues, defaultValue));
1493    instantDataFilter->connectOutput(storeFilter, 0);
1494  }
1495
1496  /*!
1497   * Transform the grid_path attribut into vector of grid.
1498   * \return the vector CGrid* containing the list of grid path for tranformation
1499   */ 
1500  vector<CGrid*> CField::getGridPath(void)
1501  {
1502    std::vector<CGrid*> gridPath;
1503
1504    if (hasDirectFieldReference() && grid_ != getDirectFieldReference()->grid_)
1505    {
1506      if (!grid_path.isEmpty())
1507      {
1508        std::string gridId;
1509        size_t start = 0, end;
1510
1511        do
1512        {
1513          end = grid_path.getValue().find(',', start);
1514          if (end != std::string::npos)
1515          {
1516            gridId = grid_path.getValue().substr(start, end - start);
1517            start = end + 1;
1518          }
1519          else gridId = grid_path.getValue().substr(start);
1520
1521          if (!CGrid::has(gridId))
1522            ERROR("void CField::solveTransformedGrid()",
1523               << "Invalid grid_path, the grid '" << gridId << "' does not exist.");
1524
1525          gridPath.push_back(CGrid::get(gridId));
1526        }
1527        while (end != std::string::npos);
1528      }
1529    }
1530    return gridPath ;
1531  }
1532
1533  /*!
1534   * Constructs the graph filter for the field, enabling or not the data output.
1535   * This method should not be called more than once with enableOutput equal to true.
1536   *
1537   * \param gc the garbage collector to use when building the filter graph
1538   * \param enableOutput must be true when the field data is to be
1539   *                     read by the client or/and written to a file
1540   */
1541   // ym obselete : to be removed later....
1542  void CField::buildFilterGraph(CGarbageCollector& gc, bool enableOutput)
1543  TRY
1544  {     
1545    if (!isReferenceSolvedAndTransformed) solveAllEnabledFieldsAndTransform();
1546    if (!isGridChecked) checkGridOfEnabledFields();
1547
1548    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1549    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1550
1551    CContext* context = CContext::getCurrent();
1552    bool hasWriterServer = context->getServiceType()==CServicesManager::OUT_SERVER ;
1553    bool hasIntermediateServer = context->getServiceType()==CServicesManager::GATHERER ;
1554
1555    if (hasWriterServer)
1556    {
1557      if (!instantDataFilter)
1558        instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, false));
1559
1560
1561      // If the field data is to be read by the client or/and written to a file
1562      if (enableOutput && !storeFilter && !fileWriterFilter)
1563      {
1564        if (getRelFile() && (getRelFile()->mode.isEmpty() || getRelFile()->mode == CFile::mode_attr::write))
1565        {
1566          fileServerWriterFilter = std::shared_ptr<CFileServerWriterFilter>(new CFileServerWriterFilter(gc, this));
1567          instantDataFilter->connectOutput(fileServerWriterFilter, 0);
1568        }
1569      }
1570    }
1571    else if (hasIntermediateServer)
1572    {
1573      if (!instantDataFilter)
1574        instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, false, false));
1575
1576      // If the field data is to be read by the client or/and written to a file
1577      if (enableOutput && !storeFilter && !fileWriterFilter)
1578      {
1579        if (getRelFile() && (getRelFile()->mode.isEmpty() || getRelFile()->mode == CFile::mode_attr::write))
1580        {
1581          fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, getRelFile()->getContextClient()));
1582          instantDataFilter->connectOutput(fileWriterFilter, 0);
1583        }
1584      }
1585    }
1586    else
1587    {
1588      // Start by building a filter which can provide the field's instant data
1589      if (!instantDataFilter)
1590      {
1591        // Check if we have an expression to parse
1592        if (hasExpression())
1593        {
1594          boost::scoped_ptr<IFilterExprNode> expr(parseExpr(getExpression() + '\0'));
1595          std::shared_ptr<COutputPin> filter = expr->reduce(gc, *this);
1596
1597          // Check if a spatial transformation is needed
1598          if (!field_ref.isEmpty())
1599          {
1600            CGrid* gridRef = CField::get(field_ref)->grid_;
1601            if (grid_ && grid_ != gridRef && grid_->hasTransform())
1602            {
1603                std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = CSpatialTransformFilter::buildFilterGraph(gc, gridRef, grid_, detectMissingValues, defaultValue); 
1604
1605              filter->connectOutput(filters.first, 0);
1606              filter = filters.second;
1607            }
1608          }
1609
1610          instantDataFilter = filter;
1611        }
1612        // Check if we have a reference on another field
1613        else if (!field_ref.isEmpty()) instantDataFilter = getFieldReference(gc);
1614        // Check if the data is to be read from a file
1615        else if (getRelFile() && !getRelFile()->mode.isEmpty() && getRelFile()->mode == CFile::mode_attr::read)
1616        {
1617          checkTimeAttributes();
1618          instantDataFilter = serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, false, freq_offset, true,
1619                                                                                                       detectMissingValues, defaultValue));
1620        }
1621        else // The data might be passed from the model
1622        {
1623          if (check_if_active.isEmpty()) check_if_active = false; 
1624          instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, false, true, NoneDu, false,
1625                                                                                                      detectMissingValues, defaultValue));
1626        }
1627      }
1628
1629      // If the field data is to be read by the client or/and written to a file
1630      if (enableOutput && !storeFilter && !fileWriterFilter)
1631      {
1632        if (!read_access.isEmpty() && read_access)
1633        {
1634          storeFilter = std::shared_ptr<CStoreFilter>(new CStoreFilter(gc, CContext::getCurrent(), grid_,
1635                                                                          detectMissingValues, defaultValue));
1636          instantDataFilter->connectOutput(storeFilter, 0);
1637        }
1638
1639        if (getRelFile() && (getRelFile()->mode.isEmpty() || getRelFile()->mode == CFile::mode_attr::write))
1640        {
1641          fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, getRelFile()->getContextClient()));
1642          getTemporalDataFilter(gc, getRelFile()->output_freq)->connectOutput(fileWriterFilter, 0);
1643        }
1644      }
1645    }
1646  }
1647  CATCH_DUMP_ATTR
1648
1649  /*!
1650   * Returns the filter needed to handle the field reference.
1651   * This method should only be called when building the filter graph corresponding to the field.
1652   *
1653   * \param gc the garbage collector to use
1654   * \return the output pin corresponding to the field reference
1655   */
1656  std::shared_ptr<COutputPin> CField::getFieldReference(CGarbageCollector& gc)
1657  TRY
1658  {
1659    if (instantDataFilter || field_ref.isEmpty())
1660      ERROR("COutputPin* CField::getFieldReference(CGarbageCollector& gc)",
1661            "Impossible to get the field reference for a field which has already been parsed or which does not have a field_ref.");
1662
1663    CField* fieldRef = CField::get(field_ref);
1664    fieldRef->buildFilterGraph(gc, false);
1665
1666    std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters;
1667    // Check if a spatial transformation is needed
1668    if (grid_ && grid_ != fieldRef->grid_ && grid_->hasTransform())
1669    {       
1670      bool hasMissingValue = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1671      double defaultValue  = hasMissingValue ? default_value : (!default_value.isEmpty() ? default_value : 0.0);                               
1672      filters = CSpatialTransformFilter::buildFilterGraph(gc, fieldRef->grid_, grid_, hasMissingValue, defaultValue);
1673    }
1674    else
1675      filters.first = filters.second = std::shared_ptr<CFilter>(new CPassThroughFilter(gc));
1676
1677    fieldRef->getInstantDataFilter()->connectOutput(filters.first, 0);
1678
1679    return filters.second;
1680  }
1681  CATCH_DUMP_ATTR
1682
1683  /*!
1684   * Returns the filter needed to handle a self reference in the field's expression.
1685   * If the needed filter does not exist, it is created, otherwise it is reused.
1686   * This method should only be called when building the filter graph corresponding
1687   * to the field's expression.
1688   *
1689   * \param gc the garbage collector to use
1690   * \return the output pin corresponding to a self reference
1691   */
1692
1693/* old version
1694  std::shared_ptr<COutputPin> CField::getSelfReference(CGarbageCollector& gc)
1695  TRY
1696  {
1697    if (instantDataFilter || !hasExpression())
1698      ERROR("COutputPin* CField::getSelfReference(CGarbageCollector& gc)",
1699            "Impossible to add a self reference to a field which has already been parsed or which does not have an expression.");
1700
1701    if (!selfReferenceFilter)
1702    {
1703      const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1704      const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1705
1706      if (file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read)
1707      {
1708        if (!serverSourceFilter)
1709        {
1710          checkTimeAttributes();
1711          serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, false, freq_offset, true,
1712                                                              detectMissingValues, defaultValue));
1713         }
1714
1715        selfReferenceFilter = serverSourceFilter;
1716      }
1717      else if (!field_ref.isEmpty())
1718      {
1719        CField* fieldRef = CField::get(field_ref);
1720        fieldRef->buildFilterGraph(gc, false);
1721        selfReferenceFilter = fieldRef->getInstantDataFilter();
1722      }
1723      else
1724      {
1725        if (!clientSourceFilter)
1726        {
1727          if (check_if_active.isEmpty()) check_if_active = false;
1728          clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, true, NoneDu, false,
1729                                                                                detectMissingValues, defaultValue));
1730        }
1731
1732        selfReferenceFilter = clientSourceFilter;
1733      }
1734    }
1735
1736    return selfReferenceFilter;
1737  }
1738  CATCH_DUMP_ATTR
1739*/
1740  std::shared_ptr<COutputPin> CField::getSelfReference(CGarbageCollector& gc)
1741  TRY
1742  {
1743    return inputFilter ;
1744  } 
1745  CATCH_DUMP_ATTR
1746
1747  /*!
1748   * Returns the temporal filter corresponding to the field's temporal operation
1749   * for the specified operation frequency. The filter is created if it does not
1750   * exist, otherwise it is reused.
1751   *
1752   * \param gc the garbage collector to use
1753   * \param outFreq the operation frequency, i.e. the frequency at which the output data will be computed
1754   * \return the output pin corresponding to the requested temporal filter
1755   */
1756  std::shared_ptr<COutputPin> CField::getTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)
1757  TRY
1758  {
1759    std::map<CDuration, std::shared_ptr<COutputPin> >::iterator it = temporalDataFilters.find(outFreq);
1760
1761    if (it == temporalDataFilters.end())
1762    {
1763      if (operation.isEmpty())
1764        ERROR("void CField::getTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)",
1765              << "An operation must be defined for field \"" << getId() << "\".");
1766
1767      checkTimeAttributes(&outFreq);
1768
1769      const bool detectMissingValues = (!detect_missing_value.isEmpty()  && detect_missing_value == true);
1770      std::shared_ptr<CTemporalFilter> temporalFilter(new CTemporalFilter(gc, operation,
1771                                                                             CContext::getCurrent()->getCalendar()->getInitDate(),
1772                                                                             freq_op, freq_offset, outFreq, detectMissingValues));
1773
1774      instantDataFilter->connectOutput(temporalFilter, 0);
1775
1776      it = temporalDataFilters.insert(std::make_pair(outFreq, temporalFilter)).first;
1777    }
1778
1779    return it->second;
1780  }
1781  CATCH_DUMP_ATTR
1782
1783  /*!
1784    * Returns the temporal filter corresponding to the field's temporal operation
1785    * for the specified operation frequency.
1786    *
1787    * \param gc the garbage collector to use
1788    * \param outFreq the operation frequency, i.e. the frequency at which the output data will be computed
1789    * \return the output pin corresponding to the requested temporal filter
1790    */
1791
1792  std::shared_ptr<COutputPin> CField::getSelfTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)
1793  TRY
1794  {
1795    if (instantDataFilter || !hasExpression())
1796      ERROR("COutputPin* CField::getSelfTemporalDataFilter(CGarbageCollector& gc)",
1797            "Impossible to add a self reference to a field which has already been parsed or which does not have an expression.");
1798   
1799    if (selfTemporalDataFilter) return selfTemporalDataFilter;
1800
1801    if (hasDirectFieldReference())
1802    {
1803      CField* fieldRef=getDirectFieldReference();
1804      return fieldRef->getTemporalDataFilter(gc, outFreq) ;
1805    }
1806    else
1807    {
1808      if (selfTemporalDataFilter) return selfTemporalDataFilter ;
1809
1810      if (operation.isEmpty())
1811        ERROR("void CField::getSelfTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)",
1812              << "An operation must be defined for field \"" << getId() << "\".");
1813
1814      checkTimeAttributes(&outFreq); //bof
1815
1816      const bool detectMissingValues = (!detect_missing_value.isEmpty() && detect_missing_value == true);
1817      selfTemporalDataFilter = std::shared_ptr<CTemporalFilter>(new CTemporalFilter(gc, operation,
1818                                                                CContext::getCurrent()->getCalendar()->getInitDate(),
1819                                                                freq_op, freq_offset, outFreq, detectMissingValues));
1820
1821      inputFilter->connectOutput(selfTemporalDataFilter, 0);
1822      return selfTemporalDataFilter ;
1823    }
1824  }
1825  CATCH_DUMP_ATTR
1826
1827/* old version   
1828  std::shared_ptr<COutputPin> CField::getSelfTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)
1829  TRY
1830  {
1831    if (instantDataFilter || !hasExpression())
1832      ERROR("COutputPin* CField::getSelfTemporalDataFilter(CGarbageCollector& gc)",
1833            "Impossible to add a self reference to a field which has already been parsed or which does not have an expression.");
1834
1835    if (!selfReferenceFilter) getSelfReference(gc) ;
1836
1837    if (serverSourceFilter || clientSourceFilter)
1838    {
1839      if (operation.isEmpty())
1840        ERROR("void CField::getSelfTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)",
1841              << "An operation must be defined for field \"" << getId() << "\".");
1842
1843      checkTimeAttributes(&outFreq);
1844
1845      const bool detectMissingValues = (!detect_missing_value.isEmpty() && detect_missing_value == true);
1846      std::shared_ptr<CTemporalFilter> temporalFilter(new CTemporalFilter(gc, operation,
1847                                                                          CContext::getCurrent()->getCalendar()->getInitDate(),
1848                                                                          freq_op, freq_offset, outFreq, detectMissingValues));
1849
1850      selfReferenceFilter->connectOutput(temporalFilter, 0);
1851      return temporalFilter ;
1852    }
1853    else if (!field_ref.isEmpty())
1854    {
1855      CField* fieldRef = CField::get(field_ref);
1856      fieldRef->buildFilterGraph(gc, false);
1857      return fieldRef->getTemporalDataFilter(gc, outFreq) ;
1858    }
1859  }
1860  CATCH_DUMP_ATTR
1861*/
1862
1863  //----------------------------------------------------------------
1864/*
1865   void CField::fromBinary(StdIStream& is)
1866   {
1867      SuperClass::fromBinary(is);
1868#define CLEAR_ATT(name_)\
1869      SuperClassAttribute::operator[](#name_)->reset()
1870
1871         CLEAR_ATT(domain_ref);
1872         CLEAR_ATT(axis_ref);
1873#undef CLEAR_ATT
1874
1875   }
1876*/
1877   //----------------------------------------------------------------
1878
1879  void CField::solveGridReference(void)
1880  TRY
1881  {
1882    if (grid_!=nullptr) return ; // already done
1883
1884    if (grid_ref.isEmpty() && domain_ref.isEmpty() && axis_ref.isEmpty() && scalar_ref.isEmpty())
1885    {
1886      ERROR("CField::solveGridReference(void)",
1887            << "A grid must be defined for field '" << getFieldOutputName() << "' .");
1888    }
1889    else if (!grid_ref.isEmpty() && (!domain_ref.isEmpty() || !axis_ref.isEmpty() || !scalar_ref.isEmpty()))
1890    {
1891      ERROR("CField::solveGridReference(void)",
1892            << "Field '" << getFieldOutputName() << "' has both a grid and a domain/axis/scalar." << std::endl
1893            << "Please define either 'grid_ref' or 'domain_ref'/'axis_ref'/'scalar_ref'.");
1894    }
1895
1896    if (grid_ref.isEmpty())
1897    {
1898      std::vector<CDomain*> vecDom;
1899      std::vector<CAxis*> vecAxis;
1900      std::vector<CScalar*> vecScalar;
1901      std::vector<int> axisDomainOrderTmp;
1902
1903      std::vector<CDomain*> vecDomRef;
1904      std::vector<CAxis*> vecAxisRef;
1905      std::vector<CScalar*> vecScalarRef;
1906       
1907      if (!domain_ref.isEmpty())
1908      {
1909        StdString tmp = domain_ref.getValue();
1910        if (CDomain::has(domain_ref))
1911        {
1912          vecDom.push_back(CDomain::get(domain_ref));
1913          vecDomRef.push_back(CDomain::createDomain());
1914          vecDomRef.back()->domain_ref=domain_ref;
1915          axisDomainOrderTmp.push_back(2);
1916        }
1917        else  ERROR("CField::solveGridReference(void)",
1918                    << "Invalid reference to domain '" << domain_ref.getValue() << "'.");
1919      }
1920
1921      if (!axis_ref.isEmpty())
1922      {
1923        if (CAxis::has(axis_ref))
1924        {
1925          vecAxis.push_back(CAxis::get(axis_ref));
1926          vecAxisRef.push_back(CAxis::createAxis());
1927          vecAxisRef.back()->axis_ref=axis_ref;
1928          axisDomainOrderTmp.push_back(1);
1929        }
1930        else  ERROR("CField::solveGridReference(void)",
1931                    << "Invalid reference to axis '" << axis_ref.getValue() << "'.");
1932      }
1933
1934      if (!scalar_ref.isEmpty())
1935      {
1936        if (CScalar::has(scalar_ref))
1937        {
1938          vecScalar.push_back(CScalar::get(scalar_ref));
1939          vecScalarRef.push_back(CScalar::createScalar());
1940          vecScalarRef.back()->scalar_ref=scalar_ref;
1941          axisDomainOrderTmp.push_back(0);
1942        }
1943        else ERROR("CField::solveGridReference(void)",
1944                   << "Invalid reference to scalar '" << scalar_ref.getValue() << "'.");
1945      }
1946       
1947      CArray<int,1> axisDomainOrder(axisDomainOrderTmp.size());
1948      for (int idx = 0; idx < axisDomainOrderTmp.size(); ++idx)
1949      {
1950        axisDomainOrder(idx) = axisDomainOrderTmp[idx];
1951      }
1952
1953      // Warning: the gridId shouldn't be set as the grid_ref since it could be inherited
1954      StdString gridId = CGrid::generateId(vecDom, vecAxis, vecScalar,axisDomainOrder);
1955      if (CGrid::has(gridId)) this->grid_ = CGrid::get(gridId);
1956      else  this->grid_ = CGrid::createGrid(gridId, vecDomRef, vecAxisRef, vecScalarRef,axisDomainOrder);
1957    }
1958    else
1959    {
1960      if (CGrid::has(grid_ref)) this->grid_ = CGrid::get(grid_ref);
1961      else  ERROR("CField::solveGridReference(void)",
1962                   << "Invalid reference to grid '" << grid_ref.getValue() << "'.");
1963    }
1964  }
1965  CATCH_DUMP_ATTR
1966
1967  void CField::solveGridDomainAxisRef(bool checkAtt)
1968  TRY
1969  {
1970    grid_->solveDomainAxisRef(checkAtt);
1971  }
1972  CATCH_DUMP_ATTR
1973
1974  void CField::solveCheckMaskIndex(bool doSendingIndex)
1975  TRY
1976  {
1977    grid_->checkMaskIndex(doSendingIndex);
1978  }
1979  CATCH_DUMP_ATTR
1980
1981  void CField::solveTransformedGrid()
1982  TRY
1983  {
1984    if (grid_ && !grid_->isTransformed() && hasDirectFieldReference() && grid_ != getDirectFieldReference()->grid_)
1985    {
1986      std::vector<CGrid*> grids;
1987      // Source grid
1988      grids.push_back(getDirectFieldReference()->grid_);
1989      auto gridPath = getGridPath() ;
1990      grids.insert(grids.begin(), gridPath.begin(), gridPath.end());
1991
1992      for (size_t i = 0, count = grids.size() - 1; i < count; ++i)
1993      {
1994        CGrid *gridSrc  = grids[i];
1995        CGrid *gridDest = grids[i + 1];
1996        if (!gridDest->isTransformed()) gridDest->transformGrid(gridSrc);
1997      }
1998    }
1999    else if (grid_ && grid_->hasTransform() && !grid_->isTransformed())
2000    {
2001      // Temporarily deactivate the self-transformation of grid
2002      // grid_->transformGrid(grid_);
2003    }
2004  }
2005  CATCH_DUMP_ATTR
2006
2007  void CField::solveGenerateGrid()
2008  TRY
2009  {
2010    if (grid_ && !grid_->isTransformed() && hasDirectFieldReference() && grid_ != getDirectFieldReference()->grid_)
2011      grid_->completeGrid(getDirectFieldReference()->grid_);
2012    else grid_->completeGrid();
2013  }
2014  CATCH_DUMP_ATTR
2015
2016  void CField::solveGridDomainAxisBaseRef()
2017  TRY
2018  {
2019    grid_->solveDomainAxisRef(false);
2020    grid_->solveDomainAxisBaseRef();
2021  }
2022  CATCH_DUMP_ATTR
2023
2024  ///-------------------------------------------------------------------
2025
2026  template <>
2027  void CGroupTemplate<CField, CFieldGroup, CFieldAttributes>::solveRefInheritance(void)
2028  TRY
2029  {
2030    if (this->group_ref.isEmpty()) return;
2031    StdString gref = this->group_ref.getValue();
2032
2033    if (!CFieldGroup::has(gref))
2034      ERROR("CGroupTemplate<CField, CFieldGroup, CFieldAttributes>::solveRefInheritance(void)",
2035         << "[ gref = " << gref << "]"
2036         << " invalid group name !");
2037
2038    CFieldGroup* group = CFieldGroup::get(gref);
2039    CFieldGroup* owner = CFieldGroup::get(boost::polymorphic_downcast<CFieldGroup*>(this));
2040    owner->setAttributes(group); // inherite of attributes of group reference
2041     
2042    std::vector<CField*> allChildren  = group->getAllChildren();
2043    std::vector<CField*>::iterator it = allChildren.begin(), end = allChildren.end();
2044
2045    for (; it != end; it++)
2046    {
2047      CField* child = *it;
2048     if (child->hasId()) owner->createChild()->field_ref.setValue(child->getId());
2049    }
2050  }
2051  CATCH_DUMP_ATTR
2052
2053  void CField::scaleFactorAddOffset(CArray<double,1>& data, double scaleFactor, double addOffset)
2054  TRY
2055  {
2056    data = (data - addOffset) / scaleFactor;
2057  }
2058  CATCH_DUMP_ATTR
2059
2060  void CField::invertScaleFactorAddOffset(CArray<double,1>& data, double scaleFactor, double addOffset)
2061  TRY
2062  {
2063    data = data * scaleFactor + addOffset;
2064  }
2065  CATCH_DUMP_ATTR
2066
2067  void CField::outputField(const CArray<double,1>& dataIn, CArray<double,1>& dataOut)
2068  TRY
2069  { 
2070    CArray<size_t,1>& outIndexClient = grid_->localIndexToWriteOnClient_;
2071    CArray<size_t,1>& outIndexServer = grid_->localIndexToWriteOnServer_;
2072    for (size_t idx = 0; idx < outIndexServer.numElements(); ++idx)
2073    {
2074      dataOut(outIndexServer(idx)) = dataIn(outIndexClient(idx));
2075    }
2076  }
2077  CATCH_DUMP_ATTR
2078
2079  void CField::inputField(const CArray<double,1>& dataIn, CArray<double,1>& dataOut)
2080  TRY
2081  {
2082    CArray<size_t,1>& outIndexClient = grid_->localIndexToWriteOnClient_;
2083    CArray<size_t,1>& outIndexServer = grid_->localIndexToWriteOnServer_;
2084    for (size_t idx = 0; idx < outIndexServer.numElements(); ++idx)
2085    {
2086      dataOut(outIndexClient(idx)) = dataIn(outIndexServer(idx));
2087    }
2088  }
2089  CATCH_DUMP_ATTR
2090
2091 void CField::outputCompressedField(const CArray<double,1>& dataIn, CArray<double,1>& dataOut)
2092 TRY
2093 {
2094    CArray<size_t,1>& outIndexClient = grid_->localIndexToWriteOnClient_;
2095    CArray<size_t,1>& outIndexServer = grid_->localIndexToWriteOnServer_;
2096    for (size_t idx = 0; idx < outIndexServer.numElements(); ++idx)
2097    {
2098      dataOut((idx)) = dataIn(outIndexClient(idx));
2099    }
2100  }
2101  CATCH_DUMP_ATTR
2102
2103  ///-------------------------------------------------------------------
2104
2105  void CField::parse(xml::CXMLNode& node)
2106  TRY
2107  {
2108    string newContent ;
2109    SuperClass::parse(node);
2110    if (node.goToChildElement())
2111    {
2112      do
2113      {
2114        if (node.getElementName() == "variable" || node.getElementName() == "variable_group") this->getVirtualVariableGroup()->parseChild(node);
2115        else if (node.getElementName() == "expr") if (node.getContent(newContent)) content+=newContent ;
2116      } while (node.goToNextElement());
2117      node.goToParentElement();
2118    }
2119    if (node.getContent(newContent)) content=newContent ;
2120  }
2121  CATCH_DUMP_ATTR
2122
2123 /*!
2124   This function retrieves Id of corresponding domain_ref and axis_ref (if any)
2125   of a field. In some cases, only domain exists but axis doesn't
2126   \return pair of Domain and Axis id
2127  */
2128  const std::vector<StdString>& CField::getRefDomainAxisIds()
2129  TRY
2130  {
2131    CGrid* cgPtr = getRelGrid();
2132    if (NULL != cgPtr)
2133    {
2134      std::vector<StdString>::iterator it;
2135      if (!domain_ref.isEmpty())
2136      {
2137        std::vector<StdString> domainList = cgPtr->getDomainList();
2138        it = std::find(domainList.begin(), domainList.end(), domain_ref.getValue());
2139        if (domainList.end() != it) domAxisScalarIds_[0] = *it;
2140      }
2141
2142      if (!axis_ref.isEmpty())
2143      {
2144        std::vector<StdString> axisList = cgPtr->getAxisList();
2145        it = std::find(axisList.begin(), axisList.end(), axis_ref.getValue());
2146        if (axisList.end() != it) domAxisScalarIds_[1] = *it;
2147      }
2148
2149      if (!scalar_ref.isEmpty())
2150      {
2151        std::vector<StdString> scalarList = cgPtr->getScalarList();
2152        it = std::find(scalarList.begin(), scalarList.end(), scalar_ref.getValue());
2153        if (scalarList.end() != it) domAxisScalarIds_[2] = *it;
2154      }
2155    }
2156    return (domAxisScalarIds_);
2157  }
2158  CATCH_DUMP_ATTR
2159
2160  CVariable* CField::addVariable(const string& id)
2161  TRY
2162  {
2163    return vVariableGroup->createChild(id);
2164  }
2165  CATCH
2166
2167  CVariableGroup* CField::addVariableGroup(const string& id)
2168  TRY
2169  {
2170    return vVariableGroup->createChildGroup(id);
2171  }
2172  CATCH
2173
2174  void CField::setContextClient(CContextClient* contextClient)
2175  TRY
2176  {
2177    CContext* context = CContext::getCurrent();
2178    client = contextClient;
2179 
2180    // A grid is sent by a client (both for read or write) or by primary server (write only)
2181    if (context->getServiceType()==CServicesManager::GATHERER)
2182    {
2183      if (getRelFile()->mode.isEmpty() || (!getRelFile()->mode.isEmpty() && getRelFile()->mode == CFile::mode_attr::write))
2184        grid_->setContextClient(contextClient);
2185    }
2186    else if (context->getServiceType()==CServicesManager::CLIENT)
2187      grid_->setContextClient(contextClient);
2188  }
2189  CATCH_DUMP_ATTR
2190
2191  void CField::sendCloseDefinition(void)
2192  {
2193    CContext::getCurrent()->sendCloseDefinition(client) ;
2194  }
2195
2196  void CField::sendFieldToFileServer(void)
2197  {
2198    CContext::getCurrent()->sendContextToFileServer(client);
2199    getRelFile()->sendFileToFileServer(client);
2200    grid_->sendGridToFileServer(client);
2201    this->sendAllAttributesToServer(client);
2202    this->sendAddAllVariables(client);
2203  }
2204
2205  void CField::sendFieldToCouplerOut(void)
2206  {
2207    if (sendFieldToCouplerOut_done_) return ;
2208    else sendFieldToCouplerOut_done_=true ;
2209    grid_->sendGridToCouplerOut(client, this->getId());
2210    this->sendGridCompleted();
2211
2212  }
2213 
2214  void CField::makeGridAliasForCoupling(void) 
2215  { 
2216    grid_->makeAliasForCoupling(this->getId()); 
2217  }
2218
2219 //! Client side: Send a message  announcing that the grid definition has been received from a coupling context
2220   void CField::sendGridCompleted(void)
2221   TRY
2222   {
2223      CEventClient event(getType(),EVENT_ID_GRID_COMPLETED);
2224
2225      if (client->isServerLeader())
2226      {
2227        CMessage msg;
2228        msg<<this->getId();
2229        for (auto& rank : client->getRanksServerLeader()) event.push(rank,1,msg);
2230        client->sendEvent(event);
2231      }
2232      else client->sendEvent(event);
2233   }
2234   CATCH_DUMP_ATTR
2235
2236   //! Server side: Receive a message announcing that the grid definition has been received from a coupling context
2237   void CField::recvGridCompleted(CEventServer& event)
2238   TRY
2239   {
2240      CBufferIn* buffer=event.subEvents.begin()->buffer;
2241      string id;
2242      *buffer>>id ;
2243      get(id)->recvGridCompleted(*buffer);
2244   }
2245   CATCH
2246
2247   //! Server side: Receive a message  message  announcing that the grid definition has been received from a coupling context
2248   void CField::recvGridCompleted(CBufferIn& buffer)
2249   TRY
2250   {
2251      setGridCompleted() ;
2252   }
2253   CATCH_DUMP_ATTR
2254
2255
2256
2257
2258
2259  void CField::sendFieldToInputFileServer(void)
2260  {
2261    CContext::getCurrent()->sendContextToFileServer(client);
2262    getRelFile()->sendFileToFileServer(client);
2263    grid_->sendGridToFileServer(client);
2264    read_access=true ; // not the best solution, but on server side, the field must be a starting point of the workflow
2265                       // must be replace by a better solution when implementing filters for reading and send to client
2266                       // on server side
2267    this->sendAllAttributesToServer(client);
2268    this->sendAddAllVariables(client);
2269  }
2270
2271  void CField::sendAddAllVariables(CContextClient* client)
2272  TRY
2273  {
2274    std::vector<CVariable*> allVar = getAllVariables();
2275    std::vector<CVariable*>::const_iterator it = allVar.begin();
2276    std::vector<CVariable*>::const_iterator itE = allVar.end();
2277
2278    for (; it != itE; ++it)
2279    {
2280      this->sendAddVariable((*it)->getId(), client);
2281      (*it)->sendAllAttributesToServer(client);
2282      (*it)->sendValue(client);
2283    }
2284  }
2285  CATCH_DUMP_ATTR
2286
2287  /*!
2288   * Send all Attributes to server. This method is overloaded, since only grid_ref attribute
2289   * must be sent to server and not domain_ref/axis_ref/scalar_ref.
2290   */
2291   
2292  void CField::sendAllAttributesToServer(CContextClient* client)
2293  TRY
2294  {
2295    if (grid_ref.isEmpty())
2296    {
2297      grid_ref=grid_->getId() ;
2298      SuperClass::sendAllAttributesToServer(client) ;
2299      domain_ref.reset() ;
2300      axis_ref.reset() ;
2301      scalar_ref.reset() ;
2302      grid_ref.reset();
2303    }
2304    else SuperClass::sendAllAttributesToServer(client) ;
2305  }
2306  CATCH_DUMP_ATTR
2307   
2308  void CField::sendAddVariable(const string& id, CContextClient* client)
2309  TRY
2310  {
2311    sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE, client);
2312  }
2313  CATCH_DUMP_ATTR
2314
2315  void CField::sendAddVariableGroup(const string& id, CContextClient* client)
2316  TRY
2317  {
2318    sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE_GROUP, client);
2319  }
2320  CATCH_DUMP_ATTR
2321
2322  void CField::recvAddVariable(CEventServer& event)
2323  TRY
2324  {
2325    CBufferIn* buffer = event.subEvents.begin()->buffer;
2326    string id;
2327    *buffer >> id;
2328    get(id)->recvAddVariable(*buffer);
2329  }
2330  CATCH
2331
2332  void CField::recvAddVariable(CBufferIn& buffer)
2333  TRY
2334  {
2335    string id;
2336    buffer >> id;
2337    addVariable(id);
2338  }
2339  CATCH_DUMP_ATTR
2340
2341  void CField::recvAddVariableGroup(CEventServer& event)
2342  TRY
2343  {
2344    CBufferIn* buffer = event.subEvents.begin()->buffer;
2345    string id;
2346    *buffer >> id;
2347    get(id)->recvAddVariableGroup(*buffer);
2348  }
2349  CATCH
2350
2351  void CField::recvAddVariableGroup(CBufferIn& buffer)
2352  TRY
2353  {
2354    string id;
2355    buffer >> id;
2356    addVariableGroup(id);
2357  }
2358  CATCH_DUMP_ATTR
2359
2360  /*!
2361   * Check on freq_off and freq_op attributes.
2362   */
2363  void CField::checkTimeAttributes(CDuration* freqOp)
2364  TRY
2365  {
2366    if (hasFileIn() && !(operation.getValue() == "instant" || operation.getValue() == "once") )     
2367      ERROR("void CField::checkTimeAttributes(void)",
2368         << "Unsupported operation for field '" << getFieldOutputName() << "'." << std::endl
2369         << "Currently only \"instant\" is supported for fields read from file.")
2370
2371    if (freq_op.isEmpty())
2372    {
2373      if (operation.getValue() == "instant")
2374      {
2375        if (hasFileIn() || hasFileOut()) freq_op.setValue(getRelFile()->output_freq.getValue());
2376        else freq_op=*freqOp ;
2377      }
2378      else freq_op.setValue(TimeStep);
2379    }
2380    if (freq_offset.isEmpty()) freq_offset.setValue(hasFileIn() ? NoneDu : (freq_op.getValue() - TimeStep));
2381  }
2382  CATCH_DUMP_ATTR
2383
2384  /*!
2385   * Returns string arithmetic expression associated to the field.
2386   * \return if content is defined return content string, otherwise, if "expr" attribute is defined, return expr string.
2387   */
2388  const string& CField::getExpression(void)
2389  TRY
2390  {
2391    if (!expr.isEmpty() && content.empty())
2392    {
2393      content = expr;
2394      expr.reset();
2395    }
2396
2397    return content;
2398  }
2399  CATCH_DUMP_ATTR
2400
2401  bool CField::hasExpression(void) const
2402  TRY
2403  {
2404    return (!expr.isEmpty() || !content.empty());
2405  }
2406  CATCH
2407
2408  bool CField::hasGridMask(void) const
2409  TRY
2410  {
2411    return (this->grid_->hasMask());
2412  }
2413  CATCH
2414
2415  DEFINE_REF_FUNC(Field,field)
2416} // namespace xios
Note: See TracBrowser for help on using the repository browser.