Changeset 1872 for XIOS


Ignore:
Timestamp:
04/23/20 16:04:22 (4 years ago)
Author:
ymipsl
Message:

Some update on XIOS_COUPLING branch...

YM

Location:
XIOS/dev/dev_ym/XIOS_COUPLING/src
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • XIOS/dev/dev_ym/XIOS_COUPLING/src/io/nc4_data_output.cpp

    r1871 r1872  
    21442144 
    21452145            if (field->compression_level.isEmpty()) 
    2146               field->compression_level = field->file->compression_level.isEmpty() ? 0 : field->file->compression_level; 
     2146              field->compression_level = field->getRelFile()->compression_level.isEmpty() ? 0 : field->getRelFile()->compression_level; 
    21472147            SuperClassWriter::setCompressionLevel(fieldid, field->compression_level); 
    21482148 
     
    23742374        if (!field->wasWritten()) 
    23752375        { 
    2376           if (appendMode && field->file->record_offset.isEmpty() &&  
     2376          if (appendMode && field->getRelFile()->record_offset.isEmpty() &&  
    23772377              field->getOperationTimeType() != func::CFunctor::once) 
    23782378          { 
    23792379            double factorUnit; 
    2380             if (!field->file->time_units.isEmpty() && field->file->time_units==CFile::time_units_attr::days) 
     2380            if (!field->getRelFile()->time_units.isEmpty() && field->getRelFile()->time_units==CFile::time_units_attr::days) 
    23812381            factorUnit=context->getCalendar()->getDayLengthInSeconds() ; 
    23822382            else factorUnit=1 ; 
     
    24392439          } 
    24402440 
    2441           if (!field->file->time_units.isEmpty() && field->file->time_units==CFile::time_units_attr::days) 
     2441          if (!field->getRelFile()->time_units.isEmpty() && field->getRelFile()->time_units==CFile::time_units_attr::days) 
    24422442          { 
    24432443            double secByDay=context->getCalendar()->getDayLengthInSeconds() ; 
     
    26782678 
    26792679         StdString strTimeUnits ; 
    2680          if (!field->file->time_units.isEmpty() && field->file->time_units==CFile::time_units_attr::days) strTimeUnits="days since " ; 
     2680         if (!field->getRelFile()->time_units.isEmpty() && field->getRelFile()->time_units==CFile::time_units_attr::days) strTimeUnits="days since " ; 
    26812681         else  strTimeUnits="seconds since " ; 
    26822682  
     
    26852685 
    26862686 
    2687          if (field->file->time_counter.isEmpty()) 
     2687         if (field->getRelFile()->time_counter.isEmpty()) 
    26882688         { 
    26892689           if (timeCounterType==none) createTimeCounterAxis=true ; 
     
    26992699           } 
    27002700         } 
    2701          else if (field->file->time_counter==CFile::time_counter_attr::instant) 
     2701         else if (field->getRelFile()->time_counter==CFile::time_counter_attr::instant) 
    27022702         { 
    27032703           if (field->hasTimeCentered) 
     
    27122712           } 
    27132713         } 
    2714          else if (field->file->time_counter==CFile::time_counter_attr::centered) 
     2714         else if (field->getRelFile()->time_counter==CFile::time_counter_attr::centered) 
    27152715         { 
    27162716           if (field->hasTimeCentered) 
     
    27252725           } 
    27262726         } 
    2727          else if (field->file->time_counter==CFile::time_counter_attr::instant_exclusive) 
     2727         else if (field->getRelFile()->time_counter==CFile::time_counter_attr::instant_exclusive) 
    27282728         { 
    27292729           if (field->hasTimeCentered) 
     
    27372737           } 
    27382738         } 
    2739          else if (field->file->time_counter==CFile::time_counter_attr::centered_exclusive) 
     2739         else if (field->getRelFile()->time_counter==CFile::time_counter_attr::centered_exclusive) 
    27402740         { 
    27412741           if (field->hasTimeCentered) 
     
    27492749           } 
    27502750         } 
    2751          else if (field->file->time_counter==CFile::time_counter_attr::exclusive) 
     2751         else if (field->getRelFile()->time_counter==CFile::time_counter_attr::exclusive) 
    27522752         { 
    27532753           if (field->hasTimeCentered) 
     
    27702770           } 
    27712771         } 
    2772          else if (field->file->time_counter==CFile::time_counter_attr::none) 
     2772         else if (field->getRelFile()->time_counter==CFile::time_counter_attr::none) 
    27732773         { 
    27742774           if (field->hasTimeCentered) 
     
    27812781           } 
    27822782         } 
    2783          else if (field->file->time_counter==CFile::time_counter_attr::record) 
     2783         else if (field->getRelFile()->time_counter==CFile::time_counter_attr::record) 
    27842784         { 
    27852785           if (timeCounterType==none) createTimeCounterAxis=true ; 
     
    28522852                SuperClassWriter::addAttribute("axis", string("T"), &axisId); 
    28532853 
    2854                 if (field->file->time_counter.isEmpty() ||  
    2855                    (field->file->time_counter != CFile::time_counter_attr::record)) 
     2854                if (field->getRelFile()->time_counter.isEmpty() ||  
     2855                   (field->getRelFile()->time_counter != CFile::time_counter_attr::record)) 
    28562856                { 
    28572857                  CDate timeOrigin = cal->getTimeOrigin(); 
     
    28652865 
    28662866             // Adding time_counter_bound dimension 
    2867              if (field->file->time_counter.isEmpty() || (field->file->time_counter != CFile::time_counter_attr::record)) 
     2867             if (field->getRelFile()->time_counter.isEmpty() || (field->getRelFile()->time_counter != CFile::time_counter_attr::record)) 
    28682868             { 
    28692869                if (!SuperClassWriter::varExist(axisBoundId)) 
  • XIOS/dev/dev_ym/XIOS_COUPLING/src/node/axis.cpp

    r1871 r1872  
    383383        for (int i = 0; i < data_n; ++i) 
    384384        { 
    385           if ((i+data_begin) >= 0 && (i+data_begin<n) && !mask(i+data_begin)) 
    386             data_index(i) = -1; 
     385           if (data_index(i) >= 0 && data_index(i)<n) 
     386             if (!mask(data_index(i))) data_index(i) = -1; 
    387387        } 
    388388      } 
     
    12811281    nonCompressedData = -1;    
    12821282    // Mask is incorporated into data_index and is not sent/received anymore 
    1283     mask.resize(0); 
     1283    mask.reset(); 
    12841284    if (hasValue) 
    12851285      value.resize(nbData); 
  • XIOS/dev/dev_ym/XIOS_COUPLING/src/node/context.cpp

    r1871 r1872  
    10911091    else if (serviceType_==CServicesManager::CLIENT) for(auto file : this->enabledWriteModeFiles) file->setContextClient(client) ; 
    10921092 
     1093    if (serviceType_==CServicesManager::CLIENT) for(auto file : this->enabledReadModeFiles) file->setContextClient(client) ; 
    10931094    
    10941095    // workflow endpoint => sent to IO/SERVER 
     
    11521153        field->connectToServerInput(garbageCollector) ; // connect the field to server filter 
    11531154        field->computeGridIndexToFileServer() ; // compute grid index for transfer to the server context 
    1154         field->sendFieldToFileServer() ; 
     1155        field->sendFieldToInputFileServer() ; 
    11551156      } 
    11561157    } 
     
    11641165 
    11651166    if (serviceType_==CServicesManager::CLIENT || serviceType_==CServicesManager::GATHERER) this->sendCloseDefinition(); 
    1166     if (serviceType_==CServicesManager::IO_SERVER || serviceType_==CServicesManager::OUT_SERVER)  createFileHeader(); 
     1167    if (serviceType_==CServicesManager::IO_SERVER || serviceType_==CServicesManager::OUT_SERVER)   
     1168    { 
     1169      createFileHeader(); 
     1170    } 
     1171 
    11671172    if (serviceType_==CServicesManager::CLIENT) startPrefetchingOfEnabledReadModeFiles(); 
    11681173    
     
    26272632      vector<CFile*>::const_iterator it; 
    26282633 
    2629       for (it=enabledFiles.begin(); it != enabledFiles.end(); it++) 
    2630       // for (it=enabledWriteModeFiles.begin(); it != enabledWriteModeFiles.end(); it++) 
     2634      //for (it=enabledFiles.begin(); it != enabledFiles.end(); it++) 
     2635      for (it=enabledWriteModeFiles.begin(); it != enabledWriteModeFiles.end(); it++) 
    26312636      { 
    26322637         (*it)->initWrite(); 
  • XIOS/dev/dev_ym/XIOS_COUPLING/src/node/field.cpp

    r1871 r1872  
    3333  CField::CField(void) 
    3434    : CObjectTemplate<CField>(), CFieldAttributes() 
    35     , file() 
    3635    , written(false) 
    3736    , nstep(0), nstepMax(0) 
     
    5150  CField::CField(const StdString& id) 
    5251    : CObjectTemplate<CField>(id), CFieldAttributes() 
    53     , file() 
    5452    , written(false) 
    5553    , nstep(0), nstepMax(0) 
     
    360358      info(20) << "currentDate : " << currentDate << endl ; 
    361359      info(20) << "lastDataRequestedFromServer : " << lastDataRequestedFromServer << endl ; 
    362       info(20) << "file->output_freq.getValue() : " << file->output_freq.getValue() << endl ; 
    363       info(20) << "lastDataRequestedFromServer + file->output_freq.getValue() : " << lastDataRequestedFromServer + file->output_freq << endl ; 
    364  
    365       dataRequested |= sendReadDataRequest(lastDataRequestedFromServer + file->output_freq, file->getContextClient()); 
     360      info(20) << "fileIn_->output_freq.getValue() : " << fileIn_->output_freq.getValue() << endl ; 
     361      info(20) << "lastDataRequestedFromServer + fileIn_->output_freq.getValue() : " << lastDataRequestedFromServer + fileIn_->output_freq << endl ; 
     362 
     363      dataRequested |= sendReadDataRequest(lastDataRequestedFromServer + fileIn_->output_freq, fileIn_->getContextClient()); 
    366364    } 
    367365 
     
    580578 
    581579    if (wasDataAlreadyReceivedFromServer) 
    582       lastDataReceivedFromServer = lastDataReceivedFromServer + file->output_freq; 
     580      lastDataReceivedFromServer = lastDataReceivedFromServer + fileIn_->output_freq; 
    583581    else 
    584582    { 
     
    616614      do 
    617615      { 
    618         const CDate nextDataDue = wasDataAlreadyReceivedFromServer ? (lastDataReceivedFromServer + file->output_freq) : context->getCalendar()->getInitDate(); 
     616        const CDate nextDataDue = wasDataAlreadyReceivedFromServer ? (lastDataReceivedFromServer + fileIn_->output_freq) : context->getCalendar()->getInitDate(); 
    619617        isDataLate = (nextDataDue <= currentDate); 
    620618 
     
    659657 
    660658  //---------------------------------------------------------------- 
    661  
     659/* 
    662660  void CField::setRelFile(CFile* _file) 
    663661  TRY 
     
    667665  } 
    668666  CATCH_DUMP_ATTR 
    669  
     667*/ 
    670668  //---------------------------------------------------------------- 
    671669 
     
    688686  TRY 
    689687  { 
    690     return this->file; 
     688    if (hasFileIn()) return this->fileIn_; 
     689    else if (hasFileOut()) return this->fileOut_ ; 
     690    else return nullptr ; 
    691691  } 
    692692  CATCH 
     
    10981098    CContext* context = CContext::getCurrent(); 
    10991099 
    1100     if (context->getServiceType()==CServicesManager::CLIENT || !hasOutputFile) return; 
    1101  
    11021100    if (freq_op.isEmpty()) freq_op.setValue(TimeStep); 
    11031101 
    11041102    if (freq_offset.isEmpty()) freq_offset.setValue(NoneDu); 
    11051103 
    1106     freq_operation_srv = file->output_freq.getValue(); 
    1107     freq_write_srv     = file->output_freq.getValue(); 
     1104    freq_operation_srv = fileOut_->output_freq.getValue(); 
     1105    freq_write_srv     = fileOut_->output_freq.getValue(); 
    11081106 
    11091107    lastlast_Write_srv = context->getCalendar()->getInitDate(); 
     
    12341232    } 
    12351233     
     1234    if (hasFileOut()) 
     1235    { 
     1236      if (fileOut_->isServerSide()) 
     1237      { 
     1238        this->solveServerOperation() ; 
     1239      } 
     1240    } 
     1241 
    12361242    buildWorkflowGraphDone_ = true ; 
    12371243    workflowEnabled_ = true ; 
     
    12881294  void CField::connectToServerInput(CGarbageCollector& gc) 
    12891295  { 
    1290     serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc,  grid_, false, false)); 
     1296    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true); 
     1297    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0); 
     1298 
     1299    checkTimeAttributes(); 
     1300    serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, false, freq_offset, true, 
     1301                                                                          detectMissingValues, defaultValue)); 
     1302    //serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc,  grid_, false, false)); 
    12911303    serverSourceFilter -> connectOutput(inputFilter,0) ; 
    12921304  }  
     
    13821394      if (enableOutput && !storeFilter && !fileWriterFilter) 
    13831395      { 
    1384         if (file && (file->mode.isEmpty() || file->mode == CFile::mode_attr::write)) 
     1396        if (getRelFile() && (getRelFile()->mode.isEmpty() || getRelFile()->mode == CFile::mode_attr::write)) 
    13851397        { 
    13861398          fileServerWriterFilter = std::shared_ptr<CFileServerWriterFilter>(new CFileServerWriterFilter(gc, this)); 
     
    13971409      if (enableOutput && !storeFilter && !fileWriterFilter) 
    13981410      { 
    1399         if (file && (file->mode.isEmpty() || file->mode == CFile::mode_attr::write)) 
     1411        if (getRelFile() && (getRelFile()->mode.isEmpty() || getRelFile()->mode == CFile::mode_attr::write)) 
    14001412        { 
    1401           fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, file->getContextClient())); 
     1413          fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, getRelFile()->getContextClient())); 
    14021414          instantDataFilter->connectOutput(fileWriterFilter, 0); 
    14031415        } 
     
    14331445        else if (!field_ref.isEmpty()) instantDataFilter = getFieldReference(gc); 
    14341446        // Check if the data is to be read from a file 
    1435         else if (file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read) 
     1447        else if (getRelFile() && !getRelFile()->mode.isEmpty() && getRelFile()->mode == CFile::mode_attr::read) 
    14361448        { 
    14371449          checkTimeAttributes(); 
     
    14571469        } 
    14581470 
    1459         if (file && (file->mode.isEmpty() || file->mode == CFile::mode_attr::write)) 
     1471        if (getRelFile() && (getRelFile()->mode.isEmpty() || getRelFile()->mode == CFile::mode_attr::write)) 
    14601472        { 
    1461           fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, file->getContextClient())); 
    1462           getTemporalDataFilter(gc, file->output_freq)->connectOutput(fileWriterFilter, 0); 
     1473          fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, getRelFile()->getContextClient())); 
     1474          getTemporalDataFilter(gc, getRelFile()->output_freq)->connectOutput(fileWriterFilter, 0); 
    14631475        } 
    14641476      } 
     
    20012013    if (context->getServiceType()==CServicesManager::GATHERER) 
    20022014    { 
    2003       if (file->mode.isEmpty() || (!file->mode.isEmpty() && file->mode == CFile::mode_attr::write)) 
     2015      if (getRelFile()->mode.isEmpty() || (!getRelFile()->mode.isEmpty() && getRelFile()->mode == CFile::mode_attr::write)) 
    20042016        grid_->setContextClient(contextClient); 
    20052017    } 
     
    20202032  { 
    20212033    CContext::getCurrent()->sendContextToFileServer(client); 
    2022     fileOut_->sendFileToFileServer(client); 
     2034    getRelFile()->sendFileToFileServer(client); 
    20232035    grid_->sendGridToFileServer(client); 
     2036    this->sendAllAttributesToServer(client); 
     2037    this->sendAddAllVariables(client); 
     2038  } 
     2039   
     2040  void CField::sendFieldToInputFileServer(void) 
     2041  { 
     2042    CContext::getCurrent()->sendContextToFileServer(client); 
     2043    getRelFile()->sendFileToFileServer(client); 
     2044    grid_->sendGridToFileServer(client); 
     2045    read_access=true ; // not the best solution, but on server side, the field must be a starting point of the workflow 
     2046                       // must be replace by a better solution when implementing filters for reading and send to client 
     2047                       // on server side 
    20242048    this->sendAllAttributesToServer(client); 
    20252049    this->sendAddAllVariables(client); 
     
    21212145  TRY 
    21222146  { 
    2123     bool isFieldRead  = file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read; 
    2124     bool isFieldWrite = file && ( file->mode.isEmpty() ||  file->mode == CFile::mode_attr::write); 
     2147    bool isFieldRead  = getRelFile() && !getRelFile()->mode.isEmpty() && getRelFile()->mode == CFile::mode_attr::read; 
     2148    bool isFieldWrite = getRelFile() && ( getRelFile()->mode.isEmpty() ||  getRelFile()->mode == CFile::mode_attr::write); 
    21252149    if (isFieldRead && !(operation.getValue() == "instant" || operation.getValue() == "once") )      
    21262150      ERROR("void CField::checkTimeAttributes(void)", 
     
    21322156      if (operation.getValue() == "instant") 
    21332157      { 
    2134         if (isFieldRead || isFieldWrite) freq_op.setValue(file->output_freq.getValue()); 
     2158        if (isFieldRead || isFieldWrite) freq_op.setValue(getRelFile()->output_freq.getValue()); 
    21352159        else freq_op=*freqOp ; 
    21362160      } 
  • XIOS/dev/dev_ym/XIOS_COUPLING/src/node/field.hpp

    r1871 r1872  
    144144 
    145145         void sendFieldToFileServer(void) ; 
     146         void sendFieldToInputFileServer(void) ; 
    146147 
    147148         /// Vérifications /// 
     
    248249 
    249250         CGrid*  grid_=nullptr; 
    250          CFile*  file; 
     251//         CFile*  file; 
    251252          
    252253         CFile* fileIn_ = nullptr ; //<! pointer to input related file 
    253          bool hasFileIn(void) { return fileIn_==nullptr ? false : true ;} 
     254         bool hasFileIn(void) const { return fileIn_==nullptr ? false : true ;}  
    254255         CFile* getFileIn(void) {return fileIn_;} 
    255256         void setFileIn(CFile* fileIn) { fileIn_ = fileIn ;} 
     
    257258 
    258259         CFile* fileOut_ = nullptr ; //<! pointer to output related file 
    259          bool hasFileOut(void) { return fileOut_==nullptr ? false : true ;} 
     260         bool hasFileOut(void) const { return fileOut_==nullptr ? false : true ;}  
    260261         CFile* getFileOut(void) {return fileOut_;} 
    261262         void setFileOut(CFile* fileOut) { fileOut_ = fileOut ;} 
     
    263264 
    264265         CCouplerIn* couplerIn_ = nullptr ; //<!pointer to input related coupler 
    265          bool hasCouplerIn(void) { return couplerIn_==nullptr ? false : true ;} 
     266         bool hasCouplerIn(void) const { return couplerIn_==nullptr ? false : true ;} 
    266267         CCouplerIn* getCouplerIn(void) {return couplerIn_;} 
    267268         void setCouplerIn(CCouplerIn* couplerIn) { couplerIn_ = couplerIn ;} 
     
    269270 
    270271         CCouplerOut* couplerOut_ = nullptr ; //<!pointer to output related coupler 
    271          bool hasCouplerOut(void) { return couplerOut_==nullptr ? false : true ;} 
     272         bool hasCouplerOut(void) const { return couplerOut_==nullptr ? false : true ;} 
    272273         CCouplerOut* getCouplerOut(void) {return couplerOut_;} 
    273274         void setCouplerOut(CCouplerOut* couplerOut) { couplerOut_ = couplerOut ;} 
  • XIOS/dev/dev_ym/XIOS_COUPLING/src/node/file.cpp

    r1871 r1872  
    276276    TRY 
    277277    { 
     278      CContext* context = CContext::getCurrent(); 
    278279      if (checkRead) return; 
    279       createSubComFile(); 
     280      //createSubComFile(); 
     281      allZoneEmpty = false;  
     282      MPI_Comm_dup(context->intraComm_, &fileComm) ; 
    280283      checkRead = true; 
    281284    } 
  • XIOS/dev/dev_ym/XIOS_COUPLING/src/node/grid.hpp

    r1871 r1872  
    253253       bool computeServerDistribution_done_=false ; 
    254254     public:  
    255        CDistributionServer* getServerDistribution(void) { if (computeServerDistribution_done_) computeServerDistribution() ; return serverDistribution_ ;} 
     255       CDistributionServer* getServerDistribution(void) { if (!computeServerDistribution_done_) computeServerDistribution() ; return serverDistribution_ ;} 
    256256 
    257257 
Note: See TracChangeset for help on using the changeset viewer.