source: tags/libIGCM_v2.0_rc3/AA_create_ts @ 998

Last change on this file since 998 was 785, checked in by labetoulle, 11 years ago

Cosmetics :

  • Convert tabs into space for better lisibility with different editors.
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 30.6 KB
Line 
1#-Q- curie ######################
2#-Q- curie ## CURIE   TGCC/CEA ##
3#-Q- curie ######################
4#-Q- curie #MSUB -r TS             # Nom du job               
5#-Q- curie #MSUB -eo
6#-Q- curie #MSUB -n 1              # Reservation du processus
7#-Q- curie #MSUB -T 86400          # Limite de temps elapsed du job
8#-Q- curie #MSUB -q xlarge
9#-Q- curie #MSUB -Q normal
10#-Q- curie set +x
11#-Q- ada #!/bin/ksh
12#-Q- ada #######################
13#-Q- ada ##   ADA   IDRIS  ##
14#-Q- ada #######################
15#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
16#-Q- ada # @ wall_clock_limit = 20:00:00
17#-Q- ada # Nom du travail LoadLeveler
18#-Q- ada # @ job_name   = TS
19#-Q- ada # Fichier de sortie standard du travail
20#-Q- ada # @ output     = $(job_name).$(jobid)
21#-Q- ada # Fichier de sortie d'erreur du travail
22#-Q- ada # @ error      =  $(job_name).$(jobid)
23#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
24#-Q- ada # @ notification = error
25#-Q- ada # @ environment  = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
26#-Q- ada # @ queue
27#-Q- sx9mercure #!/bin/ksh
28#-Q- sx9mercure ######################
29#-Q- sx9mercure ## SX9MERCURE  CCRT ##
30#-Q- sx9mercure ######################
31#-Q- sx9mercure #PBS -N TS                   # Nom du job
32#-Q- sx9mercure #PBS -j o                    # regroupement des stdout et stderr
33#-Q- sx9mercure #PBS -S /usr/bin/ksh         # shell de soumission
34#-Q- sx9mercure #PBS -l memsz_job=1gb        # Limite memoire a 1 Go
35#-Q- sx9mercure #PBS -l elapstim_req=24:00:00   # Limite temps a 2 heures
36#-Q- sx9mercure #PBS -q scalaire
37#-Q- sx9mercure #PBS -r n
38#-Q- titane #!/bin/ksh
39#-Q- titane ######################
40#-Q- titane ## TITANE   CEA ##
41#-Q- titane ######################
42#-Q- titane #MSUB -r TS             # Nom du job               
43#-Q- titane #MSUB -eo
44#-Q- titane #MSUB -n 1              # Reservation du processus
45#-Q- titane #MSUB -T 86400          # Limite de temps elapsed du job
46#-Q- titane #MSUB -q mono
47#-Q- titane #MSUB -E '-rn'
48#-Q- lxiv8 ######################
49#-Q- lxiv8 ## OBELIX      LSCE ##
50#-Q- lxiv8 ######################
51#-Q- lxiv8 #PBS -N TS
52#-Q- lxiv8 #PBS -m a
53#-Q- lxiv8 #PBS -j oe
54#-Q- lxiv8 #PBS -q medium
55#-Q- lxiv8 #PBS -o TS.$$
56#-Q- lxiv8 #PBS -S /bin/ksh
57#-Q- default #!/bin/ksh
58#-Q- default ##################
59#-Q- default ## DEFAULT HOST ##
60#-Q- default ##################
61
62#**************************************************************
63# Author: Sebastien Denvil
64# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
65# $Revision::                                          $ Revision of last commit
66# $Author::                                            $ Author of last commit
67# $Date::                                              $ Date of last commit
68# IPSL (2006)
69#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
70#
71#**************************************************************
72
73#set -eu
74#set -vx
75
76date
77
78#-Q- ada export OMP_NUM_THREADS=1
79
80#D- Task type (computing or post-processing)
81TaskType=post-processing
82
83########################################################################
84
85#D- Flag to determine if this job in a standalone mode
86#D- Default : value from AA_job if any
87StandAlone=${StandAlone:=true}
88
89#D- Path to libIGCM
90#D- Default : value from AA_job if any
91libIGCM=${libIGCM:=::modipsl::/libIGCM}
92
93#D- Flag to determine allready produced time series. Empty if you start from the beginning
94#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
95CompletedFlag=${CompletedFlag:=}
96
97#D- Flag to determine task type this script will perform.
98#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
99#D- Default : Value from AA_job if any. Usefull in StandAlone case.
100TsTask=${TsTask:=2D}
101if [ X${TsTask} = Xempty ] ; then
102  TsTask=""
103fi
104
105#D- Flag to determine if rebuild process has been performed asynchronously
106#D- Possible value true or false.
107#D- If true raw files has allready been patched by rebuild job
108#D- If false create_ts will patch the raw files
109#D- Default : Value from AA_job if any. Usefull in StandAlone case.
110RebuildFrequency=${RebuildFrequency:=true}
111
112#D- Flag to determine job's output directory
113#D- Default : value from libIGCM_post.ksh if any
114POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
115
116#D- Increased verbosity (1, 2, 3)
117#D- Default : value from AA_job if any
118Verbosity=${Verbosity:=3}
119
120#D- Low level debug : to bypass lib test checks and stack construction
121#D- Default : value from AA_job if any
122DEBUG_debug=${DEBUG_debug:=false}
123
124#D- TEMPORARY Flag to determine atmospheric resolution
125#D- Default : value from atmospheric driver if any
126RESOL_ATM=ALL
127
128#D- Flag to determine surface resolution
129#D- Default : value from surface driver if any
130RESOL_SRF=ALL
131
132#D- Flag to determine surface resolution
133#D- Default : value from surface driver if any
134RESOL_SBG=ALL
135
136#D- TEMPORARY Flag to determine ocean resolution
137#D- Default : value from ocean driver if any
138RESOL_OCE=${RESOL_OCE:=ORCA2}
139
140#D- TEMPORARY Flag to determine ice resolution
141#D- Default : value from ice driver if any
142RESOL_ICE=${RESOL_ICE:=ORCA2}
143
144#D- TEMPORARY Flag to determine marine biogeochemistry resolution
145#D- Default : value from ice driver if any
146RESOL_MBG=${RESOL_MBG:=ORCA2}
147
148########################################################################
149
150. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
151      ( ${DEBUG_debug} ) && IGCM_debug_Check
152. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
153     ( ${DEBUG_debug} ) && IGCM_card_Check
154. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
155     ( ${DEBUG_debug} ) && IGCM_date_Check
156#-------
157. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
158. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
159
160########################################################################
161
162#set -vx
163
164# Useful cleaning function
165MENAGE () 
166{
167  [ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
168}
169
170########################################################################
171
172IGCM_sys_MkdirWork ${RUN_DIR_PATH}
173IGCM_sys_Cd ${RUN_DIR_PATH}
174
175# ------------------------------------------------------------------
176# Test if all was right before proceeding further
177# ------------------------------------------------------------------
178IGCM_debug_Verif_Exit_Post
179
180if [ ${StandAlone} = true ] ; then
181  CARD_DIR=${SUBMIT_DIR}
182else
183  CARD_DIR=${RUN_DIR_PATH}
184  IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
185  IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card    ${RUN_DIR_PATH}
186  IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP        ${RUN_DIR_PATH}
187  IGCM_sys_Get_Master ${SUBMIT_DIR}/POST        ${RUN_DIR_PATH}
188fi
189
190#==================================
191# First of all
192#
193# Read libIGCM compatibility version in config.card
194# Read UserChoices section
195# Read Ensemble section
196# Read Post section
197# Define all netcdf output directories
198#==================================
199IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
200
201#==================================
202# Define default value to keep compatibility with previous card: means before changes due to TGCC
203if [ X${PackDefault} = Xtrue ] ; then
204  [ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
205else
206  config_Post_PackFrequency=NONE
207fi
208
209#==================================
210# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
211if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
212  config_UserChoices_PeriodLength=${config_Post_PackFrequency}
213fi
214
215#
216# Determine component to take care of depending on kind of task create_ts will perform
217#
218[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} || IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
219#
220# Determine period for time series. Default : value from AA_job or from launch_TS if any
221#
222[ X${CompletedFlag} = X ] || CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
223
224DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
225#
226PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
227#
228# Determine Dimension of the produced time series : 2D or 3D
229if (   [ X${TsTask} = XChunck2D ] || [ X${TsTask} = X2D ] ) ; then
230  Dimension=2D
231elif ( [ X${TsTask} = XChunck3D ] || [ X${TsTask} = X3D ] ) ; then
232  Dimension=3D
233else
234  Dimension=""
235fi
236
237# ------------------------------------------------------------------
238#D- Test if all was right before entering the loop
239# ------------------------------------------------------------------
240IGCM_debug_Verif_Exit_Post
241
242########################################################################
243#
244#      Depending on requested time series fill following variables :
245#
246#                           - LISTE_FILE_${comp}[*]
247#                           - LISTE_VARS_${file}[*]
248#                           - LISTE_AXIS_${file}[*]
249#                           - LISTE_PATCH_${file}[*]
250########################################################################
251
252#
253# For each selected component determine which files need post-processing
254#
255DoJob=false
256for comp in ${config_ListOfComponents[*]} ; do
257  #
258  ActiveComp=false
259  #
260  IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
261  eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
262  #
263  card=${CARD_DIR}/COMP/${compname}.card
264
265  IGCM_card_DefineArrayFromOption ${card} OutputFiles List
266  #
267  ListFilesName=${compname}_OutputFiles_List
268  eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
269  #
270  if [ X${FileName0} != X${NULL_STR} ] ; then
271    #
272    #IGCM_debug_Print 1 "Component      : ${compname}"
273    #
274    # INITIALISATION
275    #
276    typeset i
277    if [ ${FlagToRead} ] ; then
278      ((NbFiles=${FlagToRead}))
279      ((i=${NbFiles}))
280    else
281      eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
282      i=2
283    fi
284    #
285    until [ $i -gt $NbFiles ]; do
286      #
287      eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
288      #
289      if [ X${flag_post} != XNONE ] ; then
290        #
291        # For each comp and each file who needs post-processing
292        # determine which variables need time series.
293        #
294        FILE=$( echo ${flag_post} | awk "-FPost_" '{print $2}' )
295        IGCM_card_DefineArrayFromOption ${card}    Post_${FILE} TimeSeriesVars${Dimension}
296        IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
297        #
298        # If TimeSeriesVars list is empty we skip
299        #
300        if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
301          #
302          IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
303          IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
304          (( i=i+3 ))
305          continue
306          #
307        fi
308        #
309        # If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
310        #
311        if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
312          IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
313          IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
314          (( i=i+3 ))
315          continue
316        fi
317        #
318          # If we are not a Chunck type task, we skip files/variables handled by chunck task
319          #
320        if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
321          if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
322            IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
323            IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
324            (( i=i+3 ))
325            continue
326          fi
327        fi
328        #
329        # Now on we know we have things to do
330        #
331
332        #
333        #  We need coordinates axis to properly fill produced time series
334        #
335        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
336        #
337        if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
338          #
339          IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
340          IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
341          (( i=i+3 ))
342          continue
343          #
344        fi
345        #
346        # We need LIST of variables not allready produced (useful for standalone mode)
347        #
348        FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' )
349        case ${FlagDir} in
350          *Y)  TS_Dir=TS_YE  ;;
351          *M)  TS_Dir=TS_MO  ;;
352          *D)  TS_Dir=TS_DA  ;;
353          HF)  TS_Dir=TS_HF  ;;
354          3H)  TS_Dir=TS_HF  ;;
355          INS) TS_Dir=TS_INS ;;
356        esac
357        #
358        count=0
359        for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
360          #
361          DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
362          TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
363          #
364          IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
365          if [ ! $? = 0 ] ; then
366            eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
367            ActiveComp=true
368            (( count = count + 1 ))
369          fi
370        done
371        #
372        if [ ${count} -gt 0 ] ; then
373          #
374          eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
375          eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
376          #
377          if [ ! X${RebuildFrequency} = Xtrue ] ; then
378            IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
379            eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
380            if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) !=  X${NULL_STR} ]; then
381              for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
382                if [ Xload${Patch} != X ] ; then
383                  . ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
384                  eval load${Patch}=loaded
385                fi
386              done
387            fi
388          fi
389        fi
390      fi
391      (( i=i+3 ))
392    done
393  fi
394  #
395  if [ ${ActiveComp} = true ] ; then
396    set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
397  fi
398done
399
400set -A config_ListOfComponents ${ActiveListOfComponents[*]}
401[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
402
403########################################################################
404#
405#               IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
406#                 IF SO BRING THEM IN THE WORKING DIRECTORY
407#
408# PS : Keep in mind that IGCM_sys_Get here is a weak link :
409#      - especially for IDRiS
410#      - no special protection against "command too long" !
411#      - libIGCM_sys should handle that or modify following code part
412#
413########################################################################
414
415if [ ${StandAlone} != true ] ; then
416  Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
417  ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
418  echo \${run_PostProcessing_TimeSeriesRunning}" )
419
420  #if [ X${Running_Flag} = Xy ] ; then
421  #  # Time Series Job allready running
422  #  IGCM_debug_Print 1 "Time Series Job allready running exit"
423  #  exit
424  #fi
425  # Now run.card know we are running
426
427
428
429
430  IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
431
432
433
434
435
436  # Get information from last execution
437  #CompletedFlag=$( IGCM_sys_RshMaster \
438  # "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
439  #        echo \${run_PostProcessing_TimeSeriesCompleted}" )
440fi
441
442# Is it the first submission or not ?
443
444if ( [ X${CompletedFlag} = X ] || [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
445  # First Time Series Submission
446  FIRST_PASS=TRUE
447  Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
448  DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
449else
450  # Last Completed Time Series
451  FIRST_PASS=FALSE
452  DATE_FIN_JOB_B=${CompletedFlag}
453  date=${DateBegin}_${DATE_FIN_JOB_B}
454  for comp in ${config_ListOfComponents[*]} ; do
455    #
456    IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
457    #
458    IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
459    #
460    #set -A liste_file
461    unset liste_file
462    #
463    file_path=${R_SAVE}/${comp}/Analyse
464    #
465    i=0
466    file=${NULL_STR}
467    #
468    for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
469      #
470      # Determine in which output can we find file
471      # IE : Analyse/TS or Analyse/DA2TS or ...
472      #
473      FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
474      case ${FlagDir} in
475        *Y)  TS_Dir=TS_YE  ;;
476        *M)  TS_Dir=TS_MO  ;;
477        *D)  TS_Dir=TS_DA  ;;
478        HF)  TS_Dir=TS_HF  ;;
479        3H)  TS_Dir=TS_HF  ;;
480        INS) TS_Dir=TS_INS ;;
481      esac
482      for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
483        #
484        #set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
485        liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
486        #
487      done
488    done
489    if [ X${file} != X${NULL_STR} ] ; then
490      IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
491      # Even if some files are missing we tried to continue
492      # Either ListVar in compname.card are asked but not present in original history file
493      # Either a previous TS task failed
494      ExitFlag=false
495      IGCM_debug_Verif_Exit_Post
496    fi
497  done
498  #
499  LEVEL=0
500  #
501fi
502
503########################################################################
504#
505#                          DEFINE LOOP PARAMETERS
506#
507########################################################################
508
509DATE_COUNT=${DATE_FIN_JOB_B}
510DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
511
512( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 || NBRE_FILE_TOT=1
513
514if [ ${DoJob} = true ] ; then
515  while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
516    (( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
517    Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
518    DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
519  done
520else
521  NBRE_FILE_TOT=0
522fi
523
524if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
525  NBRE_TOUR=1
526elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
527  let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT} 
528else
529  let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
530fi
531
532CURRENT_LOOP=1
533# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
534# WE HAVE 3 LEVELS
535#    - LEVEL 1 : JUST ONE LOOP TO COMPLETE
536#    - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
537#    - LEVEL 3 : WE ARE IN THE LAST LOOP
538
539while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] ; do
540
541  if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then 
542    # - LEVEL 1 : JUST ONE LOOP TO COMPLETE
543    NBRE_FILE_LOOP=${NBRE_FILE_TOT}
544
545    if [ ${FIRST_PASS} = TRUE ] ; then
546      DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
547    else
548      Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
549      DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
550    fi
551
552    DATE_FIN=${PeriodDateEnd}
553    DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
554
555    [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
556    [ -n "${LEVEL}" ] && FLAG_B="TRUE" || FLAG_B="FALSE"
557
558    LEVEL=1
559    DEPOT="TRUE"       
560  elif [ ${CURRENT_LOOP} -ne  ${NBRE_TOUR} ] ; then
561    # - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
562    NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
563
564    if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
565      Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
566      DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
567    elif [ -z "${DATE_COURANTE}" ] ; then
568      DATE_COURANTE=${DATE_FIN_JOB_B}
569    else
570      Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
571      DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
572    fi
573
574    (( TotLength = 0 ))
575    COMPTEUR=1
576    DATE_LOOP=${DATE_COURANTE}
577
578    while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
579      #
580      Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
581      DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
582      (( TotLength = TotLength + Length ))
583      (( COMPTEUR = COMPTEUR + 1 ))
584      #
585    done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
586    DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
587
588    [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
589    [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" || [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" || FLAG_B="FALSE"
590    DEPOT="FALSE"
591    LEVEL=2
592  else
593    # - LEVEL 3 : WE ARE IN THE LAST LOOP
594    NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
595    [ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
596
597    Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
598    DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
599    DATE_FIN=${PeriodDateEnd}
600    [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
601
602    LEVEL=3
603    DEPOT="TRUE"
604    FLAG_B="TRUE"
605  fi
606
607  # THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
608  for comp in ${config_ListOfComponents[*]} ; do
609    #
610    IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
611      #
612    IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
613    #
614    eval R_OUT_${comp}=${R_SAVE}/${comp}
615    #
616    i=0
617    #
618    for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
619      #
620      # Determine in which output we can find file
621      # IE : Output/MO or Output/DA or ...
622      #
623      FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
624      case ${FlagDir} in
625        *Y)    TS_Dir=TS_YE  ; FreqDir=YE  ;;
626        *M)    TS_Dir=TS_MO  ; FreqDir=MO  ;;
627        *D)    TS_Dir=TS_DA  ; FreqDir=DA  ;;
628        HF)    TS_Dir=TS_HF  ; FreqDir=HF  ;;
629        3H)    TS_Dir=TS_HF  ; FreqDir=HF  ;;
630        INS)   TS_Dir=TS_INS ; FreqDir=INS ;;
631        *)     IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
632               IGCM_sys_Exit "Job create_ts" ;;
633      esac
634
635      # Initialize array
636      #set -A liste_file
637      #set -A liste_file_tmp
638      unset liste_file
639      unset liste_file_tmp
640      eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
641
642      COMPTEUR=1
643      DATE_LOOP=${DATE_COURANTE}
644
645      while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
646        #
647        Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
648        DATE_TAB=$(  IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
649
650        Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
651        DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
652
653        #set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
654        #set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
655        liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
656        liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
657        (( COMPTEUR = COMPTEUR + 1 ))
658        #
659      done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
660      DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
661
662      # Get selected files
663      IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
664      IGCM_debug_Verif_Exit_Post
665
666      # Apply IGCM_Patch if needed
667      if [ ! X${RebuildFrequency} = Xtrue ] ; then
668        if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) !=  X${NULL_STR} ]; then
669          for file_t in $( eval echo ${liste_file_tmp[*]} ); do         
670            for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
671              IGCM_${Patch} ${file_t}
672              IGCM_debug_Verif_Exit_Post
673            done
674          done
675        fi
676      fi
677
678      # WE CAN CONCATENATE FILES AT THIS POINT
679      [ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
680      Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
681      DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
682      DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
683      DATE_BUILD_END=${DateBegin}_${DATE_FIN}
684
685      liste_coord=" "
686      for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
687        liste_coord=${liste_coord}${axis}","
688      done
689
690      # Time axis must be the last dimension
691      time_axis=${axis}
692
693      # SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
694      for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
695        #
696        if [ "${FLAG_B}" = "TRUE" ] ; then
697          # WE CONCATENATE WITH EXISTING FILES
698          file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
699          file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
700          IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
701          #
702          IGCM_sys_Rm ${file1}
703          if [ ! "${DEPOT}" = "TRUE" ] ; then
704            eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
705            # "${file1} = file_before ?
706            [ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
707          fi
708
709          ExitFlag=false
710        else
711          # OR NOT
712          IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
713        fi
714
715        #
716        if [ "${DEPOT}" = "TRUE" ] ; then
717          # WE PUT FINAL FILE ON FILE SYSTEM
718          [ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
719          file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
720          #
721          eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
722          [ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
723          if [ ! "X${file1}" = "X" ] ; then
724            [ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
725          fi
726          #
727          if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
728            file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
729            file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
730            #
731            # Compute yearly average considering calendar
732            # and set date of the mid year to 01/07 at 00:00
733            IGCM_sys_cdo -settime,00:00 -setday,1 -setmon,7 -settunits,days -divdpy -yearsum -muldpm -selvar,${var} ${file_out} ${file_out_YE}
734            # Extract all ancillary variables except $var, they will be appended after the cdo command
735            # Needed because cdo applies calculs on all variables
736            IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
737            # This will remove the time axis because cdo rewrites severely with only considered useful variables
738            IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
739            if [ $? -eq 0 ] ; then
740              # Now append all ancillary variables if previous cdo operator ok
741              IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
742            fi
743
744            # Put global history attribut
745            IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
746            #
747            eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
748            [ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
749            IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
750          fi
751          #
752          IGCM_sys_Rm ${file_out}
753          ExitFlag=false
754          #
755        fi
756        #
757      done     # for var in ${liste_var} ; do
758      IGCM_sys_Rm ${liste_file_tmp[*]}
759    done         # for file in ${liste_file_brut} do
760  done             # for comp in ${config_ListOfComponents[*]} ; do
761  # GO AHEAD IN DEEPEST LOOP
762  (( CURRENT_LOOP = CURRENT_LOOP + 1 ))
763done                 # while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] do
764# FINISH
765
766# DODS copy
767for comp in ${config_ListOfComponents[*]} ; do
768  for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
769    IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
770    [ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
771  done
772done
773
774export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
775export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG
776
777listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
778export listVarEnv
779
780#D-
781# --------------------------------------------------------------------
782#D- Test if all was right
783# --------------------------------------------------------------------
784IGCM_debug_Verif_Exit_Post
785# --------------------------------------------------------------------
786#D- Submit MONITORING
787# --------------------------------------------------------------------
788if ( [ X${TsTask} = X2D ] || [ X${TsTask} = X3D ] || [ X${TsTask} = X ] ) ; then
789  Script_Post_Output=monitoring.${PeriodDateEnd}
790  IGCM_sys_QsubPost monitoring
791fi
792
793#D-
794# --------------------------------------------------------------------
795#D- Update the run.card
796# --------------------------------------------------------------------
797if [ ${StandAlone} != true ] ; then
798  # Put in run.card end period of time series
799  [ X"${DATE_FIN}" = X ] || IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
800  # We have finish the job !
801  IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
802fi
803
804# Clean RUN_DIR_PATH (necessary for cesium and titane only)
805IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
806
807date
Note: See TracBrowser for help on using the repository browser.