source: trunk/libIGCM/AA_create_ts @ 135

Last change on this file since 135 was 135, checked in by sdipsl, 15 years ago
  • Reflect the fact that monitoring should not depend on resolution for atmosphere and surface.
  • Some cleanup
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Date Author Revision
File size: 27.5 KB
Line 
1#-Q- platine #!/usr/bin/ksh
2#-Q- platine ##################
3#-Q- platine ## PLATINE   CEA ##
4#-Q- platine ##################
5#-Q- platine #BSUB -J TS                     # Nom du job
6#-Q- platine #BSUB -N                        # message a la fin du job
7#-Q- platine #BSUB -n 1                      # reservation des processeurs pour le job
8#-Q- platine #BSUB -W 1:00                   # Limite temps
9#-Q- platine #BSUB -q post              # Passage en queue post
10#-Q- sx8brodie #!/bin/ksh
11#-Q- sx8brodie #######################
12#-Q- sx8brodie ## SX8BRODIE   IDRIS ##
13#-Q- sx8brodie #######################
14#-Q- sx8brodie # Temps Elapsed max. d'une requete hh:mm:ss
15#-Q- sx8brodie # @ wall_clock_limit = 10:00:00
16#-Q- sx8brodie # Nom du travail LoadLeveler
17#-Q- sx8brodie # @ job_name   = TS
18#-Q- sx8brodie # Fichier de sortie standard du travail       
19#-Q- sx8brodie # @ output     = $(job_name).$(jobid)
20#-Q- sx8brodie # Fichier de sortie d'erreur du travail
21#-Q- sx8brodie # @ error      =  $(job_name).$(jobid)
22#-Q- sx8brodie # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
23#-Q- sx8brodie # @ notification = error
24#-Q- sx8brodie # @ environment  = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_SRF ; $MASTER
25#-Q- aix6 #!/bin/ksh
26#-Q- aix6 #######################
27#-Q- aix6 ##   VARGAS   IDRIS  ##
28#-Q- aix6 #######################
29#-Q- aix6 # Temps Elapsed max. d'une requete hh:mm:ss
30#-Q- aix6 # @ wall_clock_limit = 10:00:00
31#-Q- aix6 # Nom du travail LoadLeveler
32#-Q- aix6 # @ job_name   = TS
33#-Q- aix6 # Fichier de sortie standard du travail
34#-Q- aix6 # @ output     = $(job_name).$(jobid)
35#-Q- aix6 # Fichier de sortie d'erreur du travail
36#-Q- aix6 # @ error      =  $(job_name).$(jobid)
37#-Q- aix6 # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
38#-Q- aix6 # @ notification = error
39#-Q- aix6 # @ environment  = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_SRF ; $MASTER
40#-Q- aix6 # @ queue
41#-Q- sx8brodie # @ queue
42#-Q- sx8mercure #!/bin/ksh
43#-Q- sx8mercure ######################
44#-Q- sx8mercure ## SX8MERCURE   CEA ##
45#-Q- sx8mercure ######################
46#-Q- sx8mercure #PBS -N TS                   # Nom du job
47#-Q- sx8mercure #PBS -j o                    # regroupement des stdout et stderr
48#-Q- sx8mercure #PBS -S /usr/bin/ksh         # shell de soumission
49#-Q- sx8mercure #PBS -l memsz_job=1gb        # Limite memoire a 1 Go
50#-Q- sx8mercure #PBS -l cputim_job=2:00:00   # Limite temps a 2 heures
51#-Q- sx8mercure #PBS -q scalaire
52#-Q- default #!/bin/ksh
53#-Q- default ##################
54#-Q- default ## DEFAULT HOST ##
55#-Q- default ##################
56
57# $Date$
58# $Author$
59# $Revision$
60# IPSL (2006)
61#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
62
63#set -eu
64#set -vx
65
66date
67
68#-Q- sx8brodie export OMP_NUM_THREADS=1
69#-Q- aix6 export OMP_NUM_THREADS=1
70
71########################################################################
72
73#D- Flag to determine if this job in a standalone mode
74#D- Default : value from AA_job if any
75StandAlone=${StandAlone:=true}
76
77#D- Flag to determine allready produced time series.
78#D- Default : Commented. Uncomment for StandAlone case if you want to continue a time series
79#CompletedFlag=
80
81#D- Flag to determine job's output directory
82#D- Default : value from libIGCM_post.ksh if any
83POST_DIR=${POST_DIR:=${PBS_O_WORKDIR}}
84
85#D- Increased verbosity (1, 2, 3)
86#D- Default : value from AA_job if any
87Verbosity=${Verbosity:=3}
88
89#D- Low level debug : to bypass lib test checks and stack construction
90#D- Default : value from AA_job if any
91DEBUG_debug=${DEBUG_debug:=false}
92
93#D- Low level debug : to bypass lib test checks and stack construction
94#D- Default : value from AA_job if any
95libIGCM=${libIGCM:=/path/to/your/libIGCM}
96
97#D- TEMPORARY Flag to determine atmospheric resolution
98#D- Default : value from atmospheric driver if any
99RESOL_ATM=${RESOL_ATM:=ALL}
100
101#D- TEMPORARY Flag to determine ocean resolution
102#D- Default : value from ocean driver if any
103RESOL_OCE=${RESOL_OCE:=ORCA2}
104
105#D- TEMPORARY Flag to determine ice resolution
106#D- Default : value from ice driver if any
107RESOL_ICE=${RESOL_ICE:=ORCA2}
108
109#D- Flag to determine surface resolution
110#D- Default : value from surface driver if any
111RESOL_SRF=${RESOL_SRF:=ALL}
112
113########################################################################
114
115. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
116      ( ${DEBUG_debug} ) && IGCM_debug_Check
117. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
118     ( ${DEBUG_debug} ) && IGCM_card_Check
119. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
120     ( ${DEBUG_debug} ) && IGCM_date_Check
121#-------
122. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
123
124########################################################################
125
126#set -vx
127
128# Useful cleaning function
129MENAGE () 
130{
131    [ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
132}
133
134########################################################################
135
136#set -vx
137
138if [ ${DEBUG_debug} = true ] ; then
139    IGCM_sys_MkdirWork ${RUN_DIR_PATH}
140    echo "RUN_DIR_PATH ${RUN_DIR_PATH} ok."
141fi
142IGCM_sys_Cd ${RUN_DIR_PATH}
143
144if [ ${StandAlone} = true ] ; then
145    CARD_DIR=${SUBMIT_DIR}
146else
147    CARD_DIR=${RUN_DIR_PATH}/$( basename ${SUBMIT_DIR} )
148    IGCM_sys_Get_Master ${SUBMIT_DIR} ${RUN_DIR_PATH}
149fi
150
151#
152# First of all
153#
154IGCM_card_DefineArrayFromSection       ${CARD_DIR}/config.card UserChoices
155typeset option
156for option in ${config_UserChoices[*]} ; do
157    IGCM_card_DefineVariableFromOption ${CARD_DIR}/config.card UserChoices ${option}
158done
159#
160echo
161IGCM_debug_Print 1 "DefineArrayFromOption  : config_UserChoices"
162IGCM_debug_PrintVariables 3 config_UserChoices_JobName
163IGCM_debug_PrintVariables 3 config_UserChoices_LongName
164IGCM_debug_PrintVariables 3 config_UserChoices_TagName
165IGCM_debug_PrintVariables 3 config_UserChoices_CalendarType
166IGCM_debug_PrintVariables 3 config_UserChoices_DateBegin
167IGCM_debug_PrintVariables 3 config_UserChoices_DateEnd
168IGCM_debug_PrintVariables 3 config_UserChoices_PeriodLength
169echo
170
171#==================================
172
173R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
174#
175IGCM_card_DefineArrayFromSection       ${CARD_DIR}/config.card ListOfComponents
176#
177# Determine period for time series. Default : value from AA_job if any
178#
179DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
180#
181PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
182
183########################################################################
184#
185#      Depending on requested time series fill following variables :
186#
187#                           - LISTE_FILE_${comp}[*]
188#                           - LISTE_VARS_${file}[*]
189#                           - LISTE_AXIS_${file}[*]
190#                           - LISTE_PATCH_${file}[*]
191########################################################################
192
193#
194# For each selected component determine which files need post-processing
195#
196DoJob=false
197for comp in ${config_ListOfComponents[*]} ; do
198    #
199    ActiveComp=false
200    #
201    IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
202    eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
203    #
204    card=${CARD_DIR}/COMP/${compname}.card
205    IGCM_card_DefineArrayFromOption ${card} OutputFiles List
206    #
207    ListFilesName=${compname}_OutputFiles_List
208    eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
209    #
210    if [ X${FileName0} != X${NULL_STR} ] ; then
211        #
212        #IGCM_debug_Print 1 "Component      : ${compname}"
213        #
214        # INITIALISATION
215        #
216        eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
217        typeset i=0
218        #
219        until [ $i -eq $NbFiles ]; do
220            #
221            eval file_in_=\${${ListFilesName}[$i]} > /dev/null 2>&1
222            eval file_in=${file_in_}
223            let $(( i_ = i+2 ))
224            eval flag_post_=\${${ListFilesName}[$i_]} > /dev/null 2>&1
225            eval flag_post=${flag_post_}
226            #
227            if [ X${flag_post} != XNONE ] ; then
228                #
229                # Initialize ${flag_post}_${compname}_${post_freq}
230                #
231                eval export ${flag_post}_${compname}_TimeSeriesFrequency=true
232            fi
233            let $(( i=i+3 ))
234        done
235    fi
236
237    for element in $( env | grep Post_ | grep ${compname}_TimeSeriesFrequency | grep -i "=true" ) ; do
238        #
239        FILE=$( echo ${element} | awk -F "_${compname}_TimeSeriesFrequency" '{print $1}' | awk "-FPost_" '{print $2}' )
240        #
241        # For each comp and each file who needs post-processing
242        # determine which variables need time series.
243        #
244        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars
245        #
246        if [ $( eval echo \$\{${compname}_Post_${FILE}_TimeSeriesVars} ) = ${NULL_STR} ] ; then
247            #
248            IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
249            continue
250            #
251        fi
252        #
253        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
254        #
255        # So we need coordinates axis to properly fill produced time series
256        #
257        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
258        #
259        if [ $( eval echo \$\{${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
260            #
261            IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
262            IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
263            continue
264            #
265        fi
266        #
267        # CREATE LIST
268        #
269        FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' )
270        case ${FlagDir} in
271            *Y)  TS_Dir=TS_YE  ;;
272            *M)  TS_Dir=TS_MO  ;;
273            *D)  TS_Dir=TS_DA  ;;
274            HF)  TS_Dir=TS_HF  ;;
275            INS) TS_Dir=TS_INS ;;
276        esac
277
278        count=0
279        for var in $( eval echo \$\{${compname}_Post_${FILE}_TimeSeriesVars[*]} ) ; do
280            #
281            DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
282            TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
283            #
284            IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
285            if [ ! $? = 0 ] ; then
286                eval set +A LISTE_VARS_${FILE} \$\{LISTE_VARS_${FILE}[*]} ${var}
287                ActiveComp=true
288                (( count = count + 1 ))
289            fi
290        done
291
292        if [ ${count} -gt 0 ] ; then
293            #
294            eval set +A LISTE_FILE_${comp} \$\{LISTE_FILE_${comp}[*]} ${FILE}
295            eval set +A LISTE_AXIS_${FILE} \$\{${compname}_Post_${FILE}_GatherWithInternal[*]}
296            eval set +A LISTE_PATCHES_${FILE} \$\{${compname}_Post_${FILE}_Patches[*]}
297            if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) !=  X${NULL_STR} ]; then
298                for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
299                    if [ Xload${Patch} != X ] ; then
300                        . ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
301                        eval load${Patch}=loaded
302                    fi
303                done
304            fi
305            #
306        fi
307    done
308
309    if [ ${ActiveComp} = true ] ; then
310        set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
311    fi
312
313done
314
315set -A config_ListOfComponents ${ActiveListOfComponents[*]}
316[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
317
318########################################################################
319#
320#                          DEFINE DATE FUNCTIONS
321#
322########################################################################
323
324# Give the end of the next $1 period from $2 date
325function create_ts_next_date {
326    IGCM_debug_PushStack "create_ts_next_date" $@
327    typeset Length
328    case ${config_UserChoices_PeriodLength} in
329        *Y|*y) 
330            PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} | awk -F '[yY]' '{print $1}' )
331            IGCM_date_GetYearMonth ${1} year month
332            (( Length=0 ))
333            (( i=0 ))
334            until [ $i -ge $PeriodLengthInYears ] ; do
335                (( Length = Length + $( IGCM_date_DaysInYear $(( year + i + 1 )) ) )) 
336                (( i=i+1 ))
337            done
338            ;;
339        *M|*m) 
340            PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} | awk -F '[mM]' '{print $1}' )
341            IGCM_date_GetYearMonth ${1} year month
342            (( year0 = year ))
343            if [ $(( month + 1 )) -lt 13 ] ; then
344                month0=$(( month + 1 ))
345            else
346                month0=$(( month + 1 - 12 ))
347                (( year = year0 + 1 ))
348            fi
349            (( Length=0 ))
350            (( i=0 ))
351            until [ $i -ge $PeriodLengthInMonths ] ; do
352                if [ $(( month0 + i )) -lt 13 ] ; then
353                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i )) ) ))
354                else
355                    (( year = year0 + 1 ))
356                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i - 12 )) ) ))
357                fi
358                (( i=i+1 ))
359            done
360            ;;
361        *D|*d)
362            Length=$( echo ${config_UserChoices_PeriodLength} | sed -e "s/[dD]//" ) ;;
363        *)
364            IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, *M, *D."
365            IGCM_debug_Verif_Exit_Post ;;
366    esac
367    echo ${Length}
368
369    IGCM_debug_PopStack "create_ts_next_date"
370}
371
372# Give the end of the actual $1 period from $2 date
373function create_ts_begin_date {
374    IGCM_debug_PushStack "create_ts_begin_date" $@
375    typeset Length
376    case ${config_UserChoices_PeriodLength} in
377        *Y|*y) 
378            PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} | awk -F '[yY]' '{print $1}' )
379            IGCM_date_GetYearMonth ${1} year month
380            if [ X${2} = Xend ] ; then
381                (( year = year - PeriodLengthInYears + 1))
382            fi
383            (( Length=0 ))
384            (( i=0 ))
385            until [ $i -ge $PeriodLengthInYears ] ; do
386                (( Length = Length + $( IGCM_date_DaysInYear $(( year + i )) ) )) 
387                (( i=i+1 ))
388            done
389            ;;
390        *M|*m) 
391            PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} | awk -F '[mM]' '{print $1}' )
392            IGCM_date_GetYearMonth ${1} year month
393            if [ X${2} = Xend ] ; then
394                (( month = month - PeriodLengthInMonths + 1 ))
395            fi     
396            (( year0 = year ))
397            if [ $month -le 0 ] ; then
398                (( month = month + 12 ))
399                year=$( printf "%04i\n" $(( year - 1 )) )
400            fi
401            month=$( printf "%02i\n" ${month} )
402            (( Length=0 ))
403            (( i=0 ))
404            until [ $i -ge $PeriodLengthInMonths ] ; do
405                if [ $(( month + i )) -lt 13 ] ; then
406                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i )) ) ))
407                else
408                    (( year = year0 + 1 ))
409                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i - 12 )) ) ))
410                fi
411                (( i=i+1 ))
412            done
413            ;;
414        *D|*d)
415            Length=$( echo ${config_UserChoices_PeriodLength} | sed -e "s/[dD]//" ) ;;
416        *)
417            IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, *M, *D."
418            IGCM_debug_Verif_Exit_Post ;;
419    esac
420    echo ${Length}
421
422    IGCM_debug_PopStack "create_ts_begin_date"
423}
424
425########################################################################
426#
427#               IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
428#                 IF SO BRING THEM IN THE WORKING DIRECTORY
429#
430# PS : Keep in mind that IGCM_sys_Get here is a weak link :
431#      - especially for IDRiS
432#      - no special protection against "command too long" !
433#      - libIGCM_sys should handle that or modify following code part
434#
435########################################################################
436
437if [ ${StandAlone} != true ] ; then
438    Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
439       ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
440       echo \${run_PostProcessing_TimeSeriesRunning}" )
441
442    if [ X${Running_Flag} = Xy ] ; then
443        # Time Series Job allready running
444        IGCM_debug_Print 1 "Time Series Job allready running exit"
445        exit
446    fi
447    # Now run.card know we are running
448    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
449    # Get information from last execution
450    CompletedFlag=$( IGCM_sys_RshMaster \
451        "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
452        echo \${run_PostProcessing_TimeSeriesCompleted}" )
453fi
454
455# Is it the first submission or not ?
456if [ X${CompletedFlag} = X ] ; then
457    # First Time Series Submission
458    FIRST_PASS=TRUE
459    Length=$( create_ts_begin_date ${DateBegin} begin)
460    DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
461else
462    # Last Completed Time Series
463    FIRST_PASS=FALSE
464    DATE_FIN_JOB_B=${CompletedFlag}
465    date=${DateBegin}_${DATE_FIN_JOB_B}
466    for comp in ${config_ListOfComponents[*]} ; do
467        #
468        IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
469        #
470        IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
471        #
472        #set -A liste_file
473        unset liste_file
474        #
475        file_path=${R_SAVE}/${comp}/Analyse
476        #
477        i=0
478        file=${NULL_STR}
479        #
480        for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
481            #
482            # Determine in which output can we find file
483            # IE : Analyse/TS or Analyse/DA2TS or ...
484            #
485            FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
486            case ${FlagDir} in
487                *Y)  TS_Dir=TS_YE  ;;
488                *M)  TS_Dir=TS_MO  ;;
489                *D)  TS_Dir=TS_DA  ;;
490                HF)  TS_Dir=TS_HF  ;;
491                INS) TS_Dir=TS_INS ;;
492            esac
493            for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
494                #
495                #set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
496                liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
497                #
498            done
499        done
500        if [ X${file} != X${NULL_STR} ] ; then
501            IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
502            ExitFlag=false
503            IGCM_debug_Verif_Exit_Post
504        fi
505    done
506    #
507    LEVEL=0
508    #
509fi
510
511########################################################################
512#
513#                          DEFINE LOOP PARAMETERS
514#
515########################################################################
516
517DATE_COUNT=${DATE_FIN_JOB_B}
518DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
519
520( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 || NBRE_FILE_TOT=1
521
522if [ ${DoJob} = true ] ; then
523    while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
524        (( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
525        Length=$( create_ts_next_date ${DATE_COUNT} )
526        DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
527    done
528else
529    NBRE_FILE_TOT=0
530fi
531
532if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
533    NBRE_TOUR=1
534elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
535    let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT} 
536else
537    let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
538fi
539
540CURRENT_LOOP=1
541# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
542# WE HAVE 3 LEVELS
543#    - LEVEL 1 : JUST ONE LOOP TO COMPLETE
544#    - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
545#    - LEVEL 3 : WE ARE IN THE LAST LOOP
546
547while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] ; do
548     
549    if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then 
550        # - LEVEL 1 : JUST ONE LOOP TO COMPLETE
551        NBRE_FILE_LOOP=${NBRE_FILE_TOT}
552
553        if [ ${FIRST_PASS} = TRUE ] ; then
554            DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
555        else
556            Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
557            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
558        fi
559
560        DATE_FIN=${PeriodDateEnd}
561        DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
562
563        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
564        [ -n "${LEVEL}" ] && FLAG_B="TRUE" || FLAG_B="FALSE"
565
566        LEVEL=1
567        DEPOT="TRUE"     
568    elif [ ${CURRENT_LOOP} -ne  ${NBRE_TOUR} ] ; then
569        # - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
570        NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
571
572        if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
573            Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
574            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
575        elif [ -z "${DATE_COURANTE}" ] ; then
576            DATE_COURANTE=${DATE_FIN_JOB_B}
577        else
578            Length=$( create_ts_next_date ${DATE_FIN} )
579            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
580        fi
581         
582        (( TotLength = 0 ))
583        COMPTEUR=1
584        DATE_LOOP=${DATE_COURANTE}
585
586        while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
587            #
588            Length=$( create_ts_next_date ${DATE_LOOP} )
589            DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
590            (( TotLength = TotLength + Length ))
591            (( COMPTEUR = COMPTEUR + 1 ))
592            #
593        done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
594        DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
595
596        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
597        [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" || [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" || FLAG_B="FALSE"
598        DEPOT="FALSE"
599        LEVEL=2
600    else
601        # - LEVEL 3 : WE ARE IN THE LAST LOOP
602        NBRE_FILE_LOOP=$( expr ${NBRE_FILE_TOT} % $(( UNIX_MAX_LIMIT * $(( CURRENT_LOOP - 1 )) )) )
603        [ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
604
605        Length=$( create_ts_next_date ${DATE_FIN} )
606        DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
607        DATE_FIN=${PeriodDateEnd}
608        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
609
610        LEVEL=3
611        DEPOT="TRUE"
612        FLAG_B="TRUE"
613    fi
614
615    # THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
616    for comp in ${config_ListOfComponents[*]} ; do
617        #
618        IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
619        #
620        IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
621        #
622        eval R_OUT_${comp}=${R_SAVE}/${comp}
623        #
624        i=0
625        #
626        for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
627            #
628            # Determine in which output can we find file
629            # IE : Output/MO or Output/DA or ...
630            #
631            FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
632            case ${FlagDir} in
633                *Y)    TS_Dir=TS_YE  ; FreqDir=YE  ;;
634                *M)    TS_Dir=TS_MO  ; FreqDir=MO  ;;
635                *D)    TS_Dir=TS_DA  ; FreqDir=DA  ;;
636                HF)    TS_Dir=TS_HF  ; FreqDir=HF  ;;
637                INS)   TS_Dir=TS_INS ; FreqDir=INS ;;
638                *) 
639                       IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
640                       IGCM_sys_Exit "Job create_ts" ;;
641            esac
642
643            # Initialize array
644            #set -A liste_file
645            #set -A liste_file_tmp
646            unset liste_file
647            unset liste_file_tmp
648            eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
649
650            COMPTEUR=1
651            DATE_LOOP=${DATE_COURANTE}
652
653            while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
654                #
655                Length1=$( create_ts_begin_date ${DATE_LOOP} end)
656                DATE_TAB=$(  IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
657
658                Length2=$( create_ts_next_date ${DATE_LOOP} end)
659                DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
660
661                #set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
662                #set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
663                liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
664                liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
665                (( COMPTEUR = COMPTEUR + 1 ))
666                #
667            done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
668            DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
669
670            # Get selected files
671            IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
672            ExitFlag=false
673            IGCM_debug_Verif_Exit_Post
674
675            # Apply IGCM_Patch if needed
676            if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) !=  X${NULL_STR} ]; then
677                for file_t in $( eval echo ${liste_file_tmp[*]} ); do           
678                    for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
679                        IGCM_${Patch} ${file_t}
680                    done
681                done
682            fi
683
684            # WE CAN CONCATENATE FILES AT THIS POINT
685            [ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
686            Length=$( create_ts_begin_date ${DATE_COURANTE} end)
687            DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
688            DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
689            DATE_BUILD_END=${DateBegin}_${DATE_FIN}
690
691            liste_coord=" "
692            for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
693              liste_coord=${liste_coord}${axis}","
694            done
695
696            # Time axis must be the last dimension
697            time_axis=${axis}
698
699            # SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
700            for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
701                #
702                IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
703                #
704#                 # Change time axis name to be CF compliant
705#                 if [ ${time_axis} != time_counter ]; then
706#                   IGCM_sys_ncrename -O -v ${time_axis},time_counter ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
707#                 fi
708                #
709                if [ "${FLAG_B}" = "TRUE" ] ; then
710                    # WE CONCATENATE WITH EXISTING FILES
711                    file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
712                    file2=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
713                    file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
714                    IGCM_sys_ncrcat --hst ${file1} ${file2} ${file_out}
715                    #
716                    IGCM_sys_Rm ${file1} ${file2}
717                fi
718                #
719                if [ "${DEPOT}" = "TRUE" ] ; then
720                    # WE PUT FINAL FILE ON FILE SYSTEM
721                    [ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
722                    #${ncatted} -a modulo,time_counter,c,c," " ${file_out}
723                    file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
724                    eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
725                    [ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
726                    IGCM_sys_Rm ${file_out}
727                    #
728                fi
729                #
730                ExitFlag=false
731                IGCM_debug_Verif_Exit_Post
732            done     # for var in ${liste_var} ; do
733            IGCM_sys_Rm ${liste_file_tmp[*]}
734        done         # for file in ${liste_file_brut} do
735    done             # for comp in ${config_ListOfComponents[*]} ; do
736    ExitFlag=false
737    IGCM_debug_Verif_Exit_Post
738    # GO AHEAD IN DEEPEST LOOP
739    (( CURRENT_LOOP = CURRENT_LOOP + 1 ))
740done                 # while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] do
741# FINISH
742
743# DODS copy
744for comp in ${config_ListOfComponents[*]} ; do
745    for TS_Dir in TS_YE TS_MO TS_DA TS_HF TS_INS ; do
746        IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
747        [ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
748    done
749done
750
751# clean environment variables and export usefull one :
752if [ X$( hostname -s ) = Xrhodes ] ; then
753    LISTE_ENV=$( env | grep -iv MYLANG | grep -iv HOST | grep -iv LOGIN | grep -iv config_ | grep -iv R_ | grep -iv libIGCM | grep -iv PERIOD \
754        | grep -iv DMFDIR | grep -iv AN_| grep -iv YEARS | grep -iv ex | grep -iv RESOL_ | grep -iv PATH | grep -iv WORKDIR | grep -iv USER   \
755        | grep -iv LOGNAME | grep -iv PROFILE | grep -iv ENV | grep -iv TMPDIR | grep -iv POST_DIR | grep -iv StandAlone | grep -iv DB | awk -F= '{print $1}' )
756 
757    for variables in ${LISTE_ENV} ; do
758        unset ${variables}
759    done
760fi
761
762export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
763export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_SRF
764
765listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_SRF"
766export listVarEnv
767
768#D-
769# --------------------------------------------------------------------
770#D- Test if all was right
771# --------------------------------------------------------------------
772IGCM_debug_Verif_Exit_Post
773# --------------------------------------------------------------------
774#D- Submit MONITORING
775# --------------------------------------------------------------------
776IGCM_sys_QsubPost monitoring
777
778#D-
779# --------------------------------------------------------------------
780#D- Update the run.card
781# --------------------------------------------------------------------
782if [ ${StandAlone} != true ] ; then
783    # Put in run.card end period of time series
784    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
785    # We have finish the job !
786    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
787fi
788date
Note: See TracBrowser for help on using the repository browser.