source: trunk/libIGCM/AA_create_ts @ 103

Last change on this file since 103 was 88, checked in by sdipsl, 15 years ago

Bugfix with create_ts_begin_date and create_ts_next_date function du to ulam's ksh.
Do not put anymore typeset on a variable passed to a function that will modify it.
This bugs breaks leap and noleap calendar for post-processing on ulam.
Thanks to NEMO for reporting this nasty bug
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Date Author Revision
File size: 27.2 KB
Line 
1#-Q- platine #!/usr/bin/ksh
2#-Q- platine ##################
3#-Q- platine ## PLATINE   CEA ##
4#-Q- platine ##################
5#-Q- platine #BSUB -J TS                     # Nom du job
6#-Q- platine #BSUB -N                        # message a la fin du job
7#-Q- platine #BSUB -n 1                      # reservation des processeurs pour le job
8#-Q- platine #BSUB -W 1:00                   # Limite temps
9#-Q- platine #BSUB -q post              # Passage en queue post
10#-Q- sx8brodie #!/bin/ksh
11#-Q- sx8brodie #######################
12#-Q- sx8brodie ## SX8BRODIE   IDRIS ##
13#-Q- sx8brodie #######################
14#-Q- sx8brodie # Temps Elapsed max. d'une requete hh:mm:ss
15#-Q- sx8brodie # @ wall_clock_limit = 10:00:00
16#-Q- sx8brodie # Nom du travail LoadLeveler
17#-Q- sx8brodie # @ job_name   = TS
18#-Q- sx8brodie # Fichier de sortie standard du travail       
19#-Q- sx8brodie # @ output     = $(job_name).$(jobid)
20#-Q- sx8brodie # Fichier de sortie d'erreur du travail
21#-Q- sx8brodie # @ error      =  $(job_name).$(jobid)
22#-Q- sx8brodie # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
23#-Q- sx8brodie # @ notification = error
24#-Q- sx8brodie # @ environment  = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_SRF ; $MASTER
25#-Q- aix6 #!/bin/ksh
26#-Q- aix6 #######################
27#-Q- aix6 ##   VARGAS   IDRIS  ##
28#-Q- aix6 #######################
29#-Q- aix6 # Temps Elapsed max. d'une requete hh:mm:ss
30#-Q- aix6 # @ wall_clock_limit = 10:00:00
31#-Q- aix6 # Nom du travail LoadLeveler
32#-Q- aix6 # @ job_name   = TS
33#-Q- aix6 # Fichier de sortie standard du travail
34#-Q- aix6 # @ output     = $(job_name).$(jobid)
35#-Q- aix6 # Fichier de sortie d'erreur du travail
36#-Q- aix6 # @ error      =  $(job_name).$(jobid)
37#-Q- aix6 # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
38#-Q- aix6 # @ notification = error
39#-Q- aix6 # @ environment  = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_SRF ; $MASTER
40#-Q- aix6 # @ queue
41#-Q- sx8brodie # @ queue
42#-Q- sx8mercure #!/bin/ksh
43#-Q- sx8mercure ######################
44#-Q- sx8mercure ## SX8MERCURE   CEA ##
45#-Q- sx8mercure ######################
46#-Q- sx8mercure #PBS -N TS                   # Nom du job
47#-Q- sx8mercure #PBS -j o                    # regroupement des stdout et stderr
48#-Q- sx8mercure #PBS -S /usr/bin/ksh         # shell de soumission
49#-Q- sx8mercure #PBS -l memsz_job=1gb        # Limite memoire a 1 Go
50#-Q- sx8mercure #PBS -l cputim_job=2:00:00   # Limite temps a 2 heures
51#-Q- sx8mercure #PBS -q scalaire
52#-Q- default #!/bin/ksh
53#-Q- default ##################
54#-Q- default ## DEFAULT HOST ##
55#-Q- default ##################
56
57#set -vx
58
59date
60
61#-Q- sx8brodie export OMP_NUM_THREADS=1
62
63# $Date$
64# $Author$
65# $Revision$
66# IPSL (2006)
67#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
68
69########################################################################
70
71#D- Flag to determine if this job in a standalone mode
72#D- Default : value from AA_job if any
73StandAlone=${StandAlone:=true}
74
75#D- Flag to determine job's output directory
76#D- Default : value from libIGCM_post.ksh if any
77POST_DIR=${POST_DIR:=${PBS_O_WORKDIR}}
78
79#D- Increased verbosity (1, 2, 3)
80#D- Default : value from AA_job if any
81Verbosity=${Verbosity:=3}
82
83#D- Low level debug : to bypass lib test checks and stack construction
84#D- Default : value from AA_job if any
85DEBUG_debug=${DEBUG_debug:=false}
86
87#D- Low level debug : to bypass lib test checks and stack construction
88#D- Default : value from AA_job if any
89libIGCM=${libIGCM:=/work/cont003/p86denv/PARA_SX8_ORCA2xLMD144142/Prelude/modipsl/libIGCM}
90
91#D- TEMPORARY Flag to determine atmospheric resolution
92#D- Default : value from atmospheric driver if any
93RESOL_ATM=${RESOL_ATM:=LMD144142}
94
95#D- TEMPORARY Flag to determine ocean resolution
96#D- Default : value from ocean driver if any
97RESOL_OCE=${RESOL_OCE:=ORCA2}
98
99#D- TEMPORARY Flag to determine ice resolution
100#D- Default : value from ice driver if any
101RESOL_ICE=${RESOL_ICE:=ORCA2}
102
103#D- Flag to determine surface resolution
104#D- Default : value from surface driver if any
105RESOL_SRF=${RESOL_SRF:=LMD144142}
106
107########################################################################
108
109. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh ;
110. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh     #; IGCM_debug_Check
111. ${libIGCM}/libIGCM_card/libIGCM_card.ksh   #; IGCM_card_Check
112. ${libIGCM}/libIGCM_date/libIGCM_date.ksh   #; IGCM_date_Check
113
114########################################################################
115
116#set -vx
117
118# Useful cleaning function
119MENAGE () 
120{
121    [ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
122}
123
124########################################################################
125
126#set -vx
127
128if [ ${DEBUG_debug} = true ] ; then
129    IGCM_sys_MkdirWork ${RUN_DIR_PATH}
130    echo "RUN_DIR_PATH ${RUN_DIR_PATH} ok."
131fi
132IGCM_sys_Cd ${RUN_DIR_PATH}
133
134if [ ${StandAlone} = true ] ; then
135    CARD_DIR=${SUBMIT_DIR}
136else
137    CARD_DIR=${RUN_DIR_PATH}/$( basename ${SUBMIT_DIR} )
138    IGCM_sys_Get_Master ${SUBMIT_DIR} ${RUN_DIR_PATH}
139fi
140
141#
142# First of all
143#
144IGCM_card_DefineArrayFromSection       ${CARD_DIR}/config.card UserChoices
145typeset option
146for option in ${config_UserChoices[*]} ; do
147    IGCM_card_DefineVariableFromOption ${CARD_DIR}/config.card UserChoices ${option}
148done
149#
150echo
151IGCM_debug_Print 1 "DefineArrayFromOption  : config_UserChoices"
152IGCM_debug_PrintVariables 3 config_UserChoices_JobName
153IGCM_debug_PrintVariables 3 config_UserChoices_LongName
154IGCM_debug_PrintVariables 3 config_UserChoices_TagName
155IGCM_debug_PrintVariables 3 config_UserChoices_CalendarType
156IGCM_debug_PrintVariables 3 config_UserChoices_DateBegin
157IGCM_debug_PrintVariables 3 config_UserChoices_DateEnd
158IGCM_debug_PrintVariables 3 config_UserChoices_PeriodLength
159echo
160
161#==================================
162
163R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
164#
165IGCM_card_DefineArrayFromSection       ${CARD_DIR}/config.card ListOfComponents
166#
167# Determine period for time series. Default : value from AA_job if any
168#
169DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
170#
171PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
172
173########################################################################
174#
175#      Depending on requested time series fill following variables :
176#
177#                           - LISTE_FILE_${comp}[*]
178#                           - LISTE_VARS_${file}[*]
179#                           - LISTE_AXIS_${file}[*]
180#                           - LISTE_PATCH_${file}[*]
181########################################################################
182
183#
184# For each selected component determine which files need post-processing
185#
186DoJob=false
187for comp in ${config_ListOfComponents[*]} ; do
188    #
189    ActiveComp=false
190    #
191    IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
192    eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
193    #
194    card=${CARD_DIR}/COMP/${compname}.card
195    IGCM_card_DefineArrayFromOption ${card} OutputFiles List
196    #
197    ListFilesName=${compname}_OutputFiles_List
198    eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
199    #
200    if [ X${FileName0} != X${NULL_STR} ] ; then
201        #
202        #IGCM_debug_Print 1 "Component      : ${compname}"
203        #
204        # INITIALISATION
205        #
206        eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
207        typeset i=0
208        #
209        until [ $i -eq $NbFiles ]; do
210            #
211            eval file_in_=\${${ListFilesName}[$i]} > /dev/null 2>&1
212            eval file_in=${file_in_}
213            let $(( i_ = i+2 ))
214            eval flag_post_=\${${ListFilesName}[$i_]} > /dev/null 2>&1
215            eval flag_post=${flag_post_}
216            #
217            if [ X${flag_post} != XNONE ] ; then
218                #
219                # Initialize ${flag_post}_${compname}_${post_freq}
220                #
221                eval export ${flag_post}_${compname}_TimeSeriesFrequency=true
222            fi
223            let $(( i=i+3 ))
224        done
225    fi
226
227    for element in $( env | grep Post_ | grep ${compname}_TimeSeriesFrequency | grep -i "=true" ) ; do
228        #
229        FILE=$( echo ${element} | awk -F "_${compname}" '{print $1}' | awk "-FPost_" '{print $2}' )
230        #
231        # For each comp and each file who needs post-processing
232        # determine which variables need time series.
233        #
234        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars
235        #
236        if [ $( eval echo \$\{${compname}_Post_${FILE}_TimeSeriesVars} ) = ${NULL_STR} ] ; then
237            #
238            IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
239            continue
240            #
241        fi
242        #
243        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
244        #
245        # So we need coordinates axis to properly fill produced time series
246        #
247        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
248        #
249        if [ $( eval echo \$\{${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
250            #
251            IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
252            IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
253            continue
254            #
255        fi
256        #
257        # CREATE LIST
258        #
259        FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' )
260        case ${FlagDir} in
261            *Y)  TS_Dir=TS_YE  ;;
262            *M)  TS_Dir=TS_MO  ;;
263            *D)  TS_Dir=TS_DA  ;;
264            HF)  TS_Dir=TS_HF  ;;
265            INS) TS_Dir=TS_INS ;;
266        esac
267
268        count=0
269        for var in $( eval echo \$\{${compname}_Post_${FILE}_TimeSeriesVars[*]} ) ; do
270            #
271            DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
272            TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
273            #
274            IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
275            if [ ! $? = 0 ] ; then
276                eval set +A LISTE_VARS_${FILE} \$\{LISTE_VARS_${FILE}[*]} ${var}
277                ActiveComp=true
278                (( count = count + 1 ))
279            fi
280        done
281
282        if [ ${count} -gt 0 ] ; then
283            #
284            eval set +A LISTE_FILE_${comp} \$\{LISTE_FILE_${comp}[*]} ${FILE}
285            eval set +A LISTE_AXIS_${FILE} \$\{${compname}_Post_${FILE}_GatherWithInternal[*]}
286            eval set +A LISTE_PATCHES_${FILE} \$\{${compname}_Post_${FILE}_Patches[*]}
287            if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) !=  X${NULL_STR} ]; then
288                for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
289                    if [ Xload${Patch} != X ] ; then
290                        . ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
291                        eval load${Patch}=loaded
292                    fi
293                done
294            fi
295            #
296        fi
297    done
298
299    if [ ${ActiveComp} = true ] ; then
300        set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
301    fi
302
303done
304
305set -A config_ListOfComponents ${ActiveListOfComponents[*]}
306[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
307
308########################################################################
309#
310#                          DEFINE DATE FUNCTIONS
311#
312########################################################################
313
314# Give the end of the next $1 period from $2 date
315function create_ts_next_date {
316    IGCM_debug_PushStack "create_ts_next_date" $@
317    typeset Length
318    case ${config_UserChoices_PeriodLength} in
319        *Y|*y) 
320            PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} | awk -F '[yY]' '{print $1}' )
321            IGCM_date_GetYearMonth ${1} year month
322            (( Length=0 ))
323            (( i=0 ))
324            until [ $i -ge $PeriodLengthInYears ] ; do
325                (( Length = Length + $( IGCM_date_DaysInYear $(( year + i + 1 )) ) )) 
326                (( i=i+1 ))
327            done
328            ;;
329        *M|*m) 
330            PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} | awk -F '[mM]' '{print $1}' )
331            IGCM_date_GetYearMonth ${1} year month
332            (( year0 = year ))
333            if [ $(( month + 1 )) -lt 13 ] ; then
334                month0=$(( month + 1 ))
335            else
336                month0=$(( month + 1 - 12 ))
337                (( year = year0 + 1 ))
338            fi
339            (( Length=0 ))
340            (( i=0 ))
341            until [ $i -ge $PeriodLengthInMonths ] ; do
342                if [ $(( month0 + i )) -lt 13 ] ; then
343                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i )) ) ))
344                else
345                    (( year = year0 + 1 ))
346                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i - 12 )) ) ))
347                fi
348                (( i=i+1 ))
349            done
350            ;;
351        *D|*d)
352            Length=$( echo ${config_UserChoices_PeriodLength} | sed -e "s/[dD]//" ) ;;
353        *)
354            IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, *M, *D."
355            IGCM_debug_Verif_Exit_Post ;;
356    esac
357    echo ${Length}
358
359    IGCM_debug_PopStack "create_ts_next_date"
360}
361
362# Give the end of the actual $1 period from $2 date
363function create_ts_begin_date {
364    IGCM_debug_PushStack "create_ts_begin_date" $@
365    typeset Length
366    case ${config_UserChoices_PeriodLength} in
367        *Y|*y) 
368            PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} | awk -F '[yY]' '{print $1}' )
369            IGCM_date_GetYearMonth ${1} year month
370            if [ X${2} = Xend ] ; then
371                (( year = year - PeriodLengthInYears + 1))
372            fi
373            (( Length=0 ))
374            (( i=0 ))
375            until [ $i -ge $PeriodLengthInYears ] ; do
376                (( Length = Length + $( IGCM_date_DaysInYear $(( year + i )) ) )) 
377                (( i=i+1 ))
378            done
379            ;;
380        *M|*m) 
381            PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} | awk -F '[mM]' '{print $1}' )
382            IGCM_date_GetYearMonth ${1} year month
383            if [ X${2} = Xend ] ; then
384                (( month = month - PeriodLengthInMonths + 1 ))
385            fi     
386            (( year0 = year ))
387            if [ $month -le 0 ] ; then
388                (( month = month + 12 ))
389                year=$( printf "%04i\n" $(( year - 1 )) )
390            fi
391            month=$( printf "%02i\n" ${month} )
392            (( Length=0 ))
393            (( i=0 ))
394            until [ $i -ge $PeriodLengthInMonths ] ; do
395                if [ $(( month + i )) -lt 13 ] ; then
396                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i )) ) ))
397                else
398                    (( year = year0 + 1 ))
399                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i - 12 )) ) ))
400                fi
401                (( i=i+1 ))
402            done
403            ;;
404        *D|*d)
405            Length=$( echo ${config_UserChoices_PeriodLength} | sed -e "s/[dD]//" ) ;;
406        *)
407            IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, *M, *D."
408            IGCM_debug_Verif_Exit_Post ;;
409    esac
410    echo ${Length}
411
412    IGCM_debug_PopStack "create_ts_begin_date"
413}
414
415########################################################################
416#
417#               IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
418#                 IF SO BRING THEM IN THE WORKING DIRECTORY
419#
420# PS : Keep in mind that IGCM_sys_Get here is a weak link :
421#      - especially for IDRiS
422#      - no special protection against "command too long" !
423#      - libIGCM_sys should handle that or modify following code part
424#
425########################################################################
426
427if [ ${StandAlone} != true ] ; then
428    Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
429       ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
430       echo \${run_PostProcessing_TimeSeriesRunning}" )
431
432    if [ X${Running_Flag} = Xy ] ; then
433        # Time Series Job allready running
434        IGCM_debug_Print 1 "Time Series Job allready running exit"
435        exit
436    fi
437    # Now run.card know we are running
438    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
439    # Get information from last execution
440    CompletedFlag=$( IGCM_sys_RshMaster \
441        "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
442        echo \${run_PostProcessing_TimeSeriesCompleted}" )
443fi
444
445# Is it the first submission or not ?
446if [ X${CompletedFlag} = X ] ; then
447    # First Time Series Submission
448    FIRST_PASS=TRUE
449    Length=$( create_ts_begin_date ${DateBegin} begin)
450    DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
451else
452    # Last Completed Time Series
453    FIRST_PASS=FALSE
454    DATE_FIN_JOB_B=${CompletedFlag}
455    date=${DateBegin}_${DATE_FIN_JOB_B}
456    for comp in ${config_ListOfComponents[*]} ; do
457        #
458        IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
459        #
460        IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
461        #
462        #set -A liste_file
463        unset liste_file
464        #
465        file_path=${R_SAVE}/${comp}/Analyse
466        #
467        i=0
468        file=${NULL_STR}
469        #
470        for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
471            #
472            # Determine in which output can we find file
473            # IE : Analyse/TS or Analyse/DA2TS or ...
474            #
475            FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
476            case ${FlagDir} in
477                *Y)  TS_Dir=TS_YE  ;;
478                *M)  TS_Dir=TS_MO  ;;
479                *D)  TS_Dir=TS_DA  ;;
480                HF)  TS_Dir=TS_HF  ;;
481                INS) TS_Dir=TS_INS ;;
482            esac
483            for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
484                #
485                #set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
486                liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
487                #
488            done
489        done
490        if [ X${file} != X${NULL_STR} ] ; then
491            IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
492            IGCM_debug_Verif_Exit_Post
493        fi
494    done
495    #
496    LEVEL=0
497    #
498fi
499
500########################################################################
501#
502#                          DEFINE LOOP PARAMETERS
503#
504########################################################################
505
506DATE_COUNT=${DATE_FIN_JOB_B}
507DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
508
509( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 || NBRE_FILE_TOT=1
510
511if [ ${DoJob} = true ] ; then
512    while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
513        (( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
514        Length=$( create_ts_next_date ${DATE_COUNT} )
515        DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
516    done
517else
518    NBRE_FILE_TOT=0
519fi
520
521if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
522    NBRE_TOUR=1
523elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
524    let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT} 
525else
526    let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
527fi
528
529CURRENT_LOOP=1
530# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
531# WE HAVE 3 LEVELS
532#    - LEVEL 1 : JUST ONE LOOP TO COMPLETE
533#    - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
534#    - LEVEL 3 : WE ARE IN THE LAST LOOP
535
536while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] ; do
537     
538    if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then 
539        # - LEVEL 1 : JUST ONE LOOP TO COMPLETE
540        NBRE_FILE_LOOP=${NBRE_FILE_TOT}
541
542        if [ ${FIRST_PASS} = TRUE ] ; then
543            DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
544        else
545            Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
546            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
547        fi
548
549        DATE_FIN=${PeriodDateEnd}
550        DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
551
552        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
553        [ -n "${LEVEL}" ] && FLAG_B="TRUE" || FLAG_B="FALSE"
554
555        LEVEL=1
556        DEPOT="TRUE"     
557    elif [ ${CURRENT_LOOP} -ne  ${NBRE_TOUR} ] ; then
558        # - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
559        NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
560
561        if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
562            Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
563            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
564        elif [ -z "${DATE_COURANTE}" ] ; then
565            DATE_COURANTE=${DATE_FIN_JOB_B}
566        else
567            Length=$( create_ts_next_date ${DATE_FIN} )
568            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
569        fi
570         
571        (( TotLength = 0 ))
572        COMPTEUR=1
573        DATE_LOOP=${DATE_COURANTE}
574
575        while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
576            #
577            Length=$( create_ts_next_date ${DATE_LOOP} )
578            DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
579            (( TotLength = TotLength + Length ))
580            (( COMPTEUR = COMPTEUR + 1 ))
581            #
582        done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
583        DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
584
585        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
586        [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" || [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" || FLAG_B="FALSE"
587        DEPOT="FALSE"
588        LEVEL=2
589    else
590        # - LEVEL 3 : WE ARE IN THE LAST LOOP
591        NBRE_FILE_LOOP=$( expr ${NBRE_FILE_TOT} % $(( UNIX_MAX_LIMIT * $(( CURRENT_LOOP - 1 )) )) )
592        [ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
593
594        Length=$( create_ts_next_date ${DATE_FIN} )
595        DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
596        DATE_FIN=${PeriodDateEnd}
597        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
598
599        LEVEL=3
600        DEPOT="TRUE"
601        FLAG_B="TRUE"
602    fi
603
604    # THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
605    for comp in ${config_ListOfComponents[*]} ; do
606        #
607        IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
608        #
609        IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
610        #
611        eval R_OUT_${comp}=${R_SAVE}/${comp}
612        #
613        i=0
614        #
615        for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
616            #
617            # Determine in which output can we find file
618            # IE : Output/MO or Output/DA or ...
619            #
620            FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
621            case ${FlagDir} in
622                *Y)    TS_Dir=TS_YE  ; FreqDir=YE  ;;
623                *M)    TS_Dir=TS_MO  ; FreqDir=MO  ;;
624                *D)    TS_Dir=TS_DA  ; FreqDir=DA  ;;
625                HF)    TS_Dir=TS_HF  ; FreqDir=HF  ;;
626                INS)   TS_Dir=TS_INS ; FreqDir=INS ;;
627                *) 
628                       IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
629                       IGCM_sys_Exit "Job create_ts" ;;
630            esac
631
632            # Initialize array
633            #set -A liste_file
634            #set -A liste_file_tmp
635            unset liste_file
636            unset liste_file_tmp
637            eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
638
639            COMPTEUR=1
640            DATE_LOOP=${DATE_COURANTE}
641
642            while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
643                #
644                Length1=$( create_ts_begin_date ${DATE_LOOP} end)
645                DATE_TAB=$(  IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
646
647                Length2=$( create_ts_next_date ${DATE_LOOP} end)
648                DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
649
650                #set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
651                #set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
652                liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
653                liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
654                (( COMPTEUR = COMPTEUR + 1 ))
655                #
656            done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
657            DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
658
659            # Get selected files
660            IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
661            eval ExitFlag=false
662            IGCM_debug_Verif_Exit_Post
663
664            # Apply IGCM_Patch if needed
665            if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) !=  X${NULL_STR} ]; then
666                for file_t in $( eval echo ${liste_file_tmp[*]} ); do           
667                    for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
668                        IGCM_${Patch} ${file_t}
669                    done
670                done
671            fi
672
673            # WE CAN CONCATENATE FILES AT THIS POINT
674            [ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
675            Length=$( create_ts_begin_date ${DATE_COURANTE} end)
676            DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
677            DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
678            DATE_BUILD_END=${DateBegin}_${DATE_FIN}
679
680            liste_coord=" "
681            for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
682              liste_coord=${liste_coord}${axis}","
683            done
684
685            # Time axis must be the last dimension
686            time_axis=${axis}
687
688            # SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
689            for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
690                #
691                IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
692                #
693#                 # Change time axis name to be CF compliant
694#                 if [ ${time_axis} != time_counter ]; then
695#                   IGCM_sys_ncrename -O -v ${time_axis},time_counter ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
696#                 fi
697                #
698                if [ "${FLAG_B}" = "TRUE" ] ; then
699                    # WE CONCATENATE WITH EXISTING FILES
700                    file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
701                    file2=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
702                    file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
703                    IGCM_sys_ncrcat --hst ${file1} ${file2} ${file_out}
704                    #
705                    IGCM_sys_Rm ${file1} ${file2}
706                fi
707                #
708                if [ "${DEPOT}" = "TRUE" ] ; then
709                    # WE PUT FINAL FILE ON FILE SYSTEM
710                    [ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
711                    #${ncatted} -a modulo,time_counter,c,c," " ${file_out}
712                    file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
713                    eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
714                    [ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
715                    IGCM_sys_Rm ${file_out}
716                    #
717                fi
718                #
719                eval ExitFlag=false
720                IGCM_debug_Verif_Exit_Post
721            done     # for var in ${liste_var} ; do
722            IGCM_sys_Rm ${liste_file_tmp[*]}
723        done         # for file in ${liste_file_brut} do
724    done             # for comp in ${config_ListOfComponents[*]} ; do
725    eval ExitFlag=false
726    IGCM_debug_Verif_Exit_Post
727    # GO AHEAD IN DEEPEST LOOP
728    (( CURRENT_LOOP = CURRENT_LOOP + 1 ))
729done                 # while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] do
730# FINISH
731
732# DODS copy
733for comp in ${config_ListOfComponents[*]} ; do
734    for TS_Dir in TS_YE TS_MO TS_DA TS_HF TS_INS ; do
735        IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
736        [ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
737    done
738done
739
740# clean environment variables and export usefull one :
741if [ X$( hostname -s ) = Xrhodes ] ; then
742    LISTE_ENV=$( env | grep -iv MYLANG | grep -iv HOST | grep -iv LOGIN | grep -iv config_ | grep -iv R_ | grep -iv libIGCM | grep -iv PERIOD \
743        | grep -iv DMFDIR | grep -iv AN_| grep -iv YEARS | grep -iv ex | grep -iv RESOL_ | grep -iv PATH | grep -iv WORKDIR | grep -iv USER   \
744        | grep -iv LOGNAME | grep -iv PROFILE | grep -iv ENV | grep -iv TMPDIR | grep -iv POST_DIR | grep -iv StandAlone | grep -iv DB | awk -F= '{print $1}' )
745 
746    for variables in ${LISTE_ENV} ; do
747        unset ${variables}
748    done
749fi
750
751export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
752export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_SRF
753
754listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_SRF"
755export listVarEnv
756
757#D-
758# --------------------------------------------------------------------
759#D- Test if all was right
760# --------------------------------------------------------------------
761IGCM_debug_Verif_Exit_Post
762# --------------------------------------------------------------------
763#D- Submit MONITORING
764# --------------------------------------------------------------------
765IGCM_sys_QsubPost monitoring
766
767#D-
768# --------------------------------------------------------------------
769#D- Update the run.card
770# --------------------------------------------------------------------
771if [ ${StandAlone} != true ] ; then
772    # Put in run.card end period of time series
773    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
774    # We have finish the job !
775    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
776fi
777date
Note: See TracBrowser for help on using the repository browser.