source: trunk/libIGCM/AA_create_ts @ 131

Last change on this file since 131 was 131, checked in by mmaipsl, 15 years ago

Change default resolution for SRF to ALL and add STOmate component (for MONITORING).
Correct a little mistake in two loops in case of the compname equal to history filename.
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Date Author Revision
File size: 27.6 KB
Line 
1#-Q- platine #!/usr/bin/ksh
2#-Q- platine ##################
3#-Q- platine ## PLATINE   CEA ##
4#-Q- platine ##################
5#-Q- platine #BSUB -J TS                     # Nom du job
6#-Q- platine #BSUB -N                        # message a la fin du job
7#-Q- platine #BSUB -n 1                      # reservation des processeurs pour le job
8#-Q- platine #BSUB -W 1:00                   # Limite temps
9#-Q- platine #BSUB -q post              # Passage en queue post
10#-Q- sx8brodie #!/bin/ksh
11#-Q- sx8brodie #######################
12#-Q- sx8brodie ## SX8BRODIE   IDRIS ##
13#-Q- sx8brodie #######################
14#-Q- sx8brodie # Temps Elapsed max. d'une requete hh:mm:ss
15#-Q- sx8brodie # @ wall_clock_limit = 10:00:00
16#-Q- sx8brodie # Nom du travail LoadLeveler
17#-Q- sx8brodie # @ job_name   = TS
18#-Q- sx8brodie # Fichier de sortie standard du travail       
19#-Q- sx8brodie # @ output     = $(job_name).$(jobid)
20#-Q- sx8brodie # Fichier de sortie d'erreur du travail
21#-Q- sx8brodie # @ error      =  $(job_name).$(jobid)
22#-Q- sx8brodie # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
23#-Q- sx8brodie # @ notification = error
24#-Q- sx8brodie # @ environment  = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_SRF ; $MASTER
25#-Q- aix6 #!/bin/ksh
26#-Q- aix6 #######################
27#-Q- aix6 ##   VARGAS   IDRIS  ##
28#-Q- aix6 #######################
29#-Q- aix6 # Temps Elapsed max. d'une requete hh:mm:ss
30#-Q- aix6 # @ wall_clock_limit = 10:00:00
31#-Q- aix6 # Nom du travail LoadLeveler
32#-Q- aix6 # @ job_name   = TS
33#-Q- aix6 # Fichier de sortie standard du travail
34#-Q- aix6 # @ output     = $(job_name).$(jobid)
35#-Q- aix6 # Fichier de sortie d'erreur du travail
36#-Q- aix6 # @ error      =  $(job_name).$(jobid)
37#-Q- aix6 # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
38#-Q- aix6 # @ notification = error
39#-Q- aix6 # @ environment  = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_SRF ; $MASTER
40#-Q- aix6 # @ queue
41#-Q- sx8brodie # @ queue
42#-Q- sx8mercure #!/bin/ksh
43#-Q- sx8mercure ######################
44#-Q- sx8mercure ## SX8MERCURE   CEA ##
45#-Q- sx8mercure ######################
46#-Q- sx8mercure #PBS -N TS                   # Nom du job
47#-Q- sx8mercure #PBS -j o                    # regroupement des stdout et stderr
48#-Q- sx8mercure #PBS -S /usr/bin/ksh         # shell de soumission
49#-Q- sx8mercure #PBS -l memsz_job=1gb        # Limite memoire a 1 Go
50#-Q- sx8mercure #PBS -l cputim_job=2:00:00   # Limite temps a 2 heures
51#-Q- sx8mercure #PBS -q scalaire
52#-Q- default #!/bin/ksh
53#-Q- default ##################
54#-Q- default ## DEFAULT HOST ##
55#-Q- default ##################
56
57# $Date$
58# $Author$
59# $Revision$
60# IPSL (2006)
61#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
62
63#set -eu
64#set -vx
65
66date
67
68#-Q- sx8brodie export OMP_NUM_THREADS=1
69#-Q- aix6 export OMP_NUM_THREADS=1
70
71########################################################################
72
73#D- Flag to determine if this job in a standalone mode
74#D- Default : value from AA_job if any
75StandAlone=${StandAlone:=true}
76
77#D- Flag to determine job's output directory
78#D- Default : value from libIGCM_post.ksh if any
79POST_DIR=${POST_DIR:=${PBS_O_WORKDIR}}
80
81#D- Increased verbosity (1, 2, 3)
82#D- Default : value from AA_job if any
83Verbosity=${Verbosity:=3}
84
85#D- Low level debug : to bypass lib test checks and stack construction
86#D- Default : value from AA_job if any
87DEBUG_debug=${DEBUG_debug:=false}
88
89#D- Low level debug : to bypass lib test checks and stack construction
90#D- Default : value from AA_job if any
91libIGCM=${libIGCM:=/path/to/your/libIGCM}
92
93#D- TEMPORARY Flag to determine atmospheric resolution
94#D- Default : value from atmospheric driver if any
95RESOL_ATM=${RESOL_ATM:=LMD144142}
96
97#D- TEMPORARY Flag to determine ocean resolution
98#D- Default : value from ocean driver if any
99RESOL_OCE=${RESOL_OCE:=ORCA2}
100
101#D- TEMPORARY Flag to determine ice resolution
102#D- Default : value from ice driver if any
103RESOL_ICE=${RESOL_ICE:=ORCA2}
104
105#D- Flag to determine surface resolution
106#D- Default : value from surface driver if any
107RESOL_SRF=${RESOL_SRF:=ALL}
108
109#D- Flag to determine surface resolution
110#D- Default : value from surface driver if any
111RESOL_STO=${RESOL_STO:=ALL}
112
113#D- Flag to determine allready produced time series.
114#D- Default : Commented. Uncomment for StandAlone case if you want to continue a time series
115#CompletedFlag=
116
117########################################################################
118
119. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
120      ( ${DEBUG_debug} ) && IGCM_debug_Check
121. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
122     ( ${DEBUG_debug} ) && IGCM_card_Check
123. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
124     ( ${DEBUG_debug} ) && IGCM_date_Check
125#-------
126. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
127
128########################################################################
129
130#set -vx
131
132# Useful cleaning function
133MENAGE () 
134{
135    [ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
136}
137
138########################################################################
139
140#set -vx
141
142if [ ${DEBUG_debug} = true ] ; then
143    IGCM_sys_MkdirWork ${RUN_DIR_PATH}
144    echo "RUN_DIR_PATH ${RUN_DIR_PATH} ok."
145fi
146IGCM_sys_Cd ${RUN_DIR_PATH}
147
148if [ ${StandAlone} = true ] ; then
149    CARD_DIR=${SUBMIT_DIR}
150else
151    CARD_DIR=${RUN_DIR_PATH}/$( basename ${SUBMIT_DIR} )
152    IGCM_sys_Get_Master ${SUBMIT_DIR} ${RUN_DIR_PATH}
153fi
154
155#
156# First of all
157#
158IGCM_card_DefineArrayFromSection       ${CARD_DIR}/config.card UserChoices
159typeset option
160for option in ${config_UserChoices[*]} ; do
161    IGCM_card_DefineVariableFromOption ${CARD_DIR}/config.card UserChoices ${option}
162done
163#
164echo
165IGCM_debug_Print 1 "DefineArrayFromOption  : config_UserChoices"
166IGCM_debug_PrintVariables 3 config_UserChoices_JobName
167IGCM_debug_PrintVariables 3 config_UserChoices_LongName
168IGCM_debug_PrintVariables 3 config_UserChoices_TagName
169IGCM_debug_PrintVariables 3 config_UserChoices_CalendarType
170IGCM_debug_PrintVariables 3 config_UserChoices_DateBegin
171IGCM_debug_PrintVariables 3 config_UserChoices_DateEnd
172IGCM_debug_PrintVariables 3 config_UserChoices_PeriodLength
173echo
174
175#==================================
176
177R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
178#
179IGCM_card_DefineArrayFromSection       ${CARD_DIR}/config.card ListOfComponents
180#
181# Determine period for time series. Default : value from AA_job if any
182#
183DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
184#
185PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
186
187########################################################################
188#
189#      Depending on requested time series fill following variables :
190#
191#                           - LISTE_FILE_${comp}[*]
192#                           - LISTE_VARS_${file}[*]
193#                           - LISTE_AXIS_${file}[*]
194#                           - LISTE_PATCH_${file}[*]
195########################################################################
196
197#
198# For each selected component determine which files need post-processing
199#
200DoJob=false
201for comp in ${config_ListOfComponents[*]} ; do
202    #
203    ActiveComp=false
204    #
205    IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
206    eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
207    #
208    card=${CARD_DIR}/COMP/${compname}.card
209    IGCM_card_DefineArrayFromOption ${card} OutputFiles List
210    #
211    ListFilesName=${compname}_OutputFiles_List
212    eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
213    #
214    if [ X${FileName0} != X${NULL_STR} ] ; then
215        #
216        #IGCM_debug_Print 1 "Component      : ${compname}"
217        #
218        # INITIALISATION
219        #
220        eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
221        typeset i=0
222        #
223        until [ $i -eq $NbFiles ]; do
224            #
225            eval file_in_=\${${ListFilesName}[$i]} > /dev/null 2>&1
226            eval file_in=${file_in_}
227            let $(( i_ = i+2 ))
228            eval flag_post_=\${${ListFilesName}[$i_]} > /dev/null 2>&1
229            eval flag_post=${flag_post_}
230            #
231            if [ X${flag_post} != XNONE ] ; then
232                #
233                # Initialize ${flag_post}_${compname}_${post_freq}
234                #
235                eval export ${flag_post}_${compname}_TimeSeriesFrequency=true
236            fi
237            let $(( i=i+3 ))
238        done
239    fi
240
241    for element in $( env | grep Post_ | grep ${compname}_TimeSeriesFrequency | grep -i "=true" ) ; do
242        #
243        FILE=$( echo ${element} | awk -F "_${compname}_TimeSeriesFrequency" '{print $1}' | awk "-FPost_" '{print $2}' )
244        #
245        # For each comp and each file who needs post-processing
246        # determine which variables need time series.
247        #
248        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars
249        #
250        if [ $( eval echo \$\{${compname}_Post_${FILE}_TimeSeriesVars} ) = ${NULL_STR} ] ; then
251            #
252            IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
253            continue
254            #
255        fi
256        #
257        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
258        #
259        # So we need coordinates axis to properly fill produced time series
260        #
261        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
262        #
263        if [ $( eval echo \$\{${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
264            #
265            IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
266            IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
267            continue
268            #
269        fi
270        #
271        # CREATE LIST
272        #
273        FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' )
274        case ${FlagDir} in
275            *Y)  TS_Dir=TS_YE  ;;
276            *M)  TS_Dir=TS_MO  ;;
277            *D)  TS_Dir=TS_DA  ;;
278            HF)  TS_Dir=TS_HF  ;;
279            INS) TS_Dir=TS_INS ;;
280        esac
281
282        count=0
283        for var in $( eval echo \$\{${compname}_Post_${FILE}_TimeSeriesVars[*]} ) ; do
284            #
285            DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
286            TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
287            #
288            IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
289            if [ ! $? = 0 ] ; then
290                eval set +A LISTE_VARS_${FILE} \$\{LISTE_VARS_${FILE}[*]} ${var}
291                ActiveComp=true
292                (( count = count + 1 ))
293            fi
294        done
295
296        if [ ${count} -gt 0 ] ; then
297            #
298            eval set +A LISTE_FILE_${comp} \$\{LISTE_FILE_${comp}[*]} ${FILE}
299            eval set +A LISTE_AXIS_${FILE} \$\{${compname}_Post_${FILE}_GatherWithInternal[*]}
300            eval set +A LISTE_PATCHES_${FILE} \$\{${compname}_Post_${FILE}_Patches[*]}
301            if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) !=  X${NULL_STR} ]; then
302                for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
303                    if [ Xload${Patch} != X ] ; then
304                        . ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
305                        eval load${Patch}=loaded
306                    fi
307                done
308            fi
309            #
310        fi
311    done
312
313    if [ ${ActiveComp} = true ] ; then
314        set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
315    fi
316
317done
318
319set -A config_ListOfComponents ${ActiveListOfComponents[*]}
320[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
321
322########################################################################
323#
324#                          DEFINE DATE FUNCTIONS
325#
326########################################################################
327
328# Give the end of the next $1 period from $2 date
329function create_ts_next_date {
330    IGCM_debug_PushStack "create_ts_next_date" $@
331    typeset Length
332    case ${config_UserChoices_PeriodLength} in
333        *Y|*y) 
334            PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} | awk -F '[yY]' '{print $1}' )
335            IGCM_date_GetYearMonth ${1} year month
336            (( Length=0 ))
337            (( i=0 ))
338            until [ $i -ge $PeriodLengthInYears ] ; do
339                (( Length = Length + $( IGCM_date_DaysInYear $(( year + i + 1 )) ) )) 
340                (( i=i+1 ))
341            done
342            ;;
343        *M|*m) 
344            PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} | awk -F '[mM]' '{print $1}' )
345            IGCM_date_GetYearMonth ${1} year month
346            (( year0 = year ))
347            if [ $(( month + 1 )) -lt 13 ] ; then
348                month0=$(( month + 1 ))
349            else
350                month0=$(( month + 1 - 12 ))
351                (( year = year0 + 1 ))
352            fi
353            (( Length=0 ))
354            (( i=0 ))
355            until [ $i -ge $PeriodLengthInMonths ] ; do
356                if [ $(( month0 + i )) -lt 13 ] ; then
357                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i )) ) ))
358                else
359                    (( year = year0 + 1 ))
360                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i - 12 )) ) ))
361                fi
362                (( i=i+1 ))
363            done
364            ;;
365        *D|*d)
366            Length=$( echo ${config_UserChoices_PeriodLength} | sed -e "s/[dD]//" ) ;;
367        *)
368            IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, *M, *D."
369            IGCM_debug_Verif_Exit_Post ;;
370    esac
371    echo ${Length}
372
373    IGCM_debug_PopStack "create_ts_next_date"
374}
375
376# Give the end of the actual $1 period from $2 date
377function create_ts_begin_date {
378    IGCM_debug_PushStack "create_ts_begin_date" $@
379    typeset Length
380    case ${config_UserChoices_PeriodLength} in
381        *Y|*y) 
382            PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} | awk -F '[yY]' '{print $1}' )
383            IGCM_date_GetYearMonth ${1} year month
384            if [ X${2} = Xend ] ; then
385                (( year = year - PeriodLengthInYears + 1))
386            fi
387            (( Length=0 ))
388            (( i=0 ))
389            until [ $i -ge $PeriodLengthInYears ] ; do
390                (( Length = Length + $( IGCM_date_DaysInYear $(( year + i )) ) )) 
391                (( i=i+1 ))
392            done
393            ;;
394        *M|*m) 
395            PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} | awk -F '[mM]' '{print $1}' )
396            IGCM_date_GetYearMonth ${1} year month
397            if [ X${2} = Xend ] ; then
398                (( month = month - PeriodLengthInMonths + 1 ))
399            fi     
400            (( year0 = year ))
401            if [ $month -le 0 ] ; then
402                (( month = month + 12 ))
403                year=$( printf "%04i\n" $(( year - 1 )) )
404            fi
405            month=$( printf "%02i\n" ${month} )
406            (( Length=0 ))
407            (( i=0 ))
408            until [ $i -ge $PeriodLengthInMonths ] ; do
409                if [ $(( month + i )) -lt 13 ] ; then
410                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i )) ) ))
411                else
412                    (( year = year0 + 1 ))
413                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i - 12 )) ) ))
414                fi
415                (( i=i+1 ))
416            done
417            ;;
418        *D|*d)
419            Length=$( echo ${config_UserChoices_PeriodLength} | sed -e "s/[dD]//" ) ;;
420        *)
421            IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, *M, *D."
422            IGCM_debug_Verif_Exit_Post ;;
423    esac
424    echo ${Length}
425
426    IGCM_debug_PopStack "create_ts_begin_date"
427}
428
429########################################################################
430#
431#               IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
432#                 IF SO BRING THEM IN THE WORKING DIRECTORY
433#
434# PS : Keep in mind that IGCM_sys_Get here is a weak link :
435#      - especially for IDRiS
436#      - no special protection against "command too long" !
437#      - libIGCM_sys should handle that or modify following code part
438#
439########################################################################
440
441if [ ${StandAlone} != true ] ; then
442    Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
443       ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
444       echo \${run_PostProcessing_TimeSeriesRunning}" )
445
446    if [ X${Running_Flag} = Xy ] ; then
447        # Time Series Job allready running
448        IGCM_debug_Print 1 "Time Series Job allready running exit"
449        exit
450    fi
451    # Now run.card know we are running
452    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
453    # Get information from last execution
454    CompletedFlag=$( IGCM_sys_RshMaster \
455        "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
456        echo \${run_PostProcessing_TimeSeriesCompleted}" )
457fi
458
459# Is it the first submission or not ?
460if [ X${CompletedFlag} = X ] ; then
461    # First Time Series Submission
462    FIRST_PASS=TRUE
463    Length=$( create_ts_begin_date ${DateBegin} begin)
464    DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
465else
466    # Last Completed Time Series
467    FIRST_PASS=FALSE
468    DATE_FIN_JOB_B=${CompletedFlag}
469    date=${DateBegin}_${DATE_FIN_JOB_B}
470    for comp in ${config_ListOfComponents[*]} ; do
471        #
472        IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
473        #
474        IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
475        #
476        #set -A liste_file
477        unset liste_file
478        #
479        file_path=${R_SAVE}/${comp}/Analyse
480        #
481        i=0
482        file=${NULL_STR}
483        #
484        for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
485            #
486            # Determine in which output can we find file
487            # IE : Analyse/TS or Analyse/DA2TS or ...
488            #
489            FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
490            case ${FlagDir} in
491                *Y)  TS_Dir=TS_YE  ;;
492                *M)  TS_Dir=TS_MO  ;;
493                *D)  TS_Dir=TS_DA  ;;
494                HF)  TS_Dir=TS_HF  ;;
495                INS) TS_Dir=TS_INS ;;
496            esac
497            for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
498                #
499                #set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
500                liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
501                #
502            done
503        done
504        if [ X${file} != X${NULL_STR} ] ; then
505            IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
506            ExitFlag=false
507            IGCM_debug_Verif_Exit_Post
508        fi
509    done
510    #
511    LEVEL=0
512    #
513fi
514
515########################################################################
516#
517#                          DEFINE LOOP PARAMETERS
518#
519########################################################################
520
521DATE_COUNT=${DATE_FIN_JOB_B}
522DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
523
524( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 || NBRE_FILE_TOT=1
525
526if [ ${DoJob} = true ] ; then
527    while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
528        (( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
529        Length=$( create_ts_next_date ${DATE_COUNT} )
530        DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
531    done
532else
533    NBRE_FILE_TOT=0
534fi
535
536if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
537    NBRE_TOUR=1
538elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
539    let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT} 
540else
541    let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
542fi
543
544CURRENT_LOOP=1
545# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
546# WE HAVE 3 LEVELS
547#    - LEVEL 1 : JUST ONE LOOP TO COMPLETE
548#    - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
549#    - LEVEL 3 : WE ARE IN THE LAST LOOP
550
551while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] ; do
552     
553    if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then 
554        # - LEVEL 1 : JUST ONE LOOP TO COMPLETE
555        NBRE_FILE_LOOP=${NBRE_FILE_TOT}
556
557        if [ ${FIRST_PASS} = TRUE ] ; then
558            DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
559        else
560            Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
561            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
562        fi
563
564        DATE_FIN=${PeriodDateEnd}
565        DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
566
567        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
568        [ -n "${LEVEL}" ] && FLAG_B="TRUE" || FLAG_B="FALSE"
569
570        LEVEL=1
571        DEPOT="TRUE"     
572    elif [ ${CURRENT_LOOP} -ne  ${NBRE_TOUR} ] ; then
573        # - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
574        NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
575
576        if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
577            Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
578            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
579        elif [ -z "${DATE_COURANTE}" ] ; then
580            DATE_COURANTE=${DATE_FIN_JOB_B}
581        else
582            Length=$( create_ts_next_date ${DATE_FIN} )
583            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
584        fi
585         
586        (( TotLength = 0 ))
587        COMPTEUR=1
588        DATE_LOOP=${DATE_COURANTE}
589
590        while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
591            #
592            Length=$( create_ts_next_date ${DATE_LOOP} )
593            DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
594            (( TotLength = TotLength + Length ))
595            (( COMPTEUR = COMPTEUR + 1 ))
596            #
597        done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
598        DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
599
600        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
601        [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" || [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" || FLAG_B="FALSE"
602        DEPOT="FALSE"
603        LEVEL=2
604    else
605        # - LEVEL 3 : WE ARE IN THE LAST LOOP
606        NBRE_FILE_LOOP=$( expr ${NBRE_FILE_TOT} % $(( UNIX_MAX_LIMIT * $(( CURRENT_LOOP - 1 )) )) )
607        [ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
608
609        Length=$( create_ts_next_date ${DATE_FIN} )
610        DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
611        DATE_FIN=${PeriodDateEnd}
612        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
613
614        LEVEL=3
615        DEPOT="TRUE"
616        FLAG_B="TRUE"
617    fi
618
619    # THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
620    for comp in ${config_ListOfComponents[*]} ; do
621        #
622        IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
623        #
624        IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
625        #
626        eval R_OUT_${comp}=${R_SAVE}/${comp}
627        #
628        i=0
629        #
630        for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
631            #
632            # Determine in which output can we find file
633            # IE : Output/MO or Output/DA or ...
634            #
635            FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
636            case ${FlagDir} in
637                *Y)    TS_Dir=TS_YE  ; FreqDir=YE  ;;
638                *M)    TS_Dir=TS_MO  ; FreqDir=MO  ;;
639                *D)    TS_Dir=TS_DA  ; FreqDir=DA  ;;
640                HF)    TS_Dir=TS_HF  ; FreqDir=HF  ;;
641                INS)   TS_Dir=TS_INS ; FreqDir=INS ;;
642                *) 
643                       IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
644                       IGCM_sys_Exit "Job create_ts" ;;
645            esac
646
647            # Initialize array
648            #set -A liste_file
649            #set -A liste_file_tmp
650            unset liste_file
651            unset liste_file_tmp
652            eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
653
654            COMPTEUR=1
655            DATE_LOOP=${DATE_COURANTE}
656
657            while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
658                #
659                Length1=$( create_ts_begin_date ${DATE_LOOP} end)
660                DATE_TAB=$(  IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
661
662                Length2=$( create_ts_next_date ${DATE_LOOP} end)
663                DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
664
665                #set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
666                #set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
667                liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
668                liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
669                (( COMPTEUR = COMPTEUR + 1 ))
670                #
671            done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
672            DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
673
674            # Get selected files
675            IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
676            ExitFlag=false
677            IGCM_debug_Verif_Exit_Post
678
679            # Apply IGCM_Patch if needed
680            if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) !=  X${NULL_STR} ]; then
681                for file_t in $( eval echo ${liste_file_tmp[*]} ); do           
682                    for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
683                        IGCM_${Patch} ${file_t}
684                    done
685                done
686            fi
687
688            # WE CAN CONCATENATE FILES AT THIS POINT
689            [ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
690            Length=$( create_ts_begin_date ${DATE_COURANTE} end)
691            DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
692            DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
693            DATE_BUILD_END=${DateBegin}_${DATE_FIN}
694
695            liste_coord=" "
696            for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
697              liste_coord=${liste_coord}${axis}","
698            done
699
700            # Time axis must be the last dimension
701            time_axis=${axis}
702
703            # SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
704            for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
705                #
706                IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
707                #
708#                 # Change time axis name to be CF compliant
709#                 if [ ${time_axis} != time_counter ]; then
710#                   IGCM_sys_ncrename -O -v ${time_axis},time_counter ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
711#                 fi
712                #
713                if [ "${FLAG_B}" = "TRUE" ] ; then
714                    # WE CONCATENATE WITH EXISTING FILES
715                    file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
716                    file2=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
717                    file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
718                    IGCM_sys_ncrcat --hst ${file1} ${file2} ${file_out}
719                    #
720                    IGCM_sys_Rm ${file1} ${file2}
721                fi
722                #
723                if [ "${DEPOT}" = "TRUE" ] ; then
724                    # WE PUT FINAL FILE ON FILE SYSTEM
725                    [ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
726                    #${ncatted} -a modulo,time_counter,c,c," " ${file_out}
727                    file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
728                    eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
729                    [ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
730                    IGCM_sys_Rm ${file_out}
731                    #
732                fi
733                #
734                ExitFlag=false
735                IGCM_debug_Verif_Exit_Post
736            done     # for var in ${liste_var} ; do
737            IGCM_sys_Rm ${liste_file_tmp[*]}
738        done         # for file in ${liste_file_brut} do
739    done             # for comp in ${config_ListOfComponents[*]} ; do
740    ExitFlag=false
741    IGCM_debug_Verif_Exit_Post
742    # GO AHEAD IN DEEPEST LOOP
743    (( CURRENT_LOOP = CURRENT_LOOP + 1 ))
744done                 # while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] do
745# FINISH
746
747# DODS copy
748for comp in ${config_ListOfComponents[*]} ; do
749    for TS_Dir in TS_YE TS_MO TS_DA TS_HF TS_INS ; do
750        IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
751        [ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
752    done
753done
754
755# clean environment variables and export usefull one :
756if [ X$( hostname -s ) = Xrhodes ] ; then
757    LISTE_ENV=$( env | grep -iv MYLANG | grep -iv HOST | grep -iv LOGIN | grep -iv config_ | grep -iv R_ | grep -iv libIGCM | grep -iv PERIOD \
758        | grep -iv DMFDIR | grep -iv AN_| grep -iv YEARS | grep -iv ex | grep -iv RESOL_ | grep -iv PATH | grep -iv WORKDIR | grep -iv USER   \
759        | grep -iv LOGNAME | grep -iv PROFILE | grep -iv ENV | grep -iv TMPDIR | grep -iv POST_DIR | grep -iv StandAlone | grep -iv DB | awk -F= '{print $1}' )
760 
761    for variables in ${LISTE_ENV} ; do
762        unset ${variables}
763    done
764fi
765
766export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
767export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_SRF ; export RESOL_STO
768
769listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_SRF,RESOL_STO"
770export listVarEnv
771
772#D-
773# --------------------------------------------------------------------
774#D- Test if all was right
775# --------------------------------------------------------------------
776IGCM_debug_Verif_Exit_Post
777# --------------------------------------------------------------------
778#D- Submit MONITORING
779# --------------------------------------------------------------------
780IGCM_sys_QsubPost monitoring
781
782#D-
783# --------------------------------------------------------------------
784#D- Update the run.card
785# --------------------------------------------------------------------
786if [ ${StandAlone} != true ] ; then
787    # Put in run.card end period of time series
788    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
789    # We have finish the job !
790    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
791fi
792date
Note: See TracBrowser for help on using the repository browser.