source: TOOLS/AA_stomate_flux_correction @ 1322

Last change on this file since 1322 was 1322, checked in by mmaipsl, 12 years ago

Add documentation.
Add a chain of jobs for long simulation treatments :
1) You must read 'Documentation' paragraph below to parameter the chain of jobs for
specific schedulers.
2) This script must only be used in a libIGCM directory (but submit from a CONFIG directory).
3) Default parameters in this job are given for mercure CCRT X64 frontend.

  • Property svn:keywords set to Date Author Revision
File size: 16.6 KB
Line 
1#-Q- cesium #!/bin/ksh
2#-Q- cesium ######################
3#-Q- cesium ## CESIUM   CEA ##
4#-Q- cesium ######################
5#-Q- cesium #MSUB -r SBGcorr             # Nom du job               
6#-Q- cesium #MSUB -N 1              # Reservation du noeud
7#-Q- cesium #MSUB -n 1              # Reservation du processus
8#-Q- cesium #MSUB -T 86400          # Limite de temps elapsed du job
9#-Q- cesium #MSUB -E "-j o"
10#-Q- cesium #MSUB -E "-S /bin/ksh"
11#-Q- platine #!/usr/bin/ksh
12#-Q- platine ###################
13#-Q- platine ## PLATINE   CEA ##
14#-Q- platine ###################
15#-Q- platine #BSUB -J SBGcorr                     # Nom du job
16#-Q- platine #BSUB -N                        # message a la fin du job
17#-Q- platine #BSUB -n 1                      # reservation des processeurs pour le job
18#-Q- platine #BSUB -W 1:00                   # Limite temps
19#-Q- platine #BSUB -q post                   # Passage en queue post
20#-Q- sx8brodie #!/bin/ksh
21#-Q- sx8brodie #######################
22#-Q- sx8brodie ## SX8BRODIE   IDRIS ##
23#-Q- sx8brodie #######################
24#-Q- sx8brodie # Temps Elapsed max. d'une requete hh:mm:ss
25#-Q- sx8brodie # @ wall_clock_limit = 20:00:00
26#-Q- sx8brodie # Nom du travail LoadLeveler
27#-Q- sx8brodie # @ job_name   = SBGcorr
28#-Q- sx8brodie # Fichier de sortie standard du travail       
29#-Q- sx8brodie # @ output     = $(job_name).$(jobid)
30#-Q- sx8brodie # Fichier de sortie d'erreur du travail
31#-Q- sx8brodie # @ error      =  $(job_name).$(jobid)
32#-Q- sx8brodie # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
33#-Q- sx8brodie # @ notification = error
34#-Q- sx8brodie # @ environment  = $CURRENT_LOOP ; $imin ; $imax
35#-Q- sx8brodie # @ queue
36#-Q- aix6 #!/bin/ksh
37#-Q- aix6 #######################
38#-Q- aix6 ##   VARGAS   IDRIS  ##
39#-Q- aix6 #######################
40#-Q- aix6 # Temps Elapsed max. d'une requete hh:mm:ss
41#-Q- aix6 # @ wall_clock_limit = 20:00:00
42#-Q- aix6 # Nom du travail LoadLeveler
43#-Q- aix6 # @ job_name   = SBGcorr
44#-Q- aix6 # Fichier de sortie standard du travail
45#-Q- aix6 # @ output     = $(job_name).$(jobid)
46#-Q- aix6 # Fichier de sortie d'erreur du travail
47#-Q- aix6 # @ error      =  $(job_name).$(jobid)
48#-Q- aix6 # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
49#-Q- aix6 # @ notification = error
50#-Q- aix6 # @ environment  = $CURRENT_LOOP ; $imin ; $imax
51#-Q- aix6 # @ queue
52#-Q- ulam #!/bin/ksh
53#-Q- ulam #######################
54#-Q- ulam ##   ULAM   IDRIS  ##
55#-Q- ulam #######################
56#-Q- ulam # Temps Elapsed max. d'une requete hh:mm:ss
57#-Q- ulam # @ wall_clock_limit = 20:00:00
58#-Q- ulam # Nom du travail LoadLeveler
59#-Q- ulam # @ job_name   = SBGcorr
60#-Q- ulam # Fichier de sortie standard du travail
61#-Q- ulam # @ output     = $(job_name).$(jobid)
62#-Q- ulam # Fichier de sortie d'erreur du travail
63#-Q- ulam # @ error      =  $(job_name).$(jobid)
64#-Q- ulam # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
65#-Q- ulam # @ notification = error
66#-Q- ulam # @ environment  = $CURRENT_LOOP ; $imin ; $imax
67#-Q- ulam # @ queue
68#-Q- sx8mercure #!/bin/ksh
69#-Q- sx9mercure #########################
70#-Q- sx9mercure ## CESIUM FOR SX9  CEA ##
71#-Q- sx9mercure #########################
72#-Q- sx9mercure #MSUB -r SBGcorr             # Nom du job               
73#-Q- sx9mercure #MSUB -N 1              # Reservation du noeud
74#-Q- sx9mercure #MSUB -n 1              # Reservation du processus
75#-Q- sx9mercure #MSUB -T 86400          # Limite de temps elapsed du job
76#-Q- sx9mercure #MSUB -E "-j o"
77#-Q- sx9mercure #MSUB -E "-S /bin/ksh"
78#-Q- sx8mercure ######################
79#-Q- sx8mercure ## SX8MERCURE   CEA ##
80#-Q- sx8mercure ######################
81#-Q- sx8mercure #PBS -N SBGcorr                   # Nom du job
82#-Q- sx8mercure #PBS -j o                    # regroupement des stdout et stderr
83#-Q- sx8mercure #PBS -S /usr/bin/ksh         # shell de soumission
84#-Q- sx8mercure #PBS -l memsz_job=1gb        # Limite memoire a 1 Go
85#-Q- sx8mercure #PBS -l cputim_job=24:00:00   # Limite temps a 2 heures
86#-Q- sx8mercure #PBS -q scalaire
87#-Q- sx9mercure #!/bin/ksh
88#-Q- sx9mercure #########################
89#-Q- sx9mercure ## CESIUM FOR SX9  CEA ##
90#-Q- sx9mercure #########################
91#-Q- sx9mercure #MSUB -r SBGcorr             # Nom du job               
92#-Q- sx9mercure #MSUB -N 1              # Reservation du noeud
93#-Q- sx9mercure #MSUB -n 1              # Reservation du processus
94#-Q- sx9mercure #MSUB -T 86400          # Limite de temps elapsed du job
95#-Q- sx9mercure #MSUB -E "-j o"
96#-Q- sx9mercure #MSUB -E "-S /bin/ksh"
97#-Q- titane #!/bin/ksh
98#-Q- titane ######################
99#-Q- titane ## TITANE   CEA ##
100#-Q- titane ######################
101#-Q- titane #MSUB -r SBGcorr             # Nom du job               
102#-Q- titane #MSUB -N 1              # Reservation du noeud
103#-Q- titane #MSUB -n 1              # Reservation du processus
104#-Q- titane #MSUB -T 86400          # Limite de temps elapsed du job
105#-Q- titane #MSUB -E "-j o"
106#-Q- titane #MSUB -E "-S /bin/ksh"
107#-Q- lxiv8 ######################
108#-Q- lxiv8 ## OBELIX      LSCE ##
109#-Q- lxiv8 ######################
110#-Q- lxiv8 #PBS -N SBGcorr
111#-Q- lxiv8 #PBS -m a
112#-Q- lxiv8 #PBS -j oe
113#-Q- lxiv8 #PBS -q medium
114#-Q- lxiv8 #PBS -o SBGcorr.$$
115#-Q- lxiv8 #PBS -S /bin/ksh
116#-Q- default #!/bin/ksh
117#-Q- default ##################
118#-Q- default ## DEFAULT HOST ##
119#-Q- default ##################
120
121#**************************************************************
122# Author: Martial Mancip
123# Contact: Martial.Mancip__at__ipsl.jussieu.fr
124# $Revision::                                          $ Revision of last commit
125# $Author::                                            $ Author of last commit
126# $Date::                                              $ Date of last commit
127# IPSL (2006)
128#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
129#
130#**************************************************************
131
132
133#**************************************************************
134# usage :
135#
136#  This script is only used for correction of stomate_ipcc_history files created
137# with orchidee_1_9_5 version of ORCHIDEE model checkouted before 01/0   
138# Version of ORCHIDEE/src_stomate/stomate_lpj.f90 before CVS 1.26 had a bug
139# that give a wrong Land Cover Change (fLuc) variable and a wrong nbp balance variable.
140# This file will correct all those IPCC CMIP5 history files in a simulation.
141# You must read 'Documentation' paragraph below to parameter the chain of jobs for
142# specific schedulers.
143# This script must only be used in a libIGCM directory
144#     (but submit from a CONFIG directory).
145# Default parameters in this job are given for mercure CCRT X64 frontend. 
146#
147#**************************************************************
148
149
150
151
152date
153
154#-Q- sx8brodie export OMP_NUM_THREADS=1
155#-Q- aix6 export OMP_NUM_THREADS=1
156#-Q- ulam export OMP_NUM_THREADS=1
157
158########################################################################
159
160#D- Low level debug : to bypass lib test checks and stack construction
161#D- Default : value from AA_job if any
162libIGCM=${libIGCM:=::modipsl::/libIGCM}
163# WARNING for StandAlone used : To run this script on some machine,
164# you must check MirrorlibIGCM variable in sys library.
165# If this variable is true, you must use libIGCM_POST path instead
166# of your running libIGCM directory.
167
168
169#D- Flag to determine job's output directory
170#D- Default : value from libIGCM_post.ksh if any
171POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
172
173#D- Increased verbosity (1, 2, 3)
174#D- Default : value from AA_job if any
175Verbosity=${Verbosity:=3}
176
177#D- Experience type : DEB(ug), DEV(elopment), RUN (default)
178JobType=RUN
179
180
181### Documentation :
182##      1) To start again this script in StandAlone mode
183##         (because it fails with a problem with the system or a timeout, for example),
184##         you must check the initialisation of three variables after the "Start loop" line :
185##    CURRENT_LOOP=${CURRENT_LOOP:=1} give the iteration to begin the main loop
186##    imin=${imin:=0}                 give the corresponding min index in the list of stomate_history files
187##    imax=${imax:=${NbFilesLOOP}}    give the corresponding max index in the list of stomate_history files
188##         nb: the main loop is parametered by UNIX_MAX_LIMIT (in system library) value.
189##    imax is a factor of NbFilesLOOP = UNIX_MAX_LIMIT / 2 value
190##    imin is the previous imax (see previous output of the script if exist)
191##      2) This script will re-submit itself automaticly after calculate the next values of those three variables.
192
193##   To prevent timeout because of a large number of files (long simulation) and maximum elapse time
194## in the PBS queue, you must give the correct value of next line LOOP_MAX variable.
195## It gives the maximum of iterations of the MAIN LOOP during the elapse time required in the PBS.
196LOOP_MAX=15
197#
198## For example: if UNIX_MAX_LIMIT=360 and the number of stomate_history for a run if 3800,
199##              the number of files that has to be demigrated if 7600 (stomate_history + stomate_ipcc_history).
200##              The NBRE_TOUR (see previous output) calculate is 22.
201##              If the elasped time required is 24H (86400s) and each 10 years (120x2 = 240 files) take one hour,
202##              you must give LOOP_MAX= 240 * 24 / 360 = 16 max.
203##              To prevent system slow, the default value is 15.
204
205
206########################################################################
207
208. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
209      ( ${DEBUG_debug} ) && IGCM_debug_Check
210. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
211     ( ${DEBUG_debug} ) && IGCM_card_Check
212. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
213     ( ${DEBUG_debug} ) && IGCM_date_Check
214#-------
215. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
216
217########################################################################
218
219#set -vx
220
221IGCM_sys_MkdirWork ${RUN_DIR_PATH}
222IGCM_sys_Cd ${RUN_DIR_PATH}
223
224
225#
226# First of all
227#
228IGCM_card_DefineArrayFromSection       ${SUBMIT_DIR}/config.card UserChoices
229typeset option
230for option in ${config_UserChoices[*]} ; do
231    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/config.card UserChoices ${option}
232done
233#
234echo
235IGCM_debug_Print 1 "DefineArrayFromOption  : config_UserChoices"
236IGCM_debug_PrintVariables 3 config_UserChoices_JobName
237IGCM_debug_PrintVariables 3 config_UserChoices_LongName
238IGCM_debug_PrintVariables 3 config_UserChoices_TagName
239IGCM_debug_PrintVariables 3 config_UserChoices_SpaceName
240IGCM_debug_PrintVariables 3 config_UserChoices_ExperimentName
241IGCM_debug_PrintVariables 3 config_UserChoices_CalendarType
242IGCM_debug_PrintVariables 3 config_UserChoices_DateBegin
243IGCM_debug_PrintVariables 3 config_UserChoices_DateEnd
244IGCM_debug_PrintVariables 3 config_UserChoices_PeriodLength
245echo
246
247#==================================
248#R_SAVE : Job output directory
249if ( [ ! X${config_UserChoices_SpaceName} = X ] && [ ! X${config_UserChoices_ExperimentName} = X ] ) ; then
250    FreeName=$( echo ${config_UserChoices_JobName} | sed 's/.*_//' )
251    R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
252    R_DODS=${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
253else
254    R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
255    R_DODS=${config_UserChoices_TagName}/${config_UserChoices_JobName}
256fi
257
258
259########################################################################
260#
261#                          DEFINE LOOP PARAMETERS
262#
263########################################################################
264
265R_OUT_SBG=${R_SAVE}/SBG
266file_path=${R_OUT_SBG}/Output/MO
267
268set +A LISTE_FILE_SBG -- $( IGCM_sys_RshArchive "find ${file_path} -name \"*stomate_history.nc\" " )
269CountLISTE_FILE_SBG=${#LISTE_FILE_SBG[@]}
270echo "Nombre de stomate_history : " ${CountLISTE_FILE_SBG}
271
272NBRE_FILE_TOT=$(( 2 * CountLISTE_FILE_SBG ))
273echo "Nombre de fichiers à démigrer : " ${NBRE_FILE_TOT}
274
275if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
276    NBRE_TOUR=1
277    NbFilesLOOP=${CountLISTE_FILE_SBG}
278elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
279    (( NBRE_TOUR = NBRE_FILE_TOT / UNIX_MAX_LIMIT ))
280    (( NbFilesLOOP = UNIX_MAX_LIMIT / 2 ))
281else
282    (( NBRE_TOUR = NBRE_FILE_TOT / UNIX_MAX_LIMIT + 1 ))
283    (( NbFilesLOOP = UNIX_MAX_LIMIT / 2 ))
284fi
285echo "NBRE_TOUR = ${NBRE_TOUR} "
286echo "NbFilesLOOP = ${NbFilesLOOP} "
287
288########################################################################
289#
290#                          START THE MAIN LOOP
291#
292########################################################################
293
294# PARAMETERS OF THE LOOP :
295# Cumultate CURRENT LOOP number over submitions
296CURRENT_LOOP=${CURRENT_LOOP:=1}
297# min index for list of local files
298imin=${imin:=0}
299# max index for list of local files
300imax=${imax:=${NbFilesLOOP}}
301
302# DEPEND OF THE ELAPSED TIME RESERVED FOR LOOP_MAX ITERATIONS
303LOOP_MAX=${LOOP_MAX:=NBRE_TOUR}
304
305IGCM_debug_Verif_Exit_Post
306
307# MAIN LOOP
308# LOCAL CURRENT LOOP number less than LOOP_MAX
309CURRENT_LOOP0=1
310while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
311
312    echo "###############################################################"
313    echo "LOOP : " ${CURRENT_LOOP}
314    echo "imin/imax : " $imin " / " $imax
315   
316    unset liste_file
317    unset liste_file_local
318    unset liste_file_ipcc
319    unset liste_file_ipcc_local
320   
321    (( i = imin ))
322    while [ ${i} -lt ${imax} ] ; do
323        file=${LISTE_FILE_SBG[${i}]}
324        file_ipcc=$( echo ${file} | sed -e "s/stomate_history/stomate_ipcc_history/" )
325
326        liste_file[${#liste_file[*]}]=${file}
327        liste_file_local[${#liste_file_local[*]}]=$( basename ${file} )
328        RET=$?
329        if [ ${RET} -gt 0 ] ; then
330            echo "erreur basename liste_file_local " ${file}
331            echo "file = " ${file}
332            exit 1
333        fi
334        liste_file[${#liste_file[*]}]=${file_ipcc}
335
336        liste_file_ipcc[${#liste_file_ipcc[*]}]=${file_ipcc}
337        liste_file_ipcc_local[${#liste_file_ipcc_local[*]}]=$( basename ${file_ipcc} )
338        RET=$?
339        if [ ${RET} -gt 0 ] ; then
340            echo "erreur basename liste_file_ipcc_local " ${file_ipcc}
341            echo "file = " ${file_ipcc}
342            exit 3
343        fi
344
345        (( i = i + 1 ))
346    done
347    #echo ${liste_file_local[@]}
348    #echo ${liste_file[@]}
349
350    # Get selected files
351    IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}
352    IGCM_debug_Verif_Exit_Post
353
354    # chmod sur dépÃŽt
355    IGCM_sys_RshArchive "chmod u+w ${liste_file_ipcc[@]}"
356
357    # Faire le nco/ncap2
358    (( i = imin ))
359    while [ ${i} -lt ${imax} ] ; do
360        file=$( basename ${LISTE_FILE_SBG[${i}]} )
361        RET=$?
362        if [ ${RET} -gt 0 ] ; then
363            echo "erreur basename"
364            echo "i, imin, imax = " i imin imax
365            echo "file = " ${file}
366            exit 4
367        fi
368        file_ipcc=$( echo ${file} | sed -e "s/stomate_history/stomate_ipcc_history/" )
369        ls -la ${file_ipcc}
370
371        IGCM_sys_ncks -a -A -x -v GPP ${file} stomate.nc
372        IGCM_sys_ncks -a -A ${file_ipcc} stomate.nc
373
374        IGCM_sys_Cp ${file_ipcc} _${file_ipcc}
375        IGCM_sys_Chmod u+w _${file_ipcc}
376
377        IGCM_sys_ncap2 -s 'fLuc=float(fLuc*0.+(CONVFLUX+CFLUX_PROD10+CFLUX_PROD100)/1e3/86400*CONTFRAC)' \
378                       -s 'nbp=float(nbp*0.f+(gpp-ra-rh-fFire-fLuc-fHarvest))' -v stomate.nc stomate1.nc
379        IGCM_debug_Verif_Exit_Post
380
381        IGCM_sys_ncks -a -A stomate1.nc _${file_ipcc}
382        IGCM_sys_Mv -f _${file_ipcc} ${file_ipcc}
383        IGCM_sys_Chmod u-w ${file_ipcc}
384
385        ls -la ${file_ipcc}
386        IGCM_sys_Rm stomate.nc stomate1.nc
387        IGCM_debug_Verif_Exit_Post
388
389        # Put_Out des fichiers ipcc corrigés
390        IGCM_sys_Put_Out ${file_ipcc} ${file_path}/${file_ipcc}
391
392        (( i = i + 1 ))
393    done
394
395    # Nettoyage
396    IGCM_sys_Rm ${liste_file_local[*]}
397
398
399    # GO AHEAD IN DEEPEST LOOP
400    (( imin = imax ))
401    (( CURRENT_LOOP = CURRENT_LOOP + 1 ))
402    (( CURRENT_LOOP0 = CURRENT_LOOP0 + 1 ))
403    if [ ${CURRENT_LOOP} -lt ${NBRE_TOUR} ] ; then
404        (( imax = imax + ${NbFilesLOOP} ))
405    else
406        if [ ${CURRENT_LOOP} -eq ${NBRE_TOUR} ] ; then
407            echo "Dernier tour : "
408            if [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
409                echo "imin NEW = " ${imax}
410                (( imax = imax + ${NbFilesLOOP} ))
411                echo "imax NEW = " ${imax}
412            else
413                echo "imin NEW = " ${imax}
414                (( imax = imax + ( NBRE_FILE_TOT % UNIX_MAX_LIMIT ) / 2 ))
415                echo "imax NEW = " ${imax}
416            fi
417        fi
418    fi
419    if ( [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] && \
420         [ ${CURRENT_LOOP0} -gt ${LOOP_MAX} ] ) ; then
421        export CURRENT_LOOP
422        export imin
423        export imax
424        IGCM_sys_Cd ${SUBMIT_DIR}
425        Script_Post_Output=SBGcorr.${imin}.${imax}
426        listVarEnv="CURRENT_LOOP,imin,imax"
427        echo "Submit next job."
428        IGCM_sys_QsubPost stomate_flux_correction
429        break ;
430    fi
431done                 # while [ ${CURRENT_LOOP} -le  ${LOOP_MAX} ] do
432# FINISH
433
434
435#D-
436# --------------------------------------------------------------------
437#D- Test if all was right
438# --------------------------------------------------------------------
439IGCM_debug_Verif_Exit_Post
440
441# Clean RUN_DIR_PATH (necessary for cesium)
442IGCM_sys_Cd ${SUBMIT_DIR}
443IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
444
445date
Note: See TracBrowser for help on using the repository browser.