source: TOOLS/AA_stomate_flux_correction @ 1376

Last change on this file since 1376 was 1326, checked in by mmaipsl, 13 years ago

with Arnaud we have corrected some documentations and add SAVE directories
to protect original files from a script or system crash.
  • Property svn:keywords set to Date Author Revision
File size: 16.3 KB
Line 
1#-Q- cesium #!/bin/ksh
2#-Q- cesium ######################
3#-Q- cesium ## CESIUM   CEA ##
4#-Q- cesium ######################
5#-Q- cesium #MSUB -r SBGcorr             # Nom du job               
6#-Q- cesium #MSUB -N 1              # Reservation du noeud
7#-Q- cesium #MSUB -n 1              # Reservation du processus
8#-Q- cesium #MSUB -T 86400          # Limite de temps elapsed du job
9#-Q- cesium #MSUB -E "-j o"
10#-Q- cesium #MSUB -E "-S /bin/ksh"
11#-Q- platine #!/usr/bin/ksh
12#-Q- platine ###################
13#-Q- platine ## PLATINE   CEA ##
14#-Q- platine ###################
15#-Q- platine #BSUB -J SBGcorr                     # Nom du job
16#-Q- platine #BSUB -N                        # message a la fin du job
17#-Q- platine #BSUB -n 1                      # reservation des processeurs pour le job
18#-Q- platine #BSUB -W 1:00                   # Limite temps
19#-Q- platine #BSUB -q post                   # Passage en queue post
20#-Q- sx8brodie #!/bin/ksh
21#-Q- sx8brodie #######################
22#-Q- sx8brodie ## SX8BRODIE   IDRIS ##
23#-Q- sx8brodie #######################
24#-Q- sx8brodie # Temps Elapsed max. d'une requete hh:mm:ss
25#-Q- sx8brodie # @ wall_clock_limit = 20:00:00
26#-Q- sx8brodie # Nom du travail LoadLeveler
27#-Q- sx8brodie # @ job_name   = SBGcorr
28#-Q- sx8brodie # Fichier de sortie standard du travail       
29#-Q- sx8brodie # @ output     = $(job_name).$(jobid)
30#-Q- sx8brodie # Fichier de sortie d'erreur du travail
31#-Q- sx8brodie # @ error      =  $(job_name).$(jobid)
32#-Q- sx8brodie # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
33#-Q- sx8brodie # @ notification = error
34#-Q- sx8brodie # @ environment  = $CURRENT_LOOP ; $imin ; $imax
35#-Q- sx8brodie # @ queue
36#-Q- aix6 #!/bin/ksh
37#-Q- aix6 #######################
38#-Q- aix6 ##   VARGAS   IDRIS  ##
39#-Q- aix6 #######################
40#-Q- aix6 # Temps Elapsed max. d'une requete hh:mm:ss
41#-Q- aix6 # @ wall_clock_limit = 20:00:00
42#-Q- aix6 # Nom du travail LoadLeveler
43#-Q- aix6 # @ job_name   = SBGcorr
44#-Q- aix6 # Fichier de sortie standard du travail
45#-Q- aix6 # @ output     = $(job_name).$(jobid)
46#-Q- aix6 # Fichier de sortie d'erreur du travail
47#-Q- aix6 # @ error      =  $(job_name).$(jobid)
48#-Q- aix6 # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
49#-Q- aix6 # @ notification = error
50#-Q- aix6 # @ environment  = $CURRENT_LOOP ; $imin ; $imax
51#-Q- aix6 # @ queue
52#-Q- ulam #!/bin/ksh
53#-Q- ulam #######################
54#-Q- ulam ##   ULAM   IDRIS  ##
55#-Q- ulam #######################
56#-Q- ulam # Temps Elapsed max. d'une requete hh:mm:ss
57#-Q- ulam # @ wall_clock_limit = 20:00:00
58#-Q- ulam # Nom du travail LoadLeveler
59#-Q- ulam # @ job_name   = SBGcorr
60#-Q- ulam # Fichier de sortie standard du travail
61#-Q- ulam # @ output     = $(job_name).$(jobid)
62#-Q- ulam # Fichier de sortie d'erreur du travail
63#-Q- ulam # @ error      =  $(job_name).$(jobid)
64#-Q- ulam # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
65#-Q- ulam # @ notification = error
66#-Q- ulam # @ environment  = $CURRENT_LOOP ; $imin ; $imax
67#-Q- ulam # @ queue
68#-Q- sx8mercure ######################
69#-Q- sx8mercure ## SX8MERCURE   CEA ##
70#-Q- sx8mercure ######################
71#-Q- sx8mercure #PBS -N SBGcorr                   # Nom du job
72#-Q- sx8mercure #PBS -j o                    # regroupement des stdout et stderr
73#-Q- sx8mercure #PBS -S /usr/bin/ksh         # shell de soumission
74#-Q- sx8mercure #PBS -l memsz_job=1gb        # Limite memoire a 1 Go
75#-Q- sx8mercure #PBS -l cputim_job=24:00:00   # Limite temps a 2 heures
76#-Q- sx8mercure #PBS -q scalaire
77#-Q- sx9mercure #!/bin/ksh
78#-Q- sx9mercure #########################
79#-Q- sx9mercure ## CESIUM FOR SX9  CEA ##
80#-Q- sx9mercure #########################
81#-Q- sx9mercure #MSUB -r SBGcorr             # Nom du job               
82#-Q- sx9mercure #MSUB -N 1              # Reservation du noeud
83#-Q- sx9mercure #MSUB -n 1              # Reservation du processus
84#-Q- sx9mercure #MSUB -T 86400          # Limite de temps elapsed du job
85#-Q- sx9mercure #MSUB -E "-j o"
86#-Q- sx9mercure #MSUB -E "-S /bin/ksh"
87#-Q- titane #!/bin/ksh
88#-Q- titane ######################
89#-Q- titane ## TITANE   CEA ##
90#-Q- titane ######################
91#-Q- titane #MSUB -r SBGcorr             # Nom du job               
92#-Q- titane #MSUB -N 1              # Reservation du noeud
93#-Q- titane #MSUB -n 1              # Reservation du processus
94#-Q- titane #MSUB -T 86400          # Limite de temps elapsed du job
95#-Q- titane #MSUB -E "-j o"
96#-Q- titane #MSUB -E "-S /bin/ksh"
97#-Q- lxiv8 ######################
98#-Q- lxiv8 ## OBELIX      LSCE ##
99#-Q- lxiv8 ######################
100#-Q- lxiv8 #PBS -N SBGcorr
101#-Q- lxiv8 #PBS -m a
102#-Q- lxiv8 #PBS -j oe
103#-Q- lxiv8 #PBS -q medium
104#-Q- lxiv8 #PBS -o SBGcorr.$$
105#-Q- lxiv8 #PBS -S /bin/ksh
106#-Q- default #!/bin/ksh
107#-Q- default ##################
108#-Q- default ## DEFAULT HOST ##
109#-Q- default ##################
110
111#**************************************************************
112# Author: Martial Mancip
113# Contact: Martial.Mancip__at__ipsl.jussieu.fr
114# $Revision::                                          $ Revision of last commit
115# $Author::                                            $ Author of last commit
116# $Date::                                              $ Date of last commit
117# IPSL (2006)
118#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
119#
120#**************************************************************
121
122
123#**************************************************************
124# usage :
125#
126#  This script is only used for correction of stomate_ipcc_history files created
127# with orchidee_1_9_5 version of ORCHIDEE model checkouted before 05/08/10 :   
128# Version of ORCHIDEE/src_stomate/stomate_lpj.f90 before CVS 1.26 had a bug
129# that give a wrong Land Cover Change (fLuc) variable and a wrong nbp balance variable.
130# This file will correct all those IPCC CMIP5 history files in a simulation.
131# You must read 'Documentation' paragraph below to parameter the chain of jobs for
132# specific schedulers.
133#
134# To protect IPCC history files if this script crash, we first move original files
135# in SAVE_${loopmmin} directories.
136# YOU have to destroy them by hand after corrected all your run.
137#
138# This script must only be used in a libIGCM directory
139#     (but submit from a CONFIG directory).
140#
141# Default parameters in this job are given for mercure CCRT X64 frontend. 
142#
143#**************************************************************
144
145
146
147
148date
149
150#-Q- sx8brodie export OMP_NUM_THREADS=1
151#-Q- aix6 export OMP_NUM_THREADS=1
152#-Q- ulam export OMP_NUM_THREADS=1
153
154########################################################################
155
156#D- Low level debug : to bypass lib test checks and stack construction
157#D- Default : value from AA_job if any
158libIGCM=${libIGCM:=::modipsl::/libIGCM}
159# WARNING for StandAlone used : To run this script on some machine,
160# you must check MirrorlibIGCM variable in sys library.
161# If this variable is true, you must use libIGCM_POST path instead
162# of your running libIGCM directory.
163
164
165#D- Flag to determine job's output directory
166#D- Default : value from libIGCM_post.ksh if any
167POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
168
169#D- Increased verbosity (1, 2, 3)
170#D- Default : value from AA_job if any
171Verbosity=${Verbosity:=3}
172
173#D- Experience type : DEB(ug), DEV(elopment), RUN (default)
174JobType=RUN
175
176
177### Documentation :
178##      1) To start again this script in StandAlone mode
179##         (because it fails with a problem with the system or a timeout, for example),
180##         you must check the initialisation of three variables after the "Start loop" line :
181##    CURRENT_LOOP=${CURRENT_LOOP:=1} give the iteration to begin the main loop
182##    imin=${imin:=0}                 give the corresponding min index in the list of stomate_history files
183##    imax=${imax:=${NbFilesLOOP}}    give the corresponding max index in the list of stomate_history files
184##         nb: the main loop is parametered by UNIX_MAX_LIMIT (in system library) value.
185##    imax is a factor of NbFilesLOOP = UNIX_MAX_LIMIT / 2 value
186##    imin is the previous imax (see previous output of the script if exist)
187##      2) This script will re-submit itself automaticly after calculate the next values of those three variables.
188
189##   To prevent timeout because of a large number of files (long simulation) and maximum elapse time
190## in the PBS queue, you must give the correct value of next line LOOP_MAX variable.
191## It gives the maximum of iterations of the MAIN LOOP during the elapse time required in the PBS.
192LOOP_MAX=15
193#
194## For example: if UNIX_MAX_LIMIT=360 and the number of stomate_history for a run if 3800,
195##              the number of files that has to be demigrated if 7600 (stomate_history + stomate_ipcc_history).
196##              The NBRE_TOUR (see previous output) calculate is 22.
197##              If the elasped time required is 24H (86400s) and each 10 years (120x2 = 240 files) take one hour,
198##              you must give LOOP_MAX= 240 * 24 / 360 = 16 max.
199##              To prevent system slow, the default value is 15.
200
201
202########################################################################
203
204. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
205      ( ${DEBUG_debug} ) && IGCM_debug_Check
206. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
207     ( ${DEBUG_debug} ) && IGCM_card_Check
208. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
209     ( ${DEBUG_debug} ) && IGCM_date_Check
210#-------
211. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
212
213########################################################################
214
215#set -vx
216
217IGCM_sys_MkdirWork ${RUN_DIR_PATH}
218IGCM_sys_Cd ${RUN_DIR_PATH}
219
220
221#
222# First of all
223#
224IGCM_card_DefineArrayFromSection       ${SUBMIT_DIR}/config.card UserChoices
225typeset option
226for option in ${config_UserChoices[*]} ; do
227    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/config.card UserChoices ${option}
228done
229#
230echo
231IGCM_debug_Print 1 "DefineArrayFromOption  : config_UserChoices"
232IGCM_debug_PrintVariables 3 config_UserChoices_JobName
233IGCM_debug_PrintVariables 3 config_UserChoices_LongName
234IGCM_debug_PrintVariables 3 config_UserChoices_TagName
235IGCM_debug_PrintVariables 3 config_UserChoices_SpaceName
236IGCM_debug_PrintVariables 3 config_UserChoices_ExperimentName
237IGCM_debug_PrintVariables 3 config_UserChoices_CalendarType
238IGCM_debug_PrintVariables 3 config_UserChoices_DateBegin
239IGCM_debug_PrintVariables 3 config_UserChoices_DateEnd
240IGCM_debug_PrintVariables 3 config_UserChoices_PeriodLength
241echo
242
243#==================================
244#R_SAVE : Job output directory
245if ( [ ! X${config_UserChoices_SpaceName} = X ] && [ ! X${config_UserChoices_ExperimentName} = X ] ) ; then
246    FreeName=$( echo ${config_UserChoices_JobName} | sed 's/.*_//' )
247    R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
248    R_DODS=${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
249else
250    R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
251    R_DODS=${config_UserChoices_TagName}/${config_UserChoices_JobName}
252fi
253
254
255########################################################################
256#
257#                          DEFINE LOOP PARAMETERS
258#
259########################################################################
260
261R_OUT_SBG=${R_SAVE}/SBG
262file_path=${R_OUT_SBG}/Output/MO
263
264set +A LISTE_FILE_SBG -- $( IGCM_sys_RshArchive "find ${file_path} -name \"*stomate_history.nc\" " )
265CountLISTE_FILE_SBG=${#LISTE_FILE_SBG[@]}
266echo "Nombre de stomate_history : " ${CountLISTE_FILE_SBG}
267
268NBRE_FILE_TOT=$(( 2 * CountLISTE_FILE_SBG ))
269echo "Nombre de fichiers à démigrer : " ${NBRE_FILE_TOT}
270
271if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
272    NBRE_TOUR=1
273    NbFilesLOOP=${CountLISTE_FILE_SBG}
274elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
275    (( NBRE_TOUR = NBRE_FILE_TOT / UNIX_MAX_LIMIT ))
276    (( NbFilesLOOP = UNIX_MAX_LIMIT / 2 ))
277else
278    (( NBRE_TOUR = NBRE_FILE_TOT / UNIX_MAX_LIMIT + 1 ))
279    (( NbFilesLOOP = UNIX_MAX_LIMIT / 2 ))
280fi
281echo "NBRE_TOUR = ${NBRE_TOUR} "
282echo "NbFilesLOOP = ${NbFilesLOOP} "
283
284########################################################################
285#
286#                          START THE MAIN LOOP
287#
288########################################################################
289
290# PARAMETERS OF THE LOOP :
291# Cumultate CURRENT LOOP number over submitions
292CURRENT_LOOP=${CURRENT_LOOP:=1}
293# min index for list of local files
294imin=${imin:=0}
295# max index for list of local files
296imax=${imax:=${NbFilesLOOP}}
297
298# DEPEND OF THE ELAPSED TIME RESERVED FOR LOOP_MAX ITERATIONS
299LOOP_MAX=${LOOP_MAX:=NBRE_TOUR}
300
301IGCM_debug_Verif_Exit_Post
302
303# MAIN LOOP
304# LOCAL CURRENT LOOP number less than LOOP_MAX
305CURRENT_LOOP0=1
306while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
307
308    echo "###############################################################"
309    echo "CURRENT_LOOP : " ${CURRENT_LOOP}
310    echo "imin/imax : " $imin " / " $imax
311   
312    unset liste_file
313    unset liste_file_local
314    unset liste_file_ipcc
315    unset liste_file_ipcc_local
316   
317    (( i = imin ))
318    while [ ${i} -lt ${imax} ] ; do
319        file=${LISTE_FILE_SBG[${i}]}
320        file_ipcc=$( echo ${file} | sed -e "s/stomate_history/stomate_ipcc_history/" )
321
322        liste_file[${#liste_file[*]}]=${file}
323        liste_file_local[${#liste_file_local[*]}]=$( basename ${file} )
324        RET=$?
325        if [ ${RET} -gt 0 ] ; then
326            echo "erreur basename liste_file_local " ${file}
327            echo "file = " ${file}
328            exit 1
329        fi
330        liste_file[${#liste_file[*]}]=${file_ipcc}
331
332        liste_file_ipcc[${#liste_file_ipcc[*]}]=${file_ipcc}
333        liste_file_ipcc_local[${#liste_file_ipcc_local[*]}]=$( basename ${file_ipcc} )
334        RET=$?
335        if [ ${RET} -gt 0 ] ; then
336            echo "erreur basename liste_file_ipcc_local " ${file_ipcc}
337            echo "file = " ${file_ipcc}
338            exit 3
339        fi
340
341        (( i = i + 1 ))
342    done
343    #echo ${liste_file_local[@]}
344    #echo ${liste_file[@]}
345
346    # Get selected files
347    IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}
348    IGCM_debug_Verif_Exit_Post
349
350    # chmod sur dépÃŽt
351    IGCM_sys_RshArchive "mkdir ${file_path}/SAVE_${imin}"
352    IGCM_sys_RshArchive "mv ${liste_file_ipcc[@]} ${file_path}/SAVE_${imin}"
353
354    # Faire le nco/ncap2
355    (( i = imin ))
356    while [ ${i} -lt ${imax} ] ; do
357        file=$( basename ${LISTE_FILE_SBG[${i}]} )
358        RET=$?
359        if [ ${RET} -gt 0 ] ; then
360            echo "erreur basename"
361            echo "i, imin, imax = " i imin imax
362            echo "file = " ${file}
363            exit 4
364        fi
365        file_ipcc=$( echo ${file} | sed -e "s/stomate_history/stomate_ipcc_history/" )
366        ls -la ${file_ipcc}
367
368        IGCM_sys_ncks -a -A -x -v GPP ${file} stomate.nc
369        IGCM_sys_ncks -a -A ${file_ipcc} stomate.nc
370
371        IGCM_sys_Cp ${file_ipcc} _${file_ipcc}
372        IGCM_sys_Chmod u+w _${file_ipcc}
373
374        IGCM_sys_ncap2 -s 'fLuc=float(fLuc*0.+(CONVFLUX+CFLUX_PROD10+CFLUX_PROD100)/1e3/86400*CONTFRAC)' \
375                       -s 'nbp=float(nbp*0.f+(gpp-ra-rh-fFire-fLuc-fHarvest))' -v stomate.nc stomate1.nc
376        IGCM_debug_Verif_Exit_Post
377
378        IGCM_sys_ncks -a -A stomate1.nc _${file_ipcc}
379        IGCM_sys_Mv -f _${file_ipcc} ${file_ipcc}
380        IGCM_sys_Chmod u-w ${file_ipcc}
381
382        ls -la ${file_ipcc}
383        IGCM_sys_Rm stomate.nc stomate1.nc
384        IGCM_debug_Verif_Exit_Post
385
386        # Put_Out des fichiers ipcc corrigés
387        IGCM_sys_Put_Out ${file_ipcc} ${file_path}/${file_ipcc}
388
389        (( i = i + 1 ))
390    done
391
392    # Nettoyage
393    IGCM_sys_Rm ${liste_file_local[*]}
394
395
396    # GO AHEAD IN DEEPEST LOOP
397    (( imin = imax ))
398    (( CURRENT_LOOP = CURRENT_LOOP + 1 ))
399    (( CURRENT_LOOP0 = CURRENT_LOOP0 + 1 ))
400    if [ ${CURRENT_LOOP} -lt ${NBRE_TOUR} ] ; then
401        (( imax = imax + ${NbFilesLOOP} ))
402    else
403        if [ ${CURRENT_LOOP} -eq ${NBRE_TOUR} ] ; then
404            echo "Last loop : "
405            if [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
406                (( imax = imax + ${NbFilesLOOP} ))
407            else
408                (( imax = imax + ( NBRE_FILE_TOT % UNIX_MAX_LIMIT ) / 2 ))
409            fi
410        fi
411    fi
412    echo "imin NEW = " ${imin}
413    echo "imax NEW = " ${imax}
414    echo "CURRENT_LOOP = " ${CURRENT_LOOP}
415
416    if ( [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] && \
417         [ ${CURRENT_LOOP0} -gt ${LOOP_MAX} ] ) ; then
418        export CURRENT_LOOP
419        export imin
420        export imax
421        IGCM_sys_Cd ${SUBMIT_DIR}
422        Script_Post_Output=SBGcorr.${imin}.${imax}
423        listVarEnv="CURRENT_LOOP,imin,imax"
424        echo "Submit next job."
425        IGCM_sys_QsubPost stomate_flux_correction
426        break ;
427    fi
428done                 # while [ ${CURRENT_LOOP} -le  ${LOOP_MAX} ] do
429# FINISH
430
431
432#D-
433# --------------------------------------------------------------------
434#D- Test if all was right
435# --------------------------------------------------------------------
436IGCM_debug_Verif_Exit_Post
437
438# Clean RUN_DIR_PATH (necessary for cesium)
439IGCM_sys_Cd ${SUBMIT_DIR}
440IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
441
442date
Note: See TracBrowser for help on using the repository browser.