Context Navigation

source: tags/libIGCM_v1_6/AA_create_ts @ 1170

Last change on this file since 1170 was 287, checked in by mmaipsl, 14 years ago
PB, MM: Add TS_MO_YE means production of yearly mean from TS MOnthly files.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Date Author Revision`
File size: 33.2 KB

Line
1	#-Q- platine #!/usr/bin/ksh
2	#-Q- platine ###################
3	#-Q- platine ## PLATINE CEA ##
4	#-Q- platine ###################
5	#-Q- platine #BSUB -J TS # Nom du job
6	#-Q- platine #BSUB -N # message a la fin du job
7	#-Q- platine #BSUB -n 1 # reservation des processeurs pour le job
8	#-Q- platine #BSUB -W 1:00 # Limite temps
9	#-Q- platine #BSUB -q post # Passage en queue post
10	#-Q- sx8brodie #!/bin/ksh
11	#-Q- sx8brodie #######################
12	#-Q- sx8brodie ## SX8BRODIE IDRIS ##
13	#-Q- sx8brodie #######################
14	#-Q- sx8brodie # Temps Elapsed max. d'une requete hh:mm:ss
15	#-Q- sx8brodie # @ wall_clock_limit = 20:00:00
16	#-Q- sx8brodie # Nom du travail LoadLeveler
17	#-Q- sx8brodie # @ job_name = TS
18	#-Q- sx8brodie # Fichier de sortie standard du travail
19	#-Q- sx8brodie # @ output = $(job_name).$(jobid)
20	#-Q- sx8brodie # Fichier de sortie d'erreur du travail
21	#-Q- sx8brodie # @ error = $(job_name).$(jobid)
22	#-Q- sx8brodie # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
23	#-Q- sx8brodie # @ notification = error
24	#-Q- sx8brodie # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
25	#-Q- aix6 #!/bin/ksh
26	#-Q- aix6 #######################
27	#-Q- aix6 ## VARGAS IDRIS ##
28	#-Q- aix6 #######################
29	#-Q- aix6 # Temps Elapsed max. d'une requete hh:mm:ss
30	#-Q- aix6 # @ wall_clock_limit = 20:00:00
31	#-Q- aix6 # Nom du travail LoadLeveler
32	#-Q- aix6 # @ job_name = TS
33	#-Q- aix6 # Fichier de sortie standard du travail
34	#-Q- aix6 # @ output = $(job_name).$(jobid)
35	#-Q- aix6 # Fichier de sortie d'erreur du travail
36	#-Q- aix6 # @ error = $(job_name).$(jobid)
37	#-Q- aix6 # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
38	#-Q- aix6 # @ notification = error
39	#-Q- aix6 # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
40	#-Q- aix6 # @ queue
41	#-Q- sx8brodie # @ queue
42	#-Q- sx8mercure #!/bin/ksh
43	#-Q- sx8mercure ######################
44	#-Q- sx8mercure ## SX8MERCURE CEA ##
45	#-Q- sx8mercure ######################
46	#-Q- sx8mercure #PBS -N TS # Nom du job
47	#-Q- sx8mercure #PBS -j o # regroupement des stdout et stderr
48	#-Q- sx8mercure #PBS -S /usr/bin/ksh # shell de soumission
49	#-Q- sx8mercure #PBS -l memsz_job=1gb # Limite memoire a 1 Go
50	#-Q- sx8mercure #PBS -l cputim_job=24:00:00 # Limite temps a 2 heures
51	#-Q- sx8mercure #PBS -q scalaire
52	#-Q- sx9mercure #!/bin/ksh
53	#-Q- sx9mercure ######################
54	#-Q- sx9mercure ## SX9MERCURE CEA ##
55	#-Q- sx9mercure ######################
56	#-Q- sx9mercure #PBS -N TS # Nom du job
57	#-Q- sx9mercure #PBS -j o # regroupement des stdout et stderr
58	#-Q- sx9mercure #PBS -S /usr/bin/ksh # shell de soumission
59	#-Q- sx9mercure #PBS -l memsz_job=1gb # Limite memoire a 1 Go
60	#-Q- sx9mercure #PBS -l cputim_job=24:00:00 # Limite temps a 2 heures
61	#-Q- sx9mercure #PBS -q scal-new
62	#-Q- titane #!/bin/ksh
63	#-Q- titane ######################
64	#-Q- titane ## TITANE CEA ##
65	#-Q- titane ######################
66	#-Q- titane #MSUB -r TS # Nom du job
67	#-Q- titane #MSUB -N 1 # Reservation du noeud
68	#-Q- titane #MSUB -n 1 # Reservation du processus
69	#-Q- titane #MSUB -T 86400 # Limite de temps elapsed du job
70	#-Q- titane #MSUB -E "-j o"
71	#-Q- titane #MSUB -E "-S /bin/ksh"
72	#-Q- titane ##MSUB -e nco.out # Sortie standard
73	#-Q- titane ##MSUB -o nco.out # Sortie standard
74	#-Q- lxiv8 ######################
75	#-Q- lxiv8 ## OBELIX LSCE ##
76	#-Q- lxiv8 ######################
77	#-Q- lxiv8 #PBS -N TS
78	#-Q- lxiv8 #PBS -m a
79	#-Q- lxiv8 #PBS -j oe
80	#-Q- lxiv8 #PBS -q medium
81	#-Q- lxiv8 #PBS -o TS.$$
82	#-Q- lxiv8 #PBS -S /bin/ksh
83	#-Q- default #!/bin/ksh
84	#-Q- default ##################
85	#-Q- default ## DEFAULT HOST ##
86	#-Q- default ##################
87
88	# $Date$
89	# $Author$
90	# $Revision$
91	# IPSL (2006)
92	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
93
94	#set -eu
95	#set -vx
96
97	date
98
99	#-Q- sx8brodie export OMP_NUM_THREADS=1
100	#-Q- aix6 export OMP_NUM_THREADS=1
101
102	########################################################################
103
104	#D- Flag to determine if this job in a standalone mode
105	#D- Default : value from AA_job if any
106	StandAlone=${StandAlone:=true}
107
108	#D- Low level debug : to bypass lib test checks and stack construction
109	#D- Default : value from AA_job if any
110	libIGCM=${libIGCM:=::modipsl::/libIGCM}
111	# WARNING for StandAlone used : To run this script on some machine,
112	# you must check MirrorlibIGCM variable in sys library.
113	# If this variable is true, you must use libIGCM_POST path instead
114	# of your running libIGCM directory.
115
116	#D- Flag to determine allready produced time series. Empty if you start from the beginning
117	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
118	CompletedFlag=${CompletedFlag:=}
119	CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
120
121	#D- Flag to determine task type this script will perform.
122	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
123	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
124	TsTask=${TsTask:=2D}
125	if [ X${TsTask} = Xempty ] ; then
126	TsTask=""
127	fi
128
129	#D- Flag to determine if rebuild process has been performed asynchronously
130	#D- Possible value true or false.
131	#D- If true raw files has allready been patched by rebuild job
132	#D- If false create_ts will patch the raw files
133	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
134	RebuildFrequency=${RebuildFrequency:=true}
135
136	#D- Flag to determine job's output directory
137	#D- Default : value from libIGCM_post.ksh if any
138	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR}}
139
140	#D- Increased verbosity (1, 2, 3)
141	#D- Default : value from AA_job if any
142	Verbosity=${Verbosity:=3}
143
144	#D- Low level debug : to bypass lib test checks and stack construction
145	#D- Default : value from AA_job if any
146	DEBUG_debug=${DEBUG_debug:=false}
147
148	#D- TEMPORARY Flag to determine atmospheric resolution
149	#D- Default : value from atmospheric driver if any
150	RESOL_ATM=ALL
151
152	#D- Flag to determine surface resolution
153	#D- Default : value from surface driver if any
154	RESOL_SRF=ALL
155
156	#D- Flag to determine surface resolution
157	#D- Default : value from surface driver if any
158	RESOL_SBG=ALL
159
160	#D- TEMPORARY Flag to determine ocean resolution
161	#D- Default : value from ocean driver if any
162	RESOL_OCE=${RESOL_OCE:=ORCA2}
163
164	#D- TEMPORARY Flag to determine ice resolution
165	#D- Default : value from ice driver if any
166	RESOL_ICE=${RESOL_ICE:=ORCA2}
167
168	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
169	#D- Default : value from ice driver if any
170	RESOL_MBG=${RESOL_MBG:=ORCA2}
171
172	########################################################################
173
174	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
175	( ${DEBUG_debug} ) && IGCM_debug_Check
176	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
177	( ${DEBUG_debug} ) && IGCM_card_Check
178	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
179	( ${DEBUG_debug} ) && IGCM_date_Check
180	#-------
181	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
182
183	########################################################################
184
185	#set -vx
186
187	# Useful cleaning function
188	MENAGE ()
189	{
190	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
191	}
192
193	########################################################################
194
195	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
196	IGCM_sys_Cd ${RUN_DIR_PATH}
197
198	if [ ${StandAlone} = true ] ; then
199	CARD_DIR=${SUBMIT_DIR}
200	else
201	CARD_DIR=${RUN_DIR_PATH}/$( basename ${SUBMIT_DIR} )
202	IGCM_sys_Get_Master ${SUBMIT_DIR} ${RUN_DIR_PATH}
203	fi
204
205	#
206	# First of all
207	#
208	IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card UserChoices
209	typeset option
210	for option in ${config_UserChoices[*]} ; do
211	IGCM_card_DefineVariableFromOption ${CARD_DIR}/config.card UserChoices ${option}
212	done
213	#
214	echo
215	IGCM_debug_Print 1 "DefineArrayFromOption : config_UserChoices"
216	IGCM_debug_PrintVariables 3 config_UserChoices_JobName
217	IGCM_debug_PrintVariables 3 config_UserChoices_LongName
218	IGCM_debug_PrintVariables 3 config_UserChoices_TagName
219	IGCM_debug_PrintVariables 3 config_UserChoices_SpaceName
220	IGCM_debug_PrintVariables 3 config_UserChoices_ExperimentName
221	IGCM_debug_PrintVariables 3 config_UserChoices_CalendarType
222	IGCM_debug_PrintVariables 3 config_UserChoices_DateBegin
223	IGCM_debug_PrintVariables 3 config_UserChoices_DateEnd
224	IGCM_debug_PrintVariables 3 config_UserChoices_PeriodLength
225	echo
226
227	#==================================
228	#R_SAVE : Job output directory
229	if ( [ ! X${config_UserChoices_SpaceName} = X ] && [ ! X${config_UserChoices_ExperimentName} = X ] ) ; then
230	FreeName=$( echo ${config_UserChoices_JobName} \| sed 's/.*_//' )
231	R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
232	R_DODS=${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
233	else
234	R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
235	R_DODS=${config_UserChoices_TagName}/${config_UserChoices_JobName}
236	fi
237	#
238	# Determine component to take care of depending on kind of task create_ts will perform
239	#
240	( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = XChunck3D ] ) && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
241	#
242	# Determine period for time series. Default : value from AA_job or from launch_TS if any
243	#
244	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
245	#
246	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
247	#
248	# Determine Dimension of the produced time series : 2D or 3D
249	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
250	Dimension=2D
251	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
252	Dimension=3D
253	else
254	Dimension=""
255	fi
256
257
258	########################################################################
259	#
260	# Depending on requested time series fill following variables :
261	#
262	# - LISTE_FILE_${comp}[*]
263	# - LISTE_VARS_${file}[*]
264	# - LISTE_AXIS_${file}[*]
265	# - LISTE_PATCH_${file}[*]
266	########################################################################
267
268	#
269	# For each selected component determine which files need post-processing
270	#
271	DoJob=false
272	for comp in ${config_ListOfComponents[*]} ; do
273	#
274	ActiveComp=false
275	#
276	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
277	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
278	#
279	card=${CARD_DIR}/COMP/${compname}.card
280
281	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
282	#
283	ListFilesName=${compname}_OutputFiles_List
284	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
285	#
286	if [ X${FileName0} != X${NULL_STR} ] ; then
287	#
288	#IGCM_debug_Print 1 "Component : ${compname}"
289	#
290	# INITIALISATION
291	#
292	typeset i
293	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = XChunck3D ] ) ; then
294	((NbFiles=${FlagToRead}))
295	((i=${NbFiles}))
296	else
297	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
298	i=2
299	fi
300	#
301	until [ $i -gt $NbFiles ]; do
302	#
303	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
304	#
305	if [ X${flag_post} != XNONE ] ; then
306	#
307	# For each comp and each file who needs post-processing
308	# determine which variables need time series.
309	#
310	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
311	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
312	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
313	#
314	# If TimeSeriesVars list is empty we skip
315	#
316	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
317	#
318	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
319	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
320	(( i=i+3 ))
321	continue
322	#
323	fi
324	#
325	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
326	#
327	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
328	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
329	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
330	(( i=i+3 ))
331	continue
332	fi
333	#
334	# If we are not a Chunck type task, we skip files/variables handled by chunck task
335	#
336	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
337	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
338	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
339	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
340	(( i=i+3 ))
341	continue
342	fi
343	fi
344	#
345	# Now on we know we have things to do
346	#
347
348	#
349	# We need coordinates axis to properly fill produced time series
350	#
351	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
352	#
353	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
354	#
355	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
356	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
357	(( i=i+3 ))
358	continue
359	#
360	fi
361	#
362	# We need LIST of variables not allready produced (useful for standalone mode)
363	#
364	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
365	case ${FlagDir} in
366	*Y) TS_Dir=TS_YE ;;
367	*M) TS_Dir=TS_MO ;;
368	*D) TS_Dir=TS_DA ;;
369	HF) TS_Dir=TS_HF ;;
370	3H) TS_Dir=TS_HF ;;
371	INS) TS_Dir=TS_INS ;;
372	esac
373	#
374	count=0
375	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
376	#
377	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
378	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
379	#
380	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
381	if [ ! $? = 0 ] ; then
382	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
383	ActiveComp=true
384	(( count = count + 1 ))
385	fi
386	done
387	#
388	if [ ${count} -gt 0 ] ; then
389	#
390	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
391	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
392	#
393	if [ ! X${RebuildFrequency} = Xtrue ] ; then
394	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
395	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
396	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
397	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
398	if [ Xload${Patch} != X ] ; then
399	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
400	eval load${Patch}=loaded
401	fi
402	done
403	fi
404	fi
405	fi
406	fi
407	(( i=i+3 ))
408	done
409	fi
410	#
411	if [ ${ActiveComp} = true ] ; then
412	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
413	fi
414	done
415
416	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
417	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
418
419	########################################################################
420	#
421	# DEFINE DATE FUNCTIONS
422	#
423	########################################################################
424
425	# Give the end of the next $1 period from $2 date
426	function create_ts_next_date {
427	IGCM_debug_PushStack "create_ts_next_date" $@
428	typeset Length
429	case ${config_UserChoices_PeriodLength} in
430	Y\|y)
431	PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} \| awk -F '[yY]' '{print $1}' )
432	IGCM_date_GetYearMonth ${1} year month
433	year=$( IGCM_date_SupressZeros ${year} )
434	(( Length=0 ))
435	(( i=0 ))
436	until [ $i -ge $PeriodLengthInYears ] ; do
437	(( Length = Length + $( IGCM_date_DaysInYear $(( year + i + 1 )) ) ))
438	(( i=i+1 ))
439	done
440	;;
441	M\|m)
442	PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} \| awk -F '[mM]' '{print $1}' )
443	IGCM_date_GetYearMonth ${1} year month
444	year=$( IGCM_date_SupressZeros ${year} )
445	(( year0 = year ))
446	if [ $(( month + 1 )) -lt 13 ] ; then
447	month0=$(( month + 1 ))
448	else
449	month0=$(( month + 1 - 12 ))
450	(( year = year0 + 1 ))
451	fi
452	(( Length=0 ))
453	(( i=0 ))
454	until [ $i -ge $PeriodLengthInMonths ] ; do
455	if [ $(( month0 + i )) -lt 13 ] ; then
456	(( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i )) ) ))
457	else
458	(( year = year0 + 1 ))
459	(( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i - 12 )) ) ))
460	fi
461	(( i=i+1 ))
462	done
463	;;
464	D\|d)
465	Length=$( echo ${config_UserChoices_PeriodLength} \| sed -e "s/[dD]//" ) ;;
466	*)
467	IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, M, D."
468	IGCM_debug_Verif_Exit_Post ;;
469	esac
470	echo ${Length}
471
472	IGCM_debug_PopStack "create_ts_next_date"
473	}
474
475	# Give the end of the actual $1 period from $2 date
476	function create_ts_begin_date {
477	IGCM_debug_PushStack "create_ts_begin_date" $@
478	typeset Length
479	case ${config_UserChoices_PeriodLength} in
480	Y\|y)
481	PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} \| awk -F '[yY]' '{print $1}' )
482	IGCM_date_GetYearMonth ${1} year month
483	year=$( IGCM_date_SupressZeros ${year} )
484	if [ X${2} = Xend ] ; then
485	(( year = year - PeriodLengthInYears + 1))
486	fi
487	(( Length=0 ))
488	(( i=0 ))
489	until [ $i -ge $PeriodLengthInYears ] ; do
490	(( Length = Length + $( IGCM_date_DaysInYear $(( year + i )) ) ))
491	(( i=i+1 ))
492	done
493	;;
494	M\|m)
495	PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} \| awk -F '[mM]' '{print $1}' )
496	IGCM_date_GetYearMonth ${1} year month
497	year=$( IGCM_date_SupressZeros ${year} )
498	if [ X${2} = Xend ] ; then
499	(( month = month - PeriodLengthInMonths + 1 ))
500	fi
501	(( year0 = year ))
502	if [ $month -le 0 ] ; then
503	(( month = month + 12 ))
504	year=$( printf "%04i\n" $(( year - 1 )) )
505	fi
506	month=$( printf "%02i\n" ${month} )
507	(( Length=0 ))
508	(( i=0 ))
509	until [ $i -ge $PeriodLengthInMonths ] ; do
510	if [ $(( month + i )) -lt 13 ] ; then
511	(( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i )) ) ))
512	else
513	(( year = year0 + 1 ))
514	(( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i - 12 )) ) ))
515	fi
516	(( i=i+1 ))
517	done
518	;;
519	D\|d)
520	Length=$( echo ${config_UserChoices_PeriodLength} \| sed -e "s/[dD]//" ) ;;
521	*)
522	IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, M, D."
523	IGCM_debug_Verif_Exit_Post ;;
524	esac
525	echo ${Length}
526
527	IGCM_debug_PopStack "create_ts_begin_date"
528	}
529
530	########################################################################
531	#
532	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
533	# IF SO BRING THEM IN THE WORKING DIRECTORY
534	#
535	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
536	# - especially for IDRiS
537	# - no special protection against "command too long" !
538	# - libIGCM_sys should handle that or modify following code part
539	#
540	########################################################################
541
542	if [ ${StandAlone} != true ] ; then
543	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
544	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
545	echo \${run_PostProcessing_TimeSeriesRunning}" )
546
547	#if [ X${Running_Flag} = Xy ] ; then
548	# # Time Series Job allready running
549	# IGCM_debug_Print 1 "Time Series Job allready running exit"
550	# exit
551	#fi
552	# Now run.card know we are running
553
554
555
556
557	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
558
559
560
561
562
563	# Get information from last execution
564	#CompletedFlag=$( IGCM_sys_RshMaster \
565	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
566	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
567	fi
568
569	# Is it the first submission or not ?
570
571	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
572	# First Time Series Submission
573	FIRST_PASS=TRUE
574	Length=$( create_ts_begin_date ${DateBegin} begin)
575	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
576	else
577	# Last Completed Time Series
578	FIRST_PASS=FALSE
579	DATE_FIN_JOB_B=${CompletedFlag}
580	date=${DateBegin}_${DATE_FIN_JOB_B}
581	for comp in ${config_ListOfComponents[*]} ; do
582	#
583	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
584	#
585	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
586	#
587	#set -A liste_file
588	unset liste_file
589	#
590	file_path=${R_SAVE}/${comp}/Analyse
591	#
592	i=0
593	file=${NULL_STR}
594	#
595	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
596	#
597	# Determine in which output can we find file
598	# IE : Analyse/TS or Analyse/DA2TS or ...
599	#
600	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
601	case ${FlagDir} in
602	*Y) TS_Dir=TS_YE ;;
603	*M) TS_Dir=TS_MO ;;
604	*D) TS_Dir=TS_DA ;;
605	HF) TS_Dir=TS_HF ;;
606	3H) TS_Dir=TS_HF ;;
607	INS) TS_Dir=TS_INS ;;
608	esac
609	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
610	#
611	#set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
612	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
613	#
614	done
615	done
616	if [ X${file} != X${NULL_STR} ] ; then
617	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
618	ExitFlag=false
619	IGCM_debug_Verif_Exit_Post
620	fi
621	done
622	#
623	LEVEL=0
624	#
625	fi
626
627	########################################################################
628	#
629	# DEFINE LOOP PARAMETERS
630	#
631	########################################################################
632
633	DATE_COUNT=${DATE_FIN_JOB_B}
634	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
635
636	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
637
638	if [ ${DoJob} = true ] ; then
639	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
640	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
641	Length=$( create_ts_next_date ${DATE_COUNT} )
642	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
643	done
644	else
645	NBRE_FILE_TOT=0
646	fi
647
648	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
649	NBRE_TOUR=1
650	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
651	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
652	else
653	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
654	fi
655
656	CURRENT_LOOP=1
657	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
658	# WE HAVE 3 LEVELS
659	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
660	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
661	# - LEVEL 3 : WE ARE IN THE LAST LOOP
662
663	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
664
665	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
666	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
667	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
668
669	if [ ${FIRST_PASS} = TRUE ] ; then
670	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
671	else
672	Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
673	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
674	fi
675
676	DATE_FIN=${PeriodDateEnd}
677	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
678
679	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
680	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
681
682	LEVEL=1
683	DEPOT="TRUE"
684	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
685	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
686	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
687
688	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
689	Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
690	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
691	elif [ -z "${DATE_COURANTE}" ] ; then
692	DATE_COURANTE=${DATE_FIN_JOB_B}
693	else
694	Length=$( create_ts_next_date ${DATE_FIN} )
695	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
696	fi
697
698	(( TotLength = 0 ))
699	COMPTEUR=1
700	DATE_LOOP=${DATE_COURANTE}
701
702	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
703	#
704	Length=$( create_ts_next_date ${DATE_LOOP} )
705	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
706	(( TotLength = TotLength + Length ))
707	(( COMPTEUR = COMPTEUR + 1 ))
708	#
709	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
710	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
711
712	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
713	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
714	DEPOT="FALSE"
715	LEVEL=2
716	else
717	# - LEVEL 3 : WE ARE IN THE LAST LOOP
718	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
719	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
720
721	Length=$( create_ts_next_date ${DATE_FIN} )
722	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
723	DATE_FIN=${PeriodDateEnd}
724	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
725
726	LEVEL=3
727	DEPOT="TRUE"
728	FLAG_B="TRUE"
729	fi
730
731	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
732	for comp in ${config_ListOfComponents[*]} ; do
733	#
734	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
735	#
736	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
737	#
738	eval R_OUT_${comp}=${R_SAVE}/${comp}
739	#
740	i=0
741	#
742	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
743	#
744	# Determine in which output can we find file
745	# IE : Output/MO or Output/DA or ...
746	#
747	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
748	case ${FlagDir} in
749	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
750	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
751	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
752	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
753	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
754	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
755	*)
756	IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
757	IGCM_sys_Exit "Job create_ts" ;;
758	esac
759
760	# Initialize array
761	#set -A liste_file
762	#set -A liste_file_tmp
763	unset liste_file
764	unset liste_file_tmp
765	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
766
767	COMPTEUR=1
768	DATE_LOOP=${DATE_COURANTE}
769
770	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
771	#
772	Length1=$( create_ts_begin_date ${DATE_LOOP} end)
773	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
774
775	Length2=$( create_ts_next_date ${DATE_LOOP} end)
776	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
777
778	#set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
779	#set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
780	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
781	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
782	(( COMPTEUR = COMPTEUR + 1 ))
783	#
784	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
785	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
786
787	# Get selected files
788	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
789	ExitFlag=false
790	IGCM_debug_Verif_Exit_Post
791
792	# Apply IGCM_Patch if needed
793	if [ ! X${RebuildFrequency} = Xtrue ] ; then
794	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
795	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
796	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
797	IGCM_${Patch} ${file_t}
798	done
799	done
800	fi
801	fi
802
803	# WE CAN CONCATENATE FILES AT THIS POINT
804	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
805	Length=$( create_ts_begin_date ${DATE_COURANTE} end)
806	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
807	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
808	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
809
810	liste_coord=" "
811	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
812	liste_coord=${liste_coord}${axis}","
813	done
814
815	# Time axis must be the last dimension
816	time_axis=${axis}
817
818	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
819	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
820	#
821	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
822	#
823	# # Change time axis name to be CF compliant
824	# if [ ${time_axis} != time_counter ]; then
825	# IGCM_sys_ncrename -O -v ${time_axis},time_counter ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
826	# fi
827	#
828	if [ "${FLAG_B}" = "TRUE" ] ; then
829	# WE CONCATENATE WITH EXISTING FILES
830	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
831	file2=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
832	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
833	IGCM_sys_ncrcat --hst ${file1} ${file2} ${file_out}
834	#
835	IGCM_sys_Rm ${file1} ${file2}
836	fi
837
838	#
839	if [ "${DEPOT}" = "TRUE" ] ; then
840	# WE PUT FINAL FILE ON FILE SYSTEM
841	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
842	#${ncatted} -a modulo,time_counter,c,c," " ${file_out}
843	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
844	#
845	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
846	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
847	#
848	if [ ${FlagDir} = "1M" ] ; then
849	# Compute yearly average considering calendar
850	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
851	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
852	#
853	# Note that the date information in the outputfile is the date of the last contributing time step in the input file
854	IGCM_sys_cdo -settunits,days -divdpy -yearsum -muldpm ${file_out} ${file_out_YE}
855	#
856	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
857	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
858	fi
859	#
860	IGCM_sys_Rm ${file_out}
861	#
862	fi
863	#
864	ExitFlag=false
865	IGCM_debug_Verif_Exit_Post
866	done # for var in ${liste_var} ; do
867	IGCM_sys_Rm ${liste_file_tmp[*]}
868	done # for file in ${liste_file_brut} do
869	done # for comp in ${config_ListOfComponents[*]} ; do
870	ExitFlag=false
871	IGCM_debug_Verif_Exit_Post
872	# GO AHEAD IN DEEPEST LOOP
873	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
874	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
875	# FINISH
876
877	# DODS copy
878	for comp in ${config_ListOfComponents[*]} ; do
879	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
880	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
881	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
882	done
883	done
884
885	export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
886	export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG
887
888	listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
889	export listVarEnv
890
891	#D-
892	# --------------------------------------------------------------------
893	#D- Test if all was right
894	# --------------------------------------------------------------------
895	IGCM_debug_Verif_Exit_Post
896	# --------------------------------------------------------------------
897	#D- Submit MONITORING
898	# --------------------------------------------------------------------
899	( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X ] ) && IGCM_sys_QsubPost monitoring
900
901	#D-
902	# --------------------------------------------------------------------
903	#D- Update the run.card
904	# --------------------------------------------------------------------
905	if [ ${StandAlone} != true ] ; then
906	# Put in run.card end period of time series
907	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
908	# We have finish the job !
909	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
910	fi
911
912	# Clean RUN_DIR_PATH (necessary for cesium)
913	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
914
915	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: