Context Navigation

source: tags/libIGCM_v2.0/AA_create_ts @ 1170

Last change on this file since 1170 was 848, checked in by sdipsl, 11 years ago
Use sequential post-processing class
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 31.2 KB

Line
1	#-Q- curie ######################
2	#-Q- curie ## CURIE TGCC/CEA ##
3	#-Q- curie ######################
4	#-Q- curie #MSUB -r TS # Nom du job
5	#-Q- curie #MSUB -eo
6	#-Q- curie #MSUB -n 1 # Reservation du processus
7	#-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job
8	#-Q- curie #MSUB -q xlarge
9	#-Q- curie #MSUB -Q normal
10	#-Q- curie #MSUB -A ::default_project::
11	#-Q- curie set +x
12	#-Q- ada #!/bin/ksh
13	#-Q- ada #######################
14	#-Q- ada ## ADA IDRIS ##
15	#-Q- ada #######################
16	#-Q- ada # @ job_type = serial
17	#-Q- ada # @ requirements = (Feature == "prepost")
18	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
19	#-Q- ada # @ wall_clock_limit = 20:00:00
20	#-Q- ada # Nom du travail LoadLeveler
21	#-Q- ada # @ job_name = TS
22	#-Q- ada # Fichier de sortie standard du travail
23	#-Q- ada # @ output = $(job_name).$(jobid)
24	#-Q- ada # Fichier de sortie d'erreur du travail
25	#-Q- ada # @ error = $(job_name).$(jobid)
26	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
27	#-Q- ada # @ notification = error
28	#-Q- ada # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
29	#-Q- ada # @ queue
30	#-Q- sx9mercure #!/bin/ksh
31	#-Q- sx9mercure ######################
32	#-Q- sx9mercure ## SX9MERCURE CCRT ##
33	#-Q- sx9mercure ######################
34	#-Q- sx9mercure #PBS -N TS # Nom du job
35	#-Q- sx9mercure #PBS -j o # regroupement des stdout et stderr
36	#-Q- sx9mercure #PBS -S /usr/bin/ksh # shell de soumission
37	#-Q- sx9mercure #PBS -l memsz_job=1gb # Limite memoire a 1 Go
38	#-Q- sx9mercure #PBS -l elapstim_req=24:00:00 # Limite temps a 2 heures
39	#-Q- sx9mercure #PBS -q scalaire
40	#-Q- sx9mercure #PBS -r n
41	#-Q- titane #!/bin/ksh
42	#-Q- titane ######################
43	#-Q- titane ## TITANE CEA ##
44	#-Q- titane ######################
45	#-Q- titane #MSUB -r TS # Nom du job
46	#-Q- titane #MSUB -eo
47	#-Q- titane #MSUB -n 1 # Reservation du processus
48	#-Q- titane #MSUB -T 86400 # Limite de temps elapsed du job
49	#-Q- titane #MSUB -q mono
50	#-Q- titane #MSUB -E '-rn'
51	#-Q- lxiv8 ######################
52	#-Q- lxiv8 ## OBELIX LSCE ##
53	#-Q- lxiv8 ######################
54	#-Q- lxiv8 #PBS -N TS
55	#-Q- lxiv8 #PBS -m a
56	#-Q- lxiv8 #PBS -j oe
57	#-Q- lxiv8 #PBS -q medium
58	#-Q- lxiv8 #PBS -o TS.$$
59	#-Q- lxiv8 #PBS -S /bin/ksh
60	#-Q- default #!/bin/ksh
61	#-Q- default ##################
62	#-Q- default ## DEFAULT HOST ##
63	#-Q- default ##################
64
65	#**************************************************************
66	# Author: Sebastien Denvil
67	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
68	# $Revision:: $ Revision of last commit
69	# $Author:: $ Author of last commit
70	# $Date:: $ Date of last commit
71	# IPSL (2006)
72	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
73	#
74	#**************************************************************
75
76	#set -eu
77	#set -vx
78
79	date
80
81	#-Q- ada export OMP_NUM_THREADS=1
82
83	#D- Task type (computing or post-processing)
84	TaskType=post-processing
85
86	########################################################################
87
88	#D- Flag to determine if this job in a standalone mode
89	#D- Default : value from AA_job if any
90	StandAlone=${StandAlone:=true}
91
92	#D- Path to libIGCM
93	#D- Default : value from AA_job if any
94	libIGCM=${libIGCM:=::modipsl::/libIGCM}
95
96	#D- Flag to determine allready produced time series. Empty if you start from the beginning
97	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
98	CompletedFlag=${CompletedFlag:=}
99
100	#D- Flag to determine task type this script will perform.
101	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
102	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
103	TsTask=${TsTask:=2D}
104	if [ X${TsTask} = Xempty ] ; then
105	TsTask=""
106	fi
107
108	#D- Flag to determine if rebuild process has been performed asynchronously
109	#D- Possible value true or false.
110	#D- If true raw files has allready been patched by rebuild job
111	#D- If false create_ts will patch the raw files
112	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
113	RebuildFrequency=${RebuildFrequency:=true}
114
115	#D- Flag to determine job's output directory
116	#D- Default : value from libIGCM_post.ksh if any
117	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
118
119	#D- Increased verbosity (1, 2, 3)
120	#D- Default : value from AA_job if any
121	Verbosity=${Verbosity:=3}
122
123	#D- Low level debug : to bypass lib test checks and stack construction
124	#D- Default : value from AA_job if any
125	DEBUG_debug=${DEBUG_debug:=false}
126
127	#D- TEMPORARY Flag to determine atmospheric resolution
128	#D- Default : value from atmospheric driver if any
129	RESOL_ATM=ALL
130
131	#D- Flag to determine surface resolution
132	#D- Default : value from surface driver if any
133	RESOL_SRF=ALL
134
135	#D- Flag to determine surface resolution
136	#D- Default : value from surface driver if any
137	RESOL_SBG=ALL
138
139	#D- TEMPORARY Flag to determine ocean resolution
140	#D- Default : value from ocean driver if any
141	RESOL_OCE=${RESOL_OCE:=ORCA2}
142
143	#D- TEMPORARY Flag to determine ice resolution
144	#D- Default : value from ice driver if any
145	RESOL_ICE=${RESOL_ICE:=ORCA2}
146
147	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
148	#D- Default : value from ice driver if any
149	RESOL_MBG=${RESOL_MBG:=ORCA2}
150
151	########################################################################
152
153	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
154	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
155	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
156	#-------
157	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
158	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
159	#-------
160	( ${DEBUG_debug} ) && IGCM_debug_Check
161	( ${DEBUG_debug} ) && IGCM_card_Check
162	( ${DEBUG_debug} ) && IGCM_date_Check
163
164	########################################################################
165
166	#set -vx
167
168	# Useful cleaning function
169	MENAGE ()
170	{
171	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
172	}
173
174	########################################################################
175
176	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
177	IGCM_sys_Cd ${RUN_DIR_PATH}
178
179	# ------------------------------------------------------------------
180	# Test if all was right before proceeding further
181	# ------------------------------------------------------------------
182	IGCM_debug_Verif_Exit_Post
183
184	if [ ${StandAlone} = true ] ; then
185	CARD_DIR=${SUBMIT_DIR}
186	else
187	CARD_DIR=${RUN_DIR_PATH}
188	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
189	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
190	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
191	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
192	fi
193
194	#==================================
195	# First of all
196	#
197	# Read libIGCM compatibility version in config.card
198	# Read UserChoices section
199	# Read Ensemble section
200	# Read Post section
201	# Define all netcdf output directories
202	#==================================
203	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
204
205	#==================================
206	# Define default value to keep compatibility with previous card: means before changes due to TGCC
207	if [ X${PackDefault} = Xtrue ] ; then
208	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
209	else
210	config_Post_PackFrequency=NONE
211	fi
212
213	#==================================
214	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
215	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
216	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
217	fi
218
219	#
220	# Determine component to take care of depending on kind of task create_ts will perform
221	#
222	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
223	#
224	# Determine period for time series. Default : value from AA_job or from launch_TS if any
225	#
226	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
227
228	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
229	#
230	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
231	#
232	# Determine Dimension of the produced time series : 2D or 3D
233	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
234	Dimension=2D
235	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
236	Dimension=3D
237	else
238	Dimension=""
239	fi
240
241	# ------------------------------------------------------------------
242	#D- Test if all was right before entering the loop
243	# ------------------------------------------------------------------
244	IGCM_debug_Verif_Exit_Post
245
246	########################################################################
247	#
248	# Depending on requested time series fill following variables :
249	#
250	# - LISTE_FILE_${comp}[*]
251	# - LISTE_VARS_${file}[*]
252	# - LISTE_AXIS_${file}[*]
253	# - LISTE_PATCH_${file}[*]
254	########################################################################
255
256	#
257	# For each selected component determine which files need post-processing
258	#
259	DoJob=false
260	for comp in ${config_ListOfComponents[*]} ; do
261	#
262	ActiveComp=false
263	#
264	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
265	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
266	#
267	card=${CARD_DIR}/COMP/${compname}.card
268
269	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
270	#
271	ListFilesName=${compname}_OutputFiles_List
272	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
273	#
274	if [ X${FileName0} != X${NULL_STR} ] ; then
275	#
276	#IGCM_debug_Print 1 "Component : ${compname}"
277	#
278	# INITIALISATION
279	#
280	typeset i
281	if [ ${FlagToRead} ] ; then
282	((NbFiles=${FlagToRead}))
283	((i=${NbFiles}))
284	else
285	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
286	i=2
287	fi
288	#
289	until [ $i -gt $NbFiles ]; do
290	#
291	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
292	#
293	if [ X${flag_post} != XNONE ] ; then
294	#
295	# For each comp and each file who needs post-processing
296	# determine which variables need time series.
297	#
298	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
299	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
300	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
301	#
302	# If TimeSeriesVars list is empty we skip
303	#
304	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
305	#
306	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
307	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
308	(( i=i+3 ))
309	continue
310	#
311	fi
312	#
313	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
314	#
315	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
316	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
317	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
318	(( i=i+3 ))
319	continue
320	fi
321	#
322	# If we are not a Chunck type task, we skip files/variables handled by chunck task
323	#
324	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
325	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
326	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
327	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
328	(( i=i+3 ))
329	continue
330	fi
331	fi
332	#
333	# Now on we know we have things to do
334	#
335
336	#
337	# We need coordinates axis to properly fill produced time series
338	#
339	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
340	#
341	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
342	#
343	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
344	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
345	(( i=i+3 ))
346	continue
347	#
348	fi
349	#
350	# We need LIST of variables not allready produced (useful for standalone mode)
351	#
352	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
353	case ${FlagDir} in
354	*Y) TS_Dir=TS_YE ;;
355	*M) TS_Dir=TS_MO ;;
356	*D) TS_Dir=TS_DA ;;
357	HF) TS_Dir=TS_HF ;;
358	3H) TS_Dir=TS_HF ;;
359	INS) TS_Dir=TS_INS ;;
360	esac
361	#
362	count=0
363	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
364	#
365	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
366	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
367	#
368	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
369	if [ ! $? = 0 ] ; then
370	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
371	ActiveComp=true
372	(( count = count + 1 ))
373	fi
374	done
375	#
376	if [ ${count} -gt 0 ] ; then
377	#
378	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
379	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
380	#
381	if [ ! X${RebuildFrequency} = Xtrue ] ; then
382	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
383	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
384	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
385	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
386	if [ Xload${Patch} != X ] ; then
387	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
388	eval load${Patch}=loaded
389	fi
390	done
391	fi
392	fi
393	fi
394	fi
395	(( i=i+3 ))
396	done
397	fi
398	#
399	if [ ${ActiveComp} = true ] ; then
400	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
401	fi
402	done
403
404	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
405	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
406
407	########################################################################
408	#
409	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
410	# IF SO BRING THEM IN THE WORKING DIRECTORY
411	#
412	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
413	# - especially for IDRiS
414	# - no special protection against "command too long" !
415	# - libIGCM_sys should handle that or modify following code part
416	#
417	########################################################################
418
419	if [ ${StandAlone} != true ] ; then
420	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
421	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
422	echo \${run_PostProcessing_TimeSeriesRunning}" )
423
424	#if [ X${Running_Flag} = Xy ] ; then
425	# # Time Series Job allready running
426	# IGCM_debug_Print 1 "Time Series Job allready running exit"
427	# exit
428	#fi
429	# Now run.card know we are running
430
431
432
433
434	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
435
436
437
438
439
440	# Get information from last execution
441	#CompletedFlag=$( IGCM_sys_RshMaster \
442	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
443	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
444	fi
445
446	# Is it the first submission or not ?
447
448	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
449	# First Time Series Submission
450	FIRST_PASS=TRUE
451	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
452	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
453	else
454	# Last Completed Time Series
455	FIRST_PASS=FALSE
456	DATE_FIN_JOB_B=${CompletedFlag}
457	date=${DateBegin}_${DATE_FIN_JOB_B}
458	for comp in ${config_ListOfComponents[*]} ; do
459	#
460	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
461	#
462	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
463	#
464	#set -A liste_file
465	unset liste_file
466	#
467	file_path=${R_SAVE}/${comp}/Analyse
468	#
469	i=0
470	file=${NULL_STR}
471	#
472	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
473	#
474	# Determine in which output can we find file
475	# IE : Analyse/TS or Analyse/DA2TS or ...
476	#
477	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
478	case ${FlagDir} in
479	*Y) TS_Dir=TS_YE ;;
480	*M) TS_Dir=TS_MO ;;
481	*D) TS_Dir=TS_DA ;;
482	HF) TS_Dir=TS_HF ;;
483	3H) TS_Dir=TS_HF ;;
484	INS) TS_Dir=TS_INS ;;
485	esac
486	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
487	#
488	#set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
489	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
490	#
491	done
492	done
493	if [ X${file} != X${NULL_STR} ] ; then
494	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
495	# Even if some files are missing we tried to continue
496	# Either ListVar in compname.card are asked but not present in original history file
497	# Either a previous TS task failed
498	ExitFlag=false
499	IGCM_debug_Verif_Exit_Post
500	fi
501	done
502	#
503	LEVEL=0
504	#
505	fi
506
507	########################################################################
508	#
509	# DEFINE LOOP PARAMETERS
510	#
511	########################################################################
512
513	DATE_COUNT=${DATE_FIN_JOB_B}
514	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
515
516	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
517
518	if [ ${DoJob} = true ] ; then
519	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
520	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
521	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
522	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
523	done
524	else
525	NBRE_FILE_TOT=0
526	fi
527
528	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
529	NBRE_TOUR=1
530	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
531	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
532	else
533	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
534	fi
535
536	CURRENT_LOOP=1
537	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
538	# WE HAVE 3 LEVELS
539	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
540	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
541	# - LEVEL 3 : WE ARE IN THE LAST LOOP
542
543	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
544
545	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
546	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
547	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
548
549	if [ ${FIRST_PASS} = TRUE ] ; then
550	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
551	else
552	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
553	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
554	fi
555
556	DATE_FIN=${PeriodDateEnd}
557	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
558
559	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
560	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
561
562	LEVEL=1
563	DEPOT="TRUE"
564	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
565	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
566	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
567
568	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
569	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
570	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
571	elif [ -z "${DATE_COURANTE}" ] ; then
572	DATE_COURANTE=${DATE_FIN_JOB_B}
573	else
574	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
575	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
576	fi
577
578	(( TotLength = 0 ))
579	COMPTEUR=1
580	DATE_LOOP=${DATE_COURANTE}
581
582	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
583	#
584	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
585	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
586	(( TotLength = TotLength + Length ))
587	(( COMPTEUR = COMPTEUR + 1 ))
588	#
589	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
590	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
591
592	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
593	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
594	DEPOT="FALSE"
595	LEVEL=2
596	else
597	# - LEVEL 3 : WE ARE IN THE LAST LOOP
598	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
599	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
600
601	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
602	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
603	DATE_FIN=${PeriodDateEnd}
604	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
605
606	LEVEL=3
607	DEPOT="TRUE"
608	FLAG_B="TRUE"
609	fi
610
611	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
612	for comp in ${config_ListOfComponents[*]} ; do
613	#
614	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
615	#
616	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
617	#
618	eval R_OUT_${comp}=${R_SAVE}/${comp}
619	#
620	i=0
621	#
622	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
623	#
624	# Determine in which output we can find file
625	# IE : Output/MO or Output/DA or ...
626	#
627	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
628	case ${FlagDir} in
629	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
630	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
631	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
632	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
633	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
634	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
635	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
636	IGCM_sys_Exit "Job create_ts" ;;
637	esac
638
639	# Initialize array
640	#set -A liste_file
641	#set -A liste_file_tmp
642	unset liste_file
643	unset liste_file_tmp
644	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
645
646	COMPTEUR=1
647	MissingFile=FALSE
648	DATE_LOOP=${DATE_COURANTE}
649
650	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
651	#
652	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
653	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
654
655	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
656	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
657
658	#set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
659	#set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
660
661	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
662	if [ $? = 0 ] ; then
663	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
664	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
665	else
666	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
667	MissingFile=TRUE
668	fi
669	(( COMPTEUR = COMPTEUR + 1 ))
670	#
671	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
672
673	# We skip this file type if one of them is missing.
674	if [ "${MissingFile}" = "TRUE" ] ; then
675	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
676	continue
677	fi
678
679	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
680
681	# Get selected files
682	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
683	IGCM_debug_Verif_Exit_Post
684
685	# Apply IGCM_Patch if needed
686	if [ ! X${RebuildFrequency} = Xtrue ] ; then
687	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
688	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
689	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
690	IGCM_${Patch} ${file_t}
691	IGCM_debug_Verif_Exit_Post
692	done
693	done
694	fi
695	fi
696
697	# WE CAN CONCATENATE FILES AT THIS POINT
698	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
699	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
700	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
701	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
702	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
703
704	liste_coord=" "
705	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
706	liste_coord=${liste_coord}${axis}","
707	done
708
709	# Time axis must be the last dimension
710	time_axis=${axis}
711
712	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
713	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
714	#
715	if [ "${FLAG_B}" = "TRUE" ] ; then
716	# WE CONCATENATE WITH EXISTING FILES
717	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
718	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
719	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
720	#
721	IGCM_sys_Rm ${file1}
722	if [ ! "${DEPOT}" = "TRUE" ] ; then
723	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
724	# "${file1} = file_before ?
725	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
726	fi
727
728	ExitFlag=false
729	else
730	# OR NOT
731	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
732	fi
733
734	#
735	if [ "${DEPOT}" = "TRUE" ] ; then
736	# WE PUT FINAL FILE ON FILE SYSTEM
737	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
738	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
739	#
740	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
741	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
742	if [ ! "X${file1}" = "X" ] ; then
743	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
744	fi
745	#
746	if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
747	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
748	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
749	#
750	# Compute yearly average considering calendar
751	# and set date of the mid year to 01/07 at 00:00
752	IGCM_sys_cdo -settime,00:00 -setday,1 -setmon,7 -settunits,days -divdpy -yearsum -muldpm -selvar,${var} ${file_out} ${file_out_YE}
753	# Extract all ancillary variables except $var, they will be appended after the cdo command
754	# Needed because cdo applies calculs on all variables
755	IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
756	# This will remove the time axis because cdo rewrites severely with only considered useful variables
757	IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
758	if [ $? -eq 0 ] ; then
759	# Now append all ancillary variables if previous cdo operator ok
760	IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
761	fi
762
763	# Put global history attribut
764	IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
765	#
766	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
767	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
768	IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
769	fi
770	#
771	IGCM_sys_Rm ${file_out}
772	ExitFlag=false
773	#
774	fi
775	#
776	done # for var in ${liste_var} ; do
777	IGCM_sys_Rm ${liste_file_tmp[*]}
778	done # for file in ${liste_file_brut} do
779	done # for comp in ${config_ListOfComponents[*]} ; do
780	# GO AHEAD IN DEEPEST LOOP
781	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
782	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
783	# FINISH
784
785	# DODS copy
786	for comp in ${config_ListOfComponents[*]} ; do
787	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
788	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
789	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
790	done
791	done
792
793	export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
794	export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG
795
796	listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
797	export listVarEnv
798
799	#D-
800	# --------------------------------------------------------------------
801	#D- Test if all was right
802	# --------------------------------------------------------------------
803	IGCM_debug_Verif_Exit_Post
804	# --------------------------------------------------------------------
805	#D- Submit MONITORING
806	# --------------------------------------------------------------------
807	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
808	Script_Post_Output=monitoring.${PeriodDateEnd}
809	IGCM_sys_QsubPost monitoring
810	fi
811
812	#D-
813	# --------------------------------------------------------------------
814	#D- Update the run.card
815	# --------------------------------------------------------------------
816	if [ ${StandAlone} != true ] ; then
817	# Put in run.card end period of time series
818	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
819	# We have finish the job !
820	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
821	fi
822
823	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
824	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
825
826	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: