Context Navigation

source: trunk/libIGCM/AA_create_ts @ 781

Last change on this file since 781 was 770, checked in by sdipsl, 11 years ago
Adapat headers for ada. Clean up old headers (platine, cesium, brodie, ...)
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 29.6 KB

Line
1	#-Q- curie ######################
2	#-Q- curie ## CURIE TGCC/CEA ##
3	#-Q- curie ######################
4	#-Q- curie #MSUB -r TS # Nom du job
5	#-Q- curie #MSUB -eo
6	#-Q- curie #MSUB -n 1 # Reservation du processus
7	#-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job
8	#-Q- curie #MSUB -q xlarge
9	#-Q- curie #MSUB -Q normal
10	#-Q- curie set +x
11	#-Q- ada #!/bin/ksh
12	#-Q- ada #######################
13	#-Q- ada ## ADA IDRIS ##
14	#-Q- ada #######################
15	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
16	#-Q- ada # @ wall_clock_limit = 20:00:00
17	#-Q- ada # Nom du travail LoadLeveler
18	#-Q- ada # @ job_name = TS
19	#-Q- ada # Fichier de sortie standard du travail
20	#-Q- ada # @ output = $(job_name).$(jobid)
21	#-Q- ada # Fichier de sortie d'erreur du travail
22	#-Q- ada # @ error = $(job_name).$(jobid)
23	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
24	#-Q- ada # @ notification = error
25	#-Q- ada # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
26	#-Q- ada # @ queue
27	#-Q- sx9mercure #!/bin/ksh
28	#-Q- sx9mercure ######################
29	#-Q- sx9mercure ## SX9MERCURE CCRT ##
30	#-Q- sx9mercure ######################
31	#-Q- sx9mercure #PBS -N TS # Nom du job
32	#-Q- sx9mercure #PBS -j o # regroupement des stdout et stderr
33	#-Q- sx9mercure #PBS -S /usr/bin/ksh # shell de soumission
34	#-Q- sx9mercure #PBS -l memsz_job=1gb # Limite memoire a 1 Go
35	#-Q- sx9mercure #PBS -l elapstim_req=24:00:00 # Limite temps a 2 heures
36	#-Q- sx9mercure #PBS -q scalaire
37	#-Q- sx9mercure #PBS -r n
38	#-Q- titane #!/bin/ksh
39	#-Q- titane ######################
40	#-Q- titane ## TITANE CEA ##
41	#-Q- titane ######################
42	#-Q- titane #MSUB -r TS # Nom du job
43	#-Q- titane #MSUB -eo
44	#-Q- titane #MSUB -n 1 # Reservation du processus
45	#-Q- titane #MSUB -T 86400 # Limite de temps elapsed du job
46	#-Q- titane #MSUB -q mono
47	#-Q- titane #MSUB -E '-rn'
48	#-Q- lxiv8 ######################
49	#-Q- lxiv8 ## OBELIX LSCE ##
50	#-Q- lxiv8 ######################
51	#-Q- lxiv8 #PBS -N TS
52	#-Q- lxiv8 #PBS -m a
53	#-Q- lxiv8 #PBS -j oe
54	#-Q- lxiv8 #PBS -q medium
55	#-Q- lxiv8 #PBS -o TS.$$
56	#-Q- lxiv8 #PBS -S /bin/ksh
57	#-Q- default #!/bin/ksh
58	#-Q- default ##################
59	#-Q- default ## DEFAULT HOST ##
60	#-Q- default ##################
61
62	#**************************************************************
63	# Author: Sebastien Denvil
64	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
65	# $Revision:: $ Revision of last commit
66	# $Author:: $ Author of last commit
67	# $Date:: $ Date of last commit
68	# IPSL (2006)
69	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
70	#
71	#**************************************************************
72
73	#set -eu
74	#set -vx
75
76	date
77
78	#-Q- ada export OMP_NUM_THREADS=1
79
80	#D- Task type (computing or post-processing)
81	TaskType=post-processing
82
83	########################################################################
84
85	#D- Flag to determine if this job in a standalone mode
86	#D- Default : value from AA_job if any
87	StandAlone=${StandAlone:=true}
88
89	#D- Path to libIGCM
90	#D- Default : value from AA_job if any
91	libIGCM=${libIGCM:=::modipsl::/libIGCM}
92
93	#D- Flag to determine allready produced time series. Empty if you start from the beginning
94	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
95	CompletedFlag=${CompletedFlag:=}
96
97	#D- Flag to determine task type this script will perform.
98	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
99	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
100	TsTask=${TsTask:=2D}
101	if [ X${TsTask} = Xempty ] ; then
102	TsTask=""
103	fi
104
105	#D- Flag to determine if rebuild process has been performed asynchronously
106	#D- Possible value true or false.
107	#D- If true raw files has allready been patched by rebuild job
108	#D- If false create_ts will patch the raw files
109	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
110	RebuildFrequency=${RebuildFrequency:=true}
111
112	#D- Flag to determine job's output directory
113	#D- Default : value from libIGCM_post.ksh if any
114	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
115
116	#D- Increased verbosity (1, 2, 3)
117	#D- Default : value from AA_job if any
118	Verbosity=${Verbosity:=3}
119
120	#D- Low level debug : to bypass lib test checks and stack construction
121	#D- Default : value from AA_job if any
122	DEBUG_debug=${DEBUG_debug:=false}
123
124	#D- TEMPORARY Flag to determine atmospheric resolution
125	#D- Default : value from atmospheric driver if any
126	RESOL_ATM=ALL
127
128	#D- Flag to determine surface resolution
129	#D- Default : value from surface driver if any
130	RESOL_SRF=ALL
131
132	#D- Flag to determine surface resolution
133	#D- Default : value from surface driver if any
134	RESOL_SBG=ALL
135
136	#D- TEMPORARY Flag to determine ocean resolution
137	#D- Default : value from ocean driver if any
138	RESOL_OCE=${RESOL_OCE:=ORCA2}
139
140	#D- TEMPORARY Flag to determine ice resolution
141	#D- Default : value from ice driver if any
142	RESOL_ICE=${RESOL_ICE:=ORCA2}
143
144	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
145	#D- Default : value from ice driver if any
146	RESOL_MBG=${RESOL_MBG:=ORCA2}
147
148	########################################################################
149
150	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
151	( ${DEBUG_debug} ) && IGCM_debug_Check
152	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
153	( ${DEBUG_debug} ) && IGCM_card_Check
154	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
155	( ${DEBUG_debug} ) && IGCM_date_Check
156	#-------
157	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
158	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
159
160	########################################################################
161
162	#set -vx
163
164	# Useful cleaning function
165	MENAGE ()
166	{
167	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
168	}
169
170	########################################################################
171
172	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
173	IGCM_sys_Cd ${RUN_DIR_PATH}
174
175	# ------------------------------------------------------------------
176	# Test if all was right before proceeding further
177	# ------------------------------------------------------------------
178	IGCM_debug_Verif_Exit_Post
179
180	if [ ${StandAlone} = true ] ; then
181	CARD_DIR=${SUBMIT_DIR}
182	else
183	CARD_DIR=${RUN_DIR_PATH}
184	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
185	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
186	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
187	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
188	fi
189
190	#==================================
191	# First of all
192	#
193	# Read libIGCM compatibility version in config.card
194	# Read UserChoices section
195	# Read Ensemble section
196	# Read Post section
197	# Define all netcdf output directories
198	#==================================
199	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
200
201	#==================================
202	# Define default value to keep compatibility with previous card: means before changes due to TGCC
203	if [ X${PackDefault} = Xtrue ] ; then
204	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
205	else
206	config_Post_PackFrequency=NONE
207	fi
208
209	#==================================
210	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
211	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
212	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
213	fi
214
215	#
216	# Determine component to take care of depending on kind of task create_ts will perform
217	#
218	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
219	#
220	# Determine period for time series. Default : value from AA_job or from launch_TS if any
221	#
222	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
223
224	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
225	#
226	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
227	#
228	# Determine Dimension of the produced time series : 2D or 3D
229	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
230	Dimension=2D
231	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
232	Dimension=3D
233	else
234	Dimension=""
235	fi
236
237	# ------------------------------------------------------------------
238	#D- Test if all was right before entering the loop
239	# ------------------------------------------------------------------
240	IGCM_debug_Verif_Exit_Post
241
242	########################################################################
243	#
244	# Depending on requested time series fill following variables :
245	#
246	# - LISTE_FILE_${comp}[*]
247	# - LISTE_VARS_${file}[*]
248	# - LISTE_AXIS_${file}[*]
249	# - LISTE_PATCH_${file}[*]
250	########################################################################
251
252	#
253	# For each selected component determine which files need post-processing
254	#
255	DoJob=false
256	for comp in ${config_ListOfComponents[*]} ; do
257	#
258	ActiveComp=false
259	#
260	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
261	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
262	#
263	card=${CARD_DIR}/COMP/${compname}.card
264
265	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
266	#
267	ListFilesName=${compname}_OutputFiles_List
268	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
269	#
270	if [ X${FileName0} != X${NULL_STR} ] ; then
271	#
272	#IGCM_debug_Print 1 "Component : ${compname}"
273	#
274	# INITIALISATION
275	#
276	typeset i
277	if [ ${FlagToRead} ] ; then
278	((NbFiles=${FlagToRead}))
279	((i=${NbFiles}))
280	else
281	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
282	i=2
283	fi
284	#
285	until [ $i -gt $NbFiles ]; do
286	#
287	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
288	#
289	if [ X${flag_post} != XNONE ] ; then
290	#
291	# For each comp and each file who needs post-processing
292	# determine which variables need time series.
293	#
294	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
295	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
296	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
297	#
298	# If TimeSeriesVars list is empty we skip
299	#
300	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
301	#
302	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
303	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
304	(( i=i+3 ))
305	continue
306	#
307	fi
308	#
309	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
310	#
311	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
312	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
313	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
314	(( i=i+3 ))
315	continue
316	fi
317	#
318	# If we are not a Chunck type task, we skip files/variables handled by chunck task
319	#
320	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
321	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
322	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
323	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
324	(( i=i+3 ))
325	continue
326	fi
327	fi
328	#
329	# Now on we know we have things to do
330	#
331
332	#
333	# We need coordinates axis to properly fill produced time series
334	#
335	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
336	#
337	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
338	#
339	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
340	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
341	(( i=i+3 ))
342	continue
343	#
344	fi
345	#
346	# We need LIST of variables not allready produced (useful for standalone mode)
347	#
348	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
349	case ${FlagDir} in
350	*Y) TS_Dir=TS_YE ;;
351	*M) TS_Dir=TS_MO ;;
352	*D) TS_Dir=TS_DA ;;
353	HF) TS_Dir=TS_HF ;;
354	3H) TS_Dir=TS_HF ;;
355	INS) TS_Dir=TS_INS ;;
356	esac
357	#
358	count=0
359	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
360	#
361	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
362	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
363	#
364	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
365	if [ ! $? = 0 ] ; then
366	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
367	ActiveComp=true
368	(( count = count + 1 ))
369	fi
370	done
371	#
372	if [ ${count} -gt 0 ] ; then
373	#
374	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
375	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
376	#
377	if [ ! X${RebuildFrequency} = Xtrue ] ; then
378	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
379	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
380	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
381	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
382	if [ Xload${Patch} != X ] ; then
383	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
384	eval load${Patch}=loaded
385	fi
386	done
387	fi
388	fi
389	fi
390	fi
391	(( i=i+3 ))
392	done
393	fi
394	#
395	if [ ${ActiveComp} = true ] ; then
396	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
397	fi
398	done
399
400	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
401	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
402
403	########################################################################
404	#
405	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
406	# IF SO BRING THEM IN THE WORKING DIRECTORY
407	#
408	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
409	# - especially for IDRiS
410	# - no special protection against "command too long" !
411	# - libIGCM_sys should handle that or modify following code part
412	#
413	########################################################################
414
415	if [ ${StandAlone} != true ] ; then
416	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
417	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
418	echo \${run_PostProcessing_TimeSeriesRunning}" )
419
420	#if [ X${Running_Flag} = Xy ] ; then
421	# # Time Series Job allready running
422	# IGCM_debug_Print 1 "Time Series Job allready running exit"
423	# exit
424	#fi
425	# Now run.card know we are running
426
427
428
429
430	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
431
432
433
434
435
436	# Get information from last execution
437	#CompletedFlag=$( IGCM_sys_RshMaster \
438	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
439	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
440	fi
441
442	# Is it the first submission or not ?
443
444	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
445	# First Time Series Submission
446	FIRST_PASS=TRUE
447	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
448	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
449	else
450	# Last Completed Time Series
451	FIRST_PASS=FALSE
452	DATE_FIN_JOB_B=${CompletedFlag}
453	date=${DateBegin}_${DATE_FIN_JOB_B}
454	for comp in ${config_ListOfComponents[*]} ; do
455	#
456	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
457	#
458	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
459	#
460	#set -A liste_file
461	unset liste_file
462	#
463	file_path=${R_SAVE}/${comp}/Analyse
464	#
465	i=0
466	file=${NULL_STR}
467	#
468	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
469	#
470	# Determine in which output can we find file
471	# IE : Analyse/TS or Analyse/DA2TS or ...
472	#
473	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
474	case ${FlagDir} in
475	*Y) TS_Dir=TS_YE ;;
476	*M) TS_Dir=TS_MO ;;
477	*D) TS_Dir=TS_DA ;;
478	HF) TS_Dir=TS_HF ;;
479	3H) TS_Dir=TS_HF ;;
480	INS) TS_Dir=TS_INS ;;
481	esac
482	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
483	#
484	#set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
485	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
486	#
487	done
488	done
489	if [ X${file} != X${NULL_STR} ] ; then
490	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
491	# Even if some files are missing we tried to continue
492	# Either ListVar in compname.card are asked but not present in original history file
493	# Either a previous TS task failed
494	ExitFlag=false
495	IGCM_debug_Verif_Exit_Post
496	fi
497	done
498	#
499	LEVEL=0
500	#
501	fi
502
503	########################################################################
504	#
505	# DEFINE LOOP PARAMETERS
506	#
507	########################################################################
508
509	DATE_COUNT=${DATE_FIN_JOB_B}
510	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
511
512	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
513
514	if [ ${DoJob} = true ] ; then
515	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
516	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
517	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
518	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
519	done
520	else
521	NBRE_FILE_TOT=0
522	fi
523
524	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
525	NBRE_TOUR=1
526	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
527	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
528	else
529	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
530	fi
531
532	CURRENT_LOOP=1
533	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
534	# WE HAVE 3 LEVELS
535	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
536	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
537	# - LEVEL 3 : WE ARE IN THE LAST LOOP
538
539	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
540
541	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
542	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
543	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
544
545	if [ ${FIRST_PASS} = TRUE ] ; then
546	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
547	else
548	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
549	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
550	fi
551
552	DATE_FIN=${PeriodDateEnd}
553	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
554
555	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
556	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
557
558	LEVEL=1
559	DEPOT="TRUE"
560	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
561	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
562	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
563
564	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
565	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
566	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
567	elif [ -z "${DATE_COURANTE}" ] ; then
568	DATE_COURANTE=${DATE_FIN_JOB_B}
569	else
570	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
571	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
572	fi
573
574	(( TotLength = 0 ))
575	COMPTEUR=1
576	DATE_LOOP=${DATE_COURANTE}
577
578	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
579	#
580	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
581	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
582	(( TotLength = TotLength + Length ))
583	(( COMPTEUR = COMPTEUR + 1 ))
584	#
585	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
586	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
587
588	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
589	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
590	DEPOT="FALSE"
591	LEVEL=2
592	else
593	# - LEVEL 3 : WE ARE IN THE LAST LOOP
594	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
595	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
596
597	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
598	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
599	DATE_FIN=${PeriodDateEnd}
600	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
601
602	LEVEL=3
603	DEPOT="TRUE"
604	FLAG_B="TRUE"
605	fi
606
607	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
608	for comp in ${config_ListOfComponents[*]} ; do
609	#
610	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
611	#
612	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
613	#
614	eval R_OUT_${comp}=${R_SAVE}/${comp}
615	#
616	i=0
617	#
618	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
619	#
620	# Determine in which output can we find file
621	# IE : Output/MO or Output/DA or ...
622	#
623	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
624	case ${FlagDir} in
625	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
626	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
627	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
628	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
629	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
630	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
631	*)
632	IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
633	IGCM_sys_Exit "Job create_ts" ;;
634	esac
635
636	# Initialize array
637	#set -A liste_file
638	#set -A liste_file_tmp
639	unset liste_file
640	unset liste_file_tmp
641	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
642
643	COMPTEUR=1
644	DATE_LOOP=${DATE_COURANTE}
645
646	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
647	#
648	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
649	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
650
651	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
652	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
653
654	#set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
655	#set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
656	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
657	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
658	(( COMPTEUR = COMPTEUR + 1 ))
659	#
660	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
661	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
662
663	# Get selected files
664	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
665	IGCM_debug_Verif_Exit_Post
666
667	# Apply IGCM_Patch if needed
668	if [ ! X${RebuildFrequency} = Xtrue ] ; then
669	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
670	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
671	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
672	IGCM_${Patch} ${file_t}
673	IGCM_debug_Verif_Exit_Post
674	done
675	done
676	fi
677	fi
678
679	# WE CAN CONCATENATE FILES AT THIS POINT
680	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
681	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
682	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
683	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
684	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
685
686	liste_coord=" "
687	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
688	liste_coord=${liste_coord}${axis}","
689	done
690
691	# Time axis must be the last dimension
692	time_axis=${axis}
693
694	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
695	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
696	#
697	if [ "${FLAG_B}" = "TRUE" ] ; then
698	# WE CONCATENATE WITH EXISTING FILES
699	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
700	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
701	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
702	#
703	IGCM_sys_Rm ${file1}
704	if [ ! "${DEPOT}" = "TRUE" ] ; then
705	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
706	# "${file1} = file_before ?
707	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
708	fi
709
710	ExitFlag=false
711	else
712	# OR NOT
713	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
714	fi
715
716	#
717	if [ "${DEPOT}" = "TRUE" ] ; then
718	# WE PUT FINAL FILE ON FILE SYSTEM
719	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
720	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
721	#
722	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
723	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
724	if [ ! "X${file1}" = "X" ] ; then
725	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
726	fi
727	#
728	if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
729	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
730	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
731	#
732	# Compute yearly average considering calendar
733	# and set date of the mid year to 01/07 at 00:00
734	IGCM_sys_cdo -settime,00:00 -setday,1 -setmon,7 -settunits,days -divdpy -yearsum -muldpm -selvar,${var} ${file_out} ${file_out_YE}
735	# Extract all ancillary variables except $var, they will be appended after the cdo command
736	# Needed because cdo applies calculs on all variables
737	IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
738	# This will remove the time axis because cdo rewrites severely with only considered useful variables
739	IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
740	if [ $? -eq 0 ] ; then
741	# Now append all ancillary variables if previous cdo operator ok
742	IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
743	fi
744
745	# Put global history attribut
746	IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
747	#
748	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
749	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
750	IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
751	fi
752	#
753	IGCM_sys_Rm ${file_out}
754	ExitFlag=false
755	#
756	fi
757	#
758	done # for var in ${liste_var} ; do
759	IGCM_sys_Rm ${liste_file_tmp[*]}
760	done # for file in ${liste_file_brut} do
761	done # for comp in ${config_ListOfComponents[*]} ; do
762	# GO AHEAD IN DEEPEST LOOP
763	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
764	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
765	# FINISH
766
767	# DODS copy
768	for comp in ${config_ListOfComponents[*]} ; do
769	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
770	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
771	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
772	done
773	done
774
775	export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
776	export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG
777
778	listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
779	export listVarEnv
780
781	#D-
782	# --------------------------------------------------------------------
783	#D- Test if all was right
784	# --------------------------------------------------------------------
785	IGCM_debug_Verif_Exit_Post
786	# --------------------------------------------------------------------
787	#D- Submit MONITORING
788	# --------------------------------------------------------------------
789	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
790	Script_Post_Output=monitoring.${PeriodDateEnd}
791	IGCM_sys_QsubPost monitoring
792	fi
793
794	#D-
795	# --------------------------------------------------------------------
796	#D- Update the run.card
797	# --------------------------------------------------------------------
798	if [ ${StandAlone} != true ] ; then
799	# Put in run.card end period of time series
800	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
801	# We have finish the job !
802	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
803	fi
804
805	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
806	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
807
808	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: