Context Navigation

source: trunk/libIGCM/AA_create_ts @ 1206

Last change on this file since 1206 was 1206, checked in by sdipsl, 9 years ago
Remove IGCM_debug_Verif_Exit_Post. Only IGCM_debug_Verif_Exit will manage exit cases.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 31.2 KB

Line
1	#-Q- curie ######################
2	#-Q- curie ## CURIE TGCC/CEA ##
3	#-Q- curie ######################
4	#-Q- curie #MSUB -r TS # Nom du job
5	#-Q- curie #MSUB -eo
6	#-Q- curie #MSUB -n 1 # Reservation du processus
7	#-Q- curie #MSUB -T 80000 # Limite de temps elapsed du job
8	#-Q- curie #MSUB -q ::default_node::
9	#-Q- curie #MSUB -Q normal
10	#-Q- curie #MSUB -A ::default_project::
11	#-Q- curie set +x
12	#-Q- ada #!/bin/ksh
13	#-Q- ada #######################
14	#-Q- ada ## ADA IDRIS ##
15	#-Q- ada #######################
16	#-Q- ada # @ job_type = serial
17	#-Q- ada # @ requirements = (Feature == "prepost")
18	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
19	#-Q- ada # @ wall_clock_limit = 20:00:00
20	#-Q- ada # Nom du travail LoadLeveler
21	#-Q- ada # @ job_name = TS
22	#-Q- ada # Fichier de sortie standard du travail
23	#-Q- ada # @ output = $(job_name).$(jobid)
24	#-Q- ada # Fichier de sortie d'erreur du travail
25	#-Q- ada # @ error = $(job_name).$(jobid)
26	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
27	#-Q- ada # @ notification = error
28	#-Q- ada # @ environment = $DEBUG_debug ; $BigBrother ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $POST_DIR ; $Script_Post_Output ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateEnd ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; wall_clock_limit=$(wall_clock_limit)
29	#-Q- ada # @ queue
30	#-Q- lxiv8 ######################
31	#-Q- lxiv8 ## OBELIX LSCE ##
32	#-Q- lxiv8 ######################
33	#-Q- lxiv8 #PBS -N TS
34	#-Q- lxiv8 #PBS -m a
35	#-Q- lxiv8 #PBS -j oe
36	#-Q- lxiv8 #PBS -q medium
37	#-Q- lxiv8 #PBS -o TS.$$
38	#-Q- lxiv8 #PBS -S /bin/ksh
39	#-Q- ifort_CICLAD ######################
40	#-Q- ifort_CICLAD ## CICLAD IPSL ##
41	#-Q- ifort_CICLAD ######################
42	#-Q- ifort_CICLAD #PBS -N TS
43	#-Q- ifort_CICLAD #PBS -m a
44	#-Q- ifort_CICLAD #PBS -j oe
45	#-Q- ifort_CICLAD #PBS -q h12
46	#-Q- ifort_CICLAD #PBS -S /bin/ksh
47	#-Q- default #!/bin/ksh
48	#-Q- default ##################
49	#-Q- default ## DEFAULT HOST ##
50	#-Q- default ##################
51
52	#**************************************************************
53	# Author: Sebastien Denvil
54	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
55	# $Revision:: $ Revision of last commit
56	# $Author:: $ Author of last commit
57	# $Date:: $ Date of last commit
58	# IPSL (2006)
59	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
60	#
61	#**************************************************************
62
63	#set -eu
64	#set -vx
65
66	date
67
68	#-Q- ada export OMP_NUM_THREADS=1
69
70	#D- Task type (computing or post-processing)
71	TaskType=post-processing
72
73	########################################################################
74
75	#D- Flag to determine if this job in a standalone mode
76	#D- Default : value from AA_job if any
77	StandAlone=${StandAlone:=true}
78
79	#D- Path to libIGCM
80	#D- Default : value from AA_job if any
81	libIGCM=${libIGCM:=::modipsl::/libIGCM}
82
83	#D- Flag to determine allready produced time series. Empty if you start from the beginning
84	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
85	CompletedFlag=${CompletedFlag:=}
86
87	#D- Flag to determine task type this script will perform.
88	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
89	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
90	TsTask=${TsTask:=2D}
91	if [ X${TsTask} = Xempty ] ; then
92	TsTask=""
93	fi
94
95	#D- Flag to determine if rebuild process has been performed asynchronously
96	#D- Possible value true or false.
97	#D- If true raw files has allready been patched by rebuild job
98	#D- If false create_ts will patch the raw files
99	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
100	RebuildFrequency=${RebuildFrequency:=true}
101
102	#D- Flag to determine job's output directory
103	#D- Default : value from libIGCM_post.ksh if any
104	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
105
106	#D- Increased verbosity (1, 2, 3)
107	#D- Default : value from AA_job if any
108	Verbosity=${Verbosity:=3}
109
110	#D- Low level debug : to bypass lib test checks and stack construction
111	#D- Default : value from AA_job if any
112	DEBUG_debug=${DEBUG_debug:=false}
113
114	#D- TEMPORARY Flag to determine atmospheric resolution
115	#D- Default : value from atmospheric driver if any
116	RESOL_ATM=ALL
117
118	#D- Flag to determine surface resolution
119	#D- Default : value from surface driver if any
120	RESOL_SRF=ALL
121
122	#D- Flag to determine surface resolution
123	#D- Default : value from surface driver if any
124	RESOL_SBG=ALL
125
126	#D- TEMPORARY Flag to determine ocean resolution
127	#D- Default : value from ocean driver if any
128	RESOL_OCE=${RESOL_OCE:=ORCA2}
129
130	#D- TEMPORARY Flag to determine ice resolution
131	#D- Default : value from ice driver if any
132	RESOL_ICE=${RESOL_ICE:=ORCA2}
133
134	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
135	#D- Default : value from ice driver if any
136	RESOL_MBG=${RESOL_MBG:=ORCA2}
137
138	########################################################################
139
140	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
141	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
142	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
143	#-------
144	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
145	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
146	#-------
147	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
148	IGCM_sys_Cd ${RUN_DIR_PATH}
149	#-------
150	( ${DEBUG_debug} ) && IGCM_debug_Check
151	( ${DEBUG_debug} ) && IGCM_card_Check
152	( ${DEBUG_debug} ) && IGCM_date_Check
153
154	########################################################################
155
156	#set -vx
157
158	# Useful cleaning function
159	MENAGE ()
160	{
161	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
162	}
163
164	########################################################################
165
166	# ------------------------------------------------------------------
167	# Test if all was right before proceeding further
168	# ------------------------------------------------------------------
169	IGCM_debug_Verif_Exit
170
171	if [ ${StandAlone} = true ] ; then
172	CARD_DIR=${SUBMIT_DIR}
173	else
174	CARD_DIR=${RUN_DIR_PATH}
175
176	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
177
178	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
179
180	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
181
182	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
183	fi
184
185	#==================================
186	# First of all
187	#
188	# Read libIGCM compatibility version in config.card
189	# Read UserChoices section
190	# Read Ensemble section
191	# Read Post section
192	# Define all netcdf output directories
193	#==================================
194	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
195
196	# ------------------------------------------------------------------
197	# Activate BigBrother so as to supervise this job
198	# ------------------------------------------------------------------
199	IGCM_debug_BigBro_Initialize
200
201	#==================================
202	# Define default value to keep compatibility with previous card: means before changes due to TGCC
203	if [ X${PackDefault} = Xtrue ] ; then
204	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
205	else
206	config_Post_PackFrequency=NONE
207	fi
208
209	#==================================
210	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
211	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
212	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
213	fi
214
215	#
216	# Determine component to take care of depending on kind of task create_ts will perform
217	#
218	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
219	#
220	# Determine period for time series. Default : value from AA_job or from launch_TS if any
221	#
222	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
223
224	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
225	#
226	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
227	#
228	# Determine Dimension of the produced time series : 2D or 3D
229	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
230	Dimension=2D
231	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
232	Dimension=3D
233	else
234	Dimension=""
235	fi
236
237	# ------------------------------------------------------------------
238	#D- Test if all was right before entering the loop
239	# ------------------------------------------------------------------
240	IGCM_debug_Verif_Exit
241
242	########################################################################
243	# Depending on requested time series fill following variables :
244	#
245	# - LISTE_FILE_${comp}[*]
246	# - LISTE_VARS_${file}[*]
247	# - LISTE_AXIS_${file}[*]
248	# - LISTE_PATCH_${file}[*]
249	########################################################################
250
251	#
252	# For each selected component determine which files need post-processing
253	#
254	DoJob=false
255	for comp in ${config_ListOfComponents[*]} ; do
256	#
257	ActiveComp=false
258	#
259	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
260	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
261	#
262	card=${CARD_DIR}/COMP/${compname}.card
263
264	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
265	#
266	ListFilesName=${compname}_OutputFiles_List
267	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
268	#
269	if [ X${FileName0} != X${NULL_STR} ] ; then
270	#
271	#IGCM_debug_Print 1 "Component : ${compname}"
272	#
273	# INITIALISATION
274	#
275	typeset i
276	if [ ${FlagToRead} ] ; then
277	((NbFiles=${FlagToRead}))
278	((i=${NbFiles}))
279	else
280	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
281	i=2
282	fi
283	#
284	until [ $i -gt $NbFiles ]; do
285	#
286	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
287	#
288	if [ X${flag_post} != XNONE ] ; then
289	#
290	# For each comp and each file who needs post-processing
291	# determine which variables need time series.
292	#
293	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
294	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
295	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
296	#
297	# If TimeSeriesVars list is empty we skip
298	#
299	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
300	#
301	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
302	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
303	(( i=i+3 ))
304	continue
305	#
306	fi
307	#
308	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
309	#
310	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
311	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
312	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
313	(( i=i+3 ))
314	continue
315	fi
316	#
317	# If we are not a Chunck type task, we skip files/variables handled by chunck task
318	#
319	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
320	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
321	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
322	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
323	(( i=i+3 ))
324	continue
325	fi
326	fi
327	#
328	# Now on we know we have things to do
329	#
330
331	#
332	# We need coordinates axis to properly fill produced time series
333	#
334	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
335	#
336	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
337	#
338	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
339	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
340	(( i=i+3 ))
341	continue
342	#
343	fi
344	#
345	# We need LIST of variables not allready produced (useful for standalone mode)
346	#
347	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
348	case ${FlagDir} in
349	*Y) TS_Dir=TS_YE ;;
350	*M) TS_Dir=TS_MO ;;
351	*D) TS_Dir=TS_DA ;;
352	HF) TS_Dir=TS_HF ;;
353	3H) TS_Dir=TS_HF ;;
354	INS) TS_Dir=TS_INS ;;
355	esac
356	#
357	count=0
358	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
359	#
360	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
361	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
362	#
363	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
364	if [ ! $? = 0 ] ; then
365	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
366	ActiveComp=true
367	(( count = count + 1 ))
368	fi
369	done
370	#
371	if [ ${count} -gt 0 ] ; then
372	#
373	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
374	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
375	#
376	if [ ! X${RebuildFrequency} = Xtrue ] ; then
377	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
378	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
379	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
380	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
381	if [ Xload${Patch} != X ] ; then
382	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
383	eval load${Patch}=loaded
384	fi
385	done
386	fi
387	fi
388	fi
389	fi
390	(( i=i+3 ))
391	done
392	fi
393	#
394	if [ ${ActiveComp} = true ] ; then
395	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
396	fi
397	done
398
399	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
400	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
401
402	########################################################################
403	#
404	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
405	# IF SO BRING THEM IN THE WORKING DIRECTORY
406	#
407	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
408	# - especially for IDRiS
409	# - no special protection against "command too long" !
410	# - libIGCM_sys should handle that or modify following code part
411	#
412	########################################################################
413
414	if [ ${StandAlone} != true ] ; then
415	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
416	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
417	echo \${run_PostProcessing_TimeSeriesRunning}" )
418
419	#if [ X${Running_Flag} = Xy ] ; then
420	# # Time Series Job allready running
421	# IGCM_debug_Print 1 "Time Series Job allready running exit"
422	# exit
423	#fi
424	# Now run.card know we are running
425
426
427
428
429	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
430
431
432
433
434
435	# Get information from last execution
436	#CompletedFlag=$( IGCM_sys_RshMaster \
437	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
438	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
439	fi
440
441	# Is it the first submission or not ?
442
443	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
444	# First Time Series Submission
445	FIRST_PASS=TRUE
446	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
447	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
448	else
449	# Last Completed Time Series
450	FIRST_PASS=FALSE
451	DATE_FIN_JOB_B=${CompletedFlag}
452	date=${DateBegin}_${DATE_FIN_JOB_B}
453	#
454	for comp in ${config_ListOfComponents[*]} ; do
455	#
456	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
457	#
458	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
459	#
460	unset liste_file
461	#
462	file_path=${R_SAVE}/${comp}/Analyse
463	#
464	i=0
465	file=${NULL_STR}
466	#
467	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
468	#
469	# Determine in which output can we find file
470	# IE : Analyse/TS or Analyse/DA2TS or ...
471	#
472	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
473	case ${FlagDir} in
474	*Y) TS_Dir=TS_YE ;;
475	*M) TS_Dir=TS_MO ;;
476	*D) TS_Dir=TS_DA ;;
477	HF) TS_Dir=TS_HF ;;
478	3H) TS_Dir=TS_HF ;;
479	INS) TS_Dir=TS_INS ;;
480	esac
481	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
482	#
483	IGCM_sys_TestFileArchive ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
484	if [ $? = 0 ] ; then
485	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
486	else
487	IGCM_debug_Print 1 "WARNING ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc dont exist"
488	fi
489	#
490	done
491	done
492	if [ X${file} != X${NULL_STR} ] ; then
493	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
494	IGCM_debug_Verif_Exit
495	fi
496	done
497	#
498	LEVEL=0
499	#
500	fi
501
502	########################################################################
503	#
504	# DEFINE LOOP PARAMETERS
505	#
506	########################################################################
507
508	DATE_COUNT=${DATE_FIN_JOB_B}
509	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
510
511	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
512
513	if [ ${DoJob} = true ] ; then
514	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
515	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
516	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
517	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
518	done
519	else
520	NBRE_FILE_TOT=0
521	fi
522
523	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
524	NBRE_TOUR=1
525	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
526	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
527	else
528	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
529	fi
530
531	CURRENT_LOOP=1
532	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
533	# WE HAVE 3 LEVELS
534	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
535	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
536	# - LEVEL 3 : WE ARE IN THE LAST LOOP
537
538	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
539
540	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
541	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
542	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
543
544	if [ ${FIRST_PASS} = TRUE ] ; then
545	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
546	else
547	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
548	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
549	fi
550
551	DATE_FIN=${PeriodDateEnd}
552	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
553
554	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
555	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
556
557	LEVEL=1
558	DEPOT="TRUE"
559	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
560	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
561	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
562
563	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
564	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
565	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
566	elif [ -z "${DATE_COURANTE}" ] ; then
567	DATE_COURANTE=${DATE_FIN_JOB_B}
568	else
569	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
570	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
571	fi
572
573	(( TotLength = 0 ))
574	COMPTEUR=1
575	DATE_LOOP=${DATE_COURANTE}
576
577	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
578	#
579	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
580	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
581	(( TotLength = TotLength + Length ))
582	(( COMPTEUR = COMPTEUR + 1 ))
583	#
584	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
585	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
586
587	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
588	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
589	DEPOT="FALSE"
590	LEVEL=2
591	else
592	# - LEVEL 3 : WE ARE IN THE LAST LOOP
593	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
594	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
595
596	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
597	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
598	DATE_FIN=${PeriodDateEnd}
599	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
600
601	LEVEL=3
602	DEPOT="TRUE"
603	FLAG_B="TRUE"
604	fi
605
606	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
607	for comp in ${config_ListOfComponents[*]} ; do
608	#
609	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
610	#
611	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
612	#
613	eval R_OUT_${comp}=${R_SAVE}/${comp}
614	#
615	i=0
616	#
617	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
618	#
619	# Determine in which output we can find file
620	# IE : Output/MO or Output/DA or ...
621	#
622	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
623	case ${FlagDir} in
624	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
625	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
626	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
627	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
628	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
629	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
630	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
631	IGCM_debug_Exit "Job create_ts" ;;
632	esac
633
634	# Initialize array
635	unset liste_file
636	unset liste_file_tmp
637	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
638
639	COMPTEUR=1
640	MissingFile=FALSE
641	DATE_LOOP=${DATE_COURANTE}
642
643	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
644	#
645	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
646	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
647
648	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
649	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
650
651	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
652	if [ $? = 0 ] ; then
653	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
654	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
655	else
656	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
657	MissingFile=TRUE
658	fi
659	(( COMPTEUR = COMPTEUR + 1 ))
660	#
661	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
662
663	# We skip this file type if one of them is missing.
664	if [ "${MissingFile}" = "TRUE" ] ; then
665	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
666	continue
667	fi
668
669	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
670
671	# Get selected files
672	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
673	IGCM_debug_Verif_Exit
674
675	# Apply IGCM_Patch if needed
676	if [ ! X${RebuildFrequency} = Xtrue ] ; then
677	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
678	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
679	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
680	IGCM_${Patch} ${file_t}
681	IGCM_debug_Verif_Exit
682	done
683	done
684	fi
685	fi
686
687	# WE CAN CONCATENATE FILES AT THIS POINT
688	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
689	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
690	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
691	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
692	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
693
694	liste_coord=" "
695	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
696	liste_coord=${liste_coord}${axis}","
697	done
698
699	# Time axis must be the last dimension
700	time_axis=${axis}
701
702	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
703	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
704	#
705	# We skip variables not in the first file of the list.
706	# Seems overkill to test thel all... but will do if needed
707	ncdump -hv ${var} ${liste_file_tmp[0]} > /dev/null 2>&1
708	if [ ! $? = 0 ] ; then
709	IGCM_debug_Print 1 "WARNING We skip ${var}, not present within ${liste_file_tmp[0]}"
710	continue
711	fi
712	#
713	if [ "${FLAG_B}" = "TRUE" ] ; then
714	# WE CONCATENATE WITH EXISTING FILES
715	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
716	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
717	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
718	IGCM_debug_Verif_Exit
719	#
720	IGCM_sys_Rm ${file1}
721	if [ ! "${DEPOT}" = "TRUE" ] ; then
722	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
723	# "${file1} = file_before ?
724	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
725	fi
726	else
727	# OR NOT
728	file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
729	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${file_out}
730	IGCM_debug_Verif_Exit
731	fi
732
733	#
734	if [ "${DEPOT}" = "TRUE" ] ; then
735	# YEARLY AVERAGE DERIVED FROM MONTHLY MEAN
736	if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
737	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
738	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
739	#
740	# Compute yearly average considering calendar
741	IGCM_sys_cdo yearmonmean -selvar,${var} ${file_out} ${file_out_YE}
742	# Extract all ancillary variables except $var, they will be appended after the cdo command
743	# Needed because cdo applies calculs on all variables
744	IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
745	# This will remove the time axis because cdo rewrites severely with only considered useful variables
746	IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
747	if [ $? -eq 0 ] ; then
748	# Now append all ancillary variables if previous cdo operator ok
749	IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
750	fi
751
752	# Put global history attribut
753	IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
754	IGCM_debug_Verif_Exit
755	#
756	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
757	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
758	IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
759	fi
760	# WE PUT FINAL FILE ON FILE SYSTEM
761	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
762	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
763	#
764	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
765	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
766	if [ ! "X${file1}" = "X" ] ; then
767	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
768	fi
769	#
770	[ -f ${file_out} ] && IGCM_sys_Rm ${file_out}
771	#
772	fi
773	#
774	done # for var in ${liste_var} ; do
775	IGCM_sys_Rm ${liste_file_tmp[*]}
776	done # for file in ${liste_file_brut} do
777	done # for comp in ${config_ListOfComponents[*]} ; do
778	# GO AHEAD IN DEEPEST LOOP
779	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
780	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
781	# FINISH
782
783	# DODS copy
784	for comp in ${config_ListOfComponents[*]} ; do
785	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
786	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
787	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
788	done
789	done
790
791	export DEBUG_debug; export BigBrother ; export libIGCM; export SUBMIT_DIR; export POST_DIR; export Script_Output ;
792	export StandAlone ; export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ;
793	export RESOL_SBG
794	listVarEnv="DEBUG_debug,BigBrother,libIGCM,SUBMIT_DIR,POST_DIR,Script_Post_Output,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
795	export listVarEnv
796
797	#D-
798	# --------------------------------------------------------------------
799	#D- Test if all was right
800	# --------------------------------------------------------------------
801	IGCM_debug_Verif_Exit
802	# --------------------------------------------------------------------
803	#D- Submit MONITORING
804	# --------------------------------------------------------------------
805	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
806	Script_Post_Output=monitoring.${PeriodDateEnd}
807	IGCM_sys_QsubPost monitoring
808	fi
809
810	#D-
811	# --------------------------------------------------------------------
812	#D- Update the run.card
813	# --------------------------------------------------------------------
814	if [ ${StandAlone} != true ] ; then
815	# Put in run.card end period of time series
816	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
817	# We have finish the job !
818	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
819	fi
820
821	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
822	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
823
824	# ------------------------------------------------------------------
825	# Finalize BigBrother to inform that the jobs end
826	# ------------------------------------------------------------------
827	IGCM_debug_BigBro_Finalize
828
829	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: