Context Navigation

source: tags/libIGCM_v2.5/AA_create_ts @ 1456

Last change on this file since 1456 was 1154, checked in by sdipsl, 9 years ago
Add a pattern for node type. See #241
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 30.5 KB

Line
1	#-Q- curie ######################
2	#-Q- curie ## CURIE TGCC/CEA ##
3	#-Q- curie ######################
4	#-Q- curie #MSUB -r TS # Nom du job
5	#-Q- curie #MSUB -eo
6	#-Q- curie #MSUB -n 1 # Reservation du processus
7	#-Q- curie #MSUB -T 80000 # Limite de temps elapsed du job
8	#-Q- curie #MSUB -q ::default_node::
9	#-Q- curie #MSUB -Q normal
10	#-Q- curie #MSUB -A ::default_project::
11	#-Q- curie set +x
12	#-Q- ada #!/bin/ksh
13	#-Q- ada #######################
14	#-Q- ada ## ADA IDRIS ##
15	#-Q- ada #######################
16	#-Q- ada # @ job_type = serial
17	#-Q- ada # @ requirements = (Feature == "prepost")
18	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
19	#-Q- ada # @ wall_clock_limit = 20:00:00
20	#-Q- ada # Nom du travail LoadLeveler
21	#-Q- ada # @ job_name = TS
22	#-Q- ada # Fichier de sortie standard du travail
23	#-Q- ada # @ output = $(job_name).$(jobid)
24	#-Q- ada # Fichier de sortie d'erreur du travail
25	#-Q- ada # @ error = $(job_name).$(jobid)
26	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
27	#-Q- ada # @ notification = error
28	#-Q- ada # @ environment = $DEBUG_debug ; $BigBrother ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $POST_DIR ; $Script_Post_Output ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateEnd ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG
29	#-Q- ada # @ queue
30	#-Q- lxiv8 ######################
31	#-Q- lxiv8 ## OBELIX LSCE ##
32	#-Q- lxiv8 ######################
33	#-Q- lxiv8 #PBS -N TS
34	#-Q- lxiv8 #PBS -m a
35	#-Q- lxiv8 #PBS -j oe
36	#-Q- lxiv8 #PBS -q medium
37	#-Q- lxiv8 #PBS -o TS.$$
38	#-Q- lxiv8 #PBS -S /bin/ksh
39	#-Q- default #!/bin/ksh
40	#-Q- default ##################
41	#-Q- default ## DEFAULT HOST ##
42	#-Q- default ##################
43
44	#**************************************************************
45	# Author: Sebastien Denvil
46	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
47	# $Revision:: $ Revision of last commit
48	# $Author:: $ Author of last commit
49	# $Date:: $ Date of last commit
50	# IPSL (2006)
51	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
52	#
53	#**************************************************************
54
55	#set -eu
56	#set -vx
57
58	date
59
60	#-Q- ada export OMP_NUM_THREADS=1
61
62	#D- Task type (computing or post-processing)
63	TaskType=post-processing
64
65	########################################################################
66
67	#D- Flag to determine if this job in a standalone mode
68	#D- Default : value from AA_job if any
69	StandAlone=${StandAlone:=true}
70
71	#D- Path to libIGCM
72	#D- Default : value from AA_job if any
73	libIGCM=${libIGCM:=::modipsl::/libIGCM}
74
75	#D- Flag to determine allready produced time series. Empty if you start from the beginning
76	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
77	CompletedFlag=${CompletedFlag:=}
78
79	#D- Flag to determine task type this script will perform.
80	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
81	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
82	TsTask=${TsTask:=2D}
83	if [ X${TsTask} = Xempty ] ; then
84	TsTask=""
85	fi
86
87	#D- Flag to determine if rebuild process has been performed asynchronously
88	#D- Possible value true or false.
89	#D- If true raw files has allready been patched by rebuild job
90	#D- If false create_ts will patch the raw files
91	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
92	RebuildFrequency=${RebuildFrequency:=true}
93
94	#D- Flag to determine job's output directory
95	#D- Default : value from libIGCM_post.ksh if any
96	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
97
98	#D- Increased verbosity (1, 2, 3)
99	#D- Default : value from AA_job if any
100	Verbosity=${Verbosity:=3}
101
102	#D- Low level debug : to bypass lib test checks and stack construction
103	#D- Default : value from AA_job if any
104	DEBUG_debug=${DEBUG_debug:=false}
105
106	#D- TEMPORARY Flag to determine atmospheric resolution
107	#D- Default : value from atmospheric driver if any
108	RESOL_ATM=ALL
109
110	#D- Flag to determine surface resolution
111	#D- Default : value from surface driver if any
112	RESOL_SRF=ALL
113
114	#D- Flag to determine surface resolution
115	#D- Default : value from surface driver if any
116	RESOL_SBG=ALL
117
118	#D- TEMPORARY Flag to determine ocean resolution
119	#D- Default : value from ocean driver if any
120	RESOL_OCE=${RESOL_OCE:=ORCA2}
121
122	#D- TEMPORARY Flag to determine ice resolution
123	#D- Default : value from ice driver if any
124	RESOL_ICE=${RESOL_ICE:=ORCA2}
125
126	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
127	#D- Default : value from ice driver if any
128	RESOL_MBG=${RESOL_MBG:=ORCA2}
129
130	########################################################################
131
132	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
133	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
134	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
135	#-------
136	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
137	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
138	#-------
139	( ${DEBUG_debug} ) && IGCM_debug_Check
140	( ${DEBUG_debug} ) && IGCM_card_Check
141	( ${DEBUG_debug} ) && IGCM_date_Check
142
143	########################################################################
144
145	#set -vx
146
147	# Useful cleaning function
148	MENAGE ()
149	{
150	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
151	}
152
153	########################################################################
154
155	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
156	IGCM_sys_Cd ${RUN_DIR_PATH}
157
158	# ------------------------------------------------------------------
159	# Test if all was right before proceeding further
160	# ------------------------------------------------------------------
161	IGCM_debug_Verif_Exit_Post
162
163	if [ ${StandAlone} = true ] ; then
164	CARD_DIR=${SUBMIT_DIR}
165	else
166	CARD_DIR=${RUN_DIR_PATH}
167
168	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
169
170	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
171
172	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
173
174	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
175	fi
176
177	#==================================
178	# First of all
179	#
180	# Read libIGCM compatibility version in config.card
181	# Read UserChoices section
182	# Read Ensemble section
183	# Read Post section
184	# Define all netcdf output directories
185	#==================================
186	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
187
188	#==================================
189	# Define default value to keep compatibility with previous card: means before changes due to TGCC
190	if [ X${PackDefault} = Xtrue ] ; then
191	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
192	else
193	config_Post_PackFrequency=NONE
194	fi
195
196	#==================================
197	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
198	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
199	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
200	fi
201
202	#
203	# Determine component to take care of depending on kind of task create_ts will perform
204	#
205	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
206	#
207	# Determine period for time series. Default : value from AA_job or from launch_TS if any
208	#
209	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
210
211	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
212	#
213	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
214	#
215	# Determine Dimension of the produced time series : 2D or 3D
216	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
217	Dimension=2D
218	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
219	Dimension=3D
220	else
221	Dimension=""
222	fi
223
224	# ------------------------------------------------------------------
225	#D- Test if all was right before entering the loop
226	# ------------------------------------------------------------------
227	IGCM_debug_Verif_Exit_Post
228
229	########################################################################
230	# Depending on requested time series fill following variables :
231	#
232	# - LISTE_FILE_${comp}[*]
233	# - LISTE_VARS_${file}[*]
234	# - LISTE_AXIS_${file}[*]
235	# - LISTE_PATCH_${file}[*]
236	########################################################################
237
238	#
239	# For each selected component determine which files need post-processing
240	#
241	DoJob=false
242	for comp in ${config_ListOfComponents[*]} ; do
243	#
244	ActiveComp=false
245	#
246	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
247	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
248	#
249	card=${CARD_DIR}/COMP/${compname}.card
250
251	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
252	#
253	ListFilesName=${compname}_OutputFiles_List
254	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
255	#
256	if [ X${FileName0} != X${NULL_STR} ] ; then
257	#
258	#IGCM_debug_Print 1 "Component : ${compname}"
259	#
260	# INITIALISATION
261	#
262	typeset i
263	if [ ${FlagToRead} ] ; then
264	((NbFiles=${FlagToRead}))
265	((i=${NbFiles}))
266	else
267	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
268	i=2
269	fi
270	#
271	until [ $i -gt $NbFiles ]; do
272	#
273	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
274	#
275	if [ X${flag_post} != XNONE ] ; then
276	#
277	# For each comp and each file who needs post-processing
278	# determine which variables need time series.
279	#
280	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
281	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
282	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
283	#
284	# If TimeSeriesVars list is empty we skip
285	#
286	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
287	#
288	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
289	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
290	(( i=i+3 ))
291	continue
292	#
293	fi
294	#
295	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
296	#
297	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
298	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
299	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
300	(( i=i+3 ))
301	continue
302	fi
303	#
304	# If we are not a Chunck type task, we skip files/variables handled by chunck task
305	#
306	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
307	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
308	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
309	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
310	(( i=i+3 ))
311	continue
312	fi
313	fi
314	#
315	# Now on we know we have things to do
316	#
317
318	#
319	# We need coordinates axis to properly fill produced time series
320	#
321	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
322	#
323	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
324	#
325	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
326	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
327	(( i=i+3 ))
328	continue
329	#
330	fi
331	#
332	# We need LIST of variables not allready produced (useful for standalone mode)
333	#
334	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
335	case ${FlagDir} in
336	*Y) TS_Dir=TS_YE ;;
337	*M) TS_Dir=TS_MO ;;
338	*D) TS_Dir=TS_DA ;;
339	HF) TS_Dir=TS_HF ;;
340	3H) TS_Dir=TS_HF ;;
341	INS) TS_Dir=TS_INS ;;
342	esac
343	#
344	count=0
345	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
346	#
347	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
348	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
349	#
350	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
351	if [ ! $? = 0 ] ; then
352	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
353	ActiveComp=true
354	(( count = count + 1 ))
355	fi
356	done
357	#
358	if [ ${count} -gt 0 ] ; then
359	#
360	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
361	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
362	#
363	if [ ! X${RebuildFrequency} = Xtrue ] ; then
364	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
365	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
366	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
367	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
368	if [ Xload${Patch} != X ] ; then
369	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
370	eval load${Patch}=loaded
371	fi
372	done
373	fi
374	fi
375	fi
376	fi
377	(( i=i+3 ))
378	done
379	fi
380	#
381	if [ ${ActiveComp} = true ] ; then
382	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
383	fi
384	done
385
386	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
387	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
388
389	########################################################################
390	#
391	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
392	# IF SO BRING THEM IN THE WORKING DIRECTORY
393	#
394	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
395	# - especially for IDRiS
396	# - no special protection against "command too long" !
397	# - libIGCM_sys should handle that or modify following code part
398	#
399	########################################################################
400
401	if [ ${StandAlone} != true ] ; then
402	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
403	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
404	echo \${run_PostProcessing_TimeSeriesRunning}" )
405
406	#if [ X${Running_Flag} = Xy ] ; then
407	# # Time Series Job allready running
408	# IGCM_debug_Print 1 "Time Series Job allready running exit"
409	# exit
410	#fi
411	# Now run.card know we are running
412
413
414
415
416	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
417
418
419
420
421
422	# Get information from last execution
423	#CompletedFlag=$( IGCM_sys_RshMaster \
424	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
425	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
426	fi
427
428	# Is it the first submission or not ?
429
430	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
431	# First Time Series Submission
432	FIRST_PASS=TRUE
433	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
434	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
435	else
436	# Last Completed Time Series
437	FIRST_PASS=FALSE
438	DATE_FIN_JOB_B=${CompletedFlag}
439	date=${DateBegin}_${DATE_FIN_JOB_B}
440	#
441	for comp in ${config_ListOfComponents[*]} ; do
442	#
443	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
444	#
445	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
446	#
447	unset liste_file
448	#
449	file_path=${R_SAVE}/${comp}/Analyse
450	#
451	i=0
452	file=${NULL_STR}
453	#
454	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
455	#
456	# Determine in which output can we find file
457	# IE : Analyse/TS or Analyse/DA2TS or ...
458	#
459	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
460	case ${FlagDir} in
461	*Y) TS_Dir=TS_YE ;;
462	*M) TS_Dir=TS_MO ;;
463	*D) TS_Dir=TS_DA ;;
464	HF) TS_Dir=TS_HF ;;
465	3H) TS_Dir=TS_HF ;;
466	INS) TS_Dir=TS_INS ;;
467	esac
468	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
469	#
470	IGCM_sys_TestFileArchive ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
471	if [ $? = 0 ] ; then
472	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
473	else
474	IGCM_debug_Print 1 "WARNING ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc dont exist"
475	fi
476	#
477	done
478	done
479	if [ X${file} != X${NULL_STR} ] ; then
480	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
481	IGCM_debug_Verif_Exit_Post
482	fi
483	done
484	#
485	LEVEL=0
486	#
487	fi
488
489	########################################################################
490	#
491	# DEFINE LOOP PARAMETERS
492	#
493	########################################################################
494
495	DATE_COUNT=${DATE_FIN_JOB_B}
496	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
497
498	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
499
500	if [ ${DoJob} = true ] ; then
501	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
502	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
503	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
504	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
505	done
506	else
507	NBRE_FILE_TOT=0
508	fi
509
510	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
511	NBRE_TOUR=1
512	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
513	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
514	else
515	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
516	fi
517
518	CURRENT_LOOP=1
519	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
520	# WE HAVE 3 LEVELS
521	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
522	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
523	# - LEVEL 3 : WE ARE IN THE LAST LOOP
524
525	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
526
527	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
528	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
529	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
530
531	if [ ${FIRST_PASS} = TRUE ] ; then
532	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
533	else
534	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
535	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
536	fi
537
538	DATE_FIN=${PeriodDateEnd}
539	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
540
541	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
542	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
543
544	LEVEL=1
545	DEPOT="TRUE"
546	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
547	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
548	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
549
550	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
551	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
552	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
553	elif [ -z "${DATE_COURANTE}" ] ; then
554	DATE_COURANTE=${DATE_FIN_JOB_B}
555	else
556	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
557	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
558	fi
559
560	(( TotLength = 0 ))
561	COMPTEUR=1
562	DATE_LOOP=${DATE_COURANTE}
563
564	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
565	#
566	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
567	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
568	(( TotLength = TotLength + Length ))
569	(( COMPTEUR = COMPTEUR + 1 ))
570	#
571	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
572	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
573
574	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
575	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
576	DEPOT="FALSE"
577	LEVEL=2
578	else
579	# - LEVEL 3 : WE ARE IN THE LAST LOOP
580	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
581	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
582
583	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
584	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
585	DATE_FIN=${PeriodDateEnd}
586	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
587
588	LEVEL=3
589	DEPOT="TRUE"
590	FLAG_B="TRUE"
591	fi
592
593	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
594	for comp in ${config_ListOfComponents[*]} ; do
595	#
596	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
597	#
598	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
599	#
600	eval R_OUT_${comp}=${R_SAVE}/${comp}
601	#
602	i=0
603	#
604	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
605	#
606	# Determine in which output we can find file
607	# IE : Output/MO or Output/DA or ...
608	#
609	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
610	case ${FlagDir} in
611	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
612	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
613	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
614	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
615	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
616	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
617	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
618	IGCM_sys_Exit "Job create_ts" ;;
619	esac
620
621	# Initialize array
622	unset liste_file
623	unset liste_file_tmp
624	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
625
626	COMPTEUR=1
627	MissingFile=FALSE
628	DATE_LOOP=${DATE_COURANTE}
629
630	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
631	#
632	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
633	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
634
635	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
636	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
637
638	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
639	if [ $? = 0 ] ; then
640	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
641	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
642	else
643	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
644	MissingFile=TRUE
645	fi
646	(( COMPTEUR = COMPTEUR + 1 ))
647	#
648	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
649
650	# We skip this file type if one of them is missing.
651	if [ "${MissingFile}" = "TRUE" ] ; then
652	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
653	continue
654	fi
655
656	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
657
658	# Get selected files
659	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
660	IGCM_debug_Verif_Exit_Post
661
662	# Apply IGCM_Patch if needed
663	if [ ! X${RebuildFrequency} = Xtrue ] ; then
664	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
665	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
666	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
667	IGCM_${Patch} ${file_t}
668	IGCM_debug_Verif_Exit_Post
669	done
670	done
671	fi
672	fi
673
674	# WE CAN CONCATENATE FILES AT THIS POINT
675	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
676	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
677	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
678	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
679	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
680
681	liste_coord=" "
682	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
683	liste_coord=${liste_coord}${axis}","
684	done
685
686	# Time axis must be the last dimension
687	time_axis=${axis}
688
689	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
690	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
691	#
692	# We skip variables not in the first file of the list.
693	# Seems overkill to test thel all... but will do if needed
694	ncdump -hv ${var} ${liste_file_tmp[0]} > /dev/null 2>&1
695	if [ ! $? = 0 ] ; then
696	IGCM_debug_Print 1 "WARNING We skip ${var}, not present within ${liste_file_tmp[0]}"
697	continue
698	fi
699	#
700	if [ "${FLAG_B}" = "TRUE" ] ; then
701	# WE CONCATENATE WITH EXISTING FILES
702	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
703	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
704	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
705	IGCM_debug_Verif_Exit_Post
706	#
707	IGCM_sys_Rm ${file1}
708	if [ ! "${DEPOT}" = "TRUE" ] ; then
709	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
710	# "${file1} = file_before ?
711	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
712	fi
713	else
714	# OR NOT
715	file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
716	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${file_out}
717	IGCM_debug_Verif_Exit_Post
718	fi
719
720	#
721	if [ "${DEPOT}" = "TRUE" ] ; then
722	# YEARLY AVERAGE DERIVED FROM MONTHLY MEAN
723	if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
724	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
725	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
726	#
727	# Compute yearly average considering calendar
728	IGCM_sys_cdo yearmonmean -selvar,${var} ${file_out} ${file_out_YE}
729	# Extract all ancillary variables except $var, they will be appended after the cdo command
730	# Needed because cdo applies calculs on all variables
731	IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
732	# This will remove the time axis because cdo rewrites severely with only considered useful variables
733	IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
734	if [ $? -eq 0 ] ; then
735	# Now append all ancillary variables if previous cdo operator ok
736	IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
737	fi
738
739	# Put global history attribut
740	IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
741	IGCM_debug_Verif_Exit_Post
742	#
743	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
744	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
745	IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
746	fi
747	# WE PUT FINAL FILE ON FILE SYSTEM
748	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
749	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
750	#
751	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
752	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
753	if [ ! "X${file1}" = "X" ] ; then
754	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
755	fi
756	#
757	[ -f ${file_out} ] && IGCM_sys_Rm ${file_out}
758	#
759	fi
760	#
761	done # for var in ${liste_var} ; do
762	IGCM_sys_Rm ${liste_file_tmp[*]}
763	done # for file in ${liste_file_brut} do
764	done # for comp in ${config_ListOfComponents[*]} ; do
765	# GO AHEAD IN DEEPEST LOOP
766	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
767	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
768	# FINISH
769
770	# DODS copy
771	for comp in ${config_ListOfComponents[*]} ; do
772	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
773	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
774	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
775	done
776	done
777
778	export DEBUG_debug; export BigBrother ; export libIGCM; export SUBMIT_DIR; export POST_DIR; export Script_Output ;
779	export StandAlone ; export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ;
780	export RESOL_SBG
781	listVarEnv="DEBUG_debug,BigBrother,libIGCM,SUBMIT_DIR,POST_DIR,Script_Post_Output,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
782	export listVarEnv
783
784	#D-
785	# --------------------------------------------------------------------
786	#D- Test if all was right
787	# --------------------------------------------------------------------
788	IGCM_debug_Verif_Exit_Post
789	# --------------------------------------------------------------------
790	#D- Submit MONITORING
791	# --------------------------------------------------------------------
792	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
793	Script_Post_Output=monitoring.${PeriodDateEnd}
794	IGCM_sys_QsubPost monitoring
795	fi
796
797	#D-
798	# --------------------------------------------------------------------
799	#D- Update the run.card
800	# --------------------------------------------------------------------
801	if [ ${StandAlone} != true ] ; then
802	# Put in run.card end period of time series
803	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
804	# We have finish the job !
805	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
806	fi
807
808	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
809	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
810
811	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: