Context Navigation

source: trunk/libIGCM/AA_create_ts @ 1033

Last change on this file since 1033 was 1033, checked in by sdipsl, 10 years ago
See #186 issue a clear warning when requesting time series ifrom card but variable has not produced. Is an error in PROD just a WARNING in other cases.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 30.3 KB

Line
1	#-Q- curie ######################
2	#-Q- curie ## CURIE TGCC/CEA ##
3	#-Q- curie ######################
4	#-Q- curie #MSUB -r TS # Nom du job
5	#-Q- curie #MSUB -eo
6	#-Q- curie #MSUB -n 1 # Reservation du processus
7	#-Q- curie #MSUB -T 80000 # Limite de temps elapsed du job
8	#-Q- curie #MSUB -q xlarge
9	#-Q- curie #MSUB -Q normal
10	#-Q- curie #MSUB -A ::default_project::
11	#-Q- curie set +x
12	#-Q- ada #!/bin/ksh
13	#-Q- ada #######################
14	#-Q- ada ## ADA IDRIS ##
15	#-Q- ada #######################
16	#-Q- ada # @ job_type = serial
17	#-Q- ada # @ requirements = (Feature == "prepost")
18	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
19	#-Q- ada # @ wall_clock_limit = 20:00:00
20	#-Q- ada # Nom du travail LoadLeveler
21	#-Q- ada # @ job_name = TS
22	#-Q- ada # Fichier de sortie standard du travail
23	#-Q- ada # @ output = $(job_name).$(jobid)
24	#-Q- ada # Fichier de sortie d'erreur du travail
25	#-Q- ada # @ error = $(job_name).$(jobid)
26	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
27	#-Q- ada # @ notification = error
28	#-Q- ada # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
29	#-Q- ada # @ queue
30	#-Q- lxiv8 ######################
31	#-Q- lxiv8 ## OBELIX LSCE ##
32	#-Q- lxiv8 ######################
33	#-Q- lxiv8 #PBS -N TS
34	#-Q- lxiv8 #PBS -m a
35	#-Q- lxiv8 #PBS -j oe
36	#-Q- lxiv8 #PBS -q medium
37	#-Q- lxiv8 #PBS -o TS.$$
38	#-Q- lxiv8 #PBS -S /bin/ksh
39	#-Q- default #!/bin/ksh
40	#-Q- default ##################
41	#-Q- default ## DEFAULT HOST ##
42	#-Q- default ##################
43
44	#**************************************************************
45	# Author: Sebastien Denvil
46	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
47	# $Revision:: $ Revision of last commit
48	# $Author:: $ Author of last commit
49	# $Date:: $ Date of last commit
50	# IPSL (2006)
51	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
52	#
53	#**************************************************************
54
55	#set -eu
56	#set -vx
57
58	date
59
60	#-Q- ada export OMP_NUM_THREADS=1
61
62	#D- Task type (computing or post-processing)
63	TaskType=post-processing
64
65	########################################################################
66
67	#D- Flag to determine if this job in a standalone mode
68	#D- Default : value from AA_job if any
69	StandAlone=${StandAlone:=true}
70
71	#D- Path to libIGCM
72	#D- Default : value from AA_job if any
73	libIGCM=${libIGCM:=::modipsl::/libIGCM}
74
75	#D- Flag to determine allready produced time series. Empty if you start from the beginning
76	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
77	CompletedFlag=${CompletedFlag:=}
78
79	#D- Flag to determine task type this script will perform.
80	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
81	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
82	TsTask=${TsTask:=2D}
83	if [ X${TsTask} = Xempty ] ; then
84	TsTask=""
85	fi
86
87	#D- Flag to determine if rebuild process has been performed asynchronously
88	#D- Possible value true or false.
89	#D- If true raw files has allready been patched by rebuild job
90	#D- If false create_ts will patch the raw files
91	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
92	RebuildFrequency=${RebuildFrequency:=true}
93
94	#D- Flag to determine job's output directory
95	#D- Default : value from libIGCM_post.ksh if any
96	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
97
98	#D- Increased verbosity (1, 2, 3)
99	#D- Default : value from AA_job if any
100	Verbosity=${Verbosity:=3}
101
102	#D- Low level debug : to bypass lib test checks and stack construction
103	#D- Default : value from AA_job if any
104	DEBUG_debug=${DEBUG_debug:=false}
105
106	#D- TEMPORARY Flag to determine atmospheric resolution
107	#D- Default : value from atmospheric driver if any
108	RESOL_ATM=ALL
109
110	#D- Flag to determine surface resolution
111	#D- Default : value from surface driver if any
112	RESOL_SRF=ALL
113
114	#D- Flag to determine surface resolution
115	#D- Default : value from surface driver if any
116	RESOL_SBG=ALL
117
118	#D- TEMPORARY Flag to determine ocean resolution
119	#D- Default : value from ocean driver if any
120	RESOL_OCE=${RESOL_OCE:=ORCA2}
121
122	#D- TEMPORARY Flag to determine ice resolution
123	#D- Default : value from ice driver if any
124	RESOL_ICE=${RESOL_ICE:=ORCA2}
125
126	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
127	#D- Default : value from ice driver if any
128	RESOL_MBG=${RESOL_MBG:=ORCA2}
129
130	########################################################################
131
132	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
133	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
134	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
135	#-------
136	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
137	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
138	#-------
139	( ${DEBUG_debug} ) && IGCM_debug_Check
140	( ${DEBUG_debug} ) && IGCM_card_Check
141	( ${DEBUG_debug} ) && IGCM_date_Check
142
143	########################################################################
144
145	#set -vx
146
147	# Useful cleaning function
148	MENAGE ()
149	{
150	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
151	}
152
153	########################################################################
154
155	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
156	IGCM_sys_Cd ${RUN_DIR_PATH}
157
158	# ------------------------------------------------------------------
159	# Test if all was right before proceeding further
160	# ------------------------------------------------------------------
161	IGCM_debug_Verif_Exit_Post
162
163	if [ ${StandAlone} = true ] ; then
164	CARD_DIR=${SUBMIT_DIR}
165	else
166	CARD_DIR=${RUN_DIR_PATH}
167
168	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
169
170	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
171
172	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
173
174	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
175	fi
176
177	#==================================
178	# First of all
179	#
180	# Read libIGCM compatibility version in config.card
181	# Read UserChoices section
182	# Read Ensemble section
183	# Read Post section
184	# Define all netcdf output directories
185	#==================================
186	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
187
188	#==================================
189	# Define default value to keep compatibility with previous card: means before changes due to TGCC
190	if [ X${PackDefault} = Xtrue ] ; then
191	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
192	else
193	config_Post_PackFrequency=NONE
194	fi
195
196	#==================================
197	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
198	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
199	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
200	fi
201
202	#
203	# Determine component to take care of depending on kind of task create_ts will perform
204	#
205	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
206	#
207	# Determine period for time series. Default : value from AA_job or from launch_TS if any
208	#
209	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
210
211	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
212	#
213	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
214	#
215	# Determine Dimension of the produced time series : 2D or 3D
216	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
217	Dimension=2D
218	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
219	Dimension=3D
220	else
221	Dimension=""
222	fi
223
224	# ------------------------------------------------------------------
225	#D- Test if all was right before entering the loop
226	# ------------------------------------------------------------------
227	IGCM_debug_Verif_Exit_Post
228
229	########################################################################
230	# Depending on requested time series fill following variables :
231	#
232	# - LISTE_FILE_${comp}[*]
233	# - LISTE_VARS_${file}[*]
234	# - LISTE_AXIS_${file}[*]
235	# - LISTE_PATCH_${file}[*]
236	########################################################################
237
238	#
239	# For each selected component determine which files need post-processing
240	#
241	DoJob=false
242	for comp in ${config_ListOfComponents[*]} ; do
243	#
244	ActiveComp=false
245	#
246	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
247	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
248	#
249	card=${CARD_DIR}/COMP/${compname}.card
250
251	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
252	#
253	ListFilesName=${compname}_OutputFiles_List
254	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
255	#
256	if [ X${FileName0} != X${NULL_STR} ] ; then
257	#
258	#IGCM_debug_Print 1 "Component : ${compname}"
259	#
260	# INITIALISATION
261	#
262	typeset i
263	if [ ${FlagToRead} ] ; then
264	((NbFiles=${FlagToRead}))
265	((i=${NbFiles}))
266	else
267	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
268	i=2
269	fi
270	#
271	until [ $i -gt $NbFiles ]; do
272	#
273	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
274	#
275	if [ X${flag_post} != XNONE ] ; then
276	#
277	# For each comp and each file who needs post-processing
278	# determine which variables need time series.
279	#
280	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
281	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
282	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
283	#
284	# If TimeSeriesVars list is empty we skip
285	#
286	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
287	#
288	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
289	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
290	(( i=i+3 ))
291	continue
292	#
293	fi
294	#
295	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
296	#
297	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
298	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
299	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
300	(( i=i+3 ))
301	continue
302	fi
303	#
304	# If we are not a Chunck type task, we skip files/variables handled by chunck task
305	#
306	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
307	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
308	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
309	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
310	(( i=i+3 ))
311	continue
312	fi
313	fi
314	#
315	# Now on we know we have things to do
316	#
317
318	#
319	# We need coordinates axis to properly fill produced time series
320	#
321	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
322	#
323	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
324	#
325	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
326	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
327	(( i=i+3 ))
328	continue
329	#
330	fi
331	#
332	# We need LIST of variables not allready produced (useful for standalone mode)
333	#
334	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
335	case ${FlagDir} in
336	*Y) TS_Dir=TS_YE ;;
337	*M) TS_Dir=TS_MO ;;
338	*D) TS_Dir=TS_DA ;;
339	HF) TS_Dir=TS_HF ;;
340	3H) TS_Dir=TS_HF ;;
341	INS) TS_Dir=TS_INS ;;
342	esac
343	#
344	count=0
345	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
346	#
347	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
348	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
349	#
350	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
351	if [ ! $? = 0 ] ; then
352	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
353	ActiveComp=true
354	(( count = count + 1 ))
355	fi
356	done
357	#
358	if [ ${count} -gt 0 ] ; then
359	#
360	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
361	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
362	#
363	if [ ! X${RebuildFrequency} = Xtrue ] ; then
364	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
365	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
366	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
367	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
368	if [ Xload${Patch} != X ] ; then
369	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
370	eval load${Patch}=loaded
371	fi
372	done
373	fi
374	fi
375	fi
376	fi
377	(( i=i+3 ))
378	done
379	fi
380	#
381	if [ ${ActiveComp} = true ] ; then
382	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
383	fi
384	done
385
386	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
387	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
388
389	########################################################################
390	#
391	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
392	# IF SO BRING THEM IN THE WORKING DIRECTORY
393	#
394	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
395	# - especially for IDRiS
396	# - no special protection against "command too long" !
397	# - libIGCM_sys should handle that or modify following code part
398	#
399	########################################################################
400
401	if [ ${StandAlone} != true ] ; then
402	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
403	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
404	echo \${run_PostProcessing_TimeSeriesRunning}" )
405
406	#if [ X${Running_Flag} = Xy ] ; then
407	# # Time Series Job allready running
408	# IGCM_debug_Print 1 "Time Series Job allready running exit"
409	# exit
410	#fi
411	# Now run.card know we are running
412
413
414
415
416	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
417
418
419
420
421
422	# Get information from last execution
423	#CompletedFlag=$( IGCM_sys_RshMaster \
424	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
425	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
426	fi
427
428	# Is it the first submission or not ?
429
430	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
431	# First Time Series Submission
432	FIRST_PASS=TRUE
433	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
434	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
435	else
436	# Last Completed Time Series
437	FIRST_PASS=FALSE
438	DATE_FIN_JOB_B=${CompletedFlag}
439	date=${DateBegin}_${DATE_FIN_JOB_B}
440	for comp in ${config_ListOfComponents[*]} ; do
441	#
442	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
443	#
444	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
445	#
446	unset liste_file
447	#
448	file_path=${R_SAVE}/${comp}/Analyse
449	#
450	i=0
451	file=${NULL_STR}
452	#
453	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
454	#
455	# Determine in which output can we find file
456	# IE : Analyse/TS or Analyse/DA2TS or ...
457	#
458	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
459	case ${FlagDir} in
460	*Y) TS_Dir=TS_YE ;;
461	*M) TS_Dir=TS_MO ;;
462	*D) TS_Dir=TS_DA ;;
463	HF) TS_Dir=TS_HF ;;
464	3H) TS_Dir=TS_HF ;;
465	INS) TS_Dir=TS_INS ;;
466	esac
467	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
468	#
469	IGCM_sys_TestFileArchive ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
470	if [ $? = 0 ] ; then
471	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
472	else
473	IGCM_debug_Print 1 "WARNING ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc dont exist"
474	fi
475	#
476	done
477	done
478	if [ X${file} != X${NULL_STR} ] ; then
479	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
480	IGCM_debug_Verif_Exit_Post
481	fi
482	done
483	#
484	LEVEL=0
485	#
486	fi
487
488	########################################################################
489	#
490	# DEFINE LOOP PARAMETERS
491	#
492	########################################################################
493
494	DATE_COUNT=${DATE_FIN_JOB_B}
495	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
496
497	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
498
499	if [ ${DoJob} = true ] ; then
500	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
501	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
502	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
503	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
504	done
505	else
506	NBRE_FILE_TOT=0
507	fi
508
509	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
510	NBRE_TOUR=1
511	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
512	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
513	else
514	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
515	fi
516
517	CURRENT_LOOP=1
518	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
519	# WE HAVE 3 LEVELS
520	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
521	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
522	# - LEVEL 3 : WE ARE IN THE LAST LOOP
523
524	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
525
526	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
527	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
528	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
529
530	if [ ${FIRST_PASS} = TRUE ] ; then
531	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
532	else
533	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
534	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
535	fi
536
537	DATE_FIN=${PeriodDateEnd}
538	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
539
540	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
541	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
542
543	LEVEL=1
544	DEPOT="TRUE"
545	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
546	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
547	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
548
549	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
550	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
551	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
552	elif [ -z "${DATE_COURANTE}" ] ; then
553	DATE_COURANTE=${DATE_FIN_JOB_B}
554	else
555	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
556	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
557	fi
558
559	(( TotLength = 0 ))
560	COMPTEUR=1
561	DATE_LOOP=${DATE_COURANTE}
562
563	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
564	#
565	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
566	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
567	(( TotLength = TotLength + Length ))
568	(( COMPTEUR = COMPTEUR + 1 ))
569	#
570	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
571	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
572
573	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
574	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
575	DEPOT="FALSE"
576	LEVEL=2
577	else
578	# - LEVEL 3 : WE ARE IN THE LAST LOOP
579	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
580	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
581
582	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
583	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
584	DATE_FIN=${PeriodDateEnd}
585	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
586
587	LEVEL=3
588	DEPOT="TRUE"
589	FLAG_B="TRUE"
590	fi
591
592	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
593	for comp in ${config_ListOfComponents[*]} ; do
594	#
595	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
596	#
597	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
598	#
599	eval R_OUT_${comp}=${R_SAVE}/${comp}
600	#
601	i=0
602	#
603	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
604	#
605	# Determine in which output we can find file
606	# IE : Output/MO or Output/DA or ...
607	#
608	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
609	case ${FlagDir} in
610	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
611	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
612	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
613	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
614	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
615	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
616	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
617	IGCM_sys_Exit "Job create_ts" ;;
618	esac
619
620	# Initialize array
621	unset liste_file
622	unset liste_file_tmp
623	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
624
625	COMPTEUR=1
626	MissingFile=FALSE
627	DATE_LOOP=${DATE_COURANTE}
628
629	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
630	#
631	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
632	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
633
634	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
635	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
636
637	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
638	if [ $? = 0 ] ; then
639	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
640	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
641	else
642	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
643	MissingFile=TRUE
644	fi
645	(( COMPTEUR = COMPTEUR + 1 ))
646	#
647	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
648
649	# We skip this file type if one of them is missing.
650	if [ "${MissingFile}" = "TRUE" ] ; then
651	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
652	continue
653	fi
654
655	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
656
657	# Get selected files
658	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
659	IGCM_debug_Verif_Exit_Post
660
661	# Apply IGCM_Patch if needed
662	if [ ! X${RebuildFrequency} = Xtrue ] ; then
663	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
664	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
665	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
666	IGCM_${Patch} ${file_t}
667	IGCM_debug_Verif_Exit_Post
668	done
669	done
670	fi
671	fi
672
673	# WE CAN CONCATENATE FILES AT THIS POINT
674	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
675	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
676	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
677	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
678	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
679
680	liste_coord=" "
681	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
682	liste_coord=${liste_coord}${axis}","
683	done
684
685	# Time axis must be the last dimension
686	time_axis=${axis}
687
688	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
689	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
690	#
691	# We skip variables not in the first file of the list.
692	# Seems overkill to test thel all... but will do if needed
693	ncdump -hv ${var} ${liste_file_tmp[0]} > /dev/null 2>&1
694	if [ ! $? = 0 ] ; then
695	IGCM_debug_Print 1 "WARNING We skip ${var}, not present within ${liste_file_tmp[0]}"
696	continue
697	fi
698	#
699	if [ "${FLAG_B}" = "TRUE" ] ; then
700	# WE CONCATENATE WITH EXISTING FILES
701	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
702	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
703	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
704	IGCM_debug_Verif_Exit_Post
705	#
706	IGCM_sys_Rm ${file1}
707	if [ ! "${DEPOT}" = "TRUE" ] ; then
708	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
709	# "${file1} = file_before ?
710	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
711	fi
712	else
713	# OR NOT
714	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
715	IGCM_debug_Verif_Exit_Post
716	fi
717
718	#
719	if [ "${DEPOT}" = "TRUE" ] ; then
720	# WE PUT FINAL FILE ON FILE SYSTEM
721	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
722	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
723	#
724	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
725	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
726	if [ ! "X${file1}" = "X" ] ; then
727	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
728	fi
729	#
730	if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
731	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
732	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
733	#
734	# Compute yearly average considering calendar
735	# and set date of the mid year to 01/07 at 00:00
736	IGCM_sys_cdo -settime,00:00 -setday,1 -setmon,7 -settunits,days -divdpy -yearsum -muldpm -selvar,${var} ${file_out} ${file_out_YE}
737	# Extract all ancillary variables except $var, they will be appended after the cdo command
738	# Needed because cdo applies calculs on all variables
739	IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
740	# This will remove the time axis because cdo rewrites severely with only considered useful variables
741	IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
742	if [ $? -eq 0 ] ; then
743	# Now append all ancillary variables if previous cdo operator ok
744	IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
745	fi
746
747	# Put global history attribut
748	IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
749	IGCM_debug_Verif_Exit_Post
750	#
751	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
752	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
753	IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
754	fi
755	#
756	IGCM_sys_Rm ${file_out}
757	#
758	fi
759	#
760	done # for var in ${liste_var} ; do
761	IGCM_sys_Rm ${liste_file_tmp[*]}
762	done # for file in ${liste_file_brut} do
763	done # for comp in ${config_ListOfComponents[*]} ; do
764	# GO AHEAD IN DEEPEST LOOP
765	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
766	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
767	# FINISH
768
769	# DODS copy
770	for comp in ${config_ListOfComponents[*]} ; do
771	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
772	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
773	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
774	done
775	done
776
777	export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
778	export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG
779
780	listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
781	export listVarEnv
782
783	#D-
784	# --------------------------------------------------------------------
785	#D- Test if all was right
786	# --------------------------------------------------------------------
787	IGCM_debug_Verif_Exit_Post
788	# --------------------------------------------------------------------
789	#D- Submit MONITORING
790	# --------------------------------------------------------------------
791	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
792	Script_Post_Output=monitoring.${PeriodDateEnd}
793	IGCM_sys_QsubPost monitoring
794	fi
795
796	#D-
797	# --------------------------------------------------------------------
798	#D- Update the run.card
799	# --------------------------------------------------------------------
800	if [ ${StandAlone} != true ] ; then
801	# Put in run.card end period of time series
802	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
803	# We have finish the job !
804	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
805	fi
806
807	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
808	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
809
810	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: