Context Navigation

source: trunk/libIGCM/AA_create_ts @ 1184

Last change on this file since 1184 was 1184, checked in by sdipsl, 9 years ago
Add ciclad to the system. Episode II
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 30.7 KB

Line
1	#-Q- curie ######################
2	#-Q- curie ## CURIE TGCC/CEA ##
3	#-Q- curie ######################
4	#-Q- curie #MSUB -r TS # Nom du job
5	#-Q- curie #MSUB -eo
6	#-Q- curie #MSUB -n 1 # Reservation du processus
7	#-Q- curie #MSUB -T 80000 # Limite de temps elapsed du job
8	#-Q- curie #MSUB -q ::default_node::
9	#-Q- curie #MSUB -Q normal
10	#-Q- curie #MSUB -A ::default_project::
11	#-Q- curie set +x
12	#-Q- ada #!/bin/ksh
13	#-Q- ada #######################
14	#-Q- ada ## ADA IDRIS ##
15	#-Q- ada #######################
16	#-Q- ada # @ job_type = serial
17	#-Q- ada # @ requirements = (Feature == "prepost")
18	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
19	#-Q- ada # @ wall_clock_limit = 20:00:00
20	#-Q- ada # Nom du travail LoadLeveler
21	#-Q- ada # @ job_name = TS
22	#-Q- ada # Fichier de sortie standard du travail
23	#-Q- ada # @ output = $(job_name).$(jobid)
24	#-Q- ada # Fichier de sortie d'erreur du travail
25	#-Q- ada # @ error = $(job_name).$(jobid)
26	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
27	#-Q- ada # @ notification = error
28	#-Q- ada # @ environment = $DEBUG_debug ; $BigBrother ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $POST_DIR ; $Script_Post_Output ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateEnd ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG
29	#-Q- ada # @ queue
30	#-Q- lxiv8 ######################
31	#-Q- lxiv8 ## OBELIX LSCE ##
32	#-Q- lxiv8 ######################
33	#-Q- lxiv8 #PBS -N TS
34	#-Q- lxiv8 #PBS -m a
35	#-Q- lxiv8 #PBS -j oe
36	#-Q- lxiv8 #PBS -q medium
37	#-Q- lxiv8 #PBS -o TS.$$
38	#-Q- lxiv8 #PBS -S /bin/ksh
39	#-Q- ifort_CICLAD ######################
40	#-Q- ifort_CICLAD ## CICLAD IPSL ##
41	#-Q- ifort_CICLAD ######################
42	#-Q- ifort_CICLAD #PBS -N TS
43	#-Q- ifort_CICLAD #PBS -m a
44	#-Q- ifort_CICLAD #PBS -j oe
45	#-Q- ifort_CICLAD #PBS -q h12
46	#-Q- ifort_CICLAD #PBS -S /bin/ksh
47	#-Q- default #!/bin/ksh
48	#-Q- default ##################
49	#-Q- default ## DEFAULT HOST ##
50	#-Q- default ##################
51
52	#**************************************************************
53	# Author: Sebastien Denvil
54	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
55	# $Revision:: $ Revision of last commit
56	# $Author:: $ Author of last commit
57	# $Date:: $ Date of last commit
58	# IPSL (2006)
59	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
60	#
61	#**************************************************************
62
63	#set -eu
64	#set -vx
65
66	date
67
68	#-Q- ada export OMP_NUM_THREADS=1
69
70	#D- Task type (computing or post-processing)
71	TaskType=post-processing
72
73	########################################################################
74
75	#D- Flag to determine if this job in a standalone mode
76	#D- Default : value from AA_job if any
77	StandAlone=${StandAlone:=true}
78
79	#D- Path to libIGCM
80	#D- Default : value from AA_job if any
81	libIGCM=${libIGCM:=::modipsl::/libIGCM}
82
83	#D- Flag to determine allready produced time series. Empty if you start from the beginning
84	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
85	CompletedFlag=${CompletedFlag:=}
86
87	#D- Flag to determine task type this script will perform.
88	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
89	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
90	TsTask=${TsTask:=2D}
91	if [ X${TsTask} = Xempty ] ; then
92	TsTask=""
93	fi
94
95	#D- Flag to determine if rebuild process has been performed asynchronously
96	#D- Possible value true or false.
97	#D- If true raw files has allready been patched by rebuild job
98	#D- If false create_ts will patch the raw files
99	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
100	RebuildFrequency=${RebuildFrequency:=true}
101
102	#D- Flag to determine job's output directory
103	#D- Default : value from libIGCM_post.ksh if any
104	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
105
106	#D- Increased verbosity (1, 2, 3)
107	#D- Default : value from AA_job if any
108	Verbosity=${Verbosity:=3}
109
110	#D- Low level debug : to bypass lib test checks and stack construction
111	#D- Default : value from AA_job if any
112	DEBUG_debug=${DEBUG_debug:=false}
113
114	#D- TEMPORARY Flag to determine atmospheric resolution
115	#D- Default : value from atmospheric driver if any
116	RESOL_ATM=ALL
117
118	#D- Flag to determine surface resolution
119	#D- Default : value from surface driver if any
120	RESOL_SRF=ALL
121
122	#D- Flag to determine surface resolution
123	#D- Default : value from surface driver if any
124	RESOL_SBG=ALL
125
126	#D- TEMPORARY Flag to determine ocean resolution
127	#D- Default : value from ocean driver if any
128	RESOL_OCE=${RESOL_OCE:=ORCA2}
129
130	#D- TEMPORARY Flag to determine ice resolution
131	#D- Default : value from ice driver if any
132	RESOL_ICE=${RESOL_ICE:=ORCA2}
133
134	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
135	#D- Default : value from ice driver if any
136	RESOL_MBG=${RESOL_MBG:=ORCA2}
137
138	########################################################################
139
140	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
141	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
142	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
143	#-------
144	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
145	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
146	#-------
147	( ${DEBUG_debug} ) && IGCM_debug_Check
148	( ${DEBUG_debug} ) && IGCM_card_Check
149	( ${DEBUG_debug} ) && IGCM_date_Check
150
151	########################################################################
152
153	#set -vx
154
155	# Useful cleaning function
156	MENAGE ()
157	{
158	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
159	}
160
161	########################################################################
162
163	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
164	IGCM_sys_Cd ${RUN_DIR_PATH}
165
166	# ------------------------------------------------------------------
167	# Test if all was right before proceeding further
168	# ------------------------------------------------------------------
169	IGCM_debug_Verif_Exit_Post
170
171	if [ ${StandAlone} = true ] ; then
172	CARD_DIR=${SUBMIT_DIR}
173	else
174	CARD_DIR=${RUN_DIR_PATH}
175
176	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
177
178	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
179
180	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
181
182	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
183	fi
184
185	#==================================
186	# First of all
187	#
188	# Read libIGCM compatibility version in config.card
189	# Read UserChoices section
190	# Read Ensemble section
191	# Read Post section
192	# Define all netcdf output directories
193	#==================================
194	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
195
196	#==================================
197	# Define default value to keep compatibility with previous card: means before changes due to TGCC
198	if [ X${PackDefault} = Xtrue ] ; then
199	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
200	else
201	config_Post_PackFrequency=NONE
202	fi
203
204	#==================================
205	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
206	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
207	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
208	fi
209
210	#
211	# Determine component to take care of depending on kind of task create_ts will perform
212	#
213	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
214	#
215	# Determine period for time series. Default : value from AA_job or from launch_TS if any
216	#
217	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
218
219	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
220	#
221	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
222	#
223	# Determine Dimension of the produced time series : 2D or 3D
224	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
225	Dimension=2D
226	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
227	Dimension=3D
228	else
229	Dimension=""
230	fi
231
232	# ------------------------------------------------------------------
233	#D- Test if all was right before entering the loop
234	# ------------------------------------------------------------------
235	IGCM_debug_Verif_Exit_Post
236
237	########################################################################
238	# Depending on requested time series fill following variables :
239	#
240	# - LISTE_FILE_${comp}[*]
241	# - LISTE_VARS_${file}[*]
242	# - LISTE_AXIS_${file}[*]
243	# - LISTE_PATCH_${file}[*]
244	########################################################################
245
246	#
247	# For each selected component determine which files need post-processing
248	#
249	DoJob=false
250	for comp in ${config_ListOfComponents[*]} ; do
251	#
252	ActiveComp=false
253	#
254	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
255	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
256	#
257	card=${CARD_DIR}/COMP/${compname}.card
258
259	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
260	#
261	ListFilesName=${compname}_OutputFiles_List
262	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
263	#
264	if [ X${FileName0} != X${NULL_STR} ] ; then
265	#
266	#IGCM_debug_Print 1 "Component : ${compname}"
267	#
268	# INITIALISATION
269	#
270	typeset i
271	if [ ${FlagToRead} ] ; then
272	((NbFiles=${FlagToRead}))
273	((i=${NbFiles}))
274	else
275	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
276	i=2
277	fi
278	#
279	until [ $i -gt $NbFiles ]; do
280	#
281	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
282	#
283	if [ X${flag_post} != XNONE ] ; then
284	#
285	# For each comp and each file who needs post-processing
286	# determine which variables need time series.
287	#
288	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
289	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
290	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
291	#
292	# If TimeSeriesVars list is empty we skip
293	#
294	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
295	#
296	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
297	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
298	(( i=i+3 ))
299	continue
300	#
301	fi
302	#
303	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
304	#
305	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
306	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
307	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
308	(( i=i+3 ))
309	continue
310	fi
311	#
312	# If we are not a Chunck type task, we skip files/variables handled by chunck task
313	#
314	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
315	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
316	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
317	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
318	(( i=i+3 ))
319	continue
320	fi
321	fi
322	#
323	# Now on we know we have things to do
324	#
325
326	#
327	# We need coordinates axis to properly fill produced time series
328	#
329	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
330	#
331	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
332	#
333	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
334	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
335	(( i=i+3 ))
336	continue
337	#
338	fi
339	#
340	# We need LIST of variables not allready produced (useful for standalone mode)
341	#
342	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
343	case ${FlagDir} in
344	*Y) TS_Dir=TS_YE ;;
345	*M) TS_Dir=TS_MO ;;
346	*D) TS_Dir=TS_DA ;;
347	HF) TS_Dir=TS_HF ;;
348	3H) TS_Dir=TS_HF ;;
349	INS) TS_Dir=TS_INS ;;
350	esac
351	#
352	count=0
353	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
354	#
355	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
356	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
357	#
358	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
359	if [ ! $? = 0 ] ; then
360	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
361	ActiveComp=true
362	(( count = count + 1 ))
363	fi
364	done
365	#
366	if [ ${count} -gt 0 ] ; then
367	#
368	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
369	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
370	#
371	if [ ! X${RebuildFrequency} = Xtrue ] ; then
372	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
373	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
374	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
375	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
376	if [ Xload${Patch} != X ] ; then
377	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
378	eval load${Patch}=loaded
379	fi
380	done
381	fi
382	fi
383	fi
384	fi
385	(( i=i+3 ))
386	done
387	fi
388	#
389	if [ ${ActiveComp} = true ] ; then
390	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
391	fi
392	done
393
394	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
395	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
396
397	########################################################################
398	#
399	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
400	# IF SO BRING THEM IN THE WORKING DIRECTORY
401	#
402	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
403	# - especially for IDRiS
404	# - no special protection against "command too long" !
405	# - libIGCM_sys should handle that or modify following code part
406	#
407	########################################################################
408
409	if [ ${StandAlone} != true ] ; then
410	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
411	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
412	echo \${run_PostProcessing_TimeSeriesRunning}" )
413
414	#if [ X${Running_Flag} = Xy ] ; then
415	# # Time Series Job allready running
416	# IGCM_debug_Print 1 "Time Series Job allready running exit"
417	# exit
418	#fi
419	# Now run.card know we are running
420
421
422
423
424	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
425
426
427
428
429
430	# Get information from last execution
431	#CompletedFlag=$( IGCM_sys_RshMaster \
432	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
433	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
434	fi
435
436	# Is it the first submission or not ?
437
438	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
439	# First Time Series Submission
440	FIRST_PASS=TRUE
441	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
442	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
443	else
444	# Last Completed Time Series
445	FIRST_PASS=FALSE
446	DATE_FIN_JOB_B=${CompletedFlag}
447	date=${DateBegin}_${DATE_FIN_JOB_B}
448	#
449	for comp in ${config_ListOfComponents[*]} ; do
450	#
451	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
452	#
453	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
454	#
455	unset liste_file
456	#
457	file_path=${R_SAVE}/${comp}/Analyse
458	#
459	i=0
460	file=${NULL_STR}
461	#
462	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
463	#
464	# Determine in which output can we find file
465	# IE : Analyse/TS or Analyse/DA2TS or ...
466	#
467	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
468	case ${FlagDir} in
469	*Y) TS_Dir=TS_YE ;;
470	*M) TS_Dir=TS_MO ;;
471	*D) TS_Dir=TS_DA ;;
472	HF) TS_Dir=TS_HF ;;
473	3H) TS_Dir=TS_HF ;;
474	INS) TS_Dir=TS_INS ;;
475	esac
476	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
477	#
478	IGCM_sys_TestFileArchive ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
479	if [ $? = 0 ] ; then
480	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
481	else
482	IGCM_debug_Print 1 "WARNING ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc dont exist"
483	fi
484	#
485	done
486	done
487	if [ X${file} != X${NULL_STR} ] ; then
488	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
489	IGCM_debug_Verif_Exit_Post
490	fi
491	done
492	#
493	LEVEL=0
494	#
495	fi
496
497	########################################################################
498	#
499	# DEFINE LOOP PARAMETERS
500	#
501	########################################################################
502
503	DATE_COUNT=${DATE_FIN_JOB_B}
504	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
505
506	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
507
508	if [ ${DoJob} = true ] ; then
509	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
510	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
511	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
512	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
513	done
514	else
515	NBRE_FILE_TOT=0
516	fi
517
518	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
519	NBRE_TOUR=1
520	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
521	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
522	else
523	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
524	fi
525
526	CURRENT_LOOP=1
527	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
528	# WE HAVE 3 LEVELS
529	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
530	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
531	# - LEVEL 3 : WE ARE IN THE LAST LOOP
532
533	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
534
535	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
536	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
537	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
538
539	if [ ${FIRST_PASS} = TRUE ] ; then
540	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
541	else
542	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
543	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
544	fi
545
546	DATE_FIN=${PeriodDateEnd}
547	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
548
549	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
550	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
551
552	LEVEL=1
553	DEPOT="TRUE"
554	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
555	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
556	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
557
558	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
559	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
560	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
561	elif [ -z "${DATE_COURANTE}" ] ; then
562	DATE_COURANTE=${DATE_FIN_JOB_B}
563	else
564	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
565	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
566	fi
567
568	(( TotLength = 0 ))
569	COMPTEUR=1
570	DATE_LOOP=${DATE_COURANTE}
571
572	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
573	#
574	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
575	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
576	(( TotLength = TotLength + Length ))
577	(( COMPTEUR = COMPTEUR + 1 ))
578	#
579	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
580	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
581
582	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
583	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
584	DEPOT="FALSE"
585	LEVEL=2
586	else
587	# - LEVEL 3 : WE ARE IN THE LAST LOOP
588	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
589	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
590
591	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
592	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
593	DATE_FIN=${PeriodDateEnd}
594	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
595
596	LEVEL=3
597	DEPOT="TRUE"
598	FLAG_B="TRUE"
599	fi
600
601	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
602	for comp in ${config_ListOfComponents[*]} ; do
603	#
604	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
605	#
606	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
607	#
608	eval R_OUT_${comp}=${R_SAVE}/${comp}
609	#
610	i=0
611	#
612	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
613	#
614	# Determine in which output we can find file
615	# IE : Output/MO or Output/DA or ...
616	#
617	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
618	case ${FlagDir} in
619	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
620	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
621	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
622	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
623	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
624	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
625	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
626	IGCM_sys_Exit "Job create_ts" ;;
627	esac
628
629	# Initialize array
630	unset liste_file
631	unset liste_file_tmp
632	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
633
634	COMPTEUR=1
635	MissingFile=FALSE
636	DATE_LOOP=${DATE_COURANTE}
637
638	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
639	#
640	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
641	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
642
643	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
644	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
645
646	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
647	if [ $? = 0 ] ; then
648	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
649	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
650	else
651	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
652	MissingFile=TRUE
653	fi
654	(( COMPTEUR = COMPTEUR + 1 ))
655	#
656	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
657
658	# We skip this file type if one of them is missing.
659	if [ "${MissingFile}" = "TRUE" ] ; then
660	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
661	continue
662	fi
663
664	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
665
666	# Get selected files
667	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
668	IGCM_debug_Verif_Exit_Post
669
670	# Apply IGCM_Patch if needed
671	if [ ! X${RebuildFrequency} = Xtrue ] ; then
672	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
673	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
674	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
675	IGCM_${Patch} ${file_t}
676	IGCM_debug_Verif_Exit_Post
677	done
678	done
679	fi
680	fi
681
682	# WE CAN CONCATENATE FILES AT THIS POINT
683	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
684	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
685	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
686	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
687	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
688
689	liste_coord=" "
690	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
691	liste_coord=${liste_coord}${axis}","
692	done
693
694	# Time axis must be the last dimension
695	time_axis=${axis}
696
697	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
698	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
699	#
700	# We skip variables not in the first file of the list.
701	# Seems overkill to test thel all... but will do if needed
702	ncdump -hv ${var} ${liste_file_tmp[0]} > /dev/null 2>&1
703	if [ ! $? = 0 ] ; then
704	IGCM_debug_Print 1 "WARNING We skip ${var}, not present within ${liste_file_tmp[0]}"
705	continue
706	fi
707	#
708	if [ "${FLAG_B}" = "TRUE" ] ; then
709	# WE CONCATENATE WITH EXISTING FILES
710	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
711	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
712	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
713	IGCM_debug_Verif_Exit_Post
714	#
715	IGCM_sys_Rm ${file1}
716	if [ ! "${DEPOT}" = "TRUE" ] ; then
717	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
718	# "${file1} = file_before ?
719	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
720	fi
721	else
722	# OR NOT
723	file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
724	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${file_out}
725	IGCM_debug_Verif_Exit_Post
726	fi
727
728	#
729	if [ "${DEPOT}" = "TRUE" ] ; then
730	# YEARLY AVERAGE DERIVED FROM MONTHLY MEAN
731	if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
732	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
733	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
734	#
735	# Compute yearly average considering calendar
736	IGCM_sys_cdo yearmonmean -selvar,${var} ${file_out} ${file_out_YE}
737	# Extract all ancillary variables except $var, they will be appended after the cdo command
738	# Needed because cdo applies calculs on all variables
739	IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
740	# This will remove the time axis because cdo rewrites severely with only considered useful variables
741	IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
742	if [ $? -eq 0 ] ; then
743	# Now append all ancillary variables if previous cdo operator ok
744	IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
745	fi
746
747	# Put global history attribut
748	IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
749	IGCM_debug_Verif_Exit_Post
750	#
751	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
752	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
753	IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
754	fi
755	# WE PUT FINAL FILE ON FILE SYSTEM
756	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
757	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
758	#
759	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
760	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
761	if [ ! "X${file1}" = "X" ] ; then
762	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
763	fi
764	#
765	[ -f ${file_out} ] && IGCM_sys_Rm ${file_out}
766	#
767	fi
768	#
769	done # for var in ${liste_var} ; do
770	IGCM_sys_Rm ${liste_file_tmp[*]}
771	done # for file in ${liste_file_brut} do
772	done # for comp in ${config_ListOfComponents[*]} ; do
773	# GO AHEAD IN DEEPEST LOOP
774	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
775	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
776	# FINISH
777
778	# DODS copy
779	for comp in ${config_ListOfComponents[*]} ; do
780	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
781	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
782	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
783	done
784	done
785
786	export DEBUG_debug; export BigBrother ; export libIGCM; export SUBMIT_DIR; export POST_DIR; export Script_Output ;
787	export StandAlone ; export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ;
788	export RESOL_SBG
789	listVarEnv="DEBUG_debug,BigBrother,libIGCM,SUBMIT_DIR,POST_DIR,Script_Post_Output,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
790	export listVarEnv
791
792	#D-
793	# --------------------------------------------------------------------
794	#D- Test if all was right
795	# --------------------------------------------------------------------
796	IGCM_debug_Verif_Exit_Post
797	# --------------------------------------------------------------------
798	#D- Submit MONITORING
799	# --------------------------------------------------------------------
800	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
801	Script_Post_Output=monitoring.${PeriodDateEnd}
802	IGCM_sys_QsubPost monitoring
803	fi
804
805	#D-
806	# --------------------------------------------------------------------
807	#D- Update the run.card
808	# --------------------------------------------------------------------
809	if [ ${StandAlone} != true ] ; then
810	# Put in run.card end period of time series
811	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
812	# We have finish the job !
813	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
814	fi
815
816	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
817	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
818
819	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: