Context Navigation

source: trunk/libIGCM/AA_create_ts @ 935

Last change on this file since 935 was 932, checked in by sdipsl, 11 years ago
clean-up old fashion instrumentation trace
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 29.9 KB

Rev	Line
[622]	1	#-Q- curie ######################
	2	#-Q- curie ## CURIE TGCC/CEA ##
	3	#-Q- curie ######################
[906]	4	#-Q- curie #MSUB -r TS # Nom du job
[622]	5	#-Q- curie #MSUB -eo
	6	#-Q- curie #MSUB -n 1 # Reservation du processus
[880]	7	#-Q- curie #MSUB -T 80000 # Limite de temps elapsed du job
[741]	8	#-Q- curie #MSUB -q xlarge
[704]	9	#-Q- curie #MSUB -Q normal
[837]	10	#-Q- curie #MSUB -A ::default_project::
[681]	11	#-Q- curie set +x
[770]	12	#-Q- ada #!/bin/ksh
	13	#-Q- ada #######################
[929]	14	#-Q- ada ## ADA IDRIS ##
[770]	15	#-Q- ada #######################
[848]	16	#-Q- ada # @ job_type = serial
	17	#-Q- ada # @ requirements = (Feature == "prepost")
[770]	18	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
	19	#-Q- ada # @ wall_clock_limit = 20:00:00
	20	#-Q- ada # Nom du travail LoadLeveler
	21	#-Q- ada # @ job_name = TS
	22	#-Q- ada # Fichier de sortie standard du travail
	23	#-Q- ada # @ output = $(job_name).$(jobid)
	24	#-Q- ada # Fichier de sortie d'erreur du travail
	25	#-Q- ada # @ error = $(job_name).$(jobid)
	26	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
	27	#-Q- ada # @ notification = error
	28	#-Q- ada # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
	29	#-Q- ada # @ queue
[287]	30	#-Q- lxiv8 ######################
	31	#-Q- lxiv8 ## OBELIX LSCE ##
	32	#-Q- lxiv8 ######################
	33	#-Q- lxiv8 #PBS -N TS
	34	#-Q- lxiv8 #PBS -m a
	35	#-Q- lxiv8 #PBS -j oe
	36	#-Q- lxiv8 #PBS -q medium
	37	#-Q- lxiv8 #PBS -o TS.$$
	38	#-Q- lxiv8 #PBS -S /bin/ksh
[2]	39	#-Q- default #!/bin/ksh
	40	#-Q- default ##################
	41	#-Q- default ## DEFAULT HOST ##
	42	#-Q- default ##################
	43
[373]	44	#**************************************************************
	45	# Author: Sebastien Denvil
	46	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
	47	# $Revision:: $ Revision of last commit
	48	# $Author:: $ Author of last commit
	49	# $Date:: $ Date of last commit
[108]	50	# IPSL (2006)
	51	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
[373]	52	#
	53	#**************************************************************
[108]	54
	55	#set -eu
[2]	56	#set -vx
	57
	58	date
	59
[770]	60	#-Q- ada export OMP_NUM_THREADS=1
[2]	61
[712]	62	#D- Task type (computing or post-processing)
	63	TaskType=post-processing
	64
[2]	65	########################################################################
	66
	67	#D- Flag to determine if this job in a standalone mode
	68	#D- Default : value from AA_job if any
	69	StandAlone=${StandAlone:=true}
	70
[554]	71	#D- Path to libIGCM
[206]	72	#D- Default : value from AA_job if any
[229]	73	libIGCM=${libIGCM:=::modipsl::/libIGCM}
[206]	74
	75	#D- Flag to determine allready produced time series. Empty if you start from the beginning
[213]	76	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
[209]	77	CompletedFlag=${CompletedFlag:=}
[135]	78
[213]	79	#D- Flag to determine task type this script will perform.
	80	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
	81	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
[257]	82	TsTask=${TsTask:=2D}
[267]	83	if [ X${TsTask} = Xempty ] ; then
[785]	84	TsTask=""
[267]	85	fi
[213]	86
	87	#D- Flag to determine if rebuild process has been performed asynchronously
[906]	88	#D- Possible value true or false.
[213]	89	#D- If true raw files has allready been patched by rebuild job
	90	#D- If false create_ts will patch the raw files
	91	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
	92	RebuildFrequency=${RebuildFrequency:=true}
	93
[2]	94	#D- Flag to determine job's output directory
	95	#D- Default : value from libIGCM_post.ksh if any
[370]	96	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
[2]	97
	98	#D- Increased verbosity (1, 2, 3)
	99	#D- Default : value from AA_job if any
	100	Verbosity=${Verbosity:=3}
	101
	102	#D- Low level debug : to bypass lib test checks and stack construction
	103	#D- Default : value from AA_job if any
	104	DEBUG_debug=${DEBUG_debug:=false}
	105
	106	#D- TEMPORARY Flag to determine atmospheric resolution
	107	#D- Default : value from atmospheric driver if any
[254]	108	RESOL_ATM=ALL
[2]	109
[257]	110	#D- Flag to determine surface resolution
	111	#D- Default : value from surface driver if any
	112	RESOL_SRF=ALL
	113
	114	#D- Flag to determine surface resolution
	115	#D- Default : value from surface driver if any
	116	RESOL_SBG=ALL
	117
[2]	118	#D- TEMPORARY Flag to determine ocean resolution
	119	#D- Default : value from ocean driver if any
	120	RESOL_OCE=${RESOL_OCE:=ORCA2}
	121
	122	#D- TEMPORARY Flag to determine ice resolution
	123	#D- Default : value from ice driver if any
	124	RESOL_ICE=${RESOL_ICE:=ORCA2}
	125
[178]	126	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
	127	#D- Default : value from ice driver if any
	128	RESOL_MBG=${RESOL_MBG:=ORCA2}
	129
[2]	130	########################################################################
	131
[108]	132	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
	133	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
	134	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
	135	#-------
	136	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
[731]	137	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
[832]	138	#-------
	139	( ${DEBUG_debug} ) && IGCM_debug_Check
	140	( ${DEBUG_debug} ) && IGCM_card_Check
	141	( ${DEBUG_debug} ) && IGCM_date_Check
[2]	142
	143	########################################################################
	144
	145	#set -vx
	146
	147	# Useful cleaning function
[906]	148	MENAGE ()
[81]	149	{
[785]	150	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
[2]	151	}
	152
	153	########################################################################
	154
[202]	155	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
[2]	156	IGCM_sys_Cd ${RUN_DIR_PATH}
	157
[436]	158	# ------------------------------------------------------------------
	159	# Test if all was right before proceeding further
	160	# ------------------------------------------------------------------
	161	IGCM_debug_Verif_Exit_Post
	162
[2]	163	if [ ${StandAlone} = true ] ; then
[785]	164	CARD_DIR=${SUBMIT_DIR}
[2]	165	else
[785]	166	CARD_DIR=${RUN_DIR_PATH}
[903]	167
[785]	168	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
[903]	169
[785]	170	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
[903]	171
[785]	172	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
[903]	173
[785]	174	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
[2]	175	fi
	176
[591]	177	#==================================
[2]	178	# First of all
	179	#
[727]	180	# Read libIGCM compatibility version in config.card
	181	# Read UserChoices section
	182	# Read Ensemble section
	183	# Read Post section
	184	# Define all netcdf output directories
[591]	185	#==================================
[727]	186	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
[591]	187
	188	#==================================
[633]	189	# Define default value to keep compatibility with previous card: means before changes due to TGCC
	190	if [ X${PackDefault} = Xtrue ] ; then
	191	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
	192	else
	193	config_Post_PackFrequency=NONE
	194	fi
[591]	195
	196	#==================================
	197	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
	198	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
	199	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
	200	fi
	201
[2]	202	#
[149]	203	# Determine component to take care of depending on kind of task create_ts will perform
[2]	204	#
[392]	205	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
[2]	206	#
[149]	207	# Determine period for time series. Default : value from AA_job or from launch_TS if any
	208	#
[309]	209	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
[301]	210
[2]	211	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
	212	#
	213	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
[149]	214	#
	215	# Determine Dimension of the produced time series : 2D or 3D
	216	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
[785]	217	Dimension=2D
[149]	218	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
[785]	219	Dimension=3D
[149]	220	else
[785]	221	Dimension=""
[149]	222	fi
[2]	223
[441]	224	# ------------------------------------------------------------------
	225	#D- Test if all was right before entering the loop
	226	# ------------------------------------------------------------------
	227	IGCM_debug_Verif_Exit_Post
[149]	228
[2]	229	########################################################################
	230	# Depending on requested time series fill following variables :
	231	#
	232	# - LISTE_FILE_${comp}[*]
	233	# - LISTE_VARS_${file}[*]
	234	# - LISTE_AXIS_${file}[*]
	235	# - LISTE_PATCH_${file}[*]
	236	########################################################################
	237
	238	#
	239	# For each selected component determine which files need post-processing
	240	#
	241	DoJob=false
	242	for comp in ${config_ListOfComponents[*]} ; do
[785]	243	#
	244	ActiveComp=false
	245	#
	246	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
	247	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
	248	#
	249	card=${CARD_DIR}/COMP/${compname}.card
	250
	251	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
	252	#
	253	ListFilesName=${compname}_OutputFiles_List
	254	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
	255	#
	256	if [ X${FileName0} != X${NULL_STR} ] ; then
[2]	257	#
[785]	258	#IGCM_debug_Print 1 "Component : ${compname}"
[2]	259	#
[785]	260	# INITIALISATION
[2]	261	#
[785]	262	typeset i
	263	if [ ${FlagToRead} ] ; then
	264	((NbFiles=${FlagToRead}))
	265	((i=${NbFiles}))
	266	else
	267	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
	268	i=2
	269	fi
[2]	270	#
[785]	271	until [ $i -gt $NbFiles ]; do
	272	#
	273	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
	274	#
	275	if [ X${flag_post} != XNONE ] ; then
	276	#
	277	# For each comp and each file who needs post-processing
	278	# determine which variables need time series.
	279	#
	280	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
	281	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
	282	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
	283	#
	284	# If TimeSeriesVars list is empty we skip
	285	#
	286	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
	287	#
	288	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
	289	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
	290	(( i=i+3 ))
	291	continue
	292	#
	293	fi
	294	#
	295	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
	296	#
	297	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
	298	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
	299	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
	300	(( i=i+3 ))
	301	continue
	302	fi
	303	#
	304	# If we are not a Chunck type task, we skip files/variables handled by chunck task
	305	#
	306	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
	307	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
	308	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
	309	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
	310	(( i=i+3 ))
	311	continue
	312	fi
	313	fi
	314	#
	315	# Now on we know we have things to do
	316	#
[236]	317
[785]	318	#
	319	# We need coordinates axis to properly fill produced time series
	320	#
	321	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
	322	#
	323	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
	324	#
	325	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
	326	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
	327	(( i=i+3 ))
	328	continue
	329	#
	330	fi
	331	#
	332	# We need LIST of variables not allready produced (useful for standalone mode)
	333	#
	334	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
	335	case ${FlagDir} in
	336	*Y) TS_Dir=TS_YE ;;
	337	*M) TS_Dir=TS_MO ;;
	338	*D) TS_Dir=TS_DA ;;
	339	HF) TS_Dir=TS_HF ;;
	340	3H) TS_Dir=TS_HF ;;
	341	INS) TS_Dir=TS_INS ;;
	342	esac
	343	#
	344	count=0
	345	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
	346	#
	347	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
	348	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
	349	#
	350	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
	351	if [ ! $? = 0 ] ; then
	352	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
	353	ActiveComp=true
	354	(( count = count + 1 ))
	355	fi
	356	done
	357	#
	358	if [ ${count} -gt 0 ] ; then
	359	#
	360	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
	361	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
	362	#
	363	if [ ! X${RebuildFrequency} = Xtrue ] ; then
	364	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
	365	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
	366	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
	367	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
	368	if [ Xload${Patch} != X ] ; then
	369	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
	370	eval load${Patch}=loaded
	371	fi
	372	done
	373	fi
	374	fi
	375	fi
	376	fi
	377	(( i=i+3 ))
	378	done
	379	fi
	380	#
	381	if [ ${ActiveComp} = true ] ; then
	382	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
	383	fi
[2]	384	done
	385
	386	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
	387	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
	388
	389	########################################################################
	390	#
	391	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
	392	# IF SO BRING THEM IN THE WORKING DIRECTORY
	393	#
	394	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
	395	# - especially for IDRiS
	396	# - no special protection against "command too long" !
	397	# - libIGCM_sys should handle that or modify following code part
	398	#
	399	########################################################################
	400
	401	if [ ${StandAlone} != true ] ; then
[785]	402	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
	403	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
	404	echo \${run_PostProcessing_TimeSeriesRunning}" )
[2]	405
[785]	406	#if [ X${Running_Flag} = Xy ] ; then
	407	# # Time Series Job allready running
	408	# IGCM_debug_Print 1 "Time Series Job allready running exit"
	409	# exit
	410	#fi
	411	# Now run.card know we are running
[149]	412
	413
	414
	415
[785]	416	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
[149]	417
	418
	419
	420
	421
[785]	422	# Get information from last execution
	423	#CompletedFlag=$( IGCM_sys_RshMaster \
	424	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
	425	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
[2]	426	fi
	427
	428	# Is it the first submission or not ?
[149]	429
	430	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
[785]	431	# First Time Series Submission
	432	FIRST_PASS=TRUE
	433	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
	434	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
[2]	435	else
[785]	436	# Last Completed Time Series
	437	FIRST_PASS=FALSE
	438	DATE_FIN_JOB_B=${CompletedFlag}
	439	date=${DateBegin}_${DATE_FIN_JOB_B}
	440	for comp in ${config_ListOfComponents[*]} ; do
	441	#
	442	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
	443	#
	444	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
	445	#
	446	unset liste_file
	447	#
	448	file_path=${R_SAVE}/${comp}/Analyse
	449	#
	450	i=0
	451	file=${NULL_STR}
	452	#
	453	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
	454	#
	455	# Determine in which output can we find file
	456	# IE : Analyse/TS or Analyse/DA2TS or ...
	457	#
	458	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
	459	case ${FlagDir} in
	460	*Y) TS_Dir=TS_YE ;;
	461	*M) TS_Dir=TS_MO ;;
	462	*D) TS_Dir=TS_DA ;;
	463	HF) TS_Dir=TS_HF ;;
	464	3H) TS_Dir=TS_HF ;;
	465	INS) TS_Dir=TS_INS ;;
	466	esac
	467	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
[2]	468	#
[785]	469	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
	470	#
	471	done
[2]	472	done
[785]	473	if [ X${file} != X${NULL_STR} ] ; then
	474	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
[903]	475
[785]	476	# Even if some files are missing we tried to continue
	477	# Either ListVar in compname.card are asked but not present in original history file
	478	# Either a previous TS task failed
	479	ExitFlag=false
	480	IGCM_debug_Verif_Exit_Post
	481	fi
	482	done
	483	#
	484	LEVEL=0
	485	#
[2]	486	fi
	487
	488	########################################################################
	489	#
	490	# DEFINE LOOP PARAMETERS
	491	#
	492	########################################################################
	493
	494	DATE_COUNT=${DATE_FIN_JOB_B}
	495	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
	496
	497	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
	498
	499	if [ ${DoJob} = true ] ; then
[785]	500	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
	501	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
	502	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
	503	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
	504	done
[2]	505	else
[785]	506	NBRE_FILE_TOT=0
[2]	507	fi
	508
	509	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
[906]	510	NBRE_TOUR=1
[2]	511	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
[906]	512	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
[2]	513	else
[785]	514	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
[2]	515	fi
	516
	517	CURRENT_LOOP=1
	518	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
[906]	519	# WE HAVE 3 LEVELS
[2]	520	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
	521	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
	522	# - LEVEL 3 : WE ARE IN THE LAST LOOP
	523
	524	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
	525
[906]	526	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
[785]	527	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
	528	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
[2]	529
[785]	530	if [ ${FIRST_PASS} = TRUE ] ; then
	531	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
	532	else
	533	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
	534	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
	535	fi
[2]	536
[785]	537	DATE_FIN=${PeriodDateEnd}
	538	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
[2]	539
[785]	540	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
	541	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
[2]	542
[785]	543	LEVEL=1
[906]	544	DEPOT="TRUE"
[785]	545	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
	546	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
	547	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
[2]	548
[785]	549	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
	550	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
	551	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
	552	elif [ -z "${DATE_COURANTE}" ] ; then
	553	DATE_COURANTE=${DATE_FIN_JOB_B}
[2]	554	else
[785]	555	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
	556	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
	557	fi
[2]	558
[785]	559	(( TotLength = 0 ))
	560	COMPTEUR=1
	561	DATE_LOOP=${DATE_COURANTE}
[2]	562
[785]	563	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
	564	#
	565	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
	566	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
	567	(( TotLength = TotLength + Length ))
	568	(( COMPTEUR = COMPTEUR + 1 ))
	569	#
	570	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
	571	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
[2]	572
[785]	573	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
	574	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
	575	DEPOT="FALSE"
	576	LEVEL=2
	577	else
	578	# - LEVEL 3 : WE ARE IN THE LAST LOOP
	579	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
	580	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
[2]	581
[785]	582	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
	583	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
	584	DATE_FIN=${PeriodDateEnd}
	585	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
[2]	586
[785]	587	LEVEL=3
	588	DEPOT="TRUE"
	589	FLAG_B="TRUE"
	590	fi
[2]	591
[785]	592	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
	593	for comp in ${config_ListOfComponents[*]} ; do
	594	#
	595	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
	596	#
	597	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
	598	#
	599	eval R_OUT_${comp}=${R_SAVE}/${comp}
	600	#
	601	i=0
	602	#
	603	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
	604	#
	605	# Determine in which output we can find file
	606	# IE : Output/MO or Output/DA or ...
	607	#
	608	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
	609	case ${FlagDir} in
	610	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
	611	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
	612	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
	613	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
	614	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
	615	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
	616	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
	617	IGCM_sys_Exit "Job create_ts" ;;
	618	esac
[2]	619
[785]	620	# Initialize array
	621	unset liste_file
	622	unset liste_file_tmp
	623	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
[2]	624
[785]	625	COMPTEUR=1
[817]	626	MissingFile=FALSE
[785]	627	DATE_LOOP=${DATE_COURANTE}
[2]	628
[785]	629	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
	630	#
	631	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
	632	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
[2]	633
[785]	634	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
	635	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
[2]	636
[817]	637	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
	638	if [ $? = 0 ] ; then
	639	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
	640	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
	641	else
	642	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
	643	MissingFile=TRUE
	644	fi
[785]	645	(( COMPTEUR = COMPTEUR + 1 ))
	646	#
	647	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
[817]	648
	649	# We skip this file type if one of them is missing.
	650	if [ "${MissingFile}" = "TRUE" ] ; then
	651	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
	652	continue
	653	fi
	654
[785]	655	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
[2]	656
[785]	657	# Get selected files
	658	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
	659	IGCM_debug_Verif_Exit_Post
[2]	660
[785]	661	# Apply IGCM_Patch if needed
	662	if [ ! X${RebuildFrequency} = Xtrue ] ; then
	663	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
[906]	664	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
[785]	665	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
	666	IGCM_${Patch} ${file_t}
	667	IGCM_debug_Verif_Exit_Post
	668	done
	669	done
	670	fi
	671	fi
[2]	672
[785]	673	# WE CAN CONCATENATE FILES AT THIS POINT
	674	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
	675	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
	676	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
	677	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
	678	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
[514]	679
[785]	680	liste_coord=" "
	681	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
	682	liste_coord=${liste_coord}${axis}","
	683	done
[287]	684
[785]	685	# Time axis must be the last dimension
	686	time_axis=${axis}
[388]	687
[785]	688	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
	689	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
	690	#
	691	if [ "${FLAG_B}" = "TRUE" ] ; then
	692	# WE CONCATENATE WITH EXISTING FILES
	693	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
	694	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
	695	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
	696	#
	697	IGCM_sys_Rm ${file1}
	698	if [ ! "${DEPOT}" = "TRUE" ] ; then
	699	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
	700	# "${file1} = file_before ?
	701	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
	702	fi
	703
	704	ExitFlag=false
	705	else
	706	# OR NOT
	707	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
	708	fi
	709
	710	#
	711	if [ "${DEPOT}" = "TRUE" ] ; then
	712	# WE PUT FINAL FILE ON FILE SYSTEM
	713	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
	714	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
	715	#
	716	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
	717	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
	718	if [ ! "X${file1}" = "X" ] ; then
	719	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
	720	fi
	721	#
	722	if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
	723	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
	724	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
	725	#
	726	# Compute yearly average considering calendar
	727	# and set date of the mid year to 01/07 at 00:00
	728	IGCM_sys_cdo -settime,00:00 -setday,1 -setmon,7 -settunits,days -divdpy -yearsum -muldpm -selvar,${var} ${file_out} ${file_out_YE}
	729	# Extract all ancillary variables except $var, they will be appended after the cdo command
	730	# Needed because cdo applies calculs on all variables
	731	IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
	732	# This will remove the time axis because cdo rewrites severely with only considered useful variables
	733	IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
	734	if [ $? -eq 0 ] ; then
	735	# Now append all ancillary variables if previous cdo operator ok
	736	IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
	737	fi
	738
	739	# Put global history attribut
	740	IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
	741	#
	742	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
	743	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
	744	IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
	745	fi
	746	#
	747	IGCM_sys_Rm ${file_out}
	748	ExitFlag=false
	749	#
	750	fi
	751	#
	752	done # for var in ${liste_var} ; do
	753	IGCM_sys_Rm ${liste_file_tmp[*]}
	754	done # for file in ${liste_file_brut} do
	755	done # for comp in ${config_ListOfComponents[*]} ; do
	756	# GO AHEAD IN DEEPEST LOOP
	757	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
[2]	758	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
	759	# FINISH
	760
	761	# DODS copy
	762	for comp in ${config_ListOfComponents[*]} ; do
[785]	763	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
	764	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
	765	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
	766	done
[2]	767	done
	768
	769	export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
[213]	770	export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG
[2]	771
[213]	772	listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
[2]	773	export listVarEnv
	774
	775	#D-
	776	# --------------------------------------------------------------------
	777	#D- Test if all was right
	778	# --------------------------------------------------------------------
	779	IGCM_debug_Verif_Exit_Post
	780	# --------------------------------------------------------------------
	781	#D- Submit MONITORING
	782	# --------------------------------------------------------------------
[414]	783	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
[785]	784	Script_Post_Output=monitoring.${PeriodDateEnd}
	785	IGCM_sys_QsubPost monitoring
[387]	786	fi
[2]	787
	788	#D-
	789	# --------------------------------------------------------------------
	790	#D- Update the run.card
	791	# --------------------------------------------------------------------
	792	if [ ${StandAlone} != true ] ; then
[785]	793	# Put in run.card end period of time series
	794	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
	795	# We have finish the job !
	796	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
[2]	797	fi
[203]	798
[590]	799	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
[203]	800	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
	801
[2]	802	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: