Context Navigation

source: trunk/libIGCM/AA_create_ts @ 1398

Last change on this file since 1398 was 1393, checked in by sdipsl, 7 years ago
remove TS_MO_YE creation. XIOS can handle this much better now. cosmetics
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 29.8 KB

Rev	Line
[622]	1	#-Q- curie ######################
	2	#-Q- curie ## CURIE TGCC/CEA ##
	3	#-Q- curie ######################
[906]	4	#-Q- curie #MSUB -r TS # Nom du job
[622]	5	#-Q- curie #MSUB -eo
	6	#-Q- curie #MSUB -n 1 # Reservation du processus
[880]	7	#-Q- curie #MSUB -T 80000 # Limite de temps elapsed du job
[1154]	8	#-Q- curie #MSUB -q ::default_node::
[1274]	9	#-Q- curie #MSUB -c ::default_core::
[704]	10	#-Q- curie #MSUB -Q normal
[837]	11	#-Q- curie #MSUB -A ::default_project::
[681]	12	#-Q- curie set +x
[770]	13	#-Q- ada #!/bin/ksh
	14	#-Q- ada #######################
[929]	15	#-Q- ada ## ADA IDRIS ##
[770]	16	#-Q- ada #######################
[848]	17	#-Q- ada # @ job_type = serial
	18	#-Q- ada # @ requirements = (Feature == "prepost")
[770]	19	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
	20	#-Q- ada # @ wall_clock_limit = 20:00:00
	21	#-Q- ada # Nom du travail LoadLeveler
	22	#-Q- ada # @ job_name = TS
	23	#-Q- ada # Fichier de sortie standard du travail
	24	#-Q- ada # @ output = $(job_name).$(jobid)
	25	#-Q- ada # Fichier de sortie d'erreur du travail
	26	#-Q- ada # @ error = $(job_name).$(jobid)
	27	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
	28	#-Q- ada # @ notification = error
[1290]	29	#-Q- ada # @ environment = $DEBUG_debug ; $BigBrother ; $postProcessingStopLevel ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $POST_DIR ; $Script_Post_Output ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateEnd ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; wall_clock_limit=$(wall_clock_limit)
[770]	30	#-Q- ada # @ queue
[287]	31	#-Q- lxiv8 ######################
	32	#-Q- lxiv8 ## OBELIX LSCE ##
	33	#-Q- lxiv8 ######################
	34	#-Q- lxiv8 #PBS -N TS
	35	#-Q- lxiv8 #PBS -m a
	36	#-Q- lxiv8 #PBS -j oe
	37	#-Q- lxiv8 #PBS -q medium
	38	#-Q- lxiv8 #PBS -o TS.$$
	39	#-Q- lxiv8 #PBS -S /bin/ksh
[1184]	40	#-Q- ifort_CICLAD ######################
	41	#-Q- ifort_CICLAD ## CICLAD IPSL ##
	42	#-Q- ifort_CICLAD ######################
	43	#-Q- ifort_CICLAD #PBS -N TS
	44	#-Q- ifort_CICLAD #PBS -m a
	45	#-Q- ifort_CICLAD #PBS -j oe
[1314]	46	#-Q- ifort_CICLAD ###PBS -q h12 # Queue for 12 hours at ciclad only
[1184]	47	#-Q- ifort_CICLAD #PBS -S /bin/ksh
[2]	48	#-Q- default #!/bin/ksh
	49	#-Q- default ##################
	50	#-Q- default ## DEFAULT HOST ##
	51	#-Q- default ##################
	52
[373]	53	#**************************************************************
	54	# Author: Sebastien Denvil
	55	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
	56	# $Revision:: $ Revision of last commit
	57	# $Author:: $ Author of last commit
	58	# $Date:: $ Date of last commit
[108]	59	# IPSL (2006)
	60	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
[373]	61	#
	62	#**************************************************************
[108]	63
	64	#set -eu
[2]	65	#set -vx
	66
	67	date
	68
[770]	69	#-Q- ada export OMP_NUM_THREADS=1
[2]	70
[1356]	71	#D- Task type DO NOT CHANGE (computing, post-processing or checking)
[712]	72	TaskType=post-processing
	73
[2]	74	########################################################################
	75
	76	#D- Flag to determine if this job in a standalone mode
	77	#D- Default : value from AA_job if any
	78	StandAlone=${StandAlone:=true}
	79
[554]	80	#D- Path to libIGCM
[206]	81	#D- Default : value from AA_job if any
[229]	82	libIGCM=${libIGCM:=::modipsl::/libIGCM}
[206]	83
	84	#D- Flag to determine allready produced time series. Empty if you start from the beginning
[213]	85	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
[209]	86	CompletedFlag=${CompletedFlag:=}
[135]	87
[213]	88	#D- Flag to determine task type this script will perform.
	89	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
	90	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
[257]	91	TsTask=${TsTask:=2D}
[267]	92	if [ X${TsTask} = Xempty ] ; then
[785]	93	TsTask=""
[267]	94	fi
[213]	95
	96	#D- Flag to determine if rebuild process has been performed asynchronously
[906]	97	#D- Possible value true or false.
[213]	98	#D- If true raw files has allready been patched by rebuild job
	99	#D- If false create_ts will patch the raw files
	100	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
	101	RebuildFrequency=${RebuildFrequency:=true}
	102
[2]	103	#D- Flag to determine job's output directory
	104	#D- Default : value from libIGCM_post.ksh if any
[370]	105	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
[2]	106
	107	#D- Increased verbosity (1, 2, 3)
	108	#D- Default : value from AA_job if any
	109	Verbosity=${Verbosity:=3}
	110
	111	#D- Low level debug : to bypass lib test checks and stack construction
	112	#D- Default : value from AA_job if any
	113	DEBUG_debug=${DEBUG_debug:=false}
	114
	115	#D- TEMPORARY Flag to determine atmospheric resolution
	116	#D- Default : value from atmospheric driver if any
[254]	117	RESOL_ATM=ALL
[2]	118
[257]	119	#D- Flag to determine surface resolution
	120	#D- Default : value from surface driver if any
	121	RESOL_SRF=ALL
	122
	123	#D- Flag to determine surface resolution
	124	#D- Default : value from surface driver if any
	125	RESOL_SBG=ALL
	126
[2]	127	#D- TEMPORARY Flag to determine ocean resolution
	128	#D- Default : value from ocean driver if any
	129	RESOL_OCE=${RESOL_OCE:=ORCA2}
	130
	131	#D- TEMPORARY Flag to determine ice resolution
	132	#D- Default : value from ice driver if any
[1326]	133	RESOL_ICE=${RESOL_OCE:=ORCA2}
[2]	134
[178]	135	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
	136	#D- Default : value from ice driver if any
[1326]	137	RESOL_MBG=${RESOL_OCE:=ORCA2}
[178]	138
[2]	139	########################################################################
	140
[108]	141	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
	142	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
	143	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
	144	#-------
	145	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
[731]	146	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
[832]	147	#-------
[1192]	148	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
	149	IGCM_sys_Cd ${RUN_DIR_PATH}
	150	#-------
[832]	151	( ${DEBUG_debug} ) && IGCM_debug_Check
	152	( ${DEBUG_debug} ) && IGCM_card_Check
	153	( ${DEBUG_debug} ) && IGCM_date_Check
[2]	154
	155	########################################################################
	156
	157	#set -vx
	158
	159	# Useful cleaning function
[906]	160	MENAGE ()
[81]	161	{
[1306]	162	typeset ExistFlag
	163	ExistFlag=$( IGCM_sys_RshArchive "[ -f $1/$3 ] && echo 0 \|\| echo 1" )
	164	[ ${ExistFlag} -eq 0 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
[2]	165	}
	166
	167	########################################################################
	168
[436]	169	# ------------------------------------------------------------------
	170	# Test if all was right before proceeding further
	171	# ------------------------------------------------------------------
[1206]	172	IGCM_debug_Verif_Exit
[436]	173
[2]	174	if [ ${StandAlone} = true ] ; then
[785]	175	CARD_DIR=${SUBMIT_DIR}
[2]	176	else
[785]	177	CARD_DIR=${RUN_DIR_PATH}
[903]	178
[785]	179	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
[903]	180
[785]	181	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
[903]	182
[785]	183	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
[903]	184
[785]	185	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
[2]	186	fi
	187
[591]	188	#==================================
[2]	189	# First of all
	190	#
[727]	191	# Read libIGCM compatibility version in config.card
	192	# Read UserChoices section
	193	# Read Ensemble section
	194	# Read Post section
	195	# Define all netcdf output directories
[591]	196	#==================================
[727]	197	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
[591]	198
[1198]	199	# ------------------------------------------------------------------
	200	# Activate BigBrother so as to supervise this job
	201	# ------------------------------------------------------------------
	202	IGCM_debug_BigBro_Initialize
	203
[591]	204	#==================================
[633]	205	# Define default value to keep compatibility with previous card: means before changes due to TGCC
	206	if [ X${PackDefault} = Xtrue ] ; then
	207	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
	208	else
	209	config_Post_PackFrequency=NONE
	210	fi
[591]	211
	212	#==================================
	213	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
	214	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
	215	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
	216	fi
	217
[2]	218	#
[149]	219	# Determine component to take care of depending on kind of task create_ts will perform
[2]	220	#
[392]	221	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
[2]	222	#
[149]	223	# Determine period for time series. Default : value from AA_job or from launch_TS if any
	224	#
[309]	225	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
[301]	226
[2]	227	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
	228	#
	229	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
[149]	230	#
	231	# Determine Dimension of the produced time series : 2D or 3D
	232	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
[785]	233	Dimension=2D
[149]	234	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
[785]	235	Dimension=3D
[149]	236	else
[785]	237	Dimension=""
[149]	238	fi
[2]	239
[441]	240	# ------------------------------------------------------------------
	241	#D- Test if all was right before entering the loop
	242	# ------------------------------------------------------------------
[1206]	243	IGCM_debug_Verif_Exit
[149]	244
[2]	245	########################################################################
	246	# Depending on requested time series fill following variables :
	247	#
	248	# - LISTE_FILE_${comp}[*]
	249	# - LISTE_VARS_${file}[*]
	250	# - LISTE_AXIS_${file}[*]
	251	# - LISTE_PATCH_${file}[*]
	252	########################################################################
	253
	254	#
	255	# For each selected component determine which files need post-processing
	256	#
	257	DoJob=false
	258	for comp in ${config_ListOfComponents[*]} ; do
[785]	259	#
	260	ActiveComp=false
	261	#
	262	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
	263	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
	264	#
	265	card=${CARD_DIR}/COMP/${compname}.card
	266
	267	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
	268	#
	269	ListFilesName=${compname}_OutputFiles_List
	270	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
	271	#
	272	if [ X${FileName0} != X${NULL_STR} ] ; then
[2]	273	#
[785]	274	#IGCM_debug_Print 1 "Component : ${compname}"
[2]	275	#
[785]	276	# INITIALISATION
[2]	277	#
[785]	278	typeset i
	279	if [ ${FlagToRead} ] ; then
	280	((NbFiles=${FlagToRead}))
	281	((i=${NbFiles}))
	282	else
	283	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
	284	i=2
	285	fi
[2]	286	#
[785]	287	until [ $i -gt $NbFiles ]; do
	288	#
	289	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
	290	#
	291	if [ X${flag_post} != XNONE ] ; then
	292	#
	293	# For each comp and each file who needs post-processing
	294	# determine which variables need time series.
	295	#
	296	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
	297	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
	298	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
	299	#
	300	# If TimeSeriesVars list is empty we skip
	301	#
	302	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
	303	#
	304	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
	305	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
	306	(( i=i+3 ))
	307	continue
	308	#
	309	fi
	310	#
	311	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
	312	#
	313	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
	314	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
	315	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
	316	(( i=i+3 ))
	317	continue
	318	fi
	319	#
	320	# If we are not a Chunck type task, we skip files/variables handled by chunck task
	321	#
	322	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
	323	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
	324	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
	325	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
	326	(( i=i+3 ))
	327	continue
	328	fi
	329	fi
	330	#
	331	# Now on we know we have things to do
	332	#
[236]	333
[785]	334	#
	335	# We need coordinates axis to properly fill produced time series
	336	#
	337	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
	338	#
	339	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
	340	#
	341	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
	342	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
	343	(( i=i+3 ))
	344	continue
	345	#
	346	fi
	347	#
	348	# We need LIST of variables not allready produced (useful for standalone mode)
	349	#
	350	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
	351	case ${FlagDir} in
	352	*Y) TS_Dir=TS_YE ;;
	353	*M) TS_Dir=TS_MO ;;
	354	*D) TS_Dir=TS_DA ;;
	355	HF) TS_Dir=TS_HF ;;
	356	3H) TS_Dir=TS_HF ;;
	357	INS) TS_Dir=TS_INS ;;
	358	esac
	359	#
	360	count=0
	361	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
	362	#
	363	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
	364	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
	365	#
	366	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
	367	if [ ! $? = 0 ] ; then
	368	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
	369	ActiveComp=true
	370	(( count = count + 1 ))
	371	fi
	372	done
	373	#
	374	if [ ${count} -gt 0 ] ; then
	375	#
	376	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
	377	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
	378	#
	379	if [ ! X${RebuildFrequency} = Xtrue ] ; then
	380	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
	381	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
	382	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
	383	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
	384	if [ Xload${Patch} != X ] ; then
	385	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
	386	eval load${Patch}=loaded
	387	fi
	388	done
	389	fi
	390	fi
	391	fi
	392	fi
	393	(( i=i+3 ))
	394	done
	395	fi
	396	#
	397	if [ ${ActiveComp} = true ] ; then
	398	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
	399	fi
[2]	400	done
	401
	402	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
	403	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
	404
	405	########################################################################
	406	#
	407	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
	408	# IF SO BRING THEM IN THE WORKING DIRECTORY
	409	#
	410	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
	411	# - especially for IDRiS
	412	# - no special protection against "command too long" !
	413	# - libIGCM_sys should handle that or modify following code part
	414	#
	415	########################################################################
	416
	417	if [ ${StandAlone} != true ] ; then
[785]	418	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
	419	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
	420	echo \${run_PostProcessing_TimeSeriesRunning}" )
[2]	421
[785]	422	#if [ X${Running_Flag} = Xy ] ; then
	423	# # Time Series Job allready running
	424	# IGCM_debug_Print 1 "Time Series Job allready running exit"
	425	# exit
	426	#fi
	427	# Now run.card know we are running
[149]	428
	429
	430
	431
[785]	432	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
[149]	433
	434
	435
	436
	437
[785]	438	# Get information from last execution
	439	#CompletedFlag=$( IGCM_sys_RshMaster \
	440	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
	441	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
[2]	442	fi
	443
	444	# Is it the first submission or not ?
[149]	445
	446	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
[785]	447	# First Time Series Submission
	448	FIRST_PASS=TRUE
	449	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
	450	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
[2]	451	else
[785]	452	# Last Completed Time Series
	453	FIRST_PASS=FALSE
	454	DATE_FIN_JOB_B=${CompletedFlag}
	455	date=${DateBegin}_${DATE_FIN_JOB_B}
[1038]	456	#
[785]	457	for comp in ${config_ListOfComponents[*]} ; do
	458	#
	459	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
	460	#
	461	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
	462	#
	463	unset liste_file
	464	#
	465	file_path=${R_SAVE}/${comp}/Analyse
	466	#
	467	i=0
	468	file=${NULL_STR}
	469	#
	470	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
	471	#
	472	# Determine in which output can we find file
	473	# IE : Analyse/TS or Analyse/DA2TS or ...
	474	#
	475	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
	476	case ${FlagDir} in
	477	*Y) TS_Dir=TS_YE ;;
	478	*M) TS_Dir=TS_MO ;;
	479	*D) TS_Dir=TS_DA ;;
	480	HF) TS_Dir=TS_HF ;;
	481	3H) TS_Dir=TS_HF ;;
	482	INS) TS_Dir=TS_INS ;;
	483	esac
	484	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
[2]	485	#
[1033]	486	IGCM_sys_TestFileArchive ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
	487	if [ $? = 0 ] ; then
	488	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
	489	else
	490	IGCM_debug_Print 1 "WARNING ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc dont exist"
	491	fi
[785]	492	#
	493	done
[2]	494	done
[785]	495	if [ X${file} != X${NULL_STR} ] ; then
	496	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
[1206]	497	IGCM_debug_Verif_Exit
[785]	498	fi
	499	done
	500	#
	501	LEVEL=0
	502	#
[2]	503	fi
	504
	505	########################################################################
	506	#
	507	# DEFINE LOOP PARAMETERS
	508	#
	509	########################################################################
	510
	511	DATE_COUNT=${DATE_FIN_JOB_B}
	512	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
	513
	514	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
	515
	516	if [ ${DoJob} = true ] ; then
[785]	517	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
	518	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
	519	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
	520	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
	521	done
[2]	522	else
[785]	523	NBRE_FILE_TOT=0
[2]	524	fi
	525
	526	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
[906]	527	NBRE_TOUR=1
[2]	528	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
[906]	529	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
[2]	530	else
[785]	531	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
[2]	532	fi
	533
	534	CURRENT_LOOP=1
	535	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
[906]	536	# WE HAVE 3 LEVELS
[2]	537	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
	538	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
	539	# - LEVEL 3 : WE ARE IN THE LAST LOOP
	540
	541	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
	542
[906]	543	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
[785]	544	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
	545	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
[2]	546
[785]	547	if [ ${FIRST_PASS} = TRUE ] ; then
	548	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
	549	else
	550	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
	551	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
	552	fi
[2]	553
[785]	554	DATE_FIN=${PeriodDateEnd}
	555	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
[2]	556
[785]	557	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
	558	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
[2]	559
[785]	560	LEVEL=1
[906]	561	DEPOT="TRUE"
[785]	562	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
	563	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
	564	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
[2]	565
[785]	566	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
	567	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
	568	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
	569	elif [ -z "${DATE_COURANTE}" ] ; then
	570	DATE_COURANTE=${DATE_FIN_JOB_B}
[2]	571	else
[785]	572	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
	573	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
	574	fi
[2]	575
[785]	576	(( TotLength = 0 ))
	577	COMPTEUR=1
	578	DATE_LOOP=${DATE_COURANTE}
[2]	579
[785]	580	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
	581	#
	582	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
	583	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
	584	(( TotLength = TotLength + Length ))
	585	(( COMPTEUR = COMPTEUR + 1 ))
	586	#
	587	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
	588	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
[2]	589
[785]	590	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
	591	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
	592	DEPOT="FALSE"
	593	LEVEL=2
	594	else
	595	# - LEVEL 3 : WE ARE IN THE LAST LOOP
	596	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
	597	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
[2]	598
[785]	599	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
	600	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
	601	DATE_FIN=${PeriodDateEnd}
	602	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
[2]	603
[785]	604	LEVEL=3
	605	DEPOT="TRUE"
	606	FLAG_B="TRUE"
	607	fi
[2]	608
[785]	609	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
	610	for comp in ${config_ListOfComponents[*]} ; do
	611	#
	612	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
	613	#
	614	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
	615	#
	616	eval R_OUT_${comp}=${R_SAVE}/${comp}
	617	#
	618	i=0
	619	#
	620	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
	621	#
	622	# Determine in which output we can find file
	623	# IE : Output/MO or Output/DA or ...
	624	#
	625	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
	626	case ${FlagDir} in
	627	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
	628	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
	629	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
	630	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
	631	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
	632	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
	633	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
[1205]	634	IGCM_debug_Exit "Job create_ts" ;;
[785]	635	esac
[2]	636
[785]	637	# Initialize array
	638	unset liste_file
	639	unset liste_file_tmp
	640	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
[2]	641
[785]	642	COMPTEUR=1
[817]	643	MissingFile=FALSE
[785]	644	DATE_LOOP=${DATE_COURANTE}
[2]	645
[785]	646	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
	647	#
	648	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
	649	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
[2]	650
[785]	651	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
	652	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
[2]	653
[817]	654	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
	655	if [ $? = 0 ] ; then
	656	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
	657	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
	658	else
	659	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
	660	MissingFile=TRUE
	661	fi
[785]	662	(( COMPTEUR = COMPTEUR + 1 ))
	663	#
	664	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
[817]	665
	666	# We skip this file type if one of them is missing.
	667	if [ "${MissingFile}" = "TRUE" ] ; then
	668	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
	669	continue
	670	fi
	671
[785]	672	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
[2]	673
[785]	674	# Get selected files
	675	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
[1206]	676	IGCM_debug_Verif_Exit
[2]	677
[785]	678	# Apply IGCM_Patch if needed
	679	if [ ! X${RebuildFrequency} = Xtrue ] ; then
	680	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
[906]	681	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
[785]	682	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
	683	IGCM_${Patch} ${file_t}
[1206]	684	IGCM_debug_Verif_Exit
[785]	685	done
	686	done
	687	fi
	688	fi
[2]	689
[785]	690	# WE CAN CONCATENATE FILES AT THIS POINT
	691	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
	692	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
	693	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
	694	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
	695	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
[514]	696
[785]	697	liste_coord=" "
	698	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
	699	liste_coord=${liste_coord}${axis}","
	700	done
[287]	701
[785]	702	# Time axis must be the last dimension
	703	time_axis=${axis}
[388]	704
[785]	705	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
	706	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
	707	#
[1033]	708	# We skip variables not in the first file of the list.
	709	# Seems overkill to test thel all... but will do if needed
	710	ncdump -hv ${var} ${liste_file_tmp[0]} > /dev/null 2>&1
	711	if [ ! $? = 0 ] ; then
	712	IGCM_debug_Print 1 "WARNING We skip ${var}, not present within ${liste_file_tmp[0]}"
	713	continue
	714	fi
	715	#
[785]	716	if [ "${FLAG_B}" = "TRUE" ] ; then
	717	# WE CONCATENATE WITH EXISTING FILES
	718	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
	719	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
	720	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
[1206]	721	IGCM_debug_Verif_Exit
[785]	722	#
	723	IGCM_sys_Rm ${file1}
	724	if [ ! "${DEPOT}" = "TRUE" ] ; then
	725	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
	726	# "${file1} = file_before ?
	727	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
	728	fi
	729	else
	730	# OR NOT
[1047]	731	file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
	732	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${file_out}
[1206]	733	IGCM_debug_Verif_Exit
[785]	734	fi
	735
	736	#
	737	if [ "${DEPOT}" = "TRUE" ] ; then
[1042]	738	# WE PUT FINAL FILE ON FILE SYSTEM
	739	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
	740	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
[785]	741	#
[1042]	742	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
	743	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
	744	if [ ! "X${file1}" = "X" ] ; then
	745	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
	746	fi
	747	#
[1039]	748	[ -f ${file_out} ] && IGCM_sys_Rm ${file_out}
[785]	749	#
	750	fi
	751	#
[1038]	752	done # for var in ${liste_var} ; do
[785]	753	IGCM_sys_Rm ${liste_file_tmp[*]}
[1038]	754	done # for file in ${liste_file_brut} do
	755	done # for comp in ${config_ListOfComponents[*]} ; do
[785]	756	# GO AHEAD IN DEEPEST LOOP
	757	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
[1038]	758	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
[2]	759	# FINISH
	760
	761	# DODS copy
	762	for comp in ${config_ListOfComponents[*]} ; do
[1393]	763	for TS_Dir in TS_YE TS_MO TS_DA TS_HF TS_INS ; do
[785]	764	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
	765	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
	766	done
[2]	767	done
	768
[1112]	769	export DEBUG_debug; export BigBrother ; export libIGCM; export SUBMIT_DIR; export POST_DIR; export Script_Output ;
	770	export StandAlone ; export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ;
	771	export RESOL_SBG
	772	listVarEnv="DEBUG_debug,BigBrother,libIGCM,SUBMIT_DIR,POST_DIR,Script_Post_Output,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
[2]	773	export listVarEnv
	774
	775	#D-
	776	# --------------------------------------------------------------------
	777	#D- Test if all was right
	778	# --------------------------------------------------------------------
[1206]	779	IGCM_debug_Verif_Exit
[2]	780	# --------------------------------------------------------------------
	781	#D- Submit MONITORING
	782	# --------------------------------------------------------------------
[414]	783	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
[785]	784	Script_Post_Output=monitoring.${PeriodDateEnd}
	785	IGCM_sys_QsubPost monitoring
[387]	786	fi
[2]	787
	788	#D-
	789	# --------------------------------------------------------------------
	790	#D- Update the run.card
	791	# --------------------------------------------------------------------
	792	if [ ${StandAlone} != true ] ; then
[785]	793	# Put in run.card end period of time series
	794	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
	795	# We have finish the job !
	796	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
[2]	797	fi
[203]	798
[590]	799	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
[203]	800	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
	801
[1198]	802	# ------------------------------------------------------------------
	803	# Finalize BigBrother to inform that the jobs end
	804	# ------------------------------------------------------------------
	805	IGCM_debug_BigBro_Finalize
	806
[2]	807	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: