Context Navigation

source: trunk/libIGCM/libIGCM_config/libIGCM_config.ksh @ 1527

Last change on this file since 1527 was 1527, checked in by flavoni, 4 years ago
cosmetic change to InitYear_m1 variable
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 55.2 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip
5	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr Martial.Mancip__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#===================================
15	function IGCM_config_CommonConfiguration
16	{
17	IGCM_debug_PushStack "IGCM_config_CommonConfiguration" $@
18
19	# Debug Print :
20	[ ${Verbosity} -gt 0 ] && echo
21	IGCM_debug_Print 1 "IGCM_config_CommonConfiguration" $@
22
23	# config.card path
24	configCardPath=$1
25
26	#==================================
27	typeset option auxprint
28
29	#==================================
30	# Read UserChoices section:
31	[ ${Verbosity} -gt 0 ] && echo
32	IGCM_debug_Print 1 "DefineArrayFromOption : config_UserChoices"
33
34	IGCM_card_DefineArrayFromSection ${configCardPath} UserChoices
35	for option in ${config_UserChoices[*]} ; do
36	IGCM_card_DefineVariableFromOption ${configCardPath} UserChoices ${option}
37	eval auxprint=\${config_UserChoices_${option}}
38	IGCM_debug_Print 3 "${option} : ${auxprint}"
39	done
40
41	# Set variables with shorter names on some variables which
42	# are used in the executable name in config.card
43	ResolAtm=${config_UserChoices_ResolAtm}
44	ResolOce=${config_UserChoices_ResolOce}
45	ConfChem=${config_UserChoices_ConfChem}
46	OptMode=${config_UserChoices_OptMode}
47
48	#==================================
49	# Read Ensemble section:
50	[ ${Verbosity} -gt 0 ] && echo
51	IGCM_debug_Print 1 "DefineArrayFromOption : config_Ensemble"
52
53	IGCM_card_DefineArrayFromSection ${configCardPath} Ensemble
54	for option in ${config_Ensemble[*]} ; do
55	IGCM_card_DefineVariableFromOption ${configCardPath} Ensemble ${option}
56	eval auxprint=\${config_Ensemble_${option}}
57	IGCM_debug_Print 3 "${option} : ${auxprint}"
58	done
59
60	#==================================
61	# Read Post section:
62	[ ${Verbosity} -gt 0 ] && echo
63	IGCM_debug_Print 1 "DefineArrayFromOption : config_Post"
64
65	IGCM_card_DefineArrayFromSection ${configCardPath} Post
66	for option in ${config_Post[*]} ; do
67	IGCM_card_DefineVariableFromOption ${configCardPath} Post ${option}
68	eval auxprint=\${config_Post_${option}}
69	IGCM_debug_Print 3 "${option} : ${auxprint}"
70	done
71	[ ${Verbosity} -gt 0 ] && echo
72
73	#==================================
74	# Define default value to keep compatibility with previous card: means before changes due to TGCC
75	# Apply some overrules to ensure proper usage of computing centres resources
76	#
77
78	# if RebuildFrequency is not define in config.card - we initilize it to NONE
79	if [ X${config_Post_RebuildFrequency} = X ] ; then
80	config_Post_RebuildFrequency=NONE
81	fi
82
83	if [ X${PackDefault} = Xtrue ] ; then
84	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
85	# TEST simulations will not be packed and will stay on SCRATCHDIR filesystem
86	IGCM_debug_Print 1 "SpaceName=TEST. OVERRULE PackFrequency to NONE"
87	config_Post_PackFrequency=NONE
88	else
89	# Default to RebuildFrequency if nothing has been set up related to PackFrequency
90	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
91	fi
92	else
93	# If we DO NOT apply pack in this computing center
94	config_Post_PackFrequency=NONE
95	fi
96
97	#====================================================
98	# Define ARCHIVE : Dedicated to large files
99	# Define STORAGE : Dedicated to small/medium files
100	# Define R_OUT : Output tree located on ARCHIVE
101	# Define R_BUF : Output tree located on STORAGE (files waiting treatment, or file lcoation when SpaceName=!PROD)
102	# Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
103	# Define R_TMP : A temporary space used by IGCM_debug_send_AMQP_msg__MAILTUNNEL. Must be persistent in between jobs
104	IGCM_sys_defineArchives
105
106	#====================================================
107	# R_SAVE : Job output directory
108	# R_BUFR : Job output buffered directory
109	# R_CMIP : Job output directory for CMIP6 standard files
110
111	if ( [ ! X${config_UserChoices_SpaceName} = X ] && [ ! X${config_UserChoices_ExperimentName} = X ] ) ; then
112	FreeName=$( echo ${config_UserChoices_JobName} \| sed 's/.*_//' )
113	if ( [ ! X${config_Ensemble_EnsembleName} = X ] && [ ! X${config_Ensemble_EnsembleDate} = X ] ) ; then
114	R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
115	R_FIGR=${R_FIG}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
116	R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
117	R_DODS=${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
118	R_CMIP=${CMIP6_BUF}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
119	else
120	R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
121	R_FIGR=${R_FIG}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
122	R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
123	R_DODS=${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
124	R_CMIP=${CMIP6_BUF}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
125	fi
126	else
127	if ( [ ! X${config_Ensemble_EnsembleName} = X ] && [ ! X${config_Ensemble_EnsembleDate} = X ] ) ; then
128	R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${config_UserChoices_JobName}
129	R_FIGR=${R_FIG}/${config_UserChoices_TagName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${config_UserChoices_JobName}
130	R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${config_UserChoices_JobName}
131	R_DODS=${config_UserChoices_TagName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${config_UserChoices_JobName}
132	else
133	R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
134	R_FIGR=${R_FIG}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
135	R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
136	R_DODS=${config_UserChoices_TagName}/${config_UserChoices_JobName}
137	fi
138	fi
139
140	#====================================================
141	# Define R_OUT_KSH : Storage place for job output
142	# Define R_OUT_EXE : Storage place for binary used during simulation
143	R_OUT_KSH=${R_SAVE}/Out
144	R_OUT_EXE=${R_SAVE}/Exe
145
146	#====================================================
147	# Define R_BUF_KSH : Buffer place for job output
148	# Define R_BUF_EXE : Buffer place for binary used during simulation
149	R_BUF_KSH=${R_BUFR}/Out
150	R_BUF_EXE=${R_BUFR}/Exe
151
152	#====================================================
153	# Define REBUILD_DIR : where we store files needing rebuild process
154	REBUILD_DIR=${R_BUFR}/REBUILD
155	if [ ! X${TaskType} = Xchecking ] ; then
156	IGCM_sys_MkdirWork ${REBUILD_DIR}
157	fi
158
159	#====================================================
160	# DodsCopy : apply default value if not defined
161	if ( [ X${config_Post_DodsCopy} = X${NULL_STR} ] \|\| [ X${config_Post_DodsCopy} = X ] ) ; then
162	config_Post_DodsCopy=TRUE
163	fi
164
165	#====================================================
166	# AtlasIPSL : apply default value if not defined
167	if ( [ X${config_Post_AtlasIPSL} = X${NULL_STR} ] \|\| [ X${config_Post_AtlasIPSL} = X ] ) ; then
168	config_Post_AtlasIPSL=FALSE
169	fi
170
171	#====================================================
172	# ParserXIOS : apply default value if not defined
173	if ( [ X${config_Post_ParserXIOS} = X${NULL_STR} ] \|\| [ X${config_Post_ParserXIOS} = X ] ) ; then
174	config_Post_ParserXIOS=FALSE
175	fi
176
177	#====================================================
178	# MetricsPCMDI : apply default value if not defined
179	if ( [ X${config_Post_MetricsPCMDI} = X${NULL_STR} ] \|\| [ X${config_Post_MetricsPCMDI} = X ] ) ; then
180	config_Post_MetricsPCMDI=FALSE
181	fi
182
183	#====================================================
184	# IgnoreNonMonotonic : apply default value if not defined
185	if ( [ X${config_Post_IgnoreNonMonotonic} = X${NULL_STR} ] \|\| [ X${config_Post_IgnoreNonMonotonic} = X ] ) ; then
186	config_Post_IgnoreNonMonotonic=FALSE
187	fi
188
189	#====================================================
190	# LightRestartPack : apply default value if not defined
191	if ( [ X${config_Post_LightRestartPack} = X${NULL_STR} ] \|\| [ X${config_Post_LightRestartPack} = X ] ) ; then
192	config_Post_LightRestartPack=FALSE
193	fi
194
195	#====================================================
196	# Define StackFileLocation : directory where we store stack files
197	# Define StackFileName : stack file containing call tree and instrumentation
198	# Stack file containing call tree will be stored there.
199	if ( $DEBUG_debug ) ; then
200	StackFileLocation=${StackFileLocation:=${R_BUF_KSH}}
201	[ ! -d ${StackFileLocation} ] && mkdir -p ${StackFileLocation}
202	if [ X${TaskType} = Xcomputing ]; then
203	StackFileName=computing.stack.$$
204	elif [ X${TaskType} = Xpost-processing ]; then
205	StackFileName=${Script_Post_Output}.stack.$$
206	elif [ X${TaskType} = Xchecking ]; then
207	StackFileName=checking.stack.$$
208	else
209	IGCM_debug_Exit "IGCM_config_CommonConfiguration unknown TaskType : ${TaskType}"
210	IGCM_debug_Verif_Exit
211	fi
212
213	# This boolean will trigger the filling of the stack
214	# Only now we know where things should be ...
215	# We don't fill the stack when we perform checking task
216	if [ ! X${TaskType} = Xchecking ] ; then
217	ActivateStackFilling=true
218	fi
219	fi
220
221	IGCM_debug_PopStack "IGCM_config_CommonConfiguration"
222	}
223
224	#===================================
225	function IGCM_config_Initialize
226	{
227	IGCM_debug_PushStack "IGCM_config_Initialize"
228
229	# Debug Print :
230	echo
231	IGCM_debug_Print 1 "IGCM_config_Initialize"
232
233	# Test modipsl tree existence.
234	IGCM_sys_TestDir ${MODIPSL}
235	[ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir"
236	IGCM_sys_TestDir ${libIGCM}
237	[ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir"
238	IGCM_sys_TestDir ${R_EXE}
239	[ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir"
240	IGCM_sys_TestDir ${SUBMIT_DIR}
241	[ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir"
242
243	if ( $DEBUG_debug ) ; then
244	echo "Keep trace of inital SUBMIT_DIR : "
245	ls -lta ${SUBMIT_DIR}
246	fi
247
248	#==================================
249	# Read ListOfComponents section:
250	echo
251	IGCM_debug_Print 1 "DefineArrayFromSection : ListOfComponents"
252
253	IGCM_card_DefineArrayFromSection ${SUBMIT_DIR}/config.card ListOfComponents
254	for comp in ${config_ListOfComponents[*]} ; do
255	IGCM_card_DefineArrayFromOption ${SUBMIT_DIR}/config.card ListOfComponents ${comp}
256	done
257	IGCM_debug_Print 3 ${config_ListOfComponents[*]}
258
259	#==================================
260	# Read Executable section:
261	IGCM_card_DefineArrayFromSection ${SUBMIT_DIR}/config.card Executable
262
263	#==================================
264	# Read Restarts section:
265	# Restarts : Gerneral rule or local for each component.
266	echo
267	IGCM_debug_Print 1 "DefineArrayFromOption : config_Restarts"
268
269	IGCM_card_DefineArrayFromSection ${SUBMIT_DIR}/config.card Restarts
270	for option in ${config_Restarts[*]} ; do
271	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/config.card Restarts ${option}
272	eval auxprint=\${config_Restarts_${option}}
273	IGCM_debug_Print 3 "${option} : ${auxprint}"
274	done
275
276	#==================================
277	# Define Job Outputs Name
278	echo
279	IGCM_debug_Print 2 "Define Script_Output_Prefix and Exe_Output"
280	Script_Output_Prefix=${config_UserChoices_Script_Output_Prefix:='Script_Output'}
281	IGCM_debug_Print 3 "Script_Output_Prefix = ${Script_Output_Prefix}"
282	Exe_Output=out_execution
283	IGCM_debug_Print 3 "Exe_Output = ${Exe_Output}"
284
285	#===================================================================#
286	# Prepare variables available for ${COMP}.card and ${COMP}.driver #
287	# But available to any son functions #
288	#===================================================================#
289
290	# Convert yyyy-mm-dd date to gregorian yyyymmdd
291	DateBegin=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )
292	DateEnd=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )
293
294	# Period Length In Days between DateBegin and DateEnd
295	(( ExperienceLengthInDays=$( IGCM_date_DaysBetweenGregorianDate ${DateEnd} ${DateBegin} ) + 1 ))
296	if [ ${ExperienceLengthInDays} -lt 0 ] ; then
297	IGCM_debug_Print 1 "Problem with dates in config.card : ${DateEnd} < ${DateBegin} ! You must check that."
298	IGCM_debug_Exit "IGCM_config_Initialize" " Wrong Dates."
299	IGCM_debug_Verif_Exit
300	fi
301
302	# Day and Year of Initial State (Given in julian format)
303	InitDay=$(( $( IGCM_date_ConvertGregorianDateToJulian $DateBegin ) % 1000 ))
304	InitYear=$(( $( IGCM_date_ConvertGregorianDateToJulian $DateBegin ) / 1000 ))
305	InitYear_m1=$(( $( IGCM_date_ConvertGregorianDateToJulian $DateBegin ) / 1000 - 1 ))
306
307	#================================================================#
308	# Test and Prepare directories #
309	#================================================================#
310
311	# ==> 4 kinds of input files :
312	# 1) R_INIT : Initial State Files (Etat0, carteveg)
313	# 2) R_BC : Boundary Conditions (Forcages, lai)
314	# 3) Parameters files (allready define through ${SUBMIT_DIR})
315	# 4) Restarts files (allready define in IGCM_config_Initialize)
316
317	# Here we offer the possibility to redefine R_INIT, R_BC
318	# and PeriodNb through config.card
319	R_INIT=${config_UserChoices_R_INIT:=${R_IN}/INIT}
320	echo
321	IGCM_debug_Print 2 "(Re)Define R_INIT, R_BC and PeriodNb"
322	IGCM_debug_Print 3 "R_IN=${R_IN}"
323	IGCM_debug_Print 3 "R_INIT=${R_INIT}"
324	R_BC=${config_UserChoices_R_BC:=${R_IN}/BC}
325	IGCM_debug_Print 3 "R_BC=${R_BC}"
326	PeriodNb=${config_UserChoices_PeriodNb:=${PeriodNb}}
327	IGCM_debug_Print 3 "Loop in main Job with ${PeriodNb} period(s)"
328
329	# SD ADA SPECIFIC #
330	# TO FIX #
331	#IGCM_sys_TestDirArchive ${R_IN}
332	#[ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDirArchive"
333
334	if ( ${FirstInitialize} ) ; then
335	IGCM_sys_MkdirArchive ${R_SAVE}
336	[ ! ${config_Post_PackFrequency} = NONE ] && IGCM_sys_Mkdir ${R_BUFR}
337	else
338	IGCM_sys_TestDirArchive ${R_SAVE}
339	[ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDirArchive ${R_SAVE}"
340
341	if [ ! ${config_Post_PackFrequency} = NONE ] ; then
342	IGCM_sys_TestDir ${R_BUFR}
343	[ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir ${R_BUFR}"
344	fi
345
346	# Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
347	IGCM_config_StateCheck
348
349	# And EXIT if not OK
350	IGCM_debug_Verif_Exit
351	fi
352
353	#====================================================
354	# Experience type : DEB(ug), DEV(elopment), RUN
355	if [ X${JobType} != XRUN ] ; then
356	echo
357	echo "===================================================="
358	echo "libIGCM JOB is NOT in RUN type mode."
359	echo "!! OUTPUT files will NOT be PROTECTED !!"
360	echo "Be carefull : you can ERASE the result of this job !"
361
362	case ${JobType} in
363	DEB)
364	echo "DEBUG mode : activation of 'set -vx' mode."
365	echo "DEBUG mode : no protection for output files."
366	echo "DEBUG mode : if active force asynchronous rebuild frequency to PeriodLength frequency."
367	;;
368	DEV)
369	echo "DEVelopment mode : no protection for output files."
370	echo "DEVelopment mode : if active force asynchronous rebuild frequency to PeriodLength frequency."
371	;;
372	esac
373
374	if ( [ X${config_Post_RebuildFrequency} != XNONE ] && [ ${DRYRUN} -eq 0 ] ) ; then
375	if [ X${config_Post_RebuildFrequency} != X${config_UserChoices_PeriodLength} ] ; then
376	echo "------------"
377	echo "WARNING : Job is NOT in RUN mode then we will force REBUILD Frequency"
378	echo " to PeriodLength : ${config_UserChoices_PeriodLength}"
379	echo "------------"
380	config_Post_RebuildFrequency=${config_UserChoices_PeriodLength}
381	fi
382	fi
383	echo "===================================================="
384	echo
385	fi
386
387	IGCM_debug_PopStack "IGCM_config_Initialize"
388	}
389
390	#===================================
391	function IGCM_config_DaysInPeriodLength
392	{
393	IGCM_debug_PushStack "IGCM_config_DaysInPeriodLength"
394
395	typeset i
396
397	# Determine number of day(s) in PeriodLength :
398	case ${config_UserChoices_PeriodLength} in
399	Y\|y)
400	PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} \| sed -e 's/[yY]//' )
401	echo
402	IGCM_debug_Print 2 "Number of years for PeriodLength : ${PeriodLengthInYears}"
403	PeriodLengthInDays=0
404	i=0
405	until [ $i -ge $PeriodLengthInYears ] ; do
406	(( PeriodLengthInDays = PeriodLengthInDays + $( IGCM_date_DaysInYear $(( year + i )) ) ))
407	(( i=i+1 ))
408	done
409	;;
410	M\|m)
411	PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} \| sed -e 's/[mM]//' )
412	echo
413	IGCM_debug_Print 2 "Number of months for PeriodLength : ${PeriodLengthInMonths}"
414	PeriodLengthInDays=0
415	i=0
416	until [ $i -ge $PeriodLengthInMonths ] ; do
417	if [ $(( 10#${month} + ${i} )) -lt 13 ] ; then
418	(( PeriodLengthInDays = PeriodLengthInDays + $( IGCM_date_DaysInMonth $year $(( 10#${month} + ${i} )) ) ))
419	else
420	(( PeriodLengthInDays = PeriodLengthInDays + $( IGCM_date_DaysInMonth $year $(( 10#${month} + ${i} - 12 )) ) ))
421	fi
422	(( i=i+1 ))
423	done
424	;;
425	D\|d)
426	PeriodLengthInMonths=0
427	PeriodLengthInDays=$( echo ${config_UserChoices_PeriodLength} \| sed -e 's/[dD]//' )
428	echo
429	IGCM_debug_Print 2 "Number of days for PeriodLength : ${PeriodLengthInDays}";;
430	*)
431	IGCM_debug_Exit "IGCM_config_DaysInPeriodLength " ${config_UserChoices_PeriodLength} " invalid period length : choose in Y, M, *D."
432	IGCM_debug_Verif_Exit ;;
433	esac
434
435	IGCM_debug_PopStack "IGCM_config_DaysInPeriodLength"
436	}
437
438	#===================================
439	function IGCM_config_DateCoherency
440	{
441	IGCM_debug_PushStack "IGCM_config_DateCoherency"
442
443	echo
444	IGCM_debug_Print 1 "IGCM_config_DateCoherency"
445	echo
446
447	typeset Length VerifiedPeriodDateBegin VerifiedPeriodDateEnd
448
449	# check coherency between (PeriodDateBegin, PeriodDateEnd) and (DateBegin, CumulPeriod, PeriodLength)
450	# DateBegin + CumulPeriod*PeriodLength = PeriodDateBegin
451	echo
452
453	case ${config_UserChoices_PeriodLength} in
454	Y\|y)
455	Length=$( IGCM_date_DaysInCurrentPeriod ${DateBegin} $(( ${CumulPeriod} * ${PeriodLengthInYears} ))Y )
456	;;
457	M\|m)
458	Length=$( IGCM_date_DaysInCurrentPeriod ${DateBegin} $(( ${CumulPeriod} * ${PeriodLengthInMonths} ))M )
459	;;
460	D\|d)
461	Length=$( IGCM_date_DaysInCurrentPeriod ${DateBegin} $(( ${CumulPeriod} * ${PeriodLengthInDays} ))D )
462	;;
463	esac
464	VerifiedPeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} ${Length}-1 )
465
466	if [ ${VerifiedPeriodDateEnd} != ${PeriodDateEnd} ] ; then
467	IGCM_debug_Print 1 "From run.card PeriodDateEnd is not consistent with DateBegin and CumulPeriod."
468	IGCM_debug_Print 1 "We have DateBegin = ${DateBegin}"
469	IGCM_debug_Print 1 "We have CumulPeriod = ${CumulPeriod}"
470	IGCM_debug_Print 1 "We have PeriodDateEnd = ${PeriodDateEnd}"
471	IGCM_debug_Print 1 "We have VerifiedPeriodDateEnd = ${VerifiedPeriodDateEnd}"
472	IGCM_debug_Print 1 "You must have change run.card in an inconsistent way."
473
474	IGCM_debug_Exit "STOP here to avoid further issues."
475	fi
476
477	# PeriodDateBegin + PeriodLength = PeriodDateEnd
478	VerifiedPeriodDateBegin=$( IGCM_date_AddDaysToGregorianDate ${VerifiedPeriodDateEnd} $(( ${PeriodLengthInDays} * -1 )) )
479
480	IGCM_debug_PopStack "IGCM_config_DateCoherency"
481	}
482
483	#===================================
484	function IGCM_config_StateCheck
485	{
486	IGCM_debug_PushStack "IGCM_config_StateCheck"
487
488	#Test state of run in run.card
489	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodState
490
491	if [ ${run_Configuration_PeriodState} = "Fatal" ] ; then
492	echo
493	IGCM_debug_Print 1 "!! Error in run.card with PeriodState : " ${run_Configuration_PeriodState} "!!"
494	IGCM_debug_Print 1 "Check the overall status of your simulation by visiting this page:"
495	IGCM_debug_Print 1 "https://hermes.ipsl.upmc.fr/static/simulation.detail.html?uid=${simuid}"
496	IGCM_debug_Print 1 "Then try running ${libIGCM}/clean_PeriodLenght.job to clean latest failed period"
497	IGCM_debug_Print 1 "Or try running ${libIGCM}/clean_latestPackperiod.job to clean lastest pack period."
498	IGCM_debug_Print 1 "Or use ${libIGCM}/purge_simulation.job to delete the full simulation."
499	IGCM_debug_Exit
500	elif [ $( echo ${run_Configuration_PeriodState} \| grep Fatal \| wc -l ) -eq 1 ] ; then
501	echo
502	IGCM_debug_Print 1 "!! Error in run.card with PeriodState : " ${run_Configuration_PeriodState} "!!"
503	IGCM_debug_Print 1 "Compute jobs has been stop because at least the above mentionned post-processing jobs fails."
504	IGCM_debug_Print 1 "Check post-processing jobs carefully by visiting this page:"
505	IGCM_debug_Print 1 "https://hermes.ipsl.upmc.fr/static/simulation.detail.html?uid=${simuid}"
506	IGCM_debug_Print 1 "Please visit that page to see how to fix issues:"
507	IGCM_debug_Print 1 "https://forge.ipsl.jussieu.fr/igcmg_doc/wiki/DocGmonitor"
508	IGCM_debug_Print 1 "Then try running ${libIGCM}/clean_PeriodLenght.job to clean latest failed period"
509	IGCM_debug_Print 1 "Or try running ${libIGCM}/clean_latestPackperiod.job to clean latest pack period."
510	IGCM_debug_Print 1 "Or use ${libIGCM}/purge_simulation.job to delete the full simulation."
511	IGCM_debug_Exit
512	fi
513
514	IGCM_debug_PopStack "IGCM_config_StateCheck"
515	}
516
517	#===================================
518	function IGCM_config_Check
519	{
520	IGCM_debug_PushStack "IGCM_config_Check"
521
522	# If one of the following modulo is not zero :
523	# we will issue an error then explain and exit in
524	# AA_job IGCM_debug_Verif_Exit call before binary submission
525
526	echo
527	IGCM_debug_Print 1 "IGCM_config_Check"
528	echo
529
530	typeset i
531
532	# Check RebuildFrequency against key frequencies : PeriodLength ; PackFrequency ; TimeSeriesFrequency ; SeasonalFrequency
533	if ( [ ! X${config_Post_RebuildFrequency} = X${NULL_STR} ] && [ ! X${config_Post_RebuildFrequency} = XNONE ] ) ; then
534	AsynchronousRebuild=true
535	IGCM_debug_Print 1 "Asynchronous rebuild has been activated."
536	echo
537	# modulo (RebuildFrequency and PeriodLength/TimeSeriesFrequency/SeasonalFrequency) must be zero
538	IGCM_debug_Print 1 "Check coherence between RebuildFrequency and PeriodLength"
539	IGCM_post_CheckModuloFrequency config_Post_RebuildFrequency config_UserChoices_PeriodLength
540	IGCM_debug_Print 1 "Check coherence between PackFrequency and RebuildFrequency"
541	IGCM_post_CheckModuloFrequency config_Post_PackFrequency config_Post_RebuildFrequency
542	IGCM_debug_Print 1 "Check coherence between TimeSeriesFrequency and RebuildFrequency"
543	IGCM_post_CheckModuloFrequency config_Post_TimeSeriesFrequency config_Post_RebuildFrequency
544	IGCM_debug_Print 1 "Check coherence between SeasonalFrequency and RebuildFrequency"
545	IGCM_post_CheckModuloFrequency config_Post_SeasonalFrequency config_Post_RebuildFrequency
546	else
547	AsynchronousRebuild=false
548	IGCM_debug_Print 1 "Asynchronous rebuild has not been activated"
549	IGCM_debug_Print 1 "Proceed with standard post-treatment pathway"
550	echo
551	#modulo (PeriodLength and TimeSeriesFrequency/SeasonalFrequency) must be zero
552	IGCM_debug_Print 1 "Check coherence between TimeSeriesFrequency and PeriodLength"
553	IGCM_post_CheckModuloFrequency config_Post_TimeSeriesFrequency config_UserChoices_PeriodLength
554	IGCM_debug_Print 1 "Check coherence between SeasonalFrequency and PeriodLength"
555	IGCM_post_CheckModuloFrequency config_Post_SeasonalFrequency config_UserChoices_PeriodLength
556	fi
557
558	# Check PackFrequency against other key frequencies
559	# Modulo (PackFrequency and TimeSeriesFrequency/SeasonalFrequency and PeriodLenght) must be zero
560	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
561	Pack=true
562	#
563	IGCM_debug_Print 1 "Check coherence between PackFrequency and PeriodLength"
564	IGCM_post_CheckModuloFrequency config_Post_PackFrequency config_UserChoices_PeriodLength
565	IGCM_debug_Print 1 "Check coherence between TimeSeriesFrequency and PackFrequency"
566	IGCM_post_CheckModuloFrequency config_Post_TimeSeriesFrequency config_Post_PackFrequency
567	IGCM_debug_Print 1 "Check coherence between SeasonalFrequency and PackFrequency"
568	IGCM_post_CheckModuloFrequency config_Post_SeasonalFrequency config_Post_PackFrequency
569	else
570	Pack=false
571	fi
572
573	# modulo (TimeSeriesFrequency and all Chunck2D) must be zero
574	NbJob=${#CHUNCK2D_SIZE[@]}
575	i=0
576	until [ $i -ge $NbJob ]; do
577	value=${CHUNCK2D_SIZE[${i}]}
578	IGCM_debug_Print 1 "Check coherence between ${CHUNCK2D_NAME[${i}]} Chunck2D frequency and TimeSeriesFrequency"
579	IGCM_post_CheckModuloFrequency value config_Post_TimeSeriesFrequency
580	case ${value} in
581	Y\|y) ;;
582	*)
583	IGCM_debug_Print 1 "All ChunckJob2D frequency must be expressed in year Y\|y in comp.card"
584	IGCM_debug_Exit "This will stop the job" ;;
585	esac
586	(( i=i+1 ))
587	done
588
589	# modulo (TimeSeriesFrequency and all Chunck3D) must be zero
590	NbJob=${#CHUNCK3D_SIZE[@]}
591	i=0
592	until [ $i -ge $NbJob ]; do
593	value=${CHUNCK3D_SIZE[${i}]}
594	IGCM_debug_Print 1 "Check coherence between ${CHUNCK3D_NAME[${i}]} Chunck3D frequency and TimeSeriesFrequency"
595	IGCM_post_CheckModuloFrequency value config_Post_TimeSeriesFrequency
596	case ${value} in
597	Y\|y) ;;
598	*)
599	IGCM_debug_Print 1 "All ChunckJob3D frequency must be expressed in year Y\|y in comp.card"
600	IGCM_debug_Exit "This will stop the job" ;;
601	esac
602	(( i=i+1 ))
603	done
604
605	# check to be sure there is enough space on temporary filesystems to run
606	echo
607	IGCM_debug_Print 1 "Check if there is enough space on temporary filesystem"
608	IGCM_sys_check_quota
609
610	# check to be sure that RUN_DIR_PATH, that will be removed is not pointing to an important directory
611	echo
612	IGCM_debug_Print 1 "Check where RUN_DIR_PATH variable is pointing to"
613	IGCM_sys_check_path
614
615
616	IGCM_debug_PopStack "IGCM_config_Check"
617	}
618
619	#===================================
620	function IGCM_config_ConfigureExecution
621	{
622	IGCM_debug_PushStack " IGCM_config_ConfigureExecution"
623
624	#echo
625	IGCM_debug_Print 1 " IGCM_config_ConfigureExecution"
626	#echo
627
628	typeset ExeNameIn ExeNameFirst CompNameFirst configCardPath comp i
629	typeset tempvar tempvarMPI tempvarNOD NbElts NbExec
630
631	# config.card path
632	configCardPath=$1
633
634	coreNumber=0
635	mpiTasks=0
636	openMPthreads=0
637	NbExec=0
638
639	OK_PARA_MPI=false
640	OK_PARA_OMP=false
641	OK_PARA_NOD=false
642	OK_PARA_MPMD=false
643
644	for comp in ${config_ListOfComponents[*]} ; do
645
646	# Manage component executable
647	IGCM_card_DefineArrayFromOption ${configCardPath} Executable ${comp}
648
649	eval ExeNameIn=\${config_Executable_${comp}[0]}
650
651	# NO order in config.card for parallelized values !
652	# just use suffix : MPI , OMP and NOD (for number of NODes.)
653
654	# NOD is the number of NODes allocated
655	eval ${comp}_PROC_NOD=0
656
657	# MPI is the number of MPI processus per nodes
658	eval ${comp}_PROC_MPI=0
659
660	# OMP is the number of OpenMP threads per MPI processus
661	eval ${comp}_PROC_OMP=0
662
663	# Only if we really have an executable for the component :
664	if ( [ "X${ExeNameIn}" != X\"\" ] && [ "X${ExeNameIn}" != "Xinca.dat" ] ) ; then
665
666	IGCM_debug_Print 1 ${comp}
667
668	# Keep the first executable found and the first CompName
669	eval ExeNameFirst=${ExeNameIn}
670	CompNameFirst=${comp}
671
672	# Are we a second executable?
673	(( NbExec = NbExec + 1 ))
674
675	# set 1 MPI task, 1 OpenMP thread and 1 node as default
676	eval ${comp}_PROC_MPI=1
677	eval ${comp}_PROC_OMP=1
678	eval ${comp}_PROC_NOD=1
679
680	eval NbElts=\${#config_Executable_${comp}[@]}
681
682	if [ ${NbElts} -gt 2 ] ; then
683	#
684	# CURRENT METHOD TO SPECIFY MPI AND OMP RESSOURCES
685	#
686	i=2
687	while [ ${i} -lt ${NbElts} ] ; do
688	eval tempvar=\${config_Executable_${comp}[${i}]}
689	IGCM_debug_Print 2 ${tempvar}
690
691	if [ X${tempvar} = X ] ; then
692	IGCM_debug_Print 2 "Error reading MPI/OMP parameters !!!"
693	IGCM_debug_Exit "Check your config.card. Exit now"
694	IGCM_debug_Verif_Exit
695	fi
696
697	case ${tempvar} in
698	[mM][pP][iI])
699	# Read MPI parameter for composante
700	eval ${comp}_PROC_MPI=$( echo ${tempvar} \| tr '[a-z]' '[A-Z]' \| sed -e "s/MPI//" )
701	OK_PARA_MPI=true;;
702	[oO][mM][pP])
703	# Read OMP parameter for composante
704	eval ${comp}_PROC_OMP=$( echo ${tempvar} \| tr '[a-z]' '[A-Z]' \| sed -e "s/OMP//" )
705	;;
706	[nN][oO][dD])
707	# Read NOD (NumBer of Nodes) parameter for composante
708	eval ${comp}_PROC_NOD=$( echo ${tempvar} \| tr '[a-z]' '[A-Z]' \| sed -e "s/NOD//" )
709	OK_PARA_NOD=true
710	OK_PARA_MPI=true
711	;;
712	esac
713	(( i = i + 1 ))
714	done
715	else
716	#
717	# BACKWARD COMPATIBILITY NOT SUPPORTED ANYMORE
718	#
719	IGCM_debug_Exit "You are using a deprecated ressources specification mechanism in your config.card"
720	IGCM_debug_Exit "Please check : https://forge.ipsl.jussieu.fr/igcmg_doc/wiki/DocEsetup#ThesectionExecutable"
721	IGCM_debug_Exit "Please modify ${configCardPath}"
722	exit
723	fi
724	if [ ${comp}_PROC_NOD -gt 1 ] ; then
725	if ( [ ${comp}_PROC_MPI -gt 1 ] && [ ${comp}_PROC_OMP -gt 1 ] ) ; then
726	IGCM_debug_Print 2 "Error using MPI/OMP/NOD parameters !!!"
727	IGCM_debug_Exit "It is not allowed to use MPI and OMP and NOD parameters for the same component"
728	IGCM_debug_Exit "Check your config.card. Exit now"
729	IGCM_debug_Verif_Exit
730	fi
731	if [ "${comp}" != "IOS" ] ; then
732	IGCM_debug_Print 2 "Error using NOD parameter !!!"
733	IGCM_debug_Exit "NOD parameter is only available for IOS component "
734	IGCM_debug_Exit "Check your config.card. Exit now"
735	IGCM_debug_Verif_Exit
736	fi
737	fi
738
739	eval tempvarMPI=\${${comp}_PROC_MPI}
740	eval tempvarNOD=\${${comp}_PROC_NOD}
741	eval tempvarOMP=\${${comp}_PROC_OMP}
742
743	# set OMP mode if more than 1 OMP thread.
744	[ ${tempvarOMP} -ge 2 ] && OK_PARA_OMP=true
745
746	# Number of OMP threads
747	[ ${openMPthreads} -lt ${tempvarOMP} ] && openMPthreads=${tempvarOMP}
748
749	# SUM UP NUMBER OF CORES
750	if [ ${comp}_PROC_NOD -gt 1 ] ; then
751	(( coreNumber = coreNumber + tempvarNOD * NB_CORE_PER_NODE ))
752	else
753	(( coreNumber = coreNumber + tempvarMPI * tempvarNOD * tempvarOMP ))
754	fi
755	# SUM UP NUMBER OF MPI TASKS
756	(( mpiTasks = mpiTasks + tempvarMPI * tempvarNOD ))
757	fi
758	done
759
760	# MANDATORY FOR THE OPA9.DRIVER. USED TO EDIT OPA NAMELIST
761	# WE SHOULD PLANIFY NUM_PROC_??? DEPRECATION
762	NUM_PROC_CPL=${CPL_PROC_MPI}
763	NUM_PROC_OCE=${OCE_PROC_MPI}
764	NUM_PROC_ATM=${ATM_PROC_MPI}
765
766	# set MPMD mode if more than 2 executable names.
767	[ ${NbExec} -ge 2 ] && OK_PARA_MPMD=true
768
769	# Define the execution type we are running in
770	if ( ${OK_PARA_MPMD} ) ; then
771	if ( ${OK_PARA_MPI} ) ; then
772	# MPMD always implies MPI
773	executionType=1
774	fi
775	if ( ${OK_PARA_OMP} ) ; then
776	# MPMD + MPI/OMP
777	executionType=2
778	fi
779	else
780	if ( ( ${OK_PARA_MPI} ) && ( ${OK_PARA_OMP} ) ) ; then
781	# SPMD + MPI/OMP
782	executionType=3
783	elif ( ( ${OK_PARA_MPI} ) && ( ! ${OK_PARA_OMP} ) ) ; then
784	# SPMD + MPI only
785	executionType=4
786	elif ( ( ! ${OK_PARA_MPI} ) && ( ${OK_PARA_OMP} ) ) ; then
787	# SPMD + OMP only
788	executionType=5
789	elif ( ( ! ${OK_PARA_MPI} ) && ( ! ${OK_PARA_OMP} ) ) ; then
790	# SEQUENTIAL THEN
791	executionType=6
792	coreNumber=1
793	fi
794	fi
795
796	IGCM_debug_Print 1 "MPI/OMP treatment coreNumber = ${coreNumber}"
797	IGCM_debug_Print 1 "MPI/OMP treatment mpiTasks = ${mpiTasks}"
798	IGCM_debug_Print 1 "MPI/OMP treatment openMPthreads = ${openMPthreads}"
799	IGCM_debug_Print 1 "MPI/OMP treatment executionType = ${executionType}"
800
801	IGCM_debug_PopStack "IGCM_config_ConfigureExecution"
802	}
803
804	#===================================
805	function IGCM_config_PeriodStart
806	{
807	IGCM_debug_PushStack "IGCM_config_PeriodStart"
808
809	echo
810	IGCM_debug_Print 1 "IGCM_config_PeriodStart"
811	echo
812
813	if ( ${FirstInitialize} ) ; then
814	#================================================#
815	# Initialize date/period information #
816	#================================================#
817
818	IGCM_date_GetYearMonthDay ${DateBegin} year month day
819	IGCM_config_DaysInPeriodLength
820
821	PeriodDateBegin=${DateBegin}
822	PeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( ${PeriodLengthInDays} - 1 )) )
823	CumulPeriod=1
824
825	#=================================================#
826	# Write updated run.card #
827	#=================================================#
828
829	#Correct run.card Configuration for this period
830	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodDateBegin ${PeriodDateBegin}
831	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodDateEnd ${PeriodDateEnd}
832	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration CumulPeriod ${CumulPeriod}
833	if [ X$( grep "SubmitPath" ${SUBMIT_DIR}/run.card ) != X ] ; then
834	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration SubmitPath ${SUBMIT_DIR}
835	fi
836
837	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Running"
838
839	else
840	#================================================#
841	# The file run.card allready exist #
842	#================================================#
843
844	# Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
845	IGCM_config_StateCheck
846	# And EXIT if not OK
847	IGCM_debug_Verif_Exit
848
849	#===================================#
850	# Read updated run.card #
851	#===================================#
852
853	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodDateBegin
854	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodDateEnd
855	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration CumulPeriod
856
857	PeriodDateBegin=$( IGCM_date_ConvertFormatToGregorian ${run_Configuration_PeriodDateBegin} )
858	PeriodDateEnd=$( IGCM_date_ConvertFormatToGregorian ${run_Configuration_PeriodDateEnd} )
859	CumulPeriod=${run_Configuration_CumulPeriod}
860
861	LastPeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate $( IGCM_date_ConvertFormatToGregorian ${PeriodDateBegin} ) -1 )
862
863	if [ ${Period} = 1 ]; then
864	# save last Job output and current run.card
865	typeset Potential
866	IGCM_sys_Cd ${SUBMIT_DIR}
867	#
868	IGCM_debug_Print 2 "Save previous ksh job output"
869	for Potential in $( ls ${Script_Output_Prefix}_${config_UserChoices_JobName}.[0-9][0-9][0-9][0-9][0-9][0-9] ) ; do
870	if [ X${Pack} = Xtrue ] ; then
871	( IGCM_sys_TestFileBuffer ${R_BUF_KSH}/${Potential} ) \|\| IGCM_sys_Cp ${Potential} ${R_BUF_KSH}/${Potential}.$$
872	else
873	( IGCM_sys_TestFileArchive ${R_OUT_KSH}/${Potential} ) \|\| IGCM_sys_Put_Out ${Potential} ${R_OUT_KSH}/${Potential}.$$ NOMOVE
874	IGCM_debug_Print 1 "Save of previous ksh job output is not done for Pack=false"
875	fi
876	done
877
878	#
879	IGCM_debug_Print 2 "Save current run.card"
880	IGCM_card_CheckConflict run.card
881
882	if [ X${Pack} = Xtrue ] ; then
883	IGCM_sys_Cp ${SUBMIT_DIR}/run.card ${R_BUF_KSH}/run.card
884	else
885	IGCM_sys_Put_Out ${SUBMIT_DIR}/run.card ${R_OUT_KSH}/run.card NOMOVE
886	IGCM_debug_Print 1 "Save of run.card is not done for Pack=false"
887	fi
888	#
889	IGCM_sys_Cd ${RUN_DIR}
890	else
891	unset FileToBeDeleted
892	fi
893
894	# Determine number of day(s) in PeriodLength
895	IGCM_date_GetYearMonthDay $PeriodDateBegin year month day
896	IGCM_config_DaysInPeriodLength
897
898	# Check coherency between (PeriodDateBegin, PeriodDateEnd) and (DateBegin, CumulPeriod, PeriodLength)
899	IGCM_config_DateCoherency
900	# And EXIT if not OK
901	IGCM_debug_Verif_Exit
902
903	# Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
904	IGCM_config_StateCheck
905	# And EXIT if not OK
906	IGCM_debug_Verif_Exit
907
908	# We can say we are "Running" now.
909	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Running"
910	fi
911
912	# BEGIN: SHOULD GO IN A FUNCTION FROM libIGCM_date.ksh
913	# Compute year_m1 and year_p1 (year minus 1Y and year plus 1Y)
914	year_m1=$(( year - 1 ))
915	year_p1=$(( year + 1 ))
916	# Compute month_m1 (month minus 1M)
917	# Compute yyyymm_m1 (yyyymm minus 1M)
918	month_m1=$(( 10#${month} - 1 ))
919	if [ ${month_m1} = 0 ]; then
920	month_m1=12
921	yyyymm_m1=${year_m1}12
922	elif [ ${month_m1} -le 9 ]; then
923	month_m1=0${month_m1}
924	yyyymm_m1=${year}${month_m1}
925	else
926	yyyymm_m1=${year}${month_m1}
927	fi
928	# Compute month_p1 (month plus 1M)
929	# Compute yyyymm_p1 (yyyymm plus 1M)
930	month_p1=$(( 10#${month} + 1 ))
931	if [ ${month_p1} = 13 ]; then
932	month_p1=01
933	yyyymm_p1=${year_p1}01
934	elif [ ${month_p1} -le 9 ]; then
935	month_p1=0${month_p1}
936	yyyymm_p1=${year}${month_p1}
937	else
938	yyyymm_p1=${year}${month_p1}
939	fi
940
941	#===================================================================#
942	# Calculate CyclicYear to be used for looping over a given forcing #
943	# period. Add CyclicBegin and CyclicEnd in config.card UserChoices. #
944	#===================================================================#
945
946	# To use the variable CyclicYear, one must add in config.card CyclicBegin and CyclicEnd.
947	# CyclicBegin is the first year in the cycle. CyclicEnd is the last year included in the cycle.
948	if ( [ ! X${config_UserChoices_CyclicBegin} = X ] && [ ! X${config_UserChoices_CyclicEnd} = X ] ) ; then
949	CycleNb=$(( ${config_UserChoices_CyclicEnd} - ${config_UserChoices_CyclicBegin} + 1 ))
950	CyclicYear_p1=NOTDEFINED
951
952	# For current year
953	yeartmp=$year
954	diffy=$(( $yeartmp - ${config_UserChoices_CyclicBegin} ))
955	while [ $diffy -lt 0 ] ; do
956	yeartmp=$(( ${yeartmp} + ${CycleNb} ))
957	diffy=$(( $yeartmp - ${config_UserChoices_CyclicBegin} ))
958	done
959	CyclicYear=$(( ( ${diffy} % ${CycleNb} ) + ${config_UserChoices_CyclicBegin} ))
960
961	# For next coming year
962	yeartmp=$(( $year + 1 ))
963	diffy=$(( $yeartmp - ${config_UserChoices_CyclicBegin} ))
964	while [ $diffy -lt 0 ] ; do
965	yeartmp=$(( ${yeartmp} + ${CycleNb} ))
966	diffy=$(( $yeartmp - ${config_UserChoices_CyclicBegin} ))
967	done
968	CyclicYear_p1=$(( ( ${diffy} % ${CycleNb} ) + ${config_UserChoices_CyclicBegin} ))
969
970	IGCM_debug_Print 1 "CyclicYear = ${CyclicYear}, CyclicYear_p1 = ${CyclicYear_p1}, current year=$year"
971	else
972	CyclicYear="ERROR_CyclicYear_Variable_Not_Defined"
973	CyclicYear_p1="ERROR_CyclicYear_p1_Variable_Not_Defined"
974	IGCM_debug_Print 1 "CyclicYear wont be use without adding CyclicBegin and CyclicEnd in config.card"
975	fi
976
977	# END: SHOULD GO IN A FUNCTION FROM libIGCM_date.ksh
978
979	#===================================================================#
980	# Prepare variables available for ${COMP}.card and ${COMP}.driver #
981	# But available to any son functions #
982	#===================================================================#
983
984	# Period Length In Days between DateBegin and DateCurrent (at end of period == PeriodDateEnd !)
985	(( SimulationLengthInDays = $( IGCM_date_DaysBetweenGregorianDate ${PeriodDateEnd} ${DateBegin} ) + 1 ))
986
987	# Debug Print :
988	echo
989	IGCM_debug_Print 1 "IGCM_config_PeriodStart : Before Execution"
990	IGCM_debug_Print 1 "Year of simulation : ${year}"
991	IGCM_debug_Print 1 "Month of simulation : ${month}"
992	IGCM_debug_Print 1 "PeriodLengthInDays : ${PeriodLengthInDays}"
993	IGCM_debug_Print 1 "PeriodDateBegin : ${PeriodDateBegin}"
994	IGCM_debug_Print 1 "PeriodDateEnd : ${PeriodDateEnd}"
995	IGCM_debug_Print 1 "SimulationLengthInDays : ${SimulationLengthInDays}"
996	IGCM_debug_Print 1 "ExperienceLengthInDays : ${ExperienceLengthInDays}"
997
998	#================================================================#
999	# Prepare variables available for comp_finalyze #
1000	#================================================================#
1001
1002	# Period for save files
1003	DatesPeriod=${PeriodDateBegin}_${PeriodDateEnd}
1004
1005	# Prefix for save files of this period
1006	PREFIX=${config_UserChoices_JobName}_${DatesPeriod}
1007
1008	# List of files that will be deleted in RUN_DIR after run
1009	[ -f stack ] && FileToBeDeleted[0]="stack"
1010
1011	# Test if the same run as already been saved :
1012	if [ X${JobType} = XRUN ] ; then
1013	if [ ${DRYRUN} -le 0 ] ; then
1014	if ( IGCM_sys_TestFileBuffer ${R_BUF_KSH}/${PREFIX}_${Exe_Output} ) ; then
1015	IGCM_debug_Exit "IGCM_config_PeriodStart" "You are currently re-running an old job."
1016	IGCM_debug_Print 1 "Because of readonly permissions, you can't re-run a job when saved files"
1017	IGCM_debug_Print 1 " are still in the ARCHIVE directory. You must deleted those files, or "
1018	IGCM_debug_Print 1 " the whole ${R_SAVE} tree. See different clean_*.job in ${libIGCM} directory."
1019	IGCM_debug_Print 1 " This exit has been initiated because at least ${R_BUF_KSH}/${PREFIX}_${Exe_Output} exists."
1020	IGCM_debug_Verif_Exit
1021	fi
1022	fi
1023	else
1024	if ( IGCM_sys_TestFileBuffer ${R_BUF_KSH}/${PREFIX}_${Exe_Output} ) ; then
1025	IGCM_debug_Print 1 "IGCM_config_PeriodStart" "RErun an old job. Allowed in DEBUG or DEV mode."
1026	fi
1027	fi
1028
1029	#================================================================#
1030	# Prepare variables available for binary execution #
1031	# Call function for creation of run script #
1032	# Only done once per job #
1033	#================================================================#
1034
1035	if [ ${Period} -eq 1 ]; then
1036	# Define the execution context (MPMD, SPMD, MPI/OMP ...)
1037	IGCM_config_ConfigureExecution ${SUBMIT_DIR}/config.card
1038	# Create the execution script for the current context
1039	IGCM_sys_build_execution_scripts
1040	fi
1041
1042	ExecutionFail=false
1043
1044	# Update the rabbitMQ queue
1045	IGCM_debug_BigBro_Update
1046
1047	IGCM_debug_PopStack "IGCM_config_PeriodStart"
1048	}
1049
1050	#===================================
1051	function IGCM_config_SaveSourceModifications
1052	{
1053	IGCM_debug_PushStack "IGCM_config_SaveSourceModifications"
1054
1055	typeset ExeOutDateMax listVarEnv
1056	ExeOutDateMax=$1
1057
1058	listVarEnv="ExeOutDateMax,R_OUT_EXE,PREFIX,SUBMIT_DIR"
1059	IGCM_sys_RshMaster "\
1060	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh; \
1061	export ExeOutDateMax=${ExeOutDateMax};\
1062	export R_OUT_EXE=${R_OUT_EXE};\
1063	export PREFIX=${PREFIX};\
1064	export SUBMIT_DIR=${SUBMIT_DIR};\
1065	export listVarEnv=${listVarEnv};\
1066	Script_Output=out_SaveSourceModifications;\
1067	IGCM_sys_Qsub ${libIGCM}/SaveSourceModifications.job ${ExeOutDateMax} ${R_OUT_EXE} ${PREFIX} ${SUBMIT_DIR}"
1068
1069	IGCM_debug_PopStack "IGCM_config_SaveSourceModifications"
1070	}
1071
1072	#===================================
1073	function IGCM_config_PeriodEnd
1074	{
1075	IGCM_debug_PushStack "IGCM_config_PeriodEnd"
1076
1077	echo
1078	IGCM_debug_Print 1 "IGCM_config_PeriodEnd"
1079	echo
1080
1081	if [ ${DRYRUN} -le 1 ] ; then
1082
1083	IGCM_debug_Print 1 "Check components binary : size and creation date"
1084
1085	typeset LS_comp LS_bin ExeDate ExeCpuLog NextExeSize LastCompExeSize
1086	typeset comp i
1087	typeset ExeNameIn ExeNameOut UpdateExe ExeSecDateMax
1088
1089	#==================================#
1090	# Get last Exe Size #
1091	#==================================#
1092
1093	(( i=0 ))
1094	if ( ${FirstInitialize} ) ; then
1095	run_Log_LastExeSize=""
1096	for comp in ${config_ListOfComponents[*]} ; do
1097	run_Log_LastExeSize[$i]=0
1098	(( i=i+1 ))
1099	done
1100	else
1101	IGCM_card_DefineArrayFromOption ${SUBMIT_DIR}/run.card Log LastExeSize
1102	fi
1103	#==================================#
1104	# And Build ExeDate #
1105	#==================================#
1106
1107	# ExeDate = ATM_Jun_12_09:34-SRF_Jun_12_09:34-OCE_Jun_12_09:34-ICE_Jun_12_09:34-CPL_Jun_12_09:33
1108	# Would be nice to have next line but no way to format ls output (need to ls -l --time-style "+%Y-%m-%dT%H:%M")
1109	# ExeDate = ATM_2009-06-12T09:34+SRF_2009-06-12T09:34+OCE_2009-06-12T09:34+ICE_2009-06-12T09:34+CPL_2009-06-12T09:34
1110	ExeDate=""
1111	NextExeSize="( "
1112	(( i=0 ))
1113	UpdateExe=false
1114	(( ExeSecDateMax = 0 ))
1115	for comp in ${config_ListOfComponents[*]} ; do
1116
1117	IGCM_debug_Print 3 ${comp}
1118
1119	eval ExeNameIn=\${config_Executable_${comp}[0]}
1120	eval ExeNameOut=\${config_Executable_${comp}[1]}
1121	# Only if we really have an executable for the component :
1122	if [ X${ExeNameIn} = X\"\" ] ; then
1123	# If there is no exe file for this component
1124	(( ExeSize=0 ))
1125	else
1126	eval ExeNameIn=${ExeNameIn}
1127	LS_bin=${R_EXE}/${ExeNameIn}
1128	IGCM_sys_FileSize ${LS_bin} ExeSize
1129
1130	set +A LS_comp -- $( LC_TIME=en_US ls -l ${LS_bin} )
1131	if [ X${ExeDate} = X ] ; then
1132	# First component exe date
1133	ExeDate=${comp}_${LS_comp[5]}_${LS_comp[6]}
1134	else
1135	ExeDate=${ExeDate}-${comp}_${LS_comp[5]}_${LS_comp[6]}
1136	fi
1137	ExeDate=${ExeDate}_${LS_comp[7]}
1138	fi
1139
1140	if [ ${i} -eq 0 ] ; then
1141	# First component
1142	NextExeSize="( "${ExeSize}
1143	else
1144	NextExeSize=${NextExeSize}", "${ExeSize}
1145	fi
1146	LastCompExeSize=${run_Log_LastExeSize[$i]}
1147	(( i=i+1 ))
1148
1149	if [ ${ExeSize} -ne ${LastCompExeSize} ] ; then
1150	if ( ${FirstInitialize} ) ; then
1151	IGCM_debug_Print 1 "Save first ${ExeNameIn} in ${R_OUT_EXE} !"
1152	else
1153	IGCM_debug_Print 1 "${ExeNameIn} has changed in ${R_EXE} !"
1154	IGCM_debug_Print 1 "Save latest ${ExeNameIn} in ${R_OUT_EXE} !"
1155	FileToBeDeleted[${#FileToBeDeleted[@]}]=${ExeNameOut}
1156	fi
1157	IGCM_sys_Put_Out ${ExeNameOut} ${R_OUT_EXE}/${PREFIX}_${ExeNameIn} rw
1158	UpdateExe=true
1159
1160	# SD : switch off for now
1161	#IGCM_sys_GetDate_FichWork ${LS_bin} ExeSecDate
1162	#if [ $ExeSecDateMax -lt $ExeSecDate ] ; then
1163	# ExeSecDateMax=$ExeSecDate
1164	#fi
1165	fi
1166	done
1167
1168	# SD : switch off for now
1169	#if ( ${UpdateExe} ) ; then
1170	# echo "Launch SaveSourceModifications."
1171	# IGCM_config_SaveSourceModifications ${ExeSecDateMax}
1172	#fi
1173
1174	NextExeSize=${NextExeSize}" )"
1175	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Log LastExeSize "${NextExeSize}"
1176
1177	if [ ${DRYRUN} -le 1 ] ; then
1178	tail -1500 ${Exe_Output} > ${Exe_Output}_tail.txt
1179	ExeCpuLog=$( gawk -f ${libIGCM}/libIGCM_sys/IGCM_add_out.awk ${Exe_Output}_tail.txt )
1180	RET=$?
1181	if [ $RET -eq 0 ] ; then
1182	# ExeCpuLog variable contents 5 fields
1183	echo "${CumulPeriod} ${PeriodDateBegin} ${PeriodDateEnd} ${ExeCpuLog} ${ExeDate}" \| \
1184	gawk '{printf("# %11d \| %15s \| %15s \| %19s \| %19s \| %15.5f \| %15.5f \| %15.5f \| %s\n", \
1185	$1,$2,$3,$4,$5,$6,$7,$8,$9)}' >> ${SUBMIT_DIR}/run.card
1186	fi
1187	FileToBeDeleted[${#FileToBeDeleted[@]}]=${Exe_Output}_tail.txt
1188	fi
1189
1190	fi
1191
1192	#==================================#
1193	# Save Job output #
1194	#==================================#
1195	if [ X${Pack} = Xtrue ] ; then
1196	IGCM_sys_PutBuffer_Out ${Exe_Output} ${R_BUF_KSH}/${PREFIX}_${Exe_Output}
1197	else
1198	IGCM_sys_Put_Out ${Exe_Output} ${R_OUT_KSH}/${PREFIX}_${Exe_Output}
1199	fi
1200	FileToBeDeleted[${#FileToBeDeleted[@]}]=${Exe_Output}
1201
1202	# All was right ? no ? then we stop.
1203	IGCM_debug_Verif_Exit
1204
1205	# If all was OK, we can delete all files not necessary for next Job
1206	echo
1207	IGCM_debug_Print 1 "Files that will be deleted before next period-run : "
1208
1209	if [ ${DRYRUN} -le 2 ] ; then
1210	for f in ${FileToBeDeleted[@]} ; do [ -f ${f} ] && ls -la $f ; [ -f ${f} ] && rm -f $f ; done
1211	else
1212	echo ${FileToBeDeleted[@]}
1213	fi
1214
1215	# Send some accounting element to the user if CumulPeriod=3
1216	if [ ${CumulPeriod} -eq 3 ] ; then
1217	echo
1218	IGCM_debug_Print 1 "Send email containing some accounting information : "
1219
1220	RealCpuTime=$( echo ${ExeCpuLog} \| gawk '{print $3}' )
1221
1222	consumeHoursPerPeriod=$( echo "scale=6;${RealCpuTime}*${coreNumber}/3600" \| bc )
1223
1224	consumeHoursPerWholeSimulation=$( echo "scale=6;${consumeHoursPerPeriod}/${PeriodLengthInDays}*${ExperienceLengthInDays}" \| bc )
1225
1226	recommendedPeriodNb=$( echo "scale=6;${jobWarningDelay}/3600/${consumeHoursPerPeriod}*${coreNumber}" \| bc )
1227
1228	IGCM_sys_SendMail Accounting
1229	fi
1230
1231	#=================================================#
1232	# Modification of libIGCM behaviour #
1233	#=================================================#
1234
1235	# To use this function, one must copy libIGCM.card from ${libIGCM} directory
1236	# and put it in ${SUBMIT_DIR} directory. After modifications of ${SUBMIT_DIR}/libIGCM.card,
1237	# variables define inside [UserChanges] will be modified for next Period of libIGCM main loop.
1238	if [ -f ${SUBMIT_DIR}/libIGCM.card ] ; then
1239	echo
1240	echo "########################################################################"
1241	echo "!!! Modification of libIGCM behaviour !!!"
1242	echo
1243
1244	IGCM_debug_Print 1 "DefineArrayFromOption : libIGCM_UserChanges in libIGCM.card"
1245	IGCM_card_DefineArrayFromSection ${SUBMIT_DIR}/libIGCM.card UserChanges
1246	IGCM_debug_Print 2 "libIGCM_UserChanges" ${libIGCM_UserChanges[*]}
1247
1248	# Special treatments for libIGCM internals
1249	for option in ${libIGCM_UserChanges[*]} ; do
1250	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/libIGCM.card UserChanges ${option}
1251
1252	echo "We will change : ${option}."
1253	eval echo "Previous value : " \${${option}}
1254	eval echo "Change to : " \${libIGCM_UserChanges_${option}}
1255
1256	eval ${option}=\${libIGCM_UserChanges_${option}}
1257
1258	case ${option} in
1259	config_UserChoices_DateEnd)
1260	IGCM_debug_PrintVariables 1 config_UserChoices_DateEnd
1261	DateEnd=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )
1262
1263	# Period Length In Days between DateBegin and DateEnd
1264	(( ExperienceLengthInDays=$( IGCM_date_DaysBetweenGregorianDate ${DateEnd} ${DateBegin} ) + 1 ))
1265	if [ ${ExperienceLengthInDays} -lt 0 ] ; then
1266	IGCM_debug_Print 1 "Problem with dates in libIGCM.card : ${DateEnd} < ${DateBegin} ! You must check that."
1267	IGCM_debug_Exit "IGCM_PeriodEnd have wrong dates."
1268	IGCM_debug_Verif_Exit
1269	fi
1270	;;
1271	config_UserChoices_PeriodLength)
1272	IGCM_debug_Print 1 "Change config_UserChoices_PeriodLength=${config_UserChoices_PeriodLength}"
1273	;;
1274	PeriodNb)
1275	IGCM_debug_Print 1 "Loop in main Job with ${PeriodNb} period(s)"
1276	;;
1277	config_Post_RebuildFrequency)
1278	IGCM_debug_Print 1 "Change config_Post_RebuildFrequency=${config_Post_RebuildFrequency} : IGCM_post_Configure"
1279	IGCM_post_Configure
1280	;;
1281	config_Post_TimeSeriesFrequency)
1282	IGCM_debug_Print 1 "Change config_Post_TimeSeriesFrequency = ${config_Post_TimeSeriesFrequency} : IGCM_post_Configure"
1283	IGCM_post_Configure
1284	;;
1285	config_Post_SeasonalFrequency)
1286	IGCM_debug_Print 1 "Change config_Post_SeasonalFrequency = ${config_Post_SeasonalFrequency} : IGCM_post_Configure"
1287	IGCM_post_Configure
1288	;;
1289	esac
1290	done
1291
1292	echo
1293	echo "########################################################################"
1294	echo
1295	fi
1296
1297	#=================================================#
1298	# Determine next computed period #
1299	#=================================================#
1300
1301	PeriodDateBegin=$( IGCM_date_AddDaysToGregorianDate ${PeriodDateEnd} 1 )
1302	IGCM_date_GetYearMonthDay $PeriodDateBegin year month day
1303	year_m1=$(( year - 1 ))
1304	year_p1=$(( year + 1 ))
1305	IGCM_config_DaysInPeriodLength
1306	PeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate ${PeriodDateBegin} $(( ${PeriodLengthInDays} - 1 )) )
1307
1308	# Debug Print :
1309	echo
1310	IGCM_debug_Print 1 "IGCM_config_PeriodEnd : Preparing Next Execution"
1311	IGCM_debug_Print 1 "PeriodDateBegin : ${PeriodDateBegin}"
1312	IGCM_debug_Print 1 "PeriodDateEnd : ${PeriodDateEnd}"
1313	IGCM_debug_Print 1 "PeriodLengthInDays : ${PeriodLengthInDays}"
1314
1315	PeriodDateBegin=$( IGCM_date_ConvertFormatToHuman ${PeriodDateBegin} )
1316	PeriodDateEnd=$( IGCM_date_ConvertFormatToHuman ${PeriodDateEnd} )
1317
1318	(( CumulPeriod = CumulPeriod + 1 ))
1319
1320	# Debug Print :
1321	echo
1322	IGCM_debug_Print 3 "PeriodDateBegin Human : ${PeriodDateBegin}"
1323	IGCM_debug_Print 3 "PeriodDateEnd Human : ${PeriodDateEnd}"
1324	IGCM_debug_Print 3 "CumulPeriod : ${CumulPeriod}"
1325
1326	#=================================================#
1327	# Write updated run.card #
1328	#=================================================#
1329
1330	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodDateBegin ${PeriodDateBegin}
1331	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodDateEnd ${PeriodDateEnd}
1332	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration CumulPeriod ${CumulPeriod}
1333
1334	if ( ${FirstInitialize} ) ; then
1335	# It's no more the first time
1336	FirstInitialize=false
1337	fi
1338
1339	IGCM_debug_PopStack "IGCM_config_PeriodEnd"
1340	}
1341
1342	#===================================
1343	function IGCM_config_Finalize
1344	{
1345	IGCM_debug_PushStack "IGCM_config_Finalize"
1346
1347	echo
1348	IGCM_debug_Print 1 "IGCM_config_Finalize"
1349	echo
1350
1351	# Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
1352	IGCM_config_StateCheck
1353
1354	# And EXIT if not OK
1355	IGCM_debug_Verif_Exit
1356
1357	if [ ${SimulationLengthInDays} -ge ${ExperienceLengthInDays} ] ; then
1358	#==========================#
1359	# End of entire simulation #
1360	#==========================#
1361	simulationIsOver=true
1362
1363	# Mail notification
1364	IGCM_sys_SendMail
1365	#
1366	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Completed"
1367	#
1368	IGCM_debug_Print 1 "Normal End of computation."
1369
1370	else
1371	#=================#
1372	# Submit next job #
1373	#=================#
1374	simulationIsOver=false
1375
1376	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "OnQueue"
1377
1378	# Name of next Ksh Script output :
1379	Script_Output=${Script_Output_Prefix}_${config_UserChoices_JobName}.$( printf "%06d" ${CumulPeriod} )
1380
1381	IGCM_debug_Print 1 "Submit next job"
1382	# SUBMIT NEXT JOB from SUBMIT_DIR and come back in RUN_DIR
1383	IGCM_sys_Cd ${SUBMIT_DIR}
1384	# Keep only the 5 latest ${Script_Output_Prefix}_${config_UserChoices_JobName}
1385	ScriptTot=$( ls ${Script_Output_Prefix}_${config_UserChoices_JobName}.?????? 2>/dev/null \| wc -l )
1386	[ ${ScriptTot} -gt 5 ] && rm -f $( ls ${Script_Output_Prefix}_${config_UserChoices_JobName}.?????? \| head -$(( ${ScriptTot} - 5 )) )
1387	# Submit next job and come back
1388	IGCM_sys_Qsub ${SUBMIT_DIR}/Job_${config_UserChoices_JobName}
1389	IGCM_sys_Cd -
1390	fi
1391
1392	# Clean ${RUN_DIR}=${RUN_DIR_PATH}/${config_UserChoices_JobName}.${$}
1393	# Only for production run (No clean up in DEV or DEB mode)
1394	# and command sent from .. directory.
1395	IGCM_sys_Cd ..
1396	[ X${JobType} = XRUN ] && IGCM_sys_RmRunDir -rf ${RUN_DIR_PATH}
1397
1398	# Inform the rabbitMQ queue
1399	IGCM_debug_BigBro_Finalize
1400
1401	IGCM_debug_PopStack "IGCM_config_Finalize"
1402	}
1403
1404	#===================================

Note: See TracBrowser for help on using the repository browser.

Download in other formats: