Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh @ 1313

Last change on this file since 1313 was 1313, checked in by aclsce, 8 years ago
Removed exports needed to use new Intel environment : not needed anymore.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 52.1 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sébastien Denvil
5	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Ada
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=ada
71	# project name
72	typeset PROJECT=$(echo ${LOADL_STEP_GROUP:=NONE})
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=$( TZ=utc date -d '1970-01-01 '${wall_clock_limit} +%s )
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=llsubmit}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=ergon
89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
93	#====================================================
94	source /smplocal/pub/Modules/default/init/ksh
95	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
96	. /smphome/rech/psl/rpsl035/.atlas_env_ada_intel_2013_0_bash > /dev/null 2>&1
97	else
98	module load intel/2016.2 > /dev/null 2>&1
99	. /smphome/rech/psl/rpsl035/.atlas_env_ada_bash > /dev/null 2>&1
100	PCMDI_MP=/workgpfs/rech/psl/rpsl035/PCMDI-MP
101	export UVCDAT_ANONYMOUS_LOG=no
102	fi
103	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
104	[ ! X${TaskType} = Xchecking ] && module list
105
106	export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddNoise/src_X64_ADA/bin
107	export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddPerturbation/src_X64_ADA/bin
108	export PATH=${PATH}:/smphome/rech/psl/rpsl035/bin/
109
110	#====================================================
111	# Host specific DIRECTORIES
112	#====================================================
113
114	#====================================================
115	#- MirrorlibIGCM for frontend
116	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
117
118	#====================================================
119	#- libIGCM_POST for frontend
120	typeset -r libIGCM_POST=${libIGCM}
121
122	#====================================================
123	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
124	typeset -r R_EXE="${MODIPSL}/bin"
125
126	#====================================================
127	#- SUBMIT_DIR : submission dir
128	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${LOADL_STEP_INITDIR}}
129
130	#====================================================
131	#- IN
132	typeset -r R_IN=${R_IN:=/workgpfs/rech/psl/rpsl035/IGCM}
133	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/u/rech/psl/rpsl376}
134
135	#====================================================
136	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
137	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${TMPDIR}}
138
139	#====================================================
140	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
141	typeset -r OUTCOMMAND_PATH=${RUN_DIR_PATH:-/tmp}
142
143	#====================================================
144	#- HOST_MPIRUN_COMMAND
145	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
146	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
147	else
148	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time mpirun"}
149	fi
150	#====================================================
151	#- Max number of arguments passed to nco operator or demigration command
152	UNIX_MAX_LIMIT=360
153
154	#====================================================
155	#- set PackDefault to true on ada
156	PackDefault=true
157
158	#====================================================
159	#- Number of cores per node
160	core_per_node=32
161
162	#====================================================
163	#- Default number of MPI task for IPSL coupled model
164	#- required for backward compatibility
165	#-
166	DEFAULT_NUM_PROC_OCE=5
167	DEFAULT_NUM_PROC_CPL=1
168	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
169
170	#D-#==================================================
171	#D-function IGCM_sys_defineArchives
172	#D-* Purpose:
173	#D-* Define ARCHIVE : Dedicated to large files
174	#D-* Define STORAGE : Dedicated to small/medium files
175	#D-* Define R_OUT : Output tree located on ARCHIVE
176	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
177	#D-* Define R_BUF : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
178	#D-* if SpaceName=TEST everything is stored on WORKDIR
179	#D-* Examples:
180	#D-
181	function IGCM_sys_defineArchives {
182	IGCM_debug_PushStack "IGCM_sys_defineArchives"
183
184	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
185	#====================================================
186	#- ARCHIVE (dedicated to large files)
187	ARCHIVE=${config_UserChoices_ARCHIVE}
188	else
189	#====================================================
190	#- ARCHIVE (dedicated to large files)
191	ARCHIVE=$(echo ${HOME} \| sed -e "s,/.*home/,/arch/home/,")
192	fi
193
194	if [ ! X${config_UserChoices_STORAGE} = X ]; then
195	#====================================================
196	#- STORAGE (dedicated to small/medium files)
197	STORAGE=${config_UserChoices_STORAGE}
198	else
199	#====================================================
200	#- STORAGE (dedicated to small/medium files)
201	STORAGE=${WORKDIR}
202	fi
203
204	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
205	#====================================================
206	#- R_OUT
207	R_OUT=${WORKDIR}/IGCM_OUT
208
209	#====================================================
210	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
211	R_FIG=${WORKDIR}/IGCM_OUT
212
213	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
214
215	else
216	#====================================================
217	#- R_OUT
218	R_OUT=${ARCHIVE}/IGCM_OUT
219
220	#====================================================
221	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
222	R_FIG=${ARCHIVE}/IGCM_OUT
223	fi
224
225	#====================================================
226	#- R_BUF (ONLY FOR double copy an scratch)
227	R_BUF=${WORKDIR}/IGCM_OUT
228
229	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
230	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
231	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
232
233	IGCM_debug_PopStack "IGCM_sys_defineArchives"
234	}
235
236	#D-#==================================================
237	#D-function IGCM_sys_RshArchive
238	#D-* Purpose: Archive rsh command
239	#D-* Examples:
240	#D-
241	function IGCM_sys_RshArchive {
242	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
243	DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
244	if [ $? = 0 ] ; then
245	rsh ${STOREHOST} exec /bin/ksh <<-EOF
246	${@}
247	EOF
248	status=$?
249	else
250	/bin/ksh <<-EOF
251	${@}
252	EOF
253	status=$?
254	fi
255	if [ ${status} -gt 0 ] ; then
256	IGCM_debug_Print 2 "IGCM_sys_RshArchive : rsh or command failed error code ${status}"
257	IGCM_debug_Exit "IGCM_sys_RshArchive"
258	fi
259	IGCM_debug_PopStack "IGCM_sys_RshArchive"
260	}
261
262	#D-#==================================================
263	#D-function IGCM_sys_RshArchive_NoError
264	#D-* Purpose: Archive rsh command, without error
265	#D-* used only in monitoring.job
266	#D-* Examples:
267	#D-
268	function IGCM_sys_RshArchive_NoError {
269	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
270	DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
271	if [ $? = 0 ] ; then
272	rsh ${STOREHOST} exec /bin/ksh <<-EOF
273	${@} 2> /dev/null
274	EOF
275	else
276	/bin/ksh <<-EOF
277	${@} 2> /dev/null
278	EOF
279	fi
280	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
281	}
282
283	#D-#==================================================
284	#D-function IGCM_sys_MkdirArchive
285	#D-* Purpose: Mkdir on Archive
286	#D-* Examples:
287	#D-
288	function IGCM_sys_MkdirArchive {
289	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
290	if ( $DEBUG_sys ) ; then
291	echo "IGCM_sys_MkdirArchive :" $@
292	fi
293	#- creation de repertoire sur le serveur fichier
294	DEBUG_sys=false IGCM_sys_IsFileArchived $1
295	if [ $? = 0 ] ; then
296	rsh ${STOREHOST} -n mkdir -p $1
297	status=$?
298	else
299	mkdir -p $1
300	status=$?
301	fi
302
303	if [ ${status} -gt 0 ] ; then
304	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : rsh or mkdir failed error code ${status}"
305	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
306	fi
307	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
308	}
309
310	#D-#==================================================
311	#D-function IGCM_sys_TestDirArchive
312	#D-* Purpose: Test Directory that must exists on Archive
313	#D-* Examples:
314	#D-
315	function IGCM_sys_TestDirArchive {
316	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
317	if ( $DEBUG_sys ) ; then
318	echo "IGCM_sys_TestDirArchive :" $@
319	fi
320	typeset ExistFlag
321	#Command depends on targeted file system
322	DEBUG_sys=false IGCM_sys_IsFileArchived $1
323	if [ $? = 0 ] ; then
324	ExistFlag=$( IGCM_sys_RshArchive "[ -d $1 ] && echo 0 \|\| echo 1" )
325	else
326	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
327	fi
328	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
329	return ${ExistFlag}
330	}
331
332	#D-#==================================================
333	#D-function IGCM_sys_IsFileArchived
334	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
335	#D-* Examples:
336	#D-
337	function IGCM_sys_IsFileArchived {
338	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
339	if ( $DEBUG_sys ) ; then
340	echo "IGCM_sys_IsFileArchived :" $@
341	fi
342	typeset IsArchivedFlag
343	IsArchivedFlag=$( [ "X$( echo $@ \| grep \/arch\/home )" != "X" ] && echo 0 \|\| echo 1 )
344	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
345
346	return ${IsArchivedFlag}
347	}
348
349	#D-#==================================================
350	#D-function IGCM_sys_TestFileArchive
351	#D-* Purpose: Test file that must NOT EXISTS on Archive
352	#D-* Examples:
353	#D-
354	function IGCM_sys_TestFileArchive {
355	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
356	typeset ExistFlag
357	ExistFlag=$( IGCM_sys_RshArchive "[ -f $1 ] && echo 0 \|\| echo 1" )
358	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
359
360	return ${ExistFlag}
361	}
362
363	#D-#==================================================
364	#D-function IGCM_sys_CountFileArchive
365	#D-* Purpose: Count files on Archive filesystem
366	#D-* Examples:
367	#D-
368	function IGCM_sys_CountFileArchive {
369	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
370	#Command depends on targeted file system
371	DEBUG_sys=false IGCM_sys_IsFileArchived $1
372	if [ $? = 0 ] ; then
373	IGCM_sys_RshArchive "ls ${@} 2>/dev/null \| wc -l"
374	status=$?
375	else
376	ls ${@} 2>/dev/null \| wc -l
377	status=$?
378	fi
379	if [ ${status} -gt 0 ] ; then
380	echo "IGCM_sys_CountFileArchive : erreur."
381	fi
382	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
383	}
384
385	#D-#==================================================
386	#D-function IGCM_sys_Tree
387	#D-* Purpose: Tree directories with files on ${ARCHIVE}
388	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
389	#D-
390	function IGCM_sys_Tree {
391	IGCM_debug_PushStack "IGCM_sys_Tree" $@
392	if ( $DEBUG_sys ) ; then
393	echo "IGCM_sys_Tree :" $@
394	fi
395
396	\mfls -R $@
397
398	IGCM_debug_PopStack "IGCM_sys_Tree"
399	}
400
401	#D-#==================================================
402	#D-function IGCM_sys_Qsub
403	#D-* Purpose: Qsub new job
404	#D-* Examples:
405	#D-
406	function IGCM_sys_Qsub {
407	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
408	if ( $DEBUG_sys ) ; then
409	echo "IGCM_sys_Qsub :" $@
410	fi
411	typeset status
412
413	# We have to change output/error file
414	[ ${#@} = 1 ] && REP_FOR_JOB=${SUBMIT_DIR}
415	[ ${#@} = 2 ] && REP_FOR_JOB=${2}
416	sed -e "s:\# \@ output = .:\# \@ output = ${Script_Output}:" \
417	-e "s:\# \@ error = .:\# \@ error = ${Script_Output}:" \
418	$1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
419	cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
420
421	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
422	if [ ${status} -gt 0 ] ; then
423	IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
424	IGCM_debug_Exit "IGCM_sys_Qsub"
425	else
426	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ \| tail -1 \| sed -e s/\"//g )
427	IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
428	fi
429	IGCM_debug_PopStack "IGCM_sys_Qsub"
430	}
431
432	#D-#==================================================
433	#D-function IGCM_sys_QsubPost
434	#D-* Purpose: Qsub new job on scalaire
435	#D-* Examples:
436	#D-
437	function IGCM_sys_QsubPost {
438	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
439	if ( $DEBUG_sys ) ; then
440	echo "IGCM_sys_QsubPost :" $@
441	fi
442	typeset status
443
444	# We have to change output/error file
445	[ ${#@} = 1 ] && REP_FOR_JOB=${POST_DIR}
446	[ ${#@} = 2 ] && REP_FOR_JOB=${2}
447
448	sed -e "s:\# \@ output = .:\# \@ output = ${Script_Post_Output}.out:" \
449	-e "s:\# \@ error = .:\# \@ error = ${Script_Post_Output}.out:" \
450	${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
451
452	cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM_$$ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
453
454	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
455	if [ ${status} -gt 0 ] ; then
456	IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
457	IGCM_debug_Exit "IGCM_sys_QsubPost"
458	else
459	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ \| tail -1 \| sed -e s/\"//g )
460	IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
461	fi
462	IGCM_debug_PopStack "IGCM_sys_QsubPost"
463	}
464
465	#D-*************************
466	#D- File transfer functions
467	#D-*************************
468	#D-
469
470	#D-#==================================================
471	#D-function IGCM_sys_RmRunDir
472	#D-* Purpose: rm tmpdir (dummy function most of the time batch
473	#D- scheduler will do the job)
474	#D-* Examples:
475	#D-
476	function IGCM_sys_RmRunDir {
477	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
478	if ( $DEBUG_sys ) ; then
479	echo "IGCM_sys_RmRunDir :" $@
480	IGCM_debug_Print 1 "Dummy call, let the scheduler do that."
481	fi
482	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
483	}
484
485	#D-#==================================================
486	#D-function IGCM_sys_Put_Dir
487	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
488	#D-* Examples:
489	#D-
490	function IGCM_sys_Put_Dir {
491	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
492	if ( $DEBUG_sys ) ; then
493	echo "IGCM_sys_Put_Dir :" $@
494	fi
495	if [ $DRYRUN = 0 ]; then
496	if [ ! -d ${1} ] ; then
497	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
498	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
499	return
500	fi
501
502	typeset status
503
504	# Only if we use rsync
505	#IGCM_sys_TestDirArchive $( dirname $2 )
506	#
507	#Command depends on targeted file system
508	DEBUG_sys=false IGCM_sys_IsFileArchived $2
509	if [ $? = 0 ] ; then
510	\rcp -r $1 ${STOREHOST}:$2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
511	status=$?
512	else
513	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
514	status=$?
515	fi
516
517	if [ ${status} -gt 0 ] ; then
518	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : rcp or cp failed error code ${status}"
519	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
520	IGCM_debug_Exit "IGCM_sys_Put_Dir"
521	else
522	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
523	fi
524	fi
525	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
526	}
527
528	#D-#==================================================
529	#D-function IGCM_sys_Get_Dir
530	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
531	#D-* Examples:
532	#D-
533	function IGCM_sys_Get_Dir {
534	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
535	if ( $DEBUG_sys ) ; then
536	echo "IGCM_sys_Get_Dir :" $@
537	fi
538	if [ $DRYRUN = 0 ]; then
539	typeset NB_ESSAI DELAI status i
540	# number of tentative
541	NB_ESSAI=3
542	# time delay between tentative
543	DELAI=2
544
545	# Only if we use rsync
546	#IGCM_sys_TestDirArchive $( dirname $2 )
547	#
548	# Command depends on targeted filesystem
549	DEBUG_sys=false IGCM_sys_IsFileArchived $1
550	if [ $? = 0 ] ; then
551	# add dmget (to demigrate all offline files) to reduce time of this command :
552	#IGCM_sys_RshArchive "dmfind $1 -state MIG -o -state OFL -o -state PAR \| dmget -q -n"
553	i=0
554	while [ $i -lt $NB_ESSAI ] ; do
555	\rcp -rp ${STOREHOST}:$1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
556	status=$?
557	if [ ${status} -gt 0 ]; then
558	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : rcp failed error code ${status} ${i}/${NB_ESSAI}"
559	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
560	sleep $DELAI
561	else
562	break
563	fi
564	(( i = i + 1 ))
565	done
566	else
567	\cp -rp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
568	status=$?
569	if [ ${status} -gt 0 ] ; then
570	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
571	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
572	IGCM_debug_Exit "IGCM_sys_Get_Dir"
573	else
574	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
575	fi
576	fi
577	if [ ${status} -gt 0 ] ; then
578	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : (r)cp failed error code ${status}"
579	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
580	IGCM_debug_Exit "IGCM_sys_Get_Dir"
581	else
582	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
583	fi
584	fi
585	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
586	}
587
588	#D-#==================================================
589	#D-function IGCM_sys_Put_Rest
590	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
591	#D- File and target directory must exist.
592	#D-* Examples:
593	#D-
594	function IGCM_sys_Put_Rest {
595	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
596	if ( $DEBUG_sys ) ; then
597	echo "IGCM_sys_Put_Rest :" $@
598	fi
599	if [ $DRYRUN = 0 ]; then
600	if [ ! -f ${1} ] ; then
601	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
602	IGCM_debug_Exit "IGCM_sys_Put_Rest"
603	fi
604
605	typeset status
606	#
607	if [ X${JobType} = XRUN ] ; then
608	IGCM_sys_Chmod 444 ${1}
609	fi
610	#
611	#
612	#Command depends on targeted file system
613	DEBUG_sys=false IGCM_sys_IsFileArchived $2
614	if [ $? = 0 ] ; then
615	mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
616	status=$?
617	else
618	IGCM_sys_MkdirArchive $( dirname $2 )
619	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
620	status=$?
621	fi
622
623	# #RSYNC WITH NETWORK RSH CALL
624	# IGCM_sys_MkdirArchive $( dirname $2 )
625	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
626	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
627
628	# #RSYNC WITH NFS USE
629	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
630	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631
632	# status=$?
633	# IGCM_sys_Rsync_out $status
634
635	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
636	# (( status=status+$? ))
637
638	if [ ${status} -gt 0 ] ; then
639	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : mfput or cp failed error code ${status}"
640	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
641	IGCM_debug_Exit "IGCM_sys_Put_Rest"
642	else
643	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
644	fi
645	fi
646	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
647	}
648
649	#D-#==================================================
650	#D-function IGCM_sys_Put_Out
651	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
652	#D-* Examples:
653	#D-
654	function IGCM_sys_Put_Out {
655	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
656	if ( $DEBUG_sys ) ; then
657	echo "IGCM_sys_Put_Out :" $@
658	fi
659
660	typeset status
661
662	if [ $DRYRUN = 0 ]; then
663	if [ ! -f ${1} ] ; then
664	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
665	IGCM_debug_PopStack "IGCM_sys_Put_Out"
666	return 1
667	fi
668	#
669	if [ X${JobType} = XRUN ] ; then
670	if [ X${3} = X ] ; then
671	IGCM_sys_Chmod 444 ${1}
672	fi
673	fi
674	#
675	#
676	#Command depends on targeted file system
677	DEBUG_sys=false IGCM_sys_IsFileArchived $2
678	if [ $? = 0 ] ; then
679	mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
680	status=$?
681	else
682	IGCM_sys_MkdirArchive $( dirname $2 )
683	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
684	status=$?
685	fi
686
687	# #RSYNC WITH NETWORK RSH CALL
688	# IGCM_sys_MkdirArchive $( dirname $2 )
689	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
690	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
691
692	# #RSYNC WITH NFS USE
693	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
694	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
695
696	# status=$?
697	# IGCM_sys_Rsync_out $status
698
699	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
700	# (( status=status+$? ))
701
702	if [ ${status} -gt 0 ] ; then
703	IGCM_debug_Print 2 "IGCM_sys_Put_Out : mfput or cp failed error code ${status}"
704	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
705	IGCM_debug_Exit "IGCM_sys_Put_Out"
706	else
707	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
708	fi
709	fi
710	IGCM_debug_PopStack "IGCM_sys_Put_Out"
711	return 0
712	}
713
714	#D-#==================================================
715	#D-function IGCM_sys_Get
716	#D-* Purpose: Get a file from ${ARCHIVE}
717	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
718	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
719	function IGCM_sys_Get {
720	IGCM_debug_PushStack "IGCM_sys_Get" $@
721
722	typeset DEST status dm_liste
723
724	if ( $DEBUG_sys ) ; then
725	echo "IGCM_sys_Get :" $@
726	fi
727	if [ $DRYRUN -le 2 ]; then
728	if [ X${1} = X'/l' ] ; then
729	eval set +A dm_liste \${${2}}
730	else
731	dm_liste=${1}
732	fi
733	eval DEST=\${${#}}
734
735	# test if the (first) file is present in the old computation :
736	DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
737	if [ $? = 0 ] ; then
738	IGCM_sys_TestFileArchive ${dm_liste[0]}
739	status=$?
740	else
741	IGCM_sys_TestFileBuffer ${dm_liste[0]}
742	status=$?
743	fi
744
745	if [ ${status} -gt 0 ] ; then
746	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
747	IGCM_debug_Exit "IGCM_sys_Get"
748	return
749	fi
750
751	#Command depends on targeted file system
752	DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
753	if [ $? = 0 ] ; then
754	mfget ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
755	status=$?
756	else
757	\cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
758	status=$?
759	fi
760
761	# #RSYNC WITH NETWORK RSH CALL
762	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
763	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
764
765	# #RSYNC WITH NFS USE
766	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
767	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
768
769	# status=$?
770	# IGCM_sys_Rsync_out $status
771
772	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
773	# (( status=status+$? ))
774
775	if [ ${status} -gt 0 ] ; then
776	IGCM_debug_Print 2 "IGCM_sys_Get : mfget or cp failed error code ${status}"
777	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
778	IGCM_debug_Exit "IGCM_sys_Get"
779	else
780	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
781	fi
782	fi
783	IGCM_debug_PopStack "IGCM_sys_Get"
784	}
785
786	#D-#==================================================
787	#D-function IGCM_sys_GetDate_Monitoring
788	#D-* Purpose: get the last year for which the monitoring has been computed
789	#D-* Examples:
790	#D-
791	function IGCM_sys_GetDate_Monitoring {
792	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
793	if ( $DEBUG_sys ) ; then
794	echo "IGCM_sys_GetDate_Monitoring :" $@
795	fi
796
797	IGCM_sys_Cd /tmp
798	# NASTY HACK. We force /tmp here. This function is called only from the front-end and interactively.
799	RUN_DIR_PATH=/tmp IGCM_sys_Get ${1} .
800	eval ${2}=$( cdo showyear $( basename ${1} ) 2> /dev/null \| gawk '{ print $NF }' )
801	# Need to erase this file to avoid collision (permission denied getting file) between users on the front-end
802	IGCM_sys_Rm -f $( basename ${1} )
803	IGCM_sys_Cd -
804
805	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
806	}
807
808	#D-#==================================================
809	#D-function IGCM_sys_Put_Dods
810	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
811	#D-* Examples:
812	#D-
813	function IGCM_sys_Put_Dods {
814	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
815	if ( $DEBUG_sys ) ; then
816	echo "IGCM_sys_Put_Dods :" $@
817	fi
818	typeset status
819	if [ $DRYRUN = 0 ]; then
820	# We take our time on that
821	sleep 10
822	IGCM_sys_TestDirArchive ${R_SAVE}/${1}
823	if [ $? != 0 ] ; then
824	echo "WARNING : IGCM_sys_Put_Dods ${R_SAVE}/${1} DOES NOT EXIST ."
825	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
826	return
827	fi
828
829	rsh ${STOREHOST} exec /bin/ksh <<EOF
830	cd ${R_SAVE}
831	/arch/local/bin/dods_rm DODS/pub/${LOGIN}/${R_DODS}/${1} > /dev/null 2>&1
832	/bin/chmod -R u+w ${R_SAVE}/${1}
833	/arch/local/bin/dods_cp ${1} DODS/pub/${LOGIN}/${R_DODS} > /dev/null 2>&1
834	/bin/chmod -R +rX ${R_SAVE}/${1}
835	/bin/chmod -R u+w ${R_SAVE}/${1}
836	EOF
837	status=$?
838
839	if [ ${status} -gt 0 ] ; then
840	echo "IGCM_sys_Put_Dods : error."
841	IGCM_debug_Exit "IGCM_sys_Put_Dods"
842	fi
843	fi
844	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
845	}
846
847	##############################################################
848	# REBUILD OPERATOR
849
850	#D-#==================================================
851	#D-function IGCM_sys_sync
852	#D-* Purpose: flush buffer on disk (dummy function on Ada)
853	#D-* Examples:
854	#D-
855	function IGCM_sys_sync {
856	IGCM_debug_PushStack "IGCM_sys_sync" $@
857	if ( $DEBUG_sys ) ; then
858	echo "IGCM_sys_sync :" $@
859	echo "Dummy call, let the system do that."
860	fi
861	IGCM_debug_PopStack "IGCM_sys_sync"
862	}
863
864	############################################################
865	# Activate Running Environnment Variables
866
867	#D-#==================================================
868	#D-function IGCM_sys_activ_variables
869	#D-* Purpose: set environement variables prior to execution
870	#D-* Examples:
871	#D-
872	function IGCM_sys_activ_variables {
873	IGCM_debug_PushStack "IGCM_sys_activ_variables"
874	if ( $DEBUG_sys ) ; then
875	echo "IGCM_sys_activ_variables"
876	fi
877
878	# --------------------------------------------------------------------
879	#D- MPI specifications
880	# --------------------------------------------------------------------
881
882	# --------------------------------------------------------------------
883	#D- Other specifications
884	# --------------------------------------------------------------------
885
886	ulimit -s unlimited
887	## to be done only one time
888	## export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
889	## echo ${LD_LIBRARY_PATH} \| grep -i netcdf >/dev/null 2>&1 \|\| export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
890	IGCM_debug_Print 1 "set LD_LIBRARY_PATH=${LD_LIBRARY_PATH}"
891
892	export MP_STDOUTMODE=combined
893	IGCM_debug_Print 1 "set MP_STDOUTMODE=${MP_STDOUTMODE}"
894
895	## fix to reduce memory usage. Required since 2014/22/04. On ada IDRIS.
896	#export MP_EUILIBPATH=/smplocal/lib/ibmhpc/pe12012/ppe.pami/gnu/lib64/pami64
897	#IGCM_debug_Print 1 "set MP_EUILIBPATH=${MP_EUILIBPATH}"
898
899	IGCM_debug_PopStack "IGCM_sys_activ_variables"
900	}
901
902	############################################################
903	# Desactivate Running Environnment Variables
904
905	#D-#==================================================
906	#D-function IGCM_sys_desactiv_variables
907	#D-* Purpose: unset environement variables after execution
908	#D-* Examples:
909	#D-
910	function IGCM_sys_desactiv_variables {
911	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
912	if ( $DEBUG_sys ) ; then
913	echo "IGCM_sys_desactiv_variables"
914	fi
915	# --------------------------------------------------------------------
916	#D- MPI specifications
917	# --------------------------------------------------------------------
918
919	# --------------------------------------------------------------------
920	#D- Other specifications
921	# --------------------------------------------------------------------
922
923	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
924	}
925
926	############################################################
927	# Update job headers to be used by the scheduler
928
929	#D-#==================================================
930	#D-function IGCM_sys_updateHeaders
931	#D-* Purpose: Update job headers to be used by the scheduler
932	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
933	#D-
934	function IGCM_sys_updateHeaders {
935	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
936	if ( $DEBUG_sys ) ; then
937	echo "IGCM_sys_updateHeaders"
938	fi
939	typeset file
940	file=$1
941
942	if [ ${executionType} -eq 1 ] ; then
943	# MPMD + MPI
944	if [ X${MPIEnvironment} = XIBM ] ; then
945	sed -e "/::openMPthreads::/d" \
946	-e "s/::JobNumProcTot::/${coreNumber}/" \
947	${file} > ${file}.tmp
948	else
949	sed -e "/::openMPthreads::/d" \
950	-e "s/@ job_type = parallel/@ job_type = mpich/" \
951	-e "s/::JobNumProcTot::/${coreNumber}/" \
952	${file} > ${file}.tmp
953	fi
954
955	elif [ ${executionType} -eq 2 ] ; then
956	# MPMD + MPI + OMP
957	if [ X${MPIEnvironment} = XIBM ] ; then
958	sed -e "s/::openMPthreads::/${openMPthreads}/" \
959	-e "s/::JobNumProcTot::/${mpiTasks}/" \
960	${file} > ${file}.tmp
961	else
962	(( nodeNumber = coreNumber / core_per_node ))
963	[ $(( ${coreNumber} % ${core_per_node} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
964	sed -e "/::openMPthreads::/d" \
965	-e "s/@ job_type = parallel/@ job_type = mpich/" \
966	-e "s/@ total_tasks = ::JobNumProcTot::/@ node = ${nodeNumber} /" \
967	-e "/@ as_limit = 3.5gb/d" \
968	-e "s/::JobNumProcTot::/${mpiTasks}/" \
969	${file} > ${file}.tmp
970	fi
971
972	elif [ ${executionType} -eq 3 ] ; then
973	# SPMD + MPI/OMP
974	if [ X${MPIEnvironment} = XIBM ] ; then
975	sed -e "s/::openMPthreads::/${openMPthreads}/" \
976	-e "s/::JobNumProcTot::/${mpiTasks}/" \
977	${file} > ${file}.tmp
978	else
979	sed -e "s/::openMPthreads::/${openMPthreads}/" \
980	-e "s/@ job_type = parallel/@ job_type = mpich/" \
981	-e "s/::JobNumProcTot::/${mpiTasks}/" \
982	${file} > ${file}.tmp
983	fi
984
985	elif [ ${executionType} -eq 4 ] ; then
986	# SPMD + MPI only
987	if [ X${MPIEnvironment} = XIBM ] ; then
988	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
989	-e "/::openMPthreads::/d" \
990	${file} > ${file}.tmp
991	else
992	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
993	-e "s/@ job_type = parallel/@ job_type = mpich/" \
994	-e "/::openMPthreads::/d" \
995	${file} > ${file}.tmp
996	fi
997
998	elif [ ${executionType} -eq 5 ] ; then
999	# SPMD + OMP only
1000	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1001	-e "s/@ job_type = parallel/@ job_type = serial/" \
1002	-e "/::JobNumProcTot::/d" \
1003	${file} > ${file}.tmp
1004
1005	elif [ ${executionType} -eq 6 ] ; then
1006	# SEQUENTIAL THEN
1007	sed -e "s/::JobNumProcTot::/1/" \
1008	-e "s/@ job_type = parallel/@ job_type = serial/" \
1009	-e "/::openMPthreads::/d" \
1010	${file} > ${file}.tmp
1011
1012	fi
1013
1014	IGCM_sys_Mv ${file}.tmp ${file}
1015
1016	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1017	}
1018
1019	############################################################
1020	# Build MPI/OMP scripts run file (dummy function)
1021
1022	#D-#==================================================
1023	#D-function IGCM_sys_build_run_file
1024	#D-* Purpose: build run file (deprecated)
1025	#D-* Examples:
1026	#D-
1027	function IGCM_sys_build_run_file {
1028
1029	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1030
1031	}
1032
1033	############################################################
1034	# Build MPI/OMP scripts
1035
1036	#D-#==================================================
1037	#D-function IGCM_sys_build_execution_scripts
1038	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1039	#D-* Examples:
1040	#D-
1041	function IGCM_sys_build_execution_scripts
1042	{
1043	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1044	if ( $DEBUG_sys ) ; then
1045	echo "IGCM_sys_build_execution_scripts " $@
1046	fi
1047
1048	EXECUTION=${HOST_MPIRUN_COMMAND}
1049
1050	# MPMD mode
1051	if ( ${OK_PARA_MPMD} ) ; then
1052
1053	# MPI IBM Environment
1054	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1055	IGCM_debug_Print 1 "You use IBM MPI environment"
1056	if [ -f run_file ] ; then
1057	IGCM_sys_Rm -f run_file
1058	fi
1059	touch run_file
1060
1061	# Build run_file
1062
1063	# First loop on the components for the coupler ie oasis (only if oasis3)
1064	# the coupler ie oasis3 must be the first one
1065	for comp in ${config_ListOfComponents[*]} ; do
1066
1067	eval ExeNameIn=\${config_Executable_${comp}[0]}
1068	eval ExeNameOut=\${config_Executable_${comp}[1]}
1069
1070	# for CPL component only
1071	if [ "X${comp}" = "XCPL" ] ; then
1072
1073	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1074	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1075
1076	if ( ${OK_PARA_MPI} ) ; then
1077	(( mpi_count = 1 ))
1078	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1079	echo "./${ExeNameOut}" >> run_file
1080	(( mpi_count = mpi_count + 1 ))
1081	done
1082	else
1083	echo "./${ExeNameOut} " >> run_file
1084	fi
1085	fi
1086
1087	done
1088
1089	# Then second loop on the components
1090	for comp in ${config_ListOfComponents[*]} ; do
1091
1092	eval ExeNameIn=\${config_Executable_${comp}[0]}
1093	eval ExeNameOut=\${config_Executable_${comp}[1]}
1094
1095	# Only if we really have an executable for the component and not the coupler ie oasis:
1096	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1097
1098	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1099	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1100
1101	if ( ${OK_PARA_OMP} ) ; then
1102	# Check if the number of threads is correct
1103	case ${comp_proc_omp_loc} in
1104	1\|2\|4)
1105	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads on IBM MPI Environment"
1106	IGCM_debug_Print 1 "Beware : it may you use more CPU than needed."
1107	IGCM_debug_Print 1 "Think to the Intel MPI Environment to do what you want to do !"
1108	;;
1109	8\|16)
1110	IGCM_debug_Exit "ERROR with OMP parameters !"
1111	IGCM_debug_Print 2 "Beware : ${comp_proc_omp_loc} is too much for MPMD mode"
1112	IGCM_debug_Print 2 "You will use more CPU than needed : try to use Intel-MPI environment to do such a thing !"
1113	IGCM_debug_Verif_Exit
1114	;;
1115	*)
1116	IGCM_debug_Exit "ERROR with OMP parameters !"
1117	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1118	IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1119	IGCM_debug_Verif_Exit
1120	;;
1121	esac
1122
1123	fi
1124
1125	if ( ${OK_PARA_MPI} ) ; then
1126	(( mpi_count = 1 ))
1127	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1128	if ( ${OK_PARA_OMP} ) ; then
1129	echo "env OMP_NUM_THREADS=$comp_proc_omp_loc ./${ExeNameOut}" >> run_file
1130	else
1131	echo "./${ExeNameOut}" >> run_file
1132	fi
1133	(( mpi_count = mpi_count + 1 ))
1134	done
1135	else
1136	echo "./${ExeNameOut} " >> run_file
1137	fi
1138	fi
1139	done
1140	if ( ${OK_PARA_OMP} ) ; then
1141	export KMP_STACKSIZE=200m
1142	fi
1143
1144	EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
1145
1146	IGCM_sys_Chmod u+x run_file
1147	if ( $DEBUG_sys ) ; then
1148	echo "run_file contains : "
1149	cat run_file
1150	fi
1151
1152
1153	# MPI Intel Environment
1154	else
1155	IGCM_debug_Print 1 "You use Intel MPI environment"
1156
1157	# Only MPI (MPMD)
1158	if ( ! ${OK_PARA_OMP} ) ; then
1159	init_exec=n
1160
1161	# First loop on the components for the coupler ie oasis (only if oasis3)
1162	# the coupler ie oasis3 must be the first one
1163	for comp in ${config_ListOfComponents[*]} ; do
1164
1165	eval ExeNameIn=\${config_Executable_${comp}[0]}
1166	eval ExeNameOut=\${config_Executable_${comp}[1]}
1167
1168	# for CPL component only
1169	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1170
1171	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1172	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1173
1174	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1175	echo "" >> script_${ExeNameOut}.ksh
1176	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1177	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1178	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1179	init_exec=y
1180	fi
1181	done
1182	# Then second loop on the components
1183	for comp in ${config_ListOfComponents[*]} ; do
1184
1185	eval ExeNameIn=\${config_Executable_${comp}[0]}
1186	eval ExeNameOut=\${config_Executable_${comp}[1]}
1187
1188	# Only if we really have an executable for the component and not the coupler ie oasis:
1189	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1190
1191	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1192	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1193
1194	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1195	echo "" >> script_${ExeNameOut}.ksh
1196	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1197	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1198
1199	if [ ${init_exec} = y ] ; then
1200	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1201	else
1202	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1203	init_exec=y
1204	fi
1205	fi
1206	done
1207	# MPI-OpenMP (MPMD)
1208	else
1209
1210	# Execution specifications
1211	EXECUTION="${EXECUTION} -configfile run_file"
1212	export KMP_STACKSIZE=200m
1213	if [ -f run_file ] ; then
1214	IGCM_sys_Rm -f run_file
1215	fi
1216	touch run_file
1217
1218	# Initialisation of variables
1219	string_final=""
1220	string_final_hexa=""
1221	current_core=0
1222	executable_nb=1
1223
1224	# Hosts treatment
1225	for nodes in `echo $LOADL_PROCESSOR_LIST`
1226	do
1227	host[$i]=$nodes
1228	i=$((i+1))
1229	done
1230
1231	# Loop on the components
1232	for comp in ${config_ListOfComponents[*]} ; do
1233
1234	eval ExeNameIn=\${config_Executable_${comp}[0]}
1235	eval ExeNameOut=\${config_Executable_${comp}[1]}
1236
1237	# Not possible if oasis has an executable (i.e old version of oasis3)
1238	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1239	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1240	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1241	IGCM_debug_Verif_Exit
1242	fi
1243
1244	# Only if we really have an executable for the component :
1245	if [ "X${ExeNameOut}" != X\"\" ] ; then
1246
1247	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1248	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1249
1250	# Check if the number of threads is correct
1251	case ${comp_proc_omp_loc} in
1252	1\|2\|4\|8\|16)
1253	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1254	;;
1255	*)
1256	IGCM_debug_Exit "ERROR with OMP parameters !"
1257	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1258	IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1259	IGCM_debug_Verif_Exit
1260	;;
1261	esac
1262
1263
1264	# Build run_file for Ada Intel-MPI environment : method used to assign cores and nodes for the MPI process by using hexadecimal mask
1265	# Example of run_file :
1266	#-host host0 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1267	#-host host1 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1268	#-host host2 -n 10 -env I_MPI_PIN_DOMAIN=[3,c,30,c0,300,c00,3000,c000,30000,c0000,100000,200000,400000,800000,1000000,2000000,4000000,8000000,10000000,20000000] ./b.out
1269	#-host host2 -n 10 ./c.out
1270	# Example of final command :
1271	# mpirun -configfile run_file
1272
1273	rank=0
1274	# For one specific executable, loop on mpi process
1275	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1276	(( index_host = current_core / core_per_node ))
1277	host_value=${host[${index_host}]}
1278	(( slot = current_core % core_per_node ))
1279	# loop on omp threads for each mpi process (set the appropriate bit to 1 and append it to previous one)
1280	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1281	string_final="1"$string_final
1282	done
1283	# convert binary mask to hexadecimal mask
1284	if [ $rank -ne 0 ] ; then
1285	string_final_hexa=$string_final_hexa","$( printf '%x\n' "$((2#$string_final))" )
1286	else
1287	string_final_hexa=$( printf '%x\n' "$((2#$string_final))" )
1288	fi
1289	# replace bit 1 by bit 0 in order to append next one (next one wil be 1)
1290	string_final=$( echo $string_final \| sed "s/1/0/g" )
1291	# mpi rank = mpi_rank + 1
1292	(( rank = rank + 1 ))
1293	# current core takes into account the number of omp threads which was previously append
1294	(( current_core = current_core + comp_proc_omp_loc ))
1295	# We write to the configuration file either we switch to the next node or we switch to the next executable
1296	if ( [ $(( current_core / core_per_node )) -ne $index_host ] \|\| [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ) ; then
1297	# I_MPI_PIN_DOMAIN variable must be given once per node
1298	if [ $executable_nb -eq 1 ] ; then
1299	echo "-host $host_value -n $rank -env I_MPI_PIN_DOMAIN=[$string_final_hexa] ./$ExeNameOut" >> run_file
1300	else
1301	sed -i "/$host_value/s/\]/\,$string_final_hexa\]/g" run_file
1302	echo "-host $host_value -n $rank ./$ExeNameOut" >> run_file
1303	fi
1304	# +1 for the number of executbale on the same node
1305	if [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ; then
1306	(( executable_nb = executable_nb + 1 ))
1307	fi
1308	# Some initializations if we switch to the next node
1309	if [ $(( current_core / core_per_node )) -ne $index_host ] ; then
1310	string_final=""
1311	string_final_hexa=""
1312	rank=0
1313	executable_nb=1
1314	fi
1315	fi
1316	done
1317
1318	fi
1319	done
1320	IGCM_sys_Chmod u+x run_file
1321	if ( $DEBUG_sys ) ; then
1322	echo "run_file contains : "
1323	cat run_file
1324	fi
1325	fi
1326	fi
1327
1328	# Only one executable (SPMD mode).
1329	else
1330
1331	for comp in ${config_ListOfComponents[*]} ; do
1332
1333	# Only if we really have an executable for the component :
1334	eval ExeNameOut=\${config_Executable_${comp}[1]}
1335	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1336
1337	if ( ${OK_PARA_OMP} ) ; then
1338	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1339	export KMP_STACKSIZE=200m
1340	#export KMP_LIBRARY=turnaround
1341	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1342	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1343	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1344	fi
1345	if ( ${OK_PARA_MPI} ) ; then
1346	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1347	EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
1348	else
1349	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1350	EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
1351	fi
1352	else
1353	EXECUTION="/usr/bin/time ./${ExeNameOut}"
1354	fi
1355	fi
1356
1357	done
1358
1359	fi
1360
1361	IGCM_debug_Print 1 "sys ada : execution command is "
1362	IGCM_debug_Print 1 "$EXECUTION"
1363
1364	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1365	}
1366
1367	#D-#==================================================
1368	#D-function IGCM_sys_check_path
1369	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1370	#D-* do not point to an important use directory. Stop immediately in that case.
1371	#D-* Examples:
1372	#D-
1373	function IGCM_sys_check_path {
1374	IGCM_debug_PushStack "IGCM_sys_check_path"
1375	if ( $DEBUG_sys ) ; then
1376	echo "IGCM_sys_check_path"
1377	fi
1378
1379	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1380	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1381	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1382	IGCM_debug_Exit "This will stop the job"
1383	fi
1384	IGCM_debug_PopStack "IGCM_sys_check_path"
1385	}
1386
1387	#D-#==================================================
1388	#D-function IGCM_sys_check_quota
1389	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1390	#D-* Examples:
1391	#D-
1392	function IGCM_sys_check_quota {
1393	IGCM_debug_PushStack "IGCM_sys_check_quota"
1394	if ( $DEBUG_sys ) ; then
1395	echo "IGCM_sys_check_quota"
1396	fi
1397	# Limit of quota (in %)
1398	limit_quota=90
1399
1400	# Check of the volume
1401	volume_quota=$(quota_u -w 2>/dev/null\| grep 'Quota soft' \| gawk '{print $5}')
1402	if [ ! X${volume_quota} = X ] ; then
1403	quota_volume=${volume_quota%%\%}
1404	# echo $quota_volume
1405	if [ $(echo "${quota_volume} > ${limit_quota}" \| bc) -eq 1 ] ; then
1406	IGCM_debug_Print 1 "Please, check your quota of volume on workgpfs"
1407	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1408	IGCM_debug_Print 1 "Use the quota_u -w command to check"
1409	IGCM_debug_Print 1 "You must have more than 10% available to run"
1410	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1411	IGCM_debug_Verif_Exit
1412	fi
1413
1414	fi
1415	IGCM_debug_PopStack "IGCM_sys_check_quota"
1416	}
1417
1418	#D-#==================================================
1419	#D-function IGCM_sys_projectAccounting
1420	#D-* Purpose: store project accounting information in a file
1421	#D-* Examples:
1422	#D-
1423	function IGCM_sys_projectAccounting {
1424	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1425	if ( $DEBUG_sys ) ; then
1426	echo "IGCM_sys_check_quota"
1427	fi
1428
1429	cpt > $1
1430
1431	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1432	}
1433
1434	#D-#==================================================
1435	#D-function IGCM_sys_getJobSchedulerID
1436	#D-* Purpose: Get the job ID during execution
1437	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1438	#D-
1439	function IGCM_sys_getJobSchedulerID {
1440	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1441	if ( $DEBUG_sys ) ; then
1442	echo "IGCM_sys_getJobSchedulerID"
1443	fi
1444
1445	eval ${1}=$( echo $LOADL_STEP_ID \| awk -F. '{print $4}' )
1446
1447	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1448	}
1449
1450	#D-#==================================================
1451	#D-function IGCM_sys_GetJobID
1452	#D-* Purpose: Get the job ID from the JobName
1453	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1454	#D-
1455	function IGCM_sys_GetJobID {
1456	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1457	if ( $DEBUG_sys ) ; then
1458	echo "IGCM_sys_GetJobID"
1459	fi
1460
1461	# Print only the full (-W) JobID (%id) and JobName (%jn)
1462	ID=$( llq -u $2 -W -f %id %jn \| \
1463	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1464
1465	eval ${3}=${ID}
1466	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1467	}
1468
1469	#D-#==================================================
1470	#D-function IGCM_sys_CountJobInQueue
1471	#D-* Purpose: Count number of users job
1472	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1473	#D-
1474	function IGCM_sys_CountJobInQueue {
1475	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1476	if ( $DEBUG_sys ) ; then
1477	echo "IGCM_sys_CountJobInQueue"
1478	fi
1479
1480	# Print only the full (-W) JobName (%jn)
1481	NbRun=$( llq -W -f %jn \| grep -c "$1" )
1482
1483	eval ${2}=${NbRun}
1484
1485	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1486	}
1487
1488	#D-#==================================================
1489	#D-function IGCM_sys_ListJobInQueue
1490	#D-* Purpose: Produce a list of users jobs
1491	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1492	#D-
1493	function IGCM_sys_ListJobInQueue {
1494	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1495	if ( $DEBUG_sys ) ; then
1496	echo "IGCM_sys_ListJobInQueue"
1497	fi
1498
1499	# With -W option, column width is as large as necessary
1500	set -A JobList $( llq -u $1 -W -f %jn \| head -n -2 \| tail -n +3 \| \
1501	gawk '( $1 != /TS/ && \
1502	$1 !~ /PACK/ && \
1503	$1 !~ /REBUILD/ && \
1504	$1 !~ /pack/ ) \
1505	{ print $1 }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1506
1507	eval set -A ${2} ${JobList[*]}
1508
1509	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1510	}
1511
1512	#D-#==================================================
1513	#D-function IGCM_sys_atlas
1514	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1515	#D-* Examples:
1516	#D-
1517	function IGCM_sys_atlas {
1518	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1519	if ( $DEBUG_sys ) ; then
1520	echo "IGCM_sys_atlas :" $@
1521	fi
1522
1523	typeset status
1524
1525	\atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1526	status=$?
1527	if [ ${status} -gt 0 ] ; then
1528	echo "IGCM_sys_atlas : error code ${status}"
1529	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1530	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1531	IGCM_debug_PopStack "IGCM_sys_atlas"
1532	return 1
1533	else
1534	IGCM_debug_PopStack "IGCM_sys_atlas"
1535	return 0
1536	fi
1537
1538	IGCM_debug_PopStack "IGCM_sys_atlas"
1539	}
1540

Note: See TracBrowser for help on using the repository browser.

Download in other formats: