Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh @ 1357

Last change on this file since 1357 was 1357, checked in by jgipsl, 8 years ago
Allow 8 OMP at ada. See ticket #309
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 52.4 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sébastien Denvil
5	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Ada
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=ada
71	# project name
72	typeset PROJECT=$(echo ${LOADL_STEP_GROUP:=NONE})
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=$( TZ=utc date -d '1970-01-01 '${wall_clock_limit} +%s )
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=llsubmit}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=ergon
89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
93	#====================================================
94	source /smplocal/pub/Modules/default/init/ksh
95	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
96	. /smphome/rech/psl/rpsl035/.atlas_env_ada_intel_2013_0_bash > /dev/null 2>&1
97	module load python/2.7.10 > /dev/null 2>&1
98	else
99	module load intel/2016.2 > /dev/null 2>&1
100	. /smphome/rech/psl/rpsl035/.atlas_env_ada_bash > /dev/null 2>&1
101	PCMDI_MP=/workgpfs/rech/psl/rpsl035/PCMDI-MP
102	export UVCDAT_ANONYMOUS_LOG=no
103	fi
104	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
105	[ ! X${TaskType} = Xchecking ] && module list
106
107	export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddNoise/src_X64_ADA/bin
108	export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddPerturbation/src_X64_ADA/bin
109	export PATH=${PATH}:/smphome/rech/psl/rpsl035/bin/
110
111	#====================================================
112	# Host specific DIRECTORIES
113	#====================================================
114
115	#====================================================
116	#- MirrorlibIGCM for frontend
117	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
118
119	#====================================================
120	#- libIGCM_POST for frontend
121	typeset -r libIGCM_POST=${libIGCM}
122
123	#====================================================
124	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
125	typeset -r R_EXE="${MODIPSL}/bin"
126
127	#====================================================
128	#- SUBMIT_DIR : submission dir
129	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${LOADL_STEP_INITDIR}}
130
131	#====================================================
132	#- IN
133	typeset -r R_IN=${R_IN:=/workgpfs/rech/psl/rpsl035/IGCM}
134	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/u/rech/psl/rpsl376}
135
136	#====================================================
137	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
138	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${TMPDIR}}
139
140	#====================================================
141	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
142	typeset -r OUTCOMMAND_PATH=${RUN_DIR_PATH:-/tmp}
143
144	#====================================================
145	#- HOST_MPIRUN_COMMAND
146	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
147	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
148	else
149	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time mpirun"}
150	fi
151	#====================================================
152	#- Max number of arguments passed to nco operator or demigration command
153	UNIX_MAX_LIMIT=360
154
155	#====================================================
156	#- set PackDefault to true on ada
157	PackDefault=true
158
159	#====================================================
160	#- Number of cores per node
161	core_per_node=32
162
163	#====================================================
164	#- Default number of MPI task for IPSL coupled model
165	#- required for backward compatibility
166	#-
167	DEFAULT_NUM_PROC_OCE=5
168	DEFAULT_NUM_PROC_CPL=1
169	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
170
171	#D-#==================================================
172	#D-function IGCM_sys_defineArchives
173	#D-* Purpose:
174	#D-* Define ARCHIVE : Dedicated to large files
175	#D-* Define STORAGE : Dedicated to small/medium files
176	#D-* Define R_OUT : Output tree located on ARCHIVE
177	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
178	#D-* Define R_BUF : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
179	#D-* if SpaceName=TEST everything is stored on WORKDIR
180	#D-* Examples:
181	#D-
182	function IGCM_sys_defineArchives {
183	IGCM_debug_PushStack "IGCM_sys_defineArchives"
184
185	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
186	#====================================================
187	#- ARCHIVE (dedicated to large files)
188	ARCHIVE=${config_UserChoices_ARCHIVE}
189	else
190	#====================================================
191	#- ARCHIVE (dedicated to large files)
192	ARCHIVE=$(echo ${HOME} \| sed -e "s,/.*home/,/arch/home/,")
193	fi
194
195	if [ ! X${config_UserChoices_STORAGE} = X ]; then
196	#====================================================
197	#- STORAGE (dedicated to small/medium files)
198	STORAGE=${config_UserChoices_STORAGE}
199	else
200	#====================================================
201	#- STORAGE (dedicated to small/medium files)
202	STORAGE=${WORKDIR}
203	fi
204
205	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
206	#====================================================
207	#- R_OUT
208	R_OUT=${WORKDIR}/IGCM_OUT
209
210	#====================================================
211	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
212	R_FIG=${WORKDIR}/IGCM_OUT
213
214	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
215
216	else
217	#====================================================
218	#- R_OUT
219	R_OUT=${ARCHIVE}/IGCM_OUT
220
221	#====================================================
222	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
223	R_FIG=${ARCHIVE}/IGCM_OUT
224	fi
225
226	#====================================================
227	#- R_BUF (ONLY FOR double copy an scratch)
228	R_BUF=${WORKDIR}/IGCM_OUT
229
230	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
231	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
232	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
233
234	IGCM_debug_PopStack "IGCM_sys_defineArchives"
235	}
236
237	#D-#==================================================
238	#D-function IGCM_sys_RshArchive
239	#D-* Purpose: Archive rsh command
240	#D-* Examples:
241	#D-
242	function IGCM_sys_RshArchive {
243	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
244	DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
245	if [ $? = 0 ] ; then
246	rsh ${STOREHOST} exec /bin/ksh <<-EOF
247	${@}
248	EOF
249	status=$?
250	else
251	/bin/ksh <<-EOF
252	${@}
253	EOF
254	status=$?
255	fi
256	if [ ${status} -gt 0 ] ; then
257	IGCM_debug_Print 2 "IGCM_sys_RshArchive : rsh or command failed error code ${status}"
258	IGCM_debug_Exit "IGCM_sys_RshArchive"
259	fi
260	IGCM_debug_PopStack "IGCM_sys_RshArchive"
261	}
262
263	#D-#==================================================
264	#D-function IGCM_sys_RshArchive_NoError
265	#D-* Purpose: Archive rsh command, without error
266	#D-* used only in monitoring.job
267	#D-* Examples:
268	#D-
269	function IGCM_sys_RshArchive_NoError {
270	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
271	DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
272	if [ $? = 0 ] ; then
273	rsh ${STOREHOST} exec /bin/ksh <<-EOF
274	${@} 2> /dev/null
275	EOF
276	else
277	/bin/ksh <<-EOF
278	${@} 2> /dev/null
279	EOF
280	fi
281	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
282	}
283
284	#D-#==================================================
285	#D-function IGCM_sys_MkdirArchive
286	#D-* Purpose: Mkdir on Archive
287	#D-* Examples:
288	#D-
289	function IGCM_sys_MkdirArchive {
290	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
291	if ( $DEBUG_sys ) ; then
292	echo "IGCM_sys_MkdirArchive :" $@
293	fi
294	#- creation de repertoire sur le serveur fichier
295	DEBUG_sys=false IGCM_sys_IsFileArchived $1
296	if [ $? = 0 ] ; then
297	rsh ${STOREHOST} -n mkdir -p $1
298	status=$?
299	else
300	mkdir -p $1
301	status=$?
302	fi
303
304	if [ ${status} -gt 0 ] ; then
305	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : rsh or mkdir failed error code ${status}"
306	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
307	fi
308	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
309	}
310
311	#D-#==================================================
312	#D-function IGCM_sys_TestDirArchive
313	#D-* Purpose: Test Directory that must exists on Archive
314	#D-* Examples:
315	#D-
316	function IGCM_sys_TestDirArchive {
317	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
318	if ( $DEBUG_sys ) ; then
319	echo "IGCM_sys_TestDirArchive :" $@
320	fi
321	typeset ExistFlag
322	#Command depends on targeted file system
323	DEBUG_sys=false IGCM_sys_IsFileArchived $1
324	if [ $? = 0 ] ; then
325	ExistFlag=$( IGCM_sys_RshArchive "[ -d $1 ] && echo 0 \|\| echo 1" )
326	else
327	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
328	fi
329	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
330	return ${ExistFlag}
331	}
332
333	#D-#==================================================
334	#D-function IGCM_sys_IsFileArchived
335	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
336	#D-* Examples:
337	#D-
338	function IGCM_sys_IsFileArchived {
339	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
340	if ( $DEBUG_sys ) ; then
341	echo "IGCM_sys_IsFileArchived :" $@
342	fi
343	typeset IsArchivedFlag
344	IsArchivedFlag=$( [ "X$( echo $@ \| grep \/arch\/home )" != "X" ] && echo 0 \|\| echo 1 )
345	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
346
347	return ${IsArchivedFlag}
348	}
349
350	#D-#==================================================
351	#D-function IGCM_sys_TestFileArchive
352	#D-* Purpose: Test file that must NOT EXISTS on Archive
353	#D-* Examples:
354	#D-
355	function IGCM_sys_TestFileArchive {
356	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
357	typeset ExistFlag
358	ExistFlag=$( IGCM_sys_RshArchive "[ -f $1 ] && echo 0 \|\| echo 1" )
359	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
360
361	return ${ExistFlag}
362	}
363
364	#D-#==================================================
365	#D-function IGCM_sys_CountFileArchive
366	#D-* Purpose: Count files on Archive filesystem
367	#D-* Examples:
368	#D-
369	function IGCM_sys_CountFileArchive {
370	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
371	#Command depends on targeted file system
372	DEBUG_sys=false IGCM_sys_IsFileArchived $1
373	if [ $? = 0 ] ; then
374	IGCM_sys_RshArchive "ls ${@} 2>/dev/null \| wc -l"
375	status=$?
376	else
377	ls ${@} 2>/dev/null \| wc -l
378	status=$?
379	fi
380	if [ ${status} -gt 0 ] ; then
381	echo "IGCM_sys_CountFileArchive : erreur."
382	fi
383	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
384	}
385
386	#D-#==================================================
387	#D-function IGCM_sys_Tree
388	#D-* Purpose: Tree directories with files on ${ARCHIVE}
389	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
390	#D-
391	function IGCM_sys_Tree {
392	IGCM_debug_PushStack "IGCM_sys_Tree" $@
393	if ( $DEBUG_sys ) ; then
394	echo "IGCM_sys_Tree :" $@
395	fi
396
397	\mfls -R $@
398
399	IGCM_debug_PopStack "IGCM_sys_Tree"
400	}
401
402	#D-#==================================================
403	#D-function IGCM_sys_Qsub
404	#D-* Purpose: Qsub new job
405	#D-* Examples:
406	#D-
407	function IGCM_sys_Qsub {
408	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
409	if ( $DEBUG_sys ) ; then
410	echo "IGCM_sys_Qsub :" $@
411	fi
412	typeset status
413
414	# We have to change output/error file
415	[ ${#@} = 1 ] && REP_FOR_JOB=${SUBMIT_DIR}
416	[ ${#@} = 2 ] && REP_FOR_JOB=${2}
417	sed -e "s:\# \@ output = .:\# \@ output = ${Script_Output}:" \
418	-e "s:\# \@ error = .:\# \@ error = ${Script_Output}:" \
419	$1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
420	cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
421
422	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
423	if [ ${status} -gt 0 ] ; then
424	IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
425	IGCM_debug_Exit "IGCM_sys_Qsub"
426	else
427	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ \| tail -1 \| sed -e s/\"//g )
428	IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
429	fi
430	IGCM_debug_PopStack "IGCM_sys_Qsub"
431	}
432
433	#D-#==================================================
434	#D-function IGCM_sys_QsubPost
435	#D-* Purpose: Qsub new job on scalaire
436	#D-* Examples:
437	#D-
438	function IGCM_sys_QsubPost {
439	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
440	if ( $DEBUG_sys ) ; then
441	echo "IGCM_sys_QsubPost :" $@
442	fi
443	typeset status
444
445	# We have to change output/error file
446	[ ${#@} = 1 ] && REP_FOR_JOB=${POST_DIR}
447	[ ${#@} = 2 ] && REP_FOR_JOB=${2}
448
449	sed -e "s:\# \@ output = .:\# \@ output = ${Script_Post_Output}.out:" \
450	-e "s:\# \@ error = .:\# \@ error = ${Script_Post_Output}.out:" \
451	${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
452
453	cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM_$$ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
454
455	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
456	if [ ${status} -gt 0 ] ; then
457	IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
458	IGCM_debug_Exit "IGCM_sys_QsubPost"
459	else
460	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ \| tail -1 \| sed -e s/\"//g )
461	IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
462	fi
463	IGCM_debug_PopStack "IGCM_sys_QsubPost"
464	}
465
466	#D-*************************
467	#D- File transfer functions
468	#D-*************************
469	#D-
470
471	#D-#==================================================
472	#D-function IGCM_sys_RmRunDir
473	#D-* Purpose: rm tmpdir (dummy function most of the time batch
474	#D- scheduler will do the job)
475	#D-* Examples:
476	#D-
477	function IGCM_sys_RmRunDir {
478	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
479	if ( $DEBUG_sys ) ; then
480	echo "IGCM_sys_RmRunDir :" $@
481	IGCM_debug_Print 1 "Dummy call, let the scheduler do that."
482	fi
483	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
484	}
485
486	#D-#==================================================
487	#D-function IGCM_sys_Put_Dir
488	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
489	#D-* Examples:
490	#D-
491	function IGCM_sys_Put_Dir {
492	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
493	if ( $DEBUG_sys ) ; then
494	echo "IGCM_sys_Put_Dir :" $@
495	fi
496	if [ $DRYRUN = 0 ]; then
497	if [ ! -d ${1} ] ; then
498	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
499	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
500	return
501	fi
502
503	typeset status
504
505	# Only if we use rsync
506	#IGCM_sys_TestDirArchive $( dirname $2 )
507	#
508	#Command depends on targeted file system
509	DEBUG_sys=false IGCM_sys_IsFileArchived $2
510	if [ $? = 0 ] ; then
511	\rcp -r $1 ${STOREHOST}:$2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
512	status=$?
513	else
514	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
515	status=$?
516	fi
517
518	if [ ${status} -gt 0 ] ; then
519	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : rcp or cp failed error code ${status}"
520	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
521	IGCM_debug_Exit "IGCM_sys_Put_Dir"
522	else
523	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
524	fi
525	fi
526	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
527	}
528
529	#D-#==================================================
530	#D-function IGCM_sys_Get_Dir
531	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
532	#D-* Examples:
533	#D-
534	function IGCM_sys_Get_Dir {
535	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
536	if ( $DEBUG_sys ) ; then
537	echo "IGCM_sys_Get_Dir :" $@
538	fi
539	if [ $DRYRUN = 0 ]; then
540	typeset NB_ESSAI DELAI status i
541	# number of tentative
542	NB_ESSAI=3
543	# time delay between tentative
544	DELAI=2
545
546	# Only if we use rsync
547	#IGCM_sys_TestDirArchive $( dirname $2 )
548	#
549	# Command depends on targeted filesystem
550	DEBUG_sys=false IGCM_sys_IsFileArchived $1
551	if [ $? = 0 ] ; then
552	# add dmget (to demigrate all offline files) to reduce time of this command :
553	#IGCM_sys_RshArchive "dmfind $1 -state MIG -o -state OFL -o -state PAR \| dmget -q -n"
554	i=0
555	while [ $i -lt $NB_ESSAI ] ; do
556	\rcp -rp ${STOREHOST}:$1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
557	status=$?
558	if [ ${status} -gt 0 ]; then
559	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : rcp failed error code ${status} ${i}/${NB_ESSAI}"
560	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
561	sleep $DELAI
562	else
563	break
564	fi
565	(( i = i + 1 ))
566	done
567	else
568	\cp -rp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
569	status=$?
570	if [ ${status} -gt 0 ] ; then
571	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
572	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
573	IGCM_debug_Exit "IGCM_sys_Get_Dir"
574	else
575	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
576	fi
577	fi
578	if [ ${status} -gt 0 ] ; then
579	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : (r)cp failed error code ${status}"
580	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
581	IGCM_debug_Exit "IGCM_sys_Get_Dir"
582	else
583	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
584	fi
585	fi
586	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
587	}
588
589	#D-#==================================================
590	#D-function IGCM_sys_Put_Rest
591	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
592	#D- File and target directory must exist.
593	#D-* Examples:
594	#D-
595	function IGCM_sys_Put_Rest {
596	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
597	if ( $DEBUG_sys ) ; then
598	echo "IGCM_sys_Put_Rest :" $@
599	fi
600	if [ $DRYRUN = 0 ]; then
601	if [ ! -f ${1} ] ; then
602	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
603	IGCM_debug_Exit "IGCM_sys_Put_Rest"
604	fi
605
606	typeset status
607	#
608	if [ X${JobType} = XRUN ] ; then
609	IGCM_sys_Chmod 444 ${1}
610	fi
611	#
612	#
613	#Command depends on targeted file system
614	DEBUG_sys=false IGCM_sys_IsFileArchived $2
615	if [ $? = 0 ] ; then
616	mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
617	status=$?
618	else
619	IGCM_sys_MkdirArchive $( dirname $2 )
620	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
621	status=$?
622	fi
623
624	# #RSYNC WITH NETWORK RSH CALL
625	# IGCM_sys_MkdirArchive $( dirname $2 )
626	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
627	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
628
629	# #RSYNC WITH NFS USE
630	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
632
633	# status=$?
634	# IGCM_sys_Rsync_out $status
635
636	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
637	# (( status=status+$? ))
638
639	if [ ${status} -gt 0 ] ; then
640	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : mfput or cp failed error code ${status}"
641	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
642	IGCM_debug_Exit "IGCM_sys_Put_Rest"
643	else
644	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
645	fi
646	fi
647	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
648	}
649
650	#D-#==================================================
651	#D-function IGCM_sys_Put_Out
652	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
653	#D-* Examples:
654	#D-
655	function IGCM_sys_Put_Out {
656	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
657	if ( $DEBUG_sys ) ; then
658	echo "IGCM_sys_Put_Out :" $@
659	fi
660
661	typeset status
662
663	if [ $DRYRUN = 0 ]; then
664	if [ ! -f ${1} ] ; then
665	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
666	IGCM_debug_PopStack "IGCM_sys_Put_Out"
667	return 1
668	fi
669	#
670	if [ X${JobType} = XRUN ] ; then
671	if [ X${3} = X ] ; then
672	IGCM_sys_Chmod 444 ${1}
673	fi
674	fi
675	#
676	#
677	#Command depends on targeted file system
678	DEBUG_sys=false IGCM_sys_IsFileArchived $2
679	if [ $? = 0 ] ; then
680	mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
681	status=$?
682	else
683	IGCM_sys_MkdirArchive $( dirname $2 )
684	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
685	status=$?
686	fi
687
688	# #RSYNC WITH NETWORK RSH CALL
689	# IGCM_sys_MkdirArchive $( dirname $2 )
690	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
691	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
692
693	# #RSYNC WITH NFS USE
694	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
695	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
696
697	# status=$?
698	# IGCM_sys_Rsync_out $status
699
700	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
701	# (( status=status+$? ))
702
703	if [ ${status} -gt 0 ] ; then
704	IGCM_debug_Print 2 "IGCM_sys_Put_Out : mfput or cp failed error code ${status}"
705	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
706	IGCM_debug_Exit "IGCM_sys_Put_Out"
707	else
708	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
709	fi
710	fi
711	IGCM_debug_PopStack "IGCM_sys_Put_Out"
712	return 0
713	}
714
715	#D-#==================================================
716	#D-function IGCM_sys_Get
717	#D-* Purpose: Get a file from ${ARCHIVE}
718	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
719	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
720	function IGCM_sys_Get {
721	IGCM_debug_PushStack "IGCM_sys_Get" $@
722
723	typeset DEST status dm_liste
724
725	if ( $DEBUG_sys ) ; then
726	echo "IGCM_sys_Get :" $@
727	fi
728	if [ $DRYRUN -le 2 ]; then
729	if [ X${1} = X'/l' ] ; then
730	eval set +A dm_liste \${${2}}
731	else
732	dm_liste=${1}
733	fi
734	eval DEST=\${${#}}
735
736	# test if the (first) file is present in the old computation :
737	DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
738	if [ $? = 0 ] ; then
739	IGCM_sys_TestFileArchive ${dm_liste[0]}
740	status=$?
741	else
742	IGCM_sys_TestFileBuffer ${dm_liste[0]}
743	status=$?
744	fi
745
746	if [ ${status} -gt 0 ] ; then
747	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
748	IGCM_debug_Exit "IGCM_sys_Get"
749	return
750	fi
751
752	#Command depends on targeted file system
753	DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
754	if [ $? = 0 ] ; then
755	mfget ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
756	status=$?
757	else
758	\cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
759	status=$?
760	fi
761
762	# #RSYNC WITH NETWORK RSH CALL
763	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
764	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
765
766	# #RSYNC WITH NFS USE
767	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
768	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
769
770	# status=$?
771	# IGCM_sys_Rsync_out $status
772
773	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
774	# (( status=status+$? ))
775
776	if [ ${status} -gt 0 ] ; then
777	IGCM_debug_Print 2 "IGCM_sys_Get : mfget or cp failed error code ${status}"
778	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
779	IGCM_debug_Exit "IGCM_sys_Get"
780	else
781	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
782	fi
783	fi
784	IGCM_debug_PopStack "IGCM_sys_Get"
785	}
786
787	#D-#==================================================
788	#D-function IGCM_sys_GetDate_Monitoring
789	#D-* Purpose: get the last year for which the monitoring has been computed
790	#D-* Examples:
791	#D-
792	function IGCM_sys_GetDate_Monitoring {
793	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
794	if ( $DEBUG_sys ) ; then
795	echo "IGCM_sys_GetDate_Monitoring :" $@
796	fi
797
798	IGCM_sys_Cd /tmp
799	# NASTY HACK. We force /tmp here. This function is called only from the front-end and interactively.
800	RUN_DIR_PATH=/tmp IGCM_sys_Get ${1} .
801	eval ${2}=$( cdo showyear $( basename ${1} ) 2> /dev/null \| gawk '{ print $NF }' )
802	# Need to erase this file to avoid collision (permission denied getting file) between users on the front-end
803	IGCM_sys_Rm -f $( basename ${1} )
804	IGCM_sys_Cd -
805
806	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
807	}
808
809	#D-#==================================================
810	#D-function IGCM_sys_Put_Dods
811	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
812	#D-* Examples:
813	#D-
814	function IGCM_sys_Put_Dods {
815	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
816	if ( $DEBUG_sys ) ; then
817	echo "IGCM_sys_Put_Dods :" $@
818	fi
819	typeset status
820	if [ $DRYRUN = 0 ]; then
821	# We take our time on that
822	sleep 10
823	IGCM_sys_TestDirArchive ${R_SAVE}/${1}
824	if [ $? != 0 ] ; then
825	echo "WARNING : IGCM_sys_Put_Dods ${R_SAVE}/${1} DOES NOT EXIST ."
826	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
827	return
828	fi
829
830	rsh ${STOREHOST} exec /bin/ksh <<EOF
831	cd ${R_SAVE}
832	/arch/local/bin/dods_rm DODS/pub/${LOGIN}/${R_DODS}/${1} > /dev/null 2>&1
833	/bin/chmod -R u+w ${R_SAVE}/${1}
834	/arch/local/bin/dods_cp ${1} DODS/pub/${LOGIN}/${R_DODS} > /dev/null 2>&1
835	/bin/chmod -R +rX ${R_SAVE}/${1}
836	/bin/chmod -R u+w ${R_SAVE}/${1}
837	EOF
838	status=$?
839
840	if [ ${status} -gt 0 ] ; then
841	echo "IGCM_sys_Put_Dods : error."
842	IGCM_debug_Exit "IGCM_sys_Put_Dods"
843	fi
844	fi
845	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
846	}
847
848	##############################################################
849	# REBUILD OPERATOR
850
851	#D-#==================================================
852	#D-function IGCM_sys_sync
853	#D-* Purpose: flush buffer on disk (dummy function on Ada)
854	#D-* Examples:
855	#D-
856	function IGCM_sys_sync {
857	IGCM_debug_PushStack "IGCM_sys_sync" $@
858	if ( $DEBUG_sys ) ; then
859	echo "IGCM_sys_sync :" $@
860	echo "Dummy call, let the system do that."
861	fi
862	IGCM_debug_PopStack "IGCM_sys_sync"
863	}
864
865	############################################################
866	# Activate Running Environnment Variables
867
868	#D-#==================================================
869	#D-function IGCM_sys_activ_variables
870	#D-* Purpose: set environement variables prior to execution
871	#D-* Examples:
872	#D-
873	function IGCM_sys_activ_variables {
874	IGCM_debug_PushStack "IGCM_sys_activ_variables"
875	if ( $DEBUG_sys ) ; then
876	echo "IGCM_sys_activ_variables"
877	fi
878
879	# --------------------------------------------------------------------
880	#D- MPI specifications
881	# --------------------------------------------------------------------
882
883	# --------------------------------------------------------------------
884	#D- Other specifications
885	# --------------------------------------------------------------------
886
887	ulimit -s unlimited
888	## to be done only one time
889	## export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
890	## echo ${LD_LIBRARY_PATH} \| grep -i netcdf >/dev/null 2>&1 \|\| export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
891	IGCM_debug_Print 1 "set LD_LIBRARY_PATH=${LD_LIBRARY_PATH}"
892
893	export MP_STDOUTMODE=combined
894	IGCM_debug_Print 1 "set MP_STDOUTMODE=${MP_STDOUTMODE}"
895
896	## fix to reduce memory usage. Required since 2014/22/04. On ada IDRIS.
897	#export MP_EUILIBPATH=/smplocal/lib/ibmhpc/pe12012/ppe.pami/gnu/lib64/pami64
898	#IGCM_debug_Print 1 "set MP_EUILIBPATH=${MP_EUILIBPATH}"
899
900	IGCM_debug_PopStack "IGCM_sys_activ_variables"
901	}
902
903	############################################################
904	# Desactivate Running Environnment Variables
905
906	#D-#==================================================
907	#D-function IGCM_sys_desactiv_variables
908	#D-* Purpose: unset environement variables after execution
909	#D-* Examples:
910	#D-
911	function IGCM_sys_desactiv_variables {
912	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
913	if ( $DEBUG_sys ) ; then
914	echo "IGCM_sys_desactiv_variables"
915	fi
916	# --------------------------------------------------------------------
917	#D- MPI specifications
918	# --------------------------------------------------------------------
919
920	# --------------------------------------------------------------------
921	#D- Other specifications
922	# --------------------------------------------------------------------
923
924	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
925	}
926
927	############################################################
928	# Update job headers to be used by the scheduler
929
930	#D-#==================================================
931	#D-function IGCM_sys_updateHeaders
932	#D-* Purpose: Update job headers to be used by the scheduler
933	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
934	#D-
935	function IGCM_sys_updateHeaders {
936	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
937	if ( $DEBUG_sys ) ; then
938	echo "IGCM_sys_updateHeaders"
939	fi
940	typeset file
941	file=$1
942
943	if [ ${executionType} -eq 1 ] ; then
944	# MPMD + MPI
945	if [ X${MPIEnvironment} = XIBM ] ; then
946	sed -e "/::openMPthreads::/d" \
947	-e "s/::JobNumProcTot::/${coreNumber}/" \
948	${file} > ${file}.tmp
949	else
950	sed -e "/::openMPthreads::/d" \
951	-e "s/@ job_type = parallel/@ job_type = mpich/" \
952	-e "s/::JobNumProcTot::/${coreNumber}/" \
953	${file} > ${file}.tmp
954	fi
955
956	elif [ ${executionType} -eq 2 ] ; then
957	# MPMD + MPI + OMP
958	if [ X${MPIEnvironment} = XIBM ] ; then
959	sed -e "s/::openMPthreads::/${openMPthreads}/" \
960	-e "s/::JobNumProcTot::/${mpiTasks}/" \
961	${file} > ${file}.tmp
962	else
963	(( nodeNumber = coreNumber / core_per_node ))
964	[ $(( ${coreNumber} % ${core_per_node} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
965	sed -e "/::openMPthreads::/d" \
966	-e "s/@ job_type = parallel/@ job_type = mpich/" \
967	-e "s/@ total_tasks = ::JobNumProcTot::/@ node = ${nodeNumber} /" \
968	-e "/@ as_limit = 3.5gb/d" \
969	-e "s/::JobNumProcTot::/${mpiTasks}/" \
970	${file} > ${file}.tmp
971	fi
972
973	elif [ ${executionType} -eq 3 ] ; then
974	# SPMD + MPI/OMP
975	if [ X${MPIEnvironment} = XIBM ] ; then
976	sed -e "s/::openMPthreads::/${openMPthreads}/" \
977	-e "s/::JobNumProcTot::/${mpiTasks}/" \
978	${file} > ${file}.tmp
979	else
980	sed -e "s/::openMPthreads::/${openMPthreads}/" \
981	-e "s/@ job_type = parallel/@ job_type = mpich/" \
982	-e "s/::JobNumProcTot::/${mpiTasks}/" \
983	${file} > ${file}.tmp
984	fi
985
986	elif [ ${executionType} -eq 4 ] ; then
987	# SPMD + MPI only
988	if [ X${MPIEnvironment} = XIBM ] ; then
989	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
990	-e "/::openMPthreads::/d" \
991	${file} > ${file}.tmp
992	else
993	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
994	-e "s/@ job_type = parallel/@ job_type = mpich/" \
995	-e "/::openMPthreads::/d" \
996	${file} > ${file}.tmp
997	fi
998
999	elif [ ${executionType} -eq 5 ] ; then
1000	# SPMD + OMP only
1001	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1002	-e "s/@ job_type = parallel/@ job_type = serial/" \
1003	-e "/::JobNumProcTot::/d" \
1004	${file} > ${file}.tmp
1005
1006	elif [ ${executionType} -eq 6 ] ; then
1007	# SEQUENTIAL THEN
1008	sed -e "s/::JobNumProcTot::/1/" \
1009	-e "s/@ job_type = parallel/@ job_type = serial/" \
1010	-e "/::openMPthreads::/d" \
1011	${file} > ${file}.tmp
1012
1013	fi
1014
1015	IGCM_sys_Mv ${file}.tmp ${file}
1016
1017	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1018	}
1019
1020	############################################################
1021	# Build MPI/OMP scripts run file (dummy function)
1022
1023	#D-#==================================================
1024	#D-function IGCM_sys_build_run_file
1025	#D-* Purpose: build run file (deprecated)
1026	#D-* Examples:
1027	#D-
1028	function IGCM_sys_build_run_file {
1029
1030	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1031
1032	}
1033
1034	############################################################
1035	# Build MPI/OMP scripts
1036
1037	#D-#==================================================
1038	#D-function IGCM_sys_build_execution_scripts
1039	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1040	#D-* Examples:
1041	#D-
1042	function IGCM_sys_build_execution_scripts
1043	{
1044	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1045	if ( $DEBUG_sys ) ; then
1046	echo "IGCM_sys_build_execution_scripts " $@
1047	fi
1048
1049	EXECUTION=${HOST_MPIRUN_COMMAND}
1050
1051	# MPMD mode
1052	if ( ${OK_PARA_MPMD} ) ; then
1053
1054	# MPI IBM Environment
1055	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1056	IGCM_debug_Print 1 "You use IBM MPI environment"
1057	if [ -f run_file ] ; then
1058	IGCM_sys_Rm -f run_file
1059	fi
1060	touch run_file
1061
1062	# Build run_file
1063
1064	# First loop on the components for the coupler ie oasis (only if oasis3)
1065	# the coupler ie oasis3 must be the first one
1066	for comp in ${config_ListOfComponents[*]} ; do
1067
1068	eval ExeNameIn=\${config_Executable_${comp}[0]}
1069	eval ExeNameOut=\${config_Executable_${comp}[1]}
1070
1071	# for CPL component only
1072	if [ "X${comp}" = "XCPL" ] ; then
1073
1074	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1075	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1076
1077	if ( ${OK_PARA_MPI} ) ; then
1078	(( mpi_count = 1 ))
1079	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1080	echo "./${ExeNameOut}" >> run_file
1081	(( mpi_count = mpi_count + 1 ))
1082	done
1083	else
1084	echo "./${ExeNameOut} " >> run_file
1085	fi
1086	fi
1087
1088	done
1089
1090	# Then second loop on the components
1091	for comp in ${config_ListOfComponents[*]} ; do
1092
1093	eval ExeNameIn=\${config_Executable_${comp}[0]}
1094	eval ExeNameOut=\${config_Executable_${comp}[1]}
1095
1096	# Only if we really have an executable for the component and not the coupler ie oasis:
1097	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1098
1099	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1100	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1101
1102	if ( ${OK_PARA_OMP} ) ; then
1103	# Check if the number of threads is correct
1104	case ${comp_proc_omp_loc} in
1105	1\|2\|4)
1106	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads on IBM MPI Environment"
1107	IGCM_debug_Print 1 "Beware : it may you use more CPU than needed."
1108	;;
1109	8)
1110	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads on IBM MPI Environment"
1111	IGCM_debug_Print 1 "Beware : it may you use more CPU than needed."
1112	IGCM_debug_Print 1 "Beware : these settings should not be used for IPSLCM coupled model!"
1113	;;
1114	16)
1115	IGCM_debug_Exit "ERROR with OMP parameters !"
1116	IGCM_debug_Print 2 "Beware : ${comp_proc_omp_loc} is too much for MPMD mode"
1117	IGCM_debug_Print 2 "You will use more CPU than needed : try to use Intel-MPI environment to do such a thing !"
1118	IGCM_debug_Verif_Exit
1119	;;
1120	*)
1121	IGCM_debug_Exit "ERROR with OMP parameters !"
1122	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1123	IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1124	IGCM_debug_Verif_Exit
1125	;;
1126	esac
1127
1128	fi
1129
1130	if ( ${OK_PARA_MPI} ) ; then
1131	(( mpi_count = 1 ))
1132	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1133	if ( ${OK_PARA_OMP} ) ; then
1134	echo "env OMP_NUM_THREADS=$comp_proc_omp_loc ./${ExeNameOut}" >> run_file
1135	else
1136	echo "./${ExeNameOut}" >> run_file
1137	fi
1138	(( mpi_count = mpi_count + 1 ))
1139	done
1140	else
1141	echo "./${ExeNameOut} " >> run_file
1142	fi
1143	fi
1144	done
1145	if ( ${OK_PARA_OMP} ) ; then
1146	export KMP_STACKSIZE=200m
1147	fi
1148
1149	EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
1150
1151	IGCM_sys_Chmod u+x run_file
1152	if ( $DEBUG_sys ) ; then
1153	echo "run_file contains : "
1154	cat run_file
1155	fi
1156
1157
1158	# MPI Intel Environment
1159	else
1160	IGCM_debug_Print 1 "You use Intel MPI environment"
1161
1162	# Only MPI (MPMD)
1163	if ( ! ${OK_PARA_OMP} ) ; then
1164	init_exec=n
1165
1166	# First loop on the components for the coupler ie oasis (only if oasis3)
1167	# the coupler ie oasis3 must be the first one
1168	for comp in ${config_ListOfComponents[*]} ; do
1169
1170	eval ExeNameIn=\${config_Executable_${comp}[0]}
1171	eval ExeNameOut=\${config_Executable_${comp}[1]}
1172
1173	# for CPL component only
1174	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1175
1176	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1177	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1178
1179	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1180	echo "" >> script_${ExeNameOut}.ksh
1181	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1182	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1183	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1184	init_exec=y
1185	fi
1186	done
1187	# Then second loop on the components
1188	for comp in ${config_ListOfComponents[*]} ; do
1189
1190	eval ExeNameIn=\${config_Executable_${comp}[0]}
1191	eval ExeNameOut=\${config_Executable_${comp}[1]}
1192
1193	# Only if we really have an executable for the component and not the coupler ie oasis:
1194	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1195
1196	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1197	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1198
1199	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1200	echo "" >> script_${ExeNameOut}.ksh
1201	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1202	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1203
1204	if [ ${init_exec} = y ] ; then
1205	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1206	else
1207	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1208	init_exec=y
1209	fi
1210	fi
1211	done
1212	# MPI-OpenMP (MPMD)
1213	else
1214
1215	# Execution specifications
1216	EXECUTION="${EXECUTION} -configfile run_file"
1217	export KMP_STACKSIZE=200m
1218	if [ -f run_file ] ; then
1219	IGCM_sys_Rm -f run_file
1220	fi
1221	touch run_file
1222
1223	# Initialisation of variables
1224	string_final=""
1225	string_final_hexa=""
1226	current_core=0
1227	executable_nb=1
1228
1229	# Hosts treatment
1230	for nodes in `echo $LOADL_PROCESSOR_LIST`
1231	do
1232	host[$i]=$nodes
1233	i=$((i+1))
1234	done
1235
1236	# Loop on the components
1237	for comp in ${config_ListOfComponents[*]} ; do
1238
1239	eval ExeNameIn=\${config_Executable_${comp}[0]}
1240	eval ExeNameOut=\${config_Executable_${comp}[1]}
1241
1242	# Not possible if oasis has an executable (i.e old version of oasis3)
1243	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1244	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1245	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1246	IGCM_debug_Verif_Exit
1247	fi
1248
1249	# Only if we really have an executable for the component :
1250	if [ "X${ExeNameOut}" != X\"\" ] ; then
1251
1252	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1253	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1254
1255	# Check if the number of threads is correct
1256	case ${comp_proc_omp_loc} in
1257	1\|2\|4\|8\|16)
1258	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1259	;;
1260	*)
1261	IGCM_debug_Exit "ERROR with OMP parameters !"
1262	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1263	IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1264	IGCM_debug_Verif_Exit
1265	;;
1266	esac
1267
1268
1269	# Build run_file for Ada Intel-MPI environment : method used to assign cores and nodes for the MPI process by using hexadecimal mask
1270	# Example of run_file :
1271	#-host host0 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1272	#-host host1 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1273	#-host host2 -n 10 -env I_MPI_PIN_DOMAIN=[3,c,30,c0,300,c00,3000,c000,30000,c0000,100000,200000,400000,800000,1000000,2000000,4000000,8000000,10000000,20000000] ./b.out
1274	#-host host2 -n 10 ./c.out
1275	# Example of final command :
1276	# mpirun -configfile run_file
1277
1278	rank=0
1279	# For one specific executable, loop on mpi process
1280	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1281	(( index_host = current_core / core_per_node ))
1282	host_value=${host[${index_host}]}
1283	(( slot = current_core % core_per_node ))
1284	# loop on omp threads for each mpi process (set the appropriate bit to 1 and append it to previous one)
1285	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1286	string_final="1"$string_final
1287	done
1288	# convert binary mask to hexadecimal mask
1289	if [ $rank -ne 0 ] ; then
1290	string_final_hexa=$string_final_hexa","$( printf '%x\n' "$((2#$string_final))" )
1291	else
1292	string_final_hexa=$( printf '%x\n' "$((2#$string_final))" )
1293	fi
1294	# replace bit 1 by bit 0 in order to append next one (next one wil be 1)
1295	string_final=$( echo $string_final \| sed "s/1/0/g" )
1296	# mpi rank = mpi_rank + 1
1297	(( rank = rank + 1 ))
1298	# current core takes into account the number of omp threads which was previously append
1299	(( current_core = current_core + comp_proc_omp_loc ))
1300	# We write to the configuration file either we switch to the next node or we switch to the next executable
1301	if ( [ $(( current_core / core_per_node )) -ne $index_host ] \|\| [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ) ; then
1302	# I_MPI_PIN_DOMAIN variable must be given once per node
1303	if [ $executable_nb -eq 1 ] ; then
1304	echo "-host $host_value -n $rank -env I_MPI_PIN_DOMAIN=[$string_final_hexa] ./$ExeNameOut" >> run_file
1305	else
1306	sed -i "/$host_value/s/\]/\,$string_final_hexa\]/g" run_file
1307	echo "-host $host_value -n $rank ./$ExeNameOut" >> run_file
1308	fi
1309	# +1 for the number of executbale on the same node
1310	if [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ; then
1311	(( executable_nb = executable_nb + 1 ))
1312	fi
1313	# Some initializations if we switch to the next node
1314	if [ $(( current_core / core_per_node )) -ne $index_host ] ; then
1315	string_final=""
1316	string_final_hexa=""
1317	rank=0
1318	executable_nb=1
1319	fi
1320	fi
1321	done
1322
1323	fi
1324	done
1325	IGCM_sys_Chmod u+x run_file
1326	if ( $DEBUG_sys ) ; then
1327	echo "run_file contains : "
1328	cat run_file
1329	fi
1330	fi
1331	fi
1332
1333	# Only one executable (SPMD mode).
1334	else
1335
1336	for comp in ${config_ListOfComponents[*]} ; do
1337
1338	# Only if we really have an executable for the component :
1339	eval ExeNameOut=\${config_Executable_${comp}[1]}
1340	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1341
1342	if ( ${OK_PARA_OMP} ) ; then
1343	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1344	export KMP_STACKSIZE=200m
1345	#export KMP_LIBRARY=turnaround
1346	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1347	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1348	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1349	fi
1350	if ( ${OK_PARA_MPI} ) ; then
1351	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1352	EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
1353	else
1354	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1355	EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
1356	fi
1357	else
1358	EXECUTION="/usr/bin/time ./${ExeNameOut}"
1359	fi
1360	fi
1361
1362	done
1363
1364	fi
1365
1366	IGCM_debug_Print 1 "sys ada : execution command is "
1367	IGCM_debug_Print 1 "$EXECUTION"
1368
1369	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1370	}
1371
1372	#D-#==================================================
1373	#D-function IGCM_sys_check_path
1374	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1375	#D-* do not point to an important use directory. Stop immediately in that case.
1376	#D-* Examples:
1377	#D-
1378	function IGCM_sys_check_path {
1379	IGCM_debug_PushStack "IGCM_sys_check_path"
1380	if ( $DEBUG_sys ) ; then
1381	echo "IGCM_sys_check_path"
1382	fi
1383
1384	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1385	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1386	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1387	IGCM_debug_Exit "This will stop the job"
1388	fi
1389	IGCM_debug_PopStack "IGCM_sys_check_path"
1390	}
1391
1392	#D-#==================================================
1393	#D-function IGCM_sys_check_quota
1394	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1395	#D-* Examples:
1396	#D-
1397	function IGCM_sys_check_quota {
1398	IGCM_debug_PushStack "IGCM_sys_check_quota"
1399	if ( $DEBUG_sys ) ; then
1400	echo "IGCM_sys_check_quota"
1401	fi
1402	# Limit of quota (in %)
1403	limit_quota=90
1404
1405	# Check of the volume
1406	volume_quota=$(quota_u -w 2>/dev/null\| grep 'Quota soft' \| gawk '{print $5}')
1407	if [ ! X${volume_quota} = X ] ; then
1408	quota_volume=${volume_quota%%\%}
1409	# echo $quota_volume
1410	if [ $(echo "${quota_volume} > ${limit_quota}" \| bc) -eq 1 ] ; then
1411	IGCM_debug_Print 1 "Please, check your quota of volume on workgpfs"
1412	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1413	IGCM_debug_Print 1 "Use the quota_u -w command to check"
1414	IGCM_debug_Print 1 "You must have more than 10% available to run"
1415	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1416	IGCM_debug_Verif_Exit
1417	fi
1418
1419	fi
1420	IGCM_debug_PopStack "IGCM_sys_check_quota"
1421	}
1422
1423	#D-#==================================================
1424	#D-function IGCM_sys_projectAccounting
1425	#D-* Purpose: store project accounting information in a file
1426	#D-* Examples:
1427	#D-
1428	function IGCM_sys_projectAccounting {
1429	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1430	if ( $DEBUG_sys ) ; then
1431	echo "IGCM_sys_check_quota"
1432	fi
1433
1434	cpt > $1
1435
1436	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1437	}
1438
1439	#D-#==================================================
1440	#D-function IGCM_sys_getJobSchedulerID
1441	#D-* Purpose: Get the job ID during execution
1442	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1443	#D-
1444	function IGCM_sys_getJobSchedulerID {
1445	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1446	if ( $DEBUG_sys ) ; then
1447	echo "IGCM_sys_getJobSchedulerID"
1448	fi
1449
1450	eval ${1}=$( echo $LOADL_STEP_ID \| awk -F. '{print $4}' )
1451
1452	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1453	}
1454
1455	#D-#==================================================
1456	#D-function IGCM_sys_GetJobID
1457	#D-* Purpose: Get the job ID from the JobName
1458	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1459	#D-
1460	function IGCM_sys_GetJobID {
1461	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1462	if ( $DEBUG_sys ) ; then
1463	echo "IGCM_sys_GetJobID"
1464	fi
1465
1466	# Print only the full (-W) JobID (%id) and JobName (%jn)
1467	ID=$( llq -u $2 -W -f %id %jn \| \
1468	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1469
1470	eval ${3}=${ID}
1471	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1472	}
1473
1474	#D-#==================================================
1475	#D-function IGCM_sys_CountJobInQueue
1476	#D-* Purpose: Count number of users job
1477	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1478	#D-
1479	function IGCM_sys_CountJobInQueue {
1480	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1481	if ( $DEBUG_sys ) ; then
1482	echo "IGCM_sys_CountJobInQueue"
1483	fi
1484
1485	# Print only the full (-W) JobName (%jn)
1486	NbRun=$( llq -W -f %jn \| grep -c "$1" )
1487
1488	eval ${2}=${NbRun}
1489
1490	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1491	}
1492
1493	#D-#==================================================
1494	#D-function IGCM_sys_ListJobInQueue
1495	#D-* Purpose: Produce a list of users jobs
1496	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1497	#D-
1498	function IGCM_sys_ListJobInQueue {
1499	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1500	if ( $DEBUG_sys ) ; then
1501	echo "IGCM_sys_ListJobInQueue"
1502	fi
1503
1504	# With -W option, column width is as large as necessary
1505	set -A JobList $( llq -u $1 -W -f %jn \| head -n -2 \| tail -n +3 \| \
1506	gawk '( $1 != /TS/ && \
1507	$1 !~ /PACK/ && \
1508	$1 !~ /REBUILD/ && \
1509	$1 !~ /pack/ ) \
1510	{ print $1 }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1511
1512	eval set -A ${2} ${JobList[*]}
1513
1514	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1515	}
1516
1517	#D-#==================================================
1518	#D-function IGCM_sys_atlas
1519	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1520	#D-* Examples:
1521	#D-
1522	function IGCM_sys_atlas {
1523	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1524	if ( $DEBUG_sys ) ; then
1525	echo "IGCM_sys_atlas :" $@
1526	fi
1527
1528	typeset status
1529
1530	\atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1531	status=$?
1532	if [ ${status} -gt 0 ] ; then
1533	echo "IGCM_sys_atlas : error code ${status}"
1534	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1535	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1536	IGCM_debug_PopStack "IGCM_sys_atlas"
1537	return 1
1538	else
1539	IGCM_debug_PopStack "IGCM_sys_atlas"
1540	return 0
1541	fi
1542
1543	IGCM_debug_PopStack "IGCM_sys_atlas"
1544	}
1545

Note: See TracBrowser for help on using the repository browser.

Download in other formats: