Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh @ 1338

Last change on this file since 1338 was 1338, checked in by sdipsl, 8 years ago
Call xios_parser.py and ensure field_compname.xml file and file_def_*_companme.xml are coherent. See #256
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 52.2 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sébastien Denvil
5	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Ada
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=ada
71	# project name
72	typeset PROJECT=$(echo ${LOADL_STEP_GROUP:=NONE})
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=$( TZ=utc date -d '1970-01-01 '${wall_clock_limit} +%s )
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=llsubmit}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=ergon
89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
93	#====================================================
94	source /smplocal/pub/Modules/default/init/ksh
95	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
96	. /smphome/rech/psl/rpsl035/.atlas_env_ada_intel_2013_0_bash > /dev/null 2>&1
97	module load python/2.7.10 > /dev/null 2>&1
98	else
99	module load intel/2016.2 > /dev/null 2>&1
100	. /smphome/rech/psl/rpsl035/.atlas_env_ada_bash > /dev/null 2>&1
101	PCMDI_MP=/workgpfs/rech/psl/rpsl035/PCMDI-MP
102	export UVCDAT_ANONYMOUS_LOG=no
103	fi
104	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
105	[ ! X${TaskType} = Xchecking ] && module list
106
107	export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddNoise/src_X64_ADA/bin
108	export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddPerturbation/src_X64_ADA/bin
109	export PATH=${PATH}:/smphome/rech/psl/rpsl035/bin/
110
111	#====================================================
112	# Host specific DIRECTORIES
113	#====================================================
114
115	#====================================================
116	#- MirrorlibIGCM for frontend
117	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
118
119	#====================================================
120	#- libIGCM_POST for frontend
121	typeset -r libIGCM_POST=${libIGCM}
122
123	#====================================================
124	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
125	typeset -r R_EXE="${MODIPSL}/bin"
126
127	#====================================================
128	#- SUBMIT_DIR : submission dir
129	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${LOADL_STEP_INITDIR}}
130
131	#====================================================
132	#- IN
133	typeset -r R_IN=${R_IN:=/workgpfs/rech/psl/rpsl035/IGCM}
134	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/u/rech/psl/rpsl376}
135
136	#====================================================
137	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
138	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${TMPDIR}}
139
140	#====================================================
141	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
142	typeset -r OUTCOMMAND_PATH=${RUN_DIR_PATH:-/tmp}
143
144	#====================================================
145	#- HOST_MPIRUN_COMMAND
146	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
147	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
148	else
149	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time mpirun"}
150	fi
151	#====================================================
152	#- Max number of arguments passed to nco operator or demigration command
153	UNIX_MAX_LIMIT=360
154
155	#====================================================
156	#- set PackDefault to true on ada
157	PackDefault=true
158
159	#====================================================
160	#- Number of cores per node
161	core_per_node=32
162
163	#====================================================
164	#- Default number of MPI task for IPSL coupled model
165	#- required for backward compatibility
166	#-
167	DEFAULT_NUM_PROC_OCE=5
168	DEFAULT_NUM_PROC_CPL=1
169	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
170
171	#D-#==================================================
172	#D-function IGCM_sys_defineArchives
173	#D-* Purpose:
174	#D-* Define ARCHIVE : Dedicated to large files
175	#D-* Define STORAGE : Dedicated to small/medium files
176	#D-* Define R_OUT : Output tree located on ARCHIVE
177	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
178	#D-* Define R_BUF : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
179	#D-* if SpaceName=TEST everything is stored on WORKDIR
180	#D-* Examples:
181	#D-
182	function IGCM_sys_defineArchives {
183	IGCM_debug_PushStack "IGCM_sys_defineArchives"
184
185	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
186	#====================================================
187	#- ARCHIVE (dedicated to large files)
188	ARCHIVE=${config_UserChoices_ARCHIVE}
189	else
190	#====================================================
191	#- ARCHIVE (dedicated to large files)
192	ARCHIVE=$(echo ${HOME} \| sed -e "s,/.*home/,/arch/home/,")
193	fi
194
195	if [ ! X${config_UserChoices_STORAGE} = X ]; then
196	#====================================================
197	#- STORAGE (dedicated to small/medium files)
198	STORAGE=${config_UserChoices_STORAGE}
199	else
200	#====================================================
201	#- STORAGE (dedicated to small/medium files)
202	STORAGE=${WORKDIR}
203	fi
204
205	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
206	#====================================================
207	#- R_OUT
208	R_OUT=${WORKDIR}/IGCM_OUT
209
210	#====================================================
211	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
212	R_FIG=${WORKDIR}/IGCM_OUT
213
214	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
215
216	else
217	#====================================================
218	#- R_OUT
219	R_OUT=${ARCHIVE}/IGCM_OUT
220
221	#====================================================
222	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
223	R_FIG=${ARCHIVE}/IGCM_OUT
224	fi
225
226	#====================================================
227	#- R_BUF (ONLY FOR double copy an scratch)
228	R_BUF=${WORKDIR}/IGCM_OUT
229
230	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
231	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
232	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
233
234	IGCM_debug_PopStack "IGCM_sys_defineArchives"
235	}
236
237	#D-#==================================================
238	#D-function IGCM_sys_RshArchive
239	#D-* Purpose: Archive rsh command
240	#D-* Examples:
241	#D-
242	function IGCM_sys_RshArchive {
243	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
244	DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
245	if [ $? = 0 ] ; then
246	rsh ${STOREHOST} exec /bin/ksh <<-EOF
247	${@}
248	EOF
249	status=$?
250	else
251	/bin/ksh <<-EOF
252	${@}
253	EOF
254	status=$?
255	fi
256	if [ ${status} -gt 0 ] ; then
257	IGCM_debug_Print 2 "IGCM_sys_RshArchive : rsh or command failed error code ${status}"
258	IGCM_debug_Exit "IGCM_sys_RshArchive"
259	fi
260	IGCM_debug_PopStack "IGCM_sys_RshArchive"
261	}
262
263	#D-#==================================================
264	#D-function IGCM_sys_RshArchive_NoError
265	#D-* Purpose: Archive rsh command, without error
266	#D-* used only in monitoring.job
267	#D-* Examples:
268	#D-
269	function IGCM_sys_RshArchive_NoError {
270	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
271	DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
272	if [ $? = 0 ] ; then
273	rsh ${STOREHOST} exec /bin/ksh <<-EOF
274	${@} 2> /dev/null
275	EOF
276	else
277	/bin/ksh <<-EOF
278	${@} 2> /dev/null
279	EOF
280	fi
281	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
282	}
283
284	#D-#==================================================
285	#D-function IGCM_sys_MkdirArchive
286	#D-* Purpose: Mkdir on Archive
287	#D-* Examples:
288	#D-
289	function IGCM_sys_MkdirArchive {
290	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
291	if ( $DEBUG_sys ) ; then
292	echo "IGCM_sys_MkdirArchive :" $@
293	fi
294	#- creation de repertoire sur le serveur fichier
295	DEBUG_sys=false IGCM_sys_IsFileArchived $1
296	if [ $? = 0 ] ; then
297	rsh ${STOREHOST} -n mkdir -p $1
298	status=$?
299	else
300	mkdir -p $1
301	status=$?
302	fi
303
304	if [ ${status} -gt 0 ] ; then
305	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : rsh or mkdir failed error code ${status}"
306	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
307	fi
308	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
309	}
310
311	#D-#==================================================
312	#D-function IGCM_sys_TestDirArchive
313	#D-* Purpose: Test Directory that must exists on Archive
314	#D-* Examples:
315	#D-
316	function IGCM_sys_TestDirArchive {
317	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
318	if ( $DEBUG_sys ) ; then
319	echo "IGCM_sys_TestDirArchive :" $@
320	fi
321	typeset ExistFlag
322	#Command depends on targeted file system
323	DEBUG_sys=false IGCM_sys_IsFileArchived $1
324	if [ $? = 0 ] ; then
325	ExistFlag=$( IGCM_sys_RshArchive "[ -d $1 ] && echo 0 \|\| echo 1" )
326	else
327	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
328	fi
329	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
330	return ${ExistFlag}
331	}
332
333	#D-#==================================================
334	#D-function IGCM_sys_IsFileArchived
335	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
336	#D-* Examples:
337	#D-
338	function IGCM_sys_IsFileArchived {
339	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
340	if ( $DEBUG_sys ) ; then
341	echo "IGCM_sys_IsFileArchived :" $@
342	fi
343	typeset IsArchivedFlag
344	IsArchivedFlag=$( [ "X$( echo $@ \| grep \/arch\/home )" != "X" ] && echo 0 \|\| echo 1 )
345	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
346
347	return ${IsArchivedFlag}
348	}
349
350	#D-#==================================================
351	#D-function IGCM_sys_TestFileArchive
352	#D-* Purpose: Test file that must NOT EXISTS on Archive
353	#D-* Examples:
354	#D-
355	function IGCM_sys_TestFileArchive {
356	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
357	typeset ExistFlag
358	ExistFlag=$( IGCM_sys_RshArchive "[ -f $1 ] && echo 0 \|\| echo 1" )
359	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
360
361	return ${ExistFlag}
362	}
363
364	#D-#==================================================
365	#D-function IGCM_sys_CountFileArchive
366	#D-* Purpose: Count files on Archive filesystem
367	#D-* Examples:
368	#D-
369	function IGCM_sys_CountFileArchive {
370	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
371	#Command depends on targeted file system
372	DEBUG_sys=false IGCM_sys_IsFileArchived $1
373	if [ $? = 0 ] ; then
374	IGCM_sys_RshArchive "ls ${@} 2>/dev/null \| wc -l"
375	status=$?
376	else
377	ls ${@} 2>/dev/null \| wc -l
378	status=$?
379	fi
380	if [ ${status} -gt 0 ] ; then
381	echo "IGCM_sys_CountFileArchive : erreur."
382	fi
383	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
384	}
385
386	#D-#==================================================
387	#D-function IGCM_sys_Tree
388	#D-* Purpose: Tree directories with files on ${ARCHIVE}
389	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
390	#D-
391	function IGCM_sys_Tree {
392	IGCM_debug_PushStack "IGCM_sys_Tree" $@
393	if ( $DEBUG_sys ) ; then
394	echo "IGCM_sys_Tree :" $@
395	fi
396
397	\mfls -R $@
398
399	IGCM_debug_PopStack "IGCM_sys_Tree"
400	}
401
402	#D-#==================================================
403	#D-function IGCM_sys_Qsub
404	#D-* Purpose: Qsub new job
405	#D-* Examples:
406	#D-
407	function IGCM_sys_Qsub {
408	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
409	if ( $DEBUG_sys ) ; then
410	echo "IGCM_sys_Qsub :" $@
411	fi
412	typeset status
413
414	# We have to change output/error file
415	[ ${#@} = 1 ] && REP_FOR_JOB=${SUBMIT_DIR}
416	[ ${#@} = 2 ] && REP_FOR_JOB=${2}
417	sed -e "s:\# \@ output = .:\# \@ output = ${Script_Output}:" \
418	-e "s:\# \@ error = .:\# \@ error = ${Script_Output}:" \
419	$1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
420	cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
421
422	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
423	if [ ${status} -gt 0 ] ; then
424	IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
425	IGCM_debug_Exit "IGCM_sys_Qsub"
426	else
427	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ \| tail -1 \| sed -e s/\"//g )
428	IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
429	fi
430	IGCM_debug_PopStack "IGCM_sys_Qsub"
431	}
432
433	#D-#==================================================
434	#D-function IGCM_sys_QsubPost
435	#D-* Purpose: Qsub new job on scalaire
436	#D-* Examples:
437	#D-
438	function IGCM_sys_QsubPost {
439	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
440	if ( $DEBUG_sys ) ; then
441	echo "IGCM_sys_QsubPost :" $@
442	fi
443	typeset status
444
445	# We have to change output/error file
446	[ ${#@} = 1 ] && REP_FOR_JOB=${POST_DIR}
447	[ ${#@} = 2 ] && REP_FOR_JOB=${2}
448
449	sed -e "s:\# \@ output = .:\# \@ output = ${Script_Post_Output}.out:" \
450	-e "s:\# \@ error = .:\# \@ error = ${Script_Post_Output}.out:" \
451	${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
452
453	cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM_$$ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
454
455	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
456	if [ ${status} -gt 0 ] ; then
457	IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
458	IGCM_debug_Exit "IGCM_sys_QsubPost"
459	else
460	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ \| tail -1 \| sed -e s/\"//g )
461	IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
462	fi
463	IGCM_debug_PopStack "IGCM_sys_QsubPost"
464	}
465
466	#D-*************************
467	#D- File transfer functions
468	#D-*************************
469	#D-
470
471	#D-#==================================================
472	#D-function IGCM_sys_RmRunDir
473	#D-* Purpose: rm tmpdir (dummy function most of the time batch
474	#D- scheduler will do the job)
475	#D-* Examples:
476	#D-
477	function IGCM_sys_RmRunDir {
478	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
479	if ( $DEBUG_sys ) ; then
480	echo "IGCM_sys_RmRunDir :" $@
481	IGCM_debug_Print 1 "Dummy call, let the scheduler do that."
482	fi
483	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
484	}
485
486	#D-#==================================================
487	#D-function IGCM_sys_Put_Dir
488	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
489	#D-* Examples:
490	#D-
491	function IGCM_sys_Put_Dir {
492	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
493	if ( $DEBUG_sys ) ; then
494	echo "IGCM_sys_Put_Dir :" $@
495	fi
496	if [ $DRYRUN = 0 ]; then
497	if [ ! -d ${1} ] ; then
498	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
499	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
500	return
501	fi
502
503	typeset status
504
505	# Only if we use rsync
506	#IGCM_sys_TestDirArchive $( dirname $2 )
507	#
508	#Command depends on targeted file system
509	DEBUG_sys=false IGCM_sys_IsFileArchived $2
510	if [ $? = 0 ] ; then
511	\rcp -r $1 ${STOREHOST}:$2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
512	status=$?
513	else
514	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
515	status=$?
516	fi
517
518	if [ ${status} -gt 0 ] ; then
519	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : rcp or cp failed error code ${status}"
520	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
521	IGCM_debug_Exit "IGCM_sys_Put_Dir"
522	else
523	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
524	fi
525	fi
526	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
527	}
528
529	#D-#==================================================
530	#D-function IGCM_sys_Get_Dir
531	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
532	#D-* Examples:
533	#D-
534	function IGCM_sys_Get_Dir {
535	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
536	if ( $DEBUG_sys ) ; then
537	echo "IGCM_sys_Get_Dir :" $@
538	fi
539	if [ $DRYRUN = 0 ]; then
540	typeset NB_ESSAI DELAI status i
541	# number of tentative
542	NB_ESSAI=3
543	# time delay between tentative
544	DELAI=2
545
546	# Only if we use rsync
547	#IGCM_sys_TestDirArchive $( dirname $2 )
548	#
549	# Command depends on targeted filesystem
550	DEBUG_sys=false IGCM_sys_IsFileArchived $1
551	if [ $? = 0 ] ; then
552	# add dmget (to demigrate all offline files) to reduce time of this command :
553	#IGCM_sys_RshArchive "dmfind $1 -state MIG -o -state OFL -o -state PAR \| dmget -q -n"
554	i=0
555	while [ $i -lt $NB_ESSAI ] ; do
556	\rcp -rp ${STOREHOST}:$1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
557	status=$?
558	if [ ${status} -gt 0 ]; then
559	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : rcp failed error code ${status} ${i}/${NB_ESSAI}"
560	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
561	sleep $DELAI
562	else
563	break
564	fi
565	(( i = i + 1 ))
566	done
567	else
568	\cp -rp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
569	status=$?
570	if [ ${status} -gt 0 ] ; then
571	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
572	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
573	IGCM_debug_Exit "IGCM_sys_Get_Dir"
574	else
575	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
576	fi
577	fi
578	if [ ${status} -gt 0 ] ; then
579	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : (r)cp failed error code ${status}"
580	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
581	IGCM_debug_Exit "IGCM_sys_Get_Dir"
582	else
583	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
584	fi
585	fi
586	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
587	}
588
589	#D-#==================================================
590	#D-function IGCM_sys_Put_Rest
591	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
592	#D- File and target directory must exist.
593	#D-* Examples:
594	#D-
595	function IGCM_sys_Put_Rest {
596	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
597	if ( $DEBUG_sys ) ; then
598	echo "IGCM_sys_Put_Rest :" $@
599	fi
600	if [ $DRYRUN = 0 ]; then
601	if [ ! -f ${1} ] ; then
602	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
603	IGCM_debug_Exit "IGCM_sys_Put_Rest"
604	fi
605
606	typeset status
607	#
608	if [ X${JobType} = XRUN ] ; then
609	IGCM_sys_Chmod 444 ${1}
610	fi
611	#
612	#
613	#Command depends on targeted file system
614	DEBUG_sys=false IGCM_sys_IsFileArchived $2
615	if [ $? = 0 ] ; then
616	mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
617	status=$?
618	else
619	IGCM_sys_MkdirArchive $( dirname $2 )
620	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
621	status=$?
622	fi
623
624	# #RSYNC WITH NETWORK RSH CALL
625	# IGCM_sys_MkdirArchive $( dirname $2 )
626	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
627	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
628
629	# #RSYNC WITH NFS USE
630	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
632
633	# status=$?
634	# IGCM_sys_Rsync_out $status
635
636	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
637	# (( status=status+$? ))
638
639	if [ ${status} -gt 0 ] ; then
640	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : mfput or cp failed error code ${status}"
641	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
642	IGCM_debug_Exit "IGCM_sys_Put_Rest"
643	else
644	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
645	fi
646	fi
647	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
648	}
649
650	#D-#==================================================
651	#D-function IGCM_sys_Put_Out
652	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
653	#D-* Examples:
654	#D-
655	function IGCM_sys_Put_Out {
656	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
657	if ( $DEBUG_sys ) ; then
658	echo "IGCM_sys_Put_Out :" $@
659	fi
660
661	typeset status
662
663	if [ $DRYRUN = 0 ]; then
664	if [ ! -f ${1} ] ; then
665	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
666	IGCM_debug_PopStack "IGCM_sys_Put_Out"
667	return 1
668	fi
669	#
670	if [ X${JobType} = XRUN ] ; then
671	if [ X${3} = X ] ; then
672	IGCM_sys_Chmod 444 ${1}
673	fi
674	fi
675	#
676	#
677	#Command depends on targeted file system
678	DEBUG_sys=false IGCM_sys_IsFileArchived $2
679	if [ $? = 0 ] ; then
680	mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
681	status=$?
682	else
683	IGCM_sys_MkdirArchive $( dirname $2 )
684	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
685	status=$?
686	fi
687
688	# #RSYNC WITH NETWORK RSH CALL
689	# IGCM_sys_MkdirArchive $( dirname $2 )
690	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
691	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
692
693	# #RSYNC WITH NFS USE
694	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
695	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
696
697	# status=$?
698	# IGCM_sys_Rsync_out $status
699
700	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
701	# (( status=status+$? ))
702
703	if [ ${status} -gt 0 ] ; then
704	IGCM_debug_Print 2 "IGCM_sys_Put_Out : mfput or cp failed error code ${status}"
705	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
706	IGCM_debug_Exit "IGCM_sys_Put_Out"
707	else
708	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
709	fi
710	fi
711	IGCM_debug_PopStack "IGCM_sys_Put_Out"
712	return 0
713	}
714
715	#D-#==================================================
716	#D-function IGCM_sys_Get
717	#D-* Purpose: Get a file from ${ARCHIVE}
718	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
719	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
720	function IGCM_sys_Get {
721	IGCM_debug_PushStack "IGCM_sys_Get" $@
722
723	typeset DEST status dm_liste
724
725	if ( $DEBUG_sys ) ; then
726	echo "IGCM_sys_Get :" $@
727	fi
728	if [ $DRYRUN -le 2 ]; then
729	if [ X${1} = X'/l' ] ; then
730	eval set +A dm_liste \${${2}}
731	else
732	dm_liste=${1}
733	fi
734	eval DEST=\${${#}}
735
736	# test if the (first) file is present in the old computation :
737	DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
738	if [ $? = 0 ] ; then
739	IGCM_sys_TestFileArchive ${dm_liste[0]}
740	status=$?
741	else
742	IGCM_sys_TestFileBuffer ${dm_liste[0]}
743	status=$?
744	fi
745
746	if [ ${status} -gt 0 ] ; then
747	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
748	IGCM_debug_Exit "IGCM_sys_Get"
749	return
750	fi
751
752	#Command depends on targeted file system
753	DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
754	if [ $? = 0 ] ; then
755	mfget ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
756	status=$?
757	else
758	\cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
759	status=$?
760	fi
761
762	# #RSYNC WITH NETWORK RSH CALL
763	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
764	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
765
766	# #RSYNC WITH NFS USE
767	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
768	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
769
770	# status=$?
771	# IGCM_sys_Rsync_out $status
772
773	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
774	# (( status=status+$? ))
775
776	if [ ${status} -gt 0 ] ; then
777	IGCM_debug_Print 2 "IGCM_sys_Get : mfget or cp failed error code ${status}"
778	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
779	IGCM_debug_Exit "IGCM_sys_Get"
780	else
781	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
782	fi
783	fi
784	IGCM_debug_PopStack "IGCM_sys_Get"
785	}
786
787	#D-#==================================================
788	#D-function IGCM_sys_GetDate_Monitoring
789	#D-* Purpose: get the last year for which the monitoring has been computed
790	#D-* Examples:
791	#D-
792	function IGCM_sys_GetDate_Monitoring {
793	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
794	if ( $DEBUG_sys ) ; then
795	echo "IGCM_sys_GetDate_Monitoring :" $@
796	fi
797
798	IGCM_sys_Cd /tmp
799	# NASTY HACK. We force /tmp here. This function is called only from the front-end and interactively.
800	RUN_DIR_PATH=/tmp IGCM_sys_Get ${1} .
801	eval ${2}=$( cdo showyear $( basename ${1} ) 2> /dev/null \| gawk '{ print $NF }' )
802	# Need to erase this file to avoid collision (permission denied getting file) between users on the front-end
803	IGCM_sys_Rm -f $( basename ${1} )
804	IGCM_sys_Cd -
805
806	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
807	}
808
809	#D-#==================================================
810	#D-function IGCM_sys_Put_Dods
811	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
812	#D-* Examples:
813	#D-
814	function IGCM_sys_Put_Dods {
815	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
816	if ( $DEBUG_sys ) ; then
817	echo "IGCM_sys_Put_Dods :" $@
818	fi
819	typeset status
820	if [ $DRYRUN = 0 ]; then
821	# We take our time on that
822	sleep 10
823	IGCM_sys_TestDirArchive ${R_SAVE}/${1}
824	if [ $? != 0 ] ; then
825	echo "WARNING : IGCM_sys_Put_Dods ${R_SAVE}/${1} DOES NOT EXIST ."
826	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
827	return
828	fi
829
830	rsh ${STOREHOST} exec /bin/ksh <<EOF
831	cd ${R_SAVE}
832	/arch/local/bin/dods_rm DODS/pub/${LOGIN}/${R_DODS}/${1} > /dev/null 2>&1
833	/bin/chmod -R u+w ${R_SAVE}/${1}
834	/arch/local/bin/dods_cp ${1} DODS/pub/${LOGIN}/${R_DODS} > /dev/null 2>&1
835	/bin/chmod -R +rX ${R_SAVE}/${1}
836	/bin/chmod -R u+w ${R_SAVE}/${1}
837	EOF
838	status=$?
839
840	if [ ${status} -gt 0 ] ; then
841	echo "IGCM_sys_Put_Dods : error."
842	IGCM_debug_Exit "IGCM_sys_Put_Dods"
843	fi
844	fi
845	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
846	}
847
848	##############################################################
849	# REBUILD OPERATOR
850
851	#D-#==================================================
852	#D-function IGCM_sys_sync
853	#D-* Purpose: flush buffer on disk (dummy function on Ada)
854	#D-* Examples:
855	#D-
856	function IGCM_sys_sync {
857	IGCM_debug_PushStack "IGCM_sys_sync" $@
858	if ( $DEBUG_sys ) ; then
859	echo "IGCM_sys_sync :" $@
860	echo "Dummy call, let the system do that."
861	fi
862	IGCM_debug_PopStack "IGCM_sys_sync"
863	}
864
865	############################################################
866	# Activate Running Environnment Variables
867
868	#D-#==================================================
869	#D-function IGCM_sys_activ_variables
870	#D-* Purpose: set environement variables prior to execution
871	#D-* Examples:
872	#D-
873	function IGCM_sys_activ_variables {
874	IGCM_debug_PushStack "IGCM_sys_activ_variables"
875	if ( $DEBUG_sys ) ; then
876	echo "IGCM_sys_activ_variables"
877	fi
878
879	# --------------------------------------------------------------------
880	#D- MPI specifications
881	# --------------------------------------------------------------------
882
883	# --------------------------------------------------------------------
884	#D- Other specifications
885	# --------------------------------------------------------------------
886
887	ulimit -s unlimited
888	## to be done only one time
889	## export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
890	## echo ${LD_LIBRARY_PATH} \| grep -i netcdf >/dev/null 2>&1 \|\| export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
891	IGCM_debug_Print 1 "set LD_LIBRARY_PATH=${LD_LIBRARY_PATH}"
892
893	export MP_STDOUTMODE=combined
894	IGCM_debug_Print 1 "set MP_STDOUTMODE=${MP_STDOUTMODE}"
895
896	## fix to reduce memory usage. Required since 2014/22/04. On ada IDRIS.
897	#export MP_EUILIBPATH=/smplocal/lib/ibmhpc/pe12012/ppe.pami/gnu/lib64/pami64
898	#IGCM_debug_Print 1 "set MP_EUILIBPATH=${MP_EUILIBPATH}"
899
900	IGCM_debug_PopStack "IGCM_sys_activ_variables"
901	}
902
903	############################################################
904	# Desactivate Running Environnment Variables
905
906	#D-#==================================================
907	#D-function IGCM_sys_desactiv_variables
908	#D-* Purpose: unset environement variables after execution
909	#D-* Examples:
910	#D-
911	function IGCM_sys_desactiv_variables {
912	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
913	if ( $DEBUG_sys ) ; then
914	echo "IGCM_sys_desactiv_variables"
915	fi
916	# --------------------------------------------------------------------
917	#D- MPI specifications
918	# --------------------------------------------------------------------
919
920	# --------------------------------------------------------------------
921	#D- Other specifications
922	# --------------------------------------------------------------------
923
924	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
925	}
926
927	############################################################
928	# Update job headers to be used by the scheduler
929
930	#D-#==================================================
931	#D-function IGCM_sys_updateHeaders
932	#D-* Purpose: Update job headers to be used by the scheduler
933	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
934	#D-
935	function IGCM_sys_updateHeaders {
936	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
937	if ( $DEBUG_sys ) ; then
938	echo "IGCM_sys_updateHeaders"
939	fi
940	typeset file
941	file=$1
942
943	if [ ${executionType} -eq 1 ] ; then
944	# MPMD + MPI
945	if [ X${MPIEnvironment} = XIBM ] ; then
946	sed -e "/::openMPthreads::/d" \
947	-e "s/::JobNumProcTot::/${coreNumber}/" \
948	${file} > ${file}.tmp
949	else
950	sed -e "/::openMPthreads::/d" \
951	-e "s/@ job_type = parallel/@ job_type = mpich/" \
952	-e "s/::JobNumProcTot::/${coreNumber}/" \
953	${file} > ${file}.tmp
954	fi
955
956	elif [ ${executionType} -eq 2 ] ; then
957	# MPMD + MPI + OMP
958	if [ X${MPIEnvironment} = XIBM ] ; then
959	sed -e "s/::openMPthreads::/${openMPthreads}/" \
960	-e "s/::JobNumProcTot::/${mpiTasks}/" \
961	${file} > ${file}.tmp
962	else
963	(( nodeNumber = coreNumber / core_per_node ))
964	[ $(( ${coreNumber} % ${core_per_node} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
965	sed -e "/::openMPthreads::/d" \
966	-e "s/@ job_type = parallel/@ job_type = mpich/" \
967	-e "s/@ total_tasks = ::JobNumProcTot::/@ node = ${nodeNumber} /" \
968	-e "/@ as_limit = 3.5gb/d" \
969	-e "s/::JobNumProcTot::/${mpiTasks}/" \
970	${file} > ${file}.tmp
971	fi
972
973	elif [ ${executionType} -eq 3 ] ; then
974	# SPMD + MPI/OMP
975	if [ X${MPIEnvironment} = XIBM ] ; then
976	sed -e "s/::openMPthreads::/${openMPthreads}/" \
977	-e "s/::JobNumProcTot::/${mpiTasks}/" \
978	${file} > ${file}.tmp
979	else
980	sed -e "s/::openMPthreads::/${openMPthreads}/" \
981	-e "s/@ job_type = parallel/@ job_type = mpich/" \
982	-e "s/::JobNumProcTot::/${mpiTasks}/" \
983	${file} > ${file}.tmp
984	fi
985
986	elif [ ${executionType} -eq 4 ] ; then
987	# SPMD + MPI only
988	if [ X${MPIEnvironment} = XIBM ] ; then
989	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
990	-e "/::openMPthreads::/d" \
991	${file} > ${file}.tmp
992	else
993	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
994	-e "s/@ job_type = parallel/@ job_type = mpich/" \
995	-e "/::openMPthreads::/d" \
996	${file} > ${file}.tmp
997	fi
998
999	elif [ ${executionType} -eq 5 ] ; then
1000	# SPMD + OMP only
1001	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1002	-e "s/@ job_type = parallel/@ job_type = serial/" \
1003	-e "/::JobNumProcTot::/d" \
1004	${file} > ${file}.tmp
1005
1006	elif [ ${executionType} -eq 6 ] ; then
1007	# SEQUENTIAL THEN
1008	sed -e "s/::JobNumProcTot::/1/" \
1009	-e "s/@ job_type = parallel/@ job_type = serial/" \
1010	-e "/::openMPthreads::/d" \
1011	${file} > ${file}.tmp
1012
1013	fi
1014
1015	IGCM_sys_Mv ${file}.tmp ${file}
1016
1017	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1018	}
1019
1020	############################################################
1021	# Build MPI/OMP scripts run file (dummy function)
1022
1023	#D-#==================================================
1024	#D-function IGCM_sys_build_run_file
1025	#D-* Purpose: build run file (deprecated)
1026	#D-* Examples:
1027	#D-
1028	function IGCM_sys_build_run_file {
1029
1030	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1031
1032	}
1033
1034	############################################################
1035	# Build MPI/OMP scripts
1036
1037	#D-#==================================================
1038	#D-function IGCM_sys_build_execution_scripts
1039	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1040	#D-* Examples:
1041	#D-
1042	function IGCM_sys_build_execution_scripts
1043	{
1044	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1045	if ( $DEBUG_sys ) ; then
1046	echo "IGCM_sys_build_execution_scripts " $@
1047	fi
1048
1049	EXECUTION=${HOST_MPIRUN_COMMAND}
1050
1051	# MPMD mode
1052	if ( ${OK_PARA_MPMD} ) ; then
1053
1054	# MPI IBM Environment
1055	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1056	IGCM_debug_Print 1 "You use IBM MPI environment"
1057	if [ -f run_file ] ; then
1058	IGCM_sys_Rm -f run_file
1059	fi
1060	touch run_file
1061
1062	# Build run_file
1063
1064	# First loop on the components for the coupler ie oasis (only if oasis3)
1065	# the coupler ie oasis3 must be the first one
1066	for comp in ${config_ListOfComponents[*]} ; do
1067
1068	eval ExeNameIn=\${config_Executable_${comp}[0]}
1069	eval ExeNameOut=\${config_Executable_${comp}[1]}
1070
1071	# for CPL component only
1072	if [ "X${comp}" = "XCPL" ] ; then
1073
1074	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1075	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1076
1077	if ( ${OK_PARA_MPI} ) ; then
1078	(( mpi_count = 1 ))
1079	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1080	echo "./${ExeNameOut}" >> run_file
1081	(( mpi_count = mpi_count + 1 ))
1082	done
1083	else
1084	echo "./${ExeNameOut} " >> run_file
1085	fi
1086	fi
1087
1088	done
1089
1090	# Then second loop on the components
1091	for comp in ${config_ListOfComponents[*]} ; do
1092
1093	eval ExeNameIn=\${config_Executable_${comp}[0]}
1094	eval ExeNameOut=\${config_Executable_${comp}[1]}
1095
1096	# Only if we really have an executable for the component and not the coupler ie oasis:
1097	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1098
1099	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1100	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1101
1102	if ( ${OK_PARA_OMP} ) ; then
1103	# Check if the number of threads is correct
1104	case ${comp_proc_omp_loc} in
1105	1\|2\|4)
1106	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads on IBM MPI Environment"
1107	IGCM_debug_Print 1 "Beware : it may you use more CPU than needed."
1108	IGCM_debug_Print 1 "Think to the Intel MPI Environment to do what you want to do !"
1109	;;
1110	8\|16)
1111	IGCM_debug_Exit "ERROR with OMP parameters !"
1112	IGCM_debug_Print 2 "Beware : ${comp_proc_omp_loc} is too much for MPMD mode"
1113	IGCM_debug_Print 2 "You will use more CPU than needed : try to use Intel-MPI environment to do such a thing !"
1114	IGCM_debug_Verif_Exit
1115	;;
1116	*)
1117	IGCM_debug_Exit "ERROR with OMP parameters !"
1118	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1119	IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1120	IGCM_debug_Verif_Exit
1121	;;
1122	esac
1123
1124	fi
1125
1126	if ( ${OK_PARA_MPI} ) ; then
1127	(( mpi_count = 1 ))
1128	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1129	if ( ${OK_PARA_OMP} ) ; then
1130	echo "env OMP_NUM_THREADS=$comp_proc_omp_loc ./${ExeNameOut}" >> run_file
1131	else
1132	echo "./${ExeNameOut}" >> run_file
1133	fi
1134	(( mpi_count = mpi_count + 1 ))
1135	done
1136	else
1137	echo "./${ExeNameOut} " >> run_file
1138	fi
1139	fi
1140	done
1141	if ( ${OK_PARA_OMP} ) ; then
1142	export KMP_STACKSIZE=200m
1143	fi
1144
1145	EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
1146
1147	IGCM_sys_Chmod u+x run_file
1148	if ( $DEBUG_sys ) ; then
1149	echo "run_file contains : "
1150	cat run_file
1151	fi
1152
1153
1154	# MPI Intel Environment
1155	else
1156	IGCM_debug_Print 1 "You use Intel MPI environment"
1157
1158	# Only MPI (MPMD)
1159	if ( ! ${OK_PARA_OMP} ) ; then
1160	init_exec=n
1161
1162	# First loop on the components for the coupler ie oasis (only if oasis3)
1163	# the coupler ie oasis3 must be the first one
1164	for comp in ${config_ListOfComponents[*]} ; do
1165
1166	eval ExeNameIn=\${config_Executable_${comp}[0]}
1167	eval ExeNameOut=\${config_Executable_${comp}[1]}
1168
1169	# for CPL component only
1170	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1171
1172	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1173	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1174
1175	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1176	echo "" >> script_${ExeNameOut}.ksh
1177	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1178	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1179	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1180	init_exec=y
1181	fi
1182	done
1183	# Then second loop on the components
1184	for comp in ${config_ListOfComponents[*]} ; do
1185
1186	eval ExeNameIn=\${config_Executable_${comp}[0]}
1187	eval ExeNameOut=\${config_Executable_${comp}[1]}
1188
1189	# Only if we really have an executable for the component and not the coupler ie oasis:
1190	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1191
1192	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1193	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1194
1195	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1196	echo "" >> script_${ExeNameOut}.ksh
1197	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1198	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1199
1200	if [ ${init_exec} = y ] ; then
1201	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1202	else
1203	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1204	init_exec=y
1205	fi
1206	fi
1207	done
1208	# MPI-OpenMP (MPMD)
1209	else
1210
1211	# Execution specifications
1212	EXECUTION="${EXECUTION} -configfile run_file"
1213	export KMP_STACKSIZE=200m
1214	if [ -f run_file ] ; then
1215	IGCM_sys_Rm -f run_file
1216	fi
1217	touch run_file
1218
1219	# Initialisation of variables
1220	string_final=""
1221	string_final_hexa=""
1222	current_core=0
1223	executable_nb=1
1224
1225	# Hosts treatment
1226	for nodes in `echo $LOADL_PROCESSOR_LIST`
1227	do
1228	host[$i]=$nodes
1229	i=$((i+1))
1230	done
1231
1232	# Loop on the components
1233	for comp in ${config_ListOfComponents[*]} ; do
1234
1235	eval ExeNameIn=\${config_Executable_${comp}[0]}
1236	eval ExeNameOut=\${config_Executable_${comp}[1]}
1237
1238	# Not possible if oasis has an executable (i.e old version of oasis3)
1239	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1240	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1241	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1242	IGCM_debug_Verif_Exit
1243	fi
1244
1245	# Only if we really have an executable for the component :
1246	if [ "X${ExeNameOut}" != X\"\" ] ; then
1247
1248	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1249	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1250
1251	# Check if the number of threads is correct
1252	case ${comp_proc_omp_loc} in
1253	1\|2\|4\|8\|16)
1254	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1255	;;
1256	*)
1257	IGCM_debug_Exit "ERROR with OMP parameters !"
1258	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1259	IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1260	IGCM_debug_Verif_Exit
1261	;;
1262	esac
1263
1264
1265	# Build run_file for Ada Intel-MPI environment : method used to assign cores and nodes for the MPI process by using hexadecimal mask
1266	# Example of run_file :
1267	#-host host0 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1268	#-host host1 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1269	#-host host2 -n 10 -env I_MPI_PIN_DOMAIN=[3,c,30,c0,300,c00,3000,c000,30000,c0000,100000,200000,400000,800000,1000000,2000000,4000000,8000000,10000000,20000000] ./b.out
1270	#-host host2 -n 10 ./c.out
1271	# Example of final command :
1272	# mpirun -configfile run_file
1273
1274	rank=0
1275	# For one specific executable, loop on mpi process
1276	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1277	(( index_host = current_core / core_per_node ))
1278	host_value=${host[${index_host}]}
1279	(( slot = current_core % core_per_node ))
1280	# loop on omp threads for each mpi process (set the appropriate bit to 1 and append it to previous one)
1281	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1282	string_final="1"$string_final
1283	done
1284	# convert binary mask to hexadecimal mask
1285	if [ $rank -ne 0 ] ; then
1286	string_final_hexa=$string_final_hexa","$( printf '%x\n' "$((2#$string_final))" )
1287	else
1288	string_final_hexa=$( printf '%x\n' "$((2#$string_final))" )
1289	fi
1290	# replace bit 1 by bit 0 in order to append next one (next one wil be 1)
1291	string_final=$( echo $string_final \| sed "s/1/0/g" )
1292	# mpi rank = mpi_rank + 1
1293	(( rank = rank + 1 ))
1294	# current core takes into account the number of omp threads which was previously append
1295	(( current_core = current_core + comp_proc_omp_loc ))
1296	# We write to the configuration file either we switch to the next node or we switch to the next executable
1297	if ( [ $(( current_core / core_per_node )) -ne $index_host ] \|\| [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ) ; then
1298	# I_MPI_PIN_DOMAIN variable must be given once per node
1299	if [ $executable_nb -eq 1 ] ; then
1300	echo "-host $host_value -n $rank -env I_MPI_PIN_DOMAIN=[$string_final_hexa] ./$ExeNameOut" >> run_file
1301	else
1302	sed -i "/$host_value/s/\]/\,$string_final_hexa\]/g" run_file
1303	echo "-host $host_value -n $rank ./$ExeNameOut" >> run_file
1304	fi
1305	# +1 for the number of executbale on the same node
1306	if [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ; then
1307	(( executable_nb = executable_nb + 1 ))
1308	fi
1309	# Some initializations if we switch to the next node
1310	if [ $(( current_core / core_per_node )) -ne $index_host ] ; then
1311	string_final=""
1312	string_final_hexa=""
1313	rank=0
1314	executable_nb=1
1315	fi
1316	fi
1317	done
1318
1319	fi
1320	done
1321	IGCM_sys_Chmod u+x run_file
1322	if ( $DEBUG_sys ) ; then
1323	echo "run_file contains : "
1324	cat run_file
1325	fi
1326	fi
1327	fi
1328
1329	# Only one executable (SPMD mode).
1330	else
1331
1332	for comp in ${config_ListOfComponents[*]} ; do
1333
1334	# Only if we really have an executable for the component :
1335	eval ExeNameOut=\${config_Executable_${comp}[1]}
1336	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1337
1338	if ( ${OK_PARA_OMP} ) ; then
1339	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1340	export KMP_STACKSIZE=200m
1341	#export KMP_LIBRARY=turnaround
1342	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1343	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1344	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1345	fi
1346	if ( ${OK_PARA_MPI} ) ; then
1347	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1348	EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
1349	else
1350	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1351	EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
1352	fi
1353	else
1354	EXECUTION="/usr/bin/time ./${ExeNameOut}"
1355	fi
1356	fi
1357
1358	done
1359
1360	fi
1361
1362	IGCM_debug_Print 1 "sys ada : execution command is "
1363	IGCM_debug_Print 1 "$EXECUTION"
1364
1365	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1366	}
1367
1368	#D-#==================================================
1369	#D-function IGCM_sys_check_path
1370	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1371	#D-* do not point to an important use directory. Stop immediately in that case.
1372	#D-* Examples:
1373	#D-
1374	function IGCM_sys_check_path {
1375	IGCM_debug_PushStack "IGCM_sys_check_path"
1376	if ( $DEBUG_sys ) ; then
1377	echo "IGCM_sys_check_path"
1378	fi
1379
1380	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1381	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1382	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1383	IGCM_debug_Exit "This will stop the job"
1384	fi
1385	IGCM_debug_PopStack "IGCM_sys_check_path"
1386	}
1387
1388	#D-#==================================================
1389	#D-function IGCM_sys_check_quota
1390	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1391	#D-* Examples:
1392	#D-
1393	function IGCM_sys_check_quota {
1394	IGCM_debug_PushStack "IGCM_sys_check_quota"
1395	if ( $DEBUG_sys ) ; then
1396	echo "IGCM_sys_check_quota"
1397	fi
1398	# Limit of quota (in %)
1399	limit_quota=90
1400
1401	# Check of the volume
1402	volume_quota=$(quota_u -w 2>/dev/null\| grep 'Quota soft' \| gawk '{print $5}')
1403	if [ ! X${volume_quota} = X ] ; then
1404	quota_volume=${volume_quota%%\%}
1405	# echo $quota_volume
1406	if [ $(echo "${quota_volume} > ${limit_quota}" \| bc) -eq 1 ] ; then
1407	IGCM_debug_Print 1 "Please, check your quota of volume on workgpfs"
1408	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1409	IGCM_debug_Print 1 "Use the quota_u -w command to check"
1410	IGCM_debug_Print 1 "You must have more than 10% available to run"
1411	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1412	IGCM_debug_Verif_Exit
1413	fi
1414
1415	fi
1416	IGCM_debug_PopStack "IGCM_sys_check_quota"
1417	}
1418
1419	#D-#==================================================
1420	#D-function IGCM_sys_projectAccounting
1421	#D-* Purpose: store project accounting information in a file
1422	#D-* Examples:
1423	#D-
1424	function IGCM_sys_projectAccounting {
1425	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1426	if ( $DEBUG_sys ) ; then
1427	echo "IGCM_sys_check_quota"
1428	fi
1429
1430	cpt > $1
1431
1432	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1433	}
1434
1435	#D-#==================================================
1436	#D-function IGCM_sys_getJobSchedulerID
1437	#D-* Purpose: Get the job ID during execution
1438	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1439	#D-
1440	function IGCM_sys_getJobSchedulerID {
1441	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1442	if ( $DEBUG_sys ) ; then
1443	echo "IGCM_sys_getJobSchedulerID"
1444	fi
1445
1446	eval ${1}=$( echo $LOADL_STEP_ID \| awk -F. '{print $4}' )
1447
1448	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1449	}
1450
1451	#D-#==================================================
1452	#D-function IGCM_sys_GetJobID
1453	#D-* Purpose: Get the job ID from the JobName
1454	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1455	#D-
1456	function IGCM_sys_GetJobID {
1457	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1458	if ( $DEBUG_sys ) ; then
1459	echo "IGCM_sys_GetJobID"
1460	fi
1461
1462	# Print only the full (-W) JobID (%id) and JobName (%jn)
1463	ID=$( llq -u $2 -W -f %id %jn \| \
1464	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1465
1466	eval ${3}=${ID}
1467	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1468	}
1469
1470	#D-#==================================================
1471	#D-function IGCM_sys_CountJobInQueue
1472	#D-* Purpose: Count number of users job
1473	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1474	#D-
1475	function IGCM_sys_CountJobInQueue {
1476	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1477	if ( $DEBUG_sys ) ; then
1478	echo "IGCM_sys_CountJobInQueue"
1479	fi
1480
1481	# Print only the full (-W) JobName (%jn)
1482	NbRun=$( llq -W -f %jn \| grep -c "$1" )
1483
1484	eval ${2}=${NbRun}
1485
1486	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1487	}
1488
1489	#D-#==================================================
1490	#D-function IGCM_sys_ListJobInQueue
1491	#D-* Purpose: Produce a list of users jobs
1492	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1493	#D-
1494	function IGCM_sys_ListJobInQueue {
1495	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1496	if ( $DEBUG_sys ) ; then
1497	echo "IGCM_sys_ListJobInQueue"
1498	fi
1499
1500	# With -W option, column width is as large as necessary
1501	set -A JobList $( llq -u $1 -W -f %jn \| head -n -2 \| tail -n +3 \| \
1502	gawk '( $1 != /TS/ && \
1503	$1 !~ /PACK/ && \
1504	$1 !~ /REBUILD/ && \
1505	$1 !~ /pack/ ) \
1506	{ print $1 }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1507
1508	eval set -A ${2} ${JobList[*]}
1509
1510	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1511	}
1512
1513	#D-#==================================================
1514	#D-function IGCM_sys_atlas
1515	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1516	#D-* Examples:
1517	#D-
1518	function IGCM_sys_atlas {
1519	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1520	if ( $DEBUG_sys ) ; then
1521	echo "IGCM_sys_atlas :" $@
1522	fi
1523
1524	typeset status
1525
1526	\atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1527	status=$?
1528	if [ ${status} -gt 0 ] ; then
1529	echo "IGCM_sys_atlas : error code ${status}"
1530	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1531	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1532	IGCM_debug_PopStack "IGCM_sys_atlas"
1533	return 1
1534	else
1535	IGCM_debug_PopStack "IGCM_sys_atlas"
1536	return 0
1537	fi
1538
1539	IGCM_debug_PopStack "IGCM_sys_atlas"
1540	}
1541

Note: See TracBrowser for help on using the repository browser.

Download in other formats: