Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh @ 1308

Last change on this file since 1308 was 1308, checked in by sdipsl, 8 years ago
Upgrade PCMDI_MP version to have more metrics available. TGCC & IDRIS. See #266
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 53.2 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sébastien Denvil
5	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Ada
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=ada
71	# project name
72	typeset PROJECT=$(echo ${LOADL_STEP_GROUP:=NONE})
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=$( TZ=utc date -d '1970-01-01 '${wall_clock_limit} +%s )
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=llsubmit}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=ergon
89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
93	#====================================================
94	source /smplocal/pub/Modules/default/init/ksh
95	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
96	. /smphome/rech/psl/rpsl035/.atlas_env_ada_intel_2013_0_bash > /dev/null 2>&1
97	else
98	module load intel/2016.2 > /dev/null 2>&1
99	. /smphome/rech/psl/rpsl035/.atlas_env_ada_bash > /dev/null 2>&1
100	PCMDI_MP=/workgpfs/rech/psl/rpsl035/PCMDI-MP
101	export UVCDAT_ANONYMOUS_LOG=no
102	fi
103	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
104	[ ! X${TaskType} = Xchecking ] && module list
105
106	export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddNoise/src_X64_ADA/bin
107	export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddPerturbation/src_X64_ADA/bin
108	export PATH=${PATH}:/smphome/rech/psl/rpsl035/bin/
109
110	#====================================================
111	# Host specific DIRECTORIES
112	#====================================================
113
114	#====================================================
115	#- MirrorlibIGCM for frontend
116	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
117
118	#====================================================
119	#- libIGCM_POST for frontend
120	typeset -r libIGCM_POST=${libIGCM}
121
122	#====================================================
123	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
124	typeset -r R_EXE="${MODIPSL}/bin"
125
126	#====================================================
127	#- SUBMIT_DIR : submission dir
128	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${LOADL_STEP_INITDIR}}
129
130	#====================================================
131	#- IN
132	typeset -r R_IN=${R_IN:=/workgpfs/rech/psl/rpsl035/IGCM}
133	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/u/rech/psl/rpsl376}
134
135	#====================================================
136	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
137	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${TMPDIR}}
138
139	#====================================================
140	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
141	typeset -r OUTCOMMAND_PATH=${RUN_DIR_PATH:-/tmp}
142
143	#====================================================
144	#- HOST_MPIRUN_COMMAND
145	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
146	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
147	else
148	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time mpirun"}
149	fi
150	#====================================================
151	#- Max number of arguments passed to nco operator or demigration command
152	UNIX_MAX_LIMIT=360
153
154	#====================================================
155	#- set PackDefault to true on ada
156	PackDefault=true
157
158	#====================================================
159	#- Number of cores per node
160	core_per_node=32
161
162	#====================================================
163	#- Default number of MPI task for IPSL coupled model
164	#- required for backward compatibility
165	#-
166	DEFAULT_NUM_PROC_OCE=5
167	DEFAULT_NUM_PROC_CPL=1
168	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
169
170	#D-#==================================================
171	#D-function IGCM_sys_defineArchives
172	#D-* Purpose:
173	#D-* Define ARCHIVE : Dedicated to large files
174	#D-* Define STORAGE : Dedicated to small/medium files
175	#D-* Define R_OUT : Output tree located on ARCHIVE
176	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
177	#D-* Define R_BUF : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
178	#D-* if SpaceName=TEST everything is stored on WORKDIR
179	#D-* Examples:
180	#D-
181	function IGCM_sys_defineArchives {
182	IGCM_debug_PushStack "IGCM_sys_defineArchives"
183
184	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
185	#====================================================
186	#- ARCHIVE (dedicated to large files)
187	ARCHIVE=${config_UserChoices_ARCHIVE}
188	else
189	#====================================================
190	#- ARCHIVE (dedicated to large files)
191	ARCHIVE=$(echo ${HOME} \| sed -e "s,/.*home/,/arch/home/,")
192	fi
193
194	if [ ! X${config_UserChoices_STORAGE} = X ]; then
195	#====================================================
196	#- STORAGE (dedicated to small/medium files)
197	STORAGE=${config_UserChoices_STORAGE}
198	else
199	#====================================================
200	#- STORAGE (dedicated to small/medium files)
201	STORAGE=${WORKDIR}
202	fi
203
204	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
205	#====================================================
206	#- R_OUT
207	R_OUT=${WORKDIR}/IGCM_OUT
208
209	#====================================================
210	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
211	R_FIG=${WORKDIR}/IGCM_OUT
212
213	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
214
215	else
216	#====================================================
217	#- R_OUT
218	R_OUT=${ARCHIVE}/IGCM_OUT
219
220	#====================================================
221	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
222	R_FIG=${ARCHIVE}/IGCM_OUT
223	fi
224
225	#====================================================
226	#- R_BUF (ONLY FOR double copy an scratch)
227	R_BUF=${WORKDIR}/IGCM_OUT
228
229	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
230	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
231	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
232
233	IGCM_debug_PopStack "IGCM_sys_defineArchives"
234	}
235
236	#D-#==================================================
237	#D-function IGCM_sys_RshArchive
238	#D-* Purpose: Archive rsh command
239	#D-* Examples:
240	#D-
241	function IGCM_sys_RshArchive {
242	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
243	DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
244	if [ $? = 0 ] ; then
245	rsh ${STOREHOST} exec /bin/ksh <<-EOF
246	${@}
247	EOF
248	status=$?
249	else
250	/bin/ksh <<-EOF
251	${@}
252	EOF
253	status=$?
254	fi
255	if [ ${status} -gt 0 ] ; then
256	IGCM_debug_Print 2 "IGCM_sys_RshArchive : rsh or command failed error code ${status}"
257	IGCM_debug_Exit "IGCM_sys_RshArchive"
258	fi
259	IGCM_debug_PopStack "IGCM_sys_RshArchive"
260	}
261
262	#D-#==================================================
263	#D-function IGCM_sys_RshArchive_NoError
264	#D-* Purpose: Archive rsh command, without error
265	#D-* used only in monitoring.job
266	#D-* Examples:
267	#D-
268	function IGCM_sys_RshArchive_NoError {
269	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
270	DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
271	if [ $? = 0 ] ; then
272	rsh ${STOREHOST} exec /bin/ksh <<-EOF
273	${@} 2> /dev/null
274	EOF
275	else
276	/bin/ksh <<-EOF
277	${@} 2> /dev/null
278	EOF
279	fi
280	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
281	}
282
283	#D-#==================================================
284	#D-function IGCM_sys_MkdirArchive
285	#D-* Purpose: Mkdir on Archive
286	#D-* Examples:
287	#D-
288	function IGCM_sys_MkdirArchive {
289	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
290	if ( $DEBUG_sys ) ; then
291	echo "IGCM_sys_MkdirArchive :" $@
292	fi
293	#- creation de repertoire sur le serveur fichier
294	DEBUG_sys=false IGCM_sys_IsFileArchived $1
295	if [ $? = 0 ] ; then
296	rsh ${STOREHOST} -n mkdir -p $1
297	status=$?
298	else
299	mkdir -p $1
300	status=$?
301	fi
302
303	if [ ${status} -gt 0 ] ; then
304	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : rsh or mkdir failed error code ${status}"
305	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
306	fi
307	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
308	}
309
310	#D-#==================================================
311	#D-function IGCM_sys_TestDirArchive
312	#D-* Purpose: Test Directory that must exists on Archive
313	#D-* Examples:
314	#D-
315	function IGCM_sys_TestDirArchive {
316	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
317	if ( $DEBUG_sys ) ; then
318	echo "IGCM_sys_TestDirArchive :" $@
319	fi
320	typeset ExistFlag
321	#Command depends on targeted file system
322	DEBUG_sys=false IGCM_sys_IsFileArchived $1
323	if [ $? = 0 ] ; then
324	ExistFlag=$( IGCM_sys_RshArchive "[ -d $1 ] && echo 0 \|\| echo 1" )
325	else
326	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
327	fi
328	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
329	return ${ExistFlag}
330	}
331
332	#D-#==================================================
333	#D-function IGCM_sys_IsFileArchived
334	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
335	#D-* Examples:
336	#D-
337	function IGCM_sys_IsFileArchived {
338	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
339	if ( $DEBUG_sys ) ; then
340	echo "IGCM_sys_IsFileArchived :" $@
341	fi
342	typeset IsArchivedFlag
343	IsArchivedFlag=$( [ "X$( echo $@ \| grep \/arch\/home )" != "X" ] && echo 0 \|\| echo 1 )
344	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
345
346	return ${IsArchivedFlag}
347	}
348
349	#D-#==================================================
350	#D-function IGCM_sys_TestFileArchive
351	#D-* Purpose: Test file that must NOT EXISTS on Archive
352	#D-* Examples:
353	#D-
354	function IGCM_sys_TestFileArchive {
355	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
356	typeset ExistFlag
357	ExistFlag=$( IGCM_sys_RshArchive "[ -f $1 ] && echo 0 \|\| echo 1" )
358	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
359
360	return ${ExistFlag}
361	}
362
363	#D-#==================================================
364	#D-function IGCM_sys_CountFileArchive
365	#D-* Purpose: Count files on Archive filesystem
366	#D-* Examples:
367	#D-
368	function IGCM_sys_CountFileArchive {
369	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
370	#Command depends on targeted file system
371	DEBUG_sys=false IGCM_sys_IsFileArchived $1
372	if [ $? = 0 ] ; then
373	IGCM_sys_RshArchive "ls ${@} 2>/dev/null \| wc -l"
374	status=$?
375	else
376	ls ${@} 2>/dev/null \| wc -l
377	status=$?
378	fi
379	if [ ${status} -gt 0 ] ; then
380	echo "IGCM_sys_CountFileArchive : erreur."
381	fi
382	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
383	}
384
385	#D-#==================================================
386	#D-function IGCM_sys_Tree
387	#D-* Purpose: Tree directories with files on ${ARCHIVE}
388	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
389	#D-
390	function IGCM_sys_Tree {
391	IGCM_debug_PushStack "IGCM_sys_Tree" $@
392	if ( $DEBUG_sys ) ; then
393	echo "IGCM_sys_Tree :" $@
394	fi
395
396	\mfls -R $@
397
398	IGCM_debug_PopStack "IGCM_sys_Tree"
399	}
400
401	#D-#==================================================
402	#D-function IGCM_sys_Qsub
403	#D-* Purpose: Qsub new job
404	#D-* Examples:
405	#D-
406	function IGCM_sys_Qsub {
407	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
408	if ( $DEBUG_sys ) ; then
409	echo "IGCM_sys_Qsub :" $@
410	fi
411	typeset status
412
413	# We have to change output/error file
414	[ ${#@} = 1 ] && REP_FOR_JOB=${SUBMIT_DIR}
415	[ ${#@} = 2 ] && REP_FOR_JOB=${2}
416	sed -e "s:\# \@ output = .:\# \@ output = ${Script_Output}:" \
417	-e "s:\# \@ error = .:\# \@ error = ${Script_Output}:" \
418	$1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
419	cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
420
421	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
422	if [ ${status} -gt 0 ] ; then
423	IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
424	IGCM_debug_Exit "IGCM_sys_Qsub"
425	else
426	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ \| tail -1 \| sed -e s/\"//g )
427	IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
428	fi
429	IGCM_debug_PopStack "IGCM_sys_Qsub"
430	}
431
432	#D-#==================================================
433	#D-function IGCM_sys_QsubPost
434	#D-* Purpose: Qsub new job on scalaire
435	#D-* Examples:
436	#D-
437	function IGCM_sys_QsubPost {
438	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
439	if ( $DEBUG_sys ) ; then
440	echo "IGCM_sys_QsubPost :" $@
441	fi
442	typeset status
443
444	# We have to change output/error file
445	[ ${#@} = 1 ] && REP_FOR_JOB=${POST_DIR}
446	[ ${#@} = 2 ] && REP_FOR_JOB=${2}
447
448	sed -e "s:\# \@ output = .:\# \@ output = ${Script_Post_Output}.out:" \
449	-e "s:\# \@ error = .:\# \@ error = ${Script_Post_Output}.out:" \
450	${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
451
452	cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM_$$ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
453
454	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
455	if [ ${status} -gt 0 ] ; then
456	IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
457	IGCM_debug_Exit "IGCM_sys_QsubPost"
458	else
459	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ \| tail -1 \| sed -e s/\"//g )
460	IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
461	fi
462	IGCM_debug_PopStack "IGCM_sys_QsubPost"
463	}
464
465	#D-*************************
466	#D- File transfer functions
467	#D-*************************
468	#D-
469
470	#D-#==================================================
471	#D-function IGCM_sys_RmRunDir
472	#D-* Purpose: rm tmpdir (dummy function most of the time batch
473	#D- scheduler will do the job)
474	#D-* Examples:
475	#D-
476	function IGCM_sys_RmRunDir {
477	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
478	if ( $DEBUG_sys ) ; then
479	echo "IGCM_sys_RmRunDir :" $@
480	IGCM_debug_Print 1 "Dummy call, let the scheduler do that."
481	fi
482	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
483	}
484
485	#D-#==================================================
486	#D-function IGCM_sys_Put_Dir
487	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
488	#D-* Examples:
489	#D-
490	function IGCM_sys_Put_Dir {
491	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
492	if ( $DEBUG_sys ) ; then
493	echo "IGCM_sys_Put_Dir :" $@
494	fi
495	if [ $DRYRUN = 0 ]; then
496	if [ ! -d ${1} ] ; then
497	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
498	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
499	return
500	fi
501
502	typeset status
503
504	# Only if we use rsync
505	#IGCM_sys_TestDirArchive $( dirname $2 )
506	#
507	#Command depends on targeted file system
508	DEBUG_sys=false IGCM_sys_IsFileArchived $2
509	if [ $? = 0 ] ; then
510	\rcp -r $1 ${STOREHOST}:$2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
511	status=$?
512	else
513	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
514	status=$?
515	fi
516
517	if [ ${status} -gt 0 ] ; then
518	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : rcp or cp failed error code ${status}"
519	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
520	IGCM_debug_Exit "IGCM_sys_Put_Dir"
521	else
522	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
523	fi
524	fi
525	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
526	}
527
528	#D-#==================================================
529	#D-function IGCM_sys_Get_Dir
530	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
531	#D-* Examples:
532	#D-
533	function IGCM_sys_Get_Dir {
534	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
535	if ( $DEBUG_sys ) ; then
536	echo "IGCM_sys_Get_Dir :" $@
537	fi
538	if [ $DRYRUN = 0 ]; then
539	typeset NB_ESSAI DELAI status i
540	# number of tentative
541	NB_ESSAI=3
542	# time delay between tentative
543	DELAI=2
544
545	# Only if we use rsync
546	#IGCM_sys_TestDirArchive $( dirname $2 )
547	#
548	# Command depends on targeted filesystem
549	DEBUG_sys=false IGCM_sys_IsFileArchived $1
550	if [ $? = 0 ] ; then
551	# add dmget (to demigrate all offline files) to reduce time of this command :
552	#IGCM_sys_RshArchive "dmfind $1 -state MIG -o -state OFL -o -state PAR \| dmget -q -n"
553	i=0
554	while [ $i -lt $NB_ESSAI ] ; do
555	\rcp -rp ${STOREHOST}:$1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
556	status=$?
557	if [ ${status} -gt 0 ]; then
558	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : rcp failed error code ${status} ${i}/${NB_ESSAI}"
559	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
560	sleep $DELAI
561	else
562	break
563	fi
564	(( i = i + 1 ))
565	done
566	else
567	\cp -rp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
568	status=$?
569	if [ ${status} -gt 0 ] ; then
570	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
571	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
572	IGCM_debug_Exit "IGCM_sys_Get_Dir"
573	else
574	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
575	fi
576	fi
577	if [ ${status} -gt 0 ] ; then
578	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : (r)cp failed error code ${status}"
579	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
580	IGCM_debug_Exit "IGCM_sys_Get_Dir"
581	else
582	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
583	fi
584	fi
585	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
586	}
587
588	#D-#==================================================
589	#D-function IGCM_sys_Put_Rest
590	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
591	#D- File and target directory must exist.
592	#D-* Examples:
593	#D-
594	function IGCM_sys_Put_Rest {
595	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
596	if ( $DEBUG_sys ) ; then
597	echo "IGCM_sys_Put_Rest :" $@
598	fi
599	if [ $DRYRUN = 0 ]; then
600	if [ ! -f ${1} ] ; then
601	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
602	IGCM_debug_Exit "IGCM_sys_Put_Rest"
603	fi
604
605	typeset status
606	#
607	if [ X${JobType} = XRUN ] ; then
608	IGCM_sys_Chmod 444 ${1}
609	fi
610	#
611	#
612	#Command depends on targeted file system
613	DEBUG_sys=false IGCM_sys_IsFileArchived $2
614	if [ $? = 0 ] ; then
615	mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
616	status=$?
617	else
618	IGCM_sys_MkdirArchive $( dirname $2 )
619	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
620	status=$?
621	fi
622
623	# #RSYNC WITH NETWORK RSH CALL
624	# IGCM_sys_MkdirArchive $( dirname $2 )
625	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
626	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
627
628	# #RSYNC WITH NFS USE
629	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
630	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631
632	# status=$?
633	# IGCM_sys_Rsync_out $status
634
635	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
636	# (( status=status+$? ))
637
638	if [ ${status} -gt 0 ] ; then
639	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : mfput or cp failed error code ${status}"
640	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
641	IGCM_debug_Exit "IGCM_sys_Put_Rest"
642	else
643	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
644	fi
645	fi
646	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
647	}
648
649	#D-#==================================================
650	#D-function IGCM_sys_Put_Out
651	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
652	#D-* Examples:
653	#D-
654	function IGCM_sys_Put_Out {
655	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
656	if ( $DEBUG_sys ) ; then
657	echo "IGCM_sys_Put_Out :" $@
658	fi
659
660	typeset status
661
662	if [ $DRYRUN = 0 ]; then
663	if [ ! -f ${1} ] ; then
664	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
665	IGCM_debug_PopStack "IGCM_sys_Put_Out"
666	return 1
667	fi
668	#
669	if [ X${JobType} = XRUN ] ; then
670	if [ X${3} = X ] ; then
671	IGCM_sys_Chmod 444 ${1}
672	fi
673	fi
674	#
675	#
676	#Command depends on targeted file system
677	DEBUG_sys=false IGCM_sys_IsFileArchived $2
678	if [ $? = 0 ] ; then
679	mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
680	status=$?
681	else
682	IGCM_sys_MkdirArchive $( dirname $2 )
683	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
684	status=$?
685	fi
686
687	# #RSYNC WITH NETWORK RSH CALL
688	# IGCM_sys_MkdirArchive $( dirname $2 )
689	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
690	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
691
692	# #RSYNC WITH NFS USE
693	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
694	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
695
696	# status=$?
697	# IGCM_sys_Rsync_out $status
698
699	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
700	# (( status=status+$? ))
701
702	if [ ${status} -gt 0 ] ; then
703	IGCM_debug_Print 2 "IGCM_sys_Put_Out : mfput or cp failed error code ${status}"
704	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
705	IGCM_debug_Exit "IGCM_sys_Put_Out"
706	else
707	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
708	fi
709	fi
710	IGCM_debug_PopStack "IGCM_sys_Put_Out"
711	return 0
712	}
713
714	#D-#==================================================
715	#D-function IGCM_sys_Get
716	#D-* Purpose: Get a file from ${ARCHIVE}
717	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
718	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
719	function IGCM_sys_Get {
720	IGCM_debug_PushStack "IGCM_sys_Get" $@
721
722	typeset DEST status dm_liste
723
724	if ( $DEBUG_sys ) ; then
725	echo "IGCM_sys_Get :" $@
726	fi
727	if [ $DRYRUN -le 2 ]; then
728	if [ X${1} = X'/l' ] ; then
729	eval set +A dm_liste \${${2}}
730	else
731	dm_liste=${1}
732	fi
733	eval DEST=\${${#}}
734
735	# test if the (first) file is present in the old computation :
736	DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
737	if [ $? = 0 ] ; then
738	IGCM_sys_TestFileArchive ${dm_liste[0]}
739	status=$?
740	else
741	IGCM_sys_TestFileBuffer ${dm_liste[0]}
742	status=$?
743	fi
744
745	if [ ${status} -gt 0 ] ; then
746	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
747	IGCM_debug_Exit "IGCM_sys_Get"
748	return
749	fi
750
751	#Command depends on targeted file system
752	DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
753	if [ $? = 0 ] ; then
754	mfget ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
755	status=$?
756	else
757	\cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
758	status=$?
759	fi
760
761	# #RSYNC WITH NETWORK RSH CALL
762	# echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
763	# ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
764
765	# #RSYNC WITH NFS USE
766	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
767	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
768
769	# status=$?
770	# IGCM_sys_Rsync_out $status
771
772	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
773	# (( status=status+$? ))
774
775	if [ ${status} -gt 0 ] ; then
776	IGCM_debug_Print 2 "IGCM_sys_Get : mfget or cp failed error code ${status}"
777	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
778	IGCM_debug_Exit "IGCM_sys_Get"
779	else
780	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
781	fi
782	fi
783	IGCM_debug_PopStack "IGCM_sys_Get"
784	}
785
786	#D-#==================================================
787	#D-function IGCM_sys_GetDate_Monitoring
788	#D-* Purpose: get the last year for which the monitoring has been computed
789	#D-* Examples:
790	#D-
791	function IGCM_sys_GetDate_Monitoring {
792	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
793	if ( $DEBUG_sys ) ; then
794	echo "IGCM_sys_GetDate_Monitoring :" $@
795	fi
796
797	IGCM_sys_Cd /tmp
798	# NASTY HACK. We force /tmp here. This function is called only from the front-end and interactively.
799	RUN_DIR_PATH=/tmp IGCM_sys_Get ${1} .
800	eval ${2}=$( cdo showyear $( basename ${1} ) 2> /dev/null \| gawk '{ print $NF }' )
801	# Need to erase this file to avoid collision (permission denied getting file) between users on the front-end
802	IGCM_sys_Rm -f $( basename ${1} )
803	IGCM_sys_Cd -
804
805	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
806	}
807
808	#D-#==================================================
809	#D-function IGCM_sys_Put_Dods
810	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
811	#D-* Examples:
812	#D-
813	function IGCM_sys_Put_Dods {
814	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
815	if ( $DEBUG_sys ) ; then
816	echo "IGCM_sys_Put_Dods :" $@
817	fi
818	typeset status
819	if [ $DRYRUN = 0 ]; then
820	# We take our time on that
821	sleep 10
822	IGCM_sys_TestDirArchive ${R_SAVE}/${1}
823	if [ $? != 0 ] ; then
824	echo "WARNING : IGCM_sys_Put_Dods ${R_SAVE}/${1} DOES NOT EXIST ."
825	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
826	return
827	fi
828
829	rsh ${STOREHOST} exec /bin/ksh <<EOF
830	cd ${R_SAVE}
831	/arch/local/bin/dods_rm DODS/pub/${LOGIN}/${R_DODS}/${1} > /dev/null 2>&1
832	/bin/chmod -R u+w ${R_SAVE}/${1}
833	/arch/local/bin/dods_cp ${1} DODS/pub/${LOGIN}/${R_DODS} > /dev/null 2>&1
834	/bin/chmod -R +rX ${R_SAVE}/${1}
835	/bin/chmod -R u+w ${R_SAVE}/${1}
836	EOF
837	status=$?
838
839	if [ ${status} -gt 0 ] ; then
840	echo "IGCM_sys_Put_Dods : error."
841	IGCM_debug_Exit "IGCM_sys_Put_Dods"
842	fi
843	fi
844	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
845	}
846
847	##############################################################
848	# REBUILD OPERATOR
849
850	#D-#==================================================
851	#D-function IGCM_sys_sync
852	#D-* Purpose: flush buffer on disk (dummy function on Ada)
853	#D-* Examples:
854	#D-
855	function IGCM_sys_sync {
856	IGCM_debug_PushStack "IGCM_sys_sync" $@
857	if ( $DEBUG_sys ) ; then
858	echo "IGCM_sys_sync :" $@
859	echo "Dummy call, let the system do that."
860	fi
861	IGCM_debug_PopStack "IGCM_sys_sync"
862	}
863
864	############################################################
865	# Activate Running Environnment Variables
866
867	#D-#==================================================
868	#D-function IGCM_sys_activ_variables
869	#D-* Purpose: set environement variables prior to execution
870	#D-* Examples:
871	#D-
872	function IGCM_sys_activ_variables {
873	IGCM_debug_PushStack "IGCM_sys_activ_variables"
874	if ( [ ! "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] && [ ! "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
875	# Temporary modifications for Intel MPI Environment
876	# To use DAPL Provider instead of OFA for inter-node communications
877	# and shm for intra-node communications
878	export I_MPI_FABRICS=shm:dapl
879	# To use UD protocol instead of RC (default with DAPL)
880	export I_MPI_DAPL_UD=1
881	# Some optimizations
882	export DAPL_UCM_REP_TIME=2000 # REQUEST timer, waiting for REPLY in millisecs
883	export DAPL_UCM_RTU_TIME=2000 # REPLY timer, waiting for RTU in millisecs
884	export DAPL_UCM_CQ_SIZE=2000 # CM completion queue
885	export DAPL_UCM_QP_SIZE=2000 # CM message queue
886	export DAPL_UCM_RETRY=7 # REQUEST and REPLY retries
887	export DAPL_ACK_RETRY=7 # IB RC Ack retry count
888	export DAPL_ACK_TIMER=20 # IB RC Ack retry timer
889	# To enable RDMA
890	export I_MPI_DAPL_UD_RDMA_MIXED=enable
891	# To use DAPL 2.1.7
892	export DAT_OVERRIDE=/smplocal/pub/DAPL/2.1.7/etc/dat.conf
893	export LD_LIBRARY_PATH=/smplocal/pub/DAPL/2.1.7/lib:${LD_LIBRARY_PATH}
894	fi
895	if ( $DEBUG_sys ) ; then
896	echo "IGCM_sys_activ_variables"
897	fi
898
899	# --------------------------------------------------------------------
900	#D- MPI specifications
901	# --------------------------------------------------------------------
902
903	# --------------------------------------------------------------------
904	#D- Other specifications
905	# --------------------------------------------------------------------
906
907	ulimit -s unlimited
908	## to be done only one time
909	## export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
910	## echo ${LD_LIBRARY_PATH} \| grep -i netcdf >/dev/null 2>&1 \|\| export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
911	IGCM_debug_Print 1 "set LD_LIBRARY_PATH=${LD_LIBRARY_PATH}"
912
913	export MP_STDOUTMODE=combined
914	IGCM_debug_Print 1 "set MP_STDOUTMODE=${MP_STDOUTMODE}"
915
916	## fix to reduce memory usage. Required since 2014/22/04. On ada IDRIS.
917	#export MP_EUILIBPATH=/smplocal/lib/ibmhpc/pe12012/ppe.pami/gnu/lib64/pami64
918	#IGCM_debug_Print 1 "set MP_EUILIBPATH=${MP_EUILIBPATH}"
919
920	IGCM_debug_PopStack "IGCM_sys_activ_variables"
921	}
922
923	############################################################
924	# Desactivate Running Environnment Variables
925
926	#D-#==================================================
927	#D-function IGCM_sys_desactiv_variables
928	#D-* Purpose: unset environement variables after execution
929	#D-* Examples:
930	#D-
931	function IGCM_sys_desactiv_variables {
932	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
933	if ( $DEBUG_sys ) ; then
934	echo "IGCM_sys_desactiv_variables"
935	fi
936	# --------------------------------------------------------------------
937	#D- MPI specifications
938	# --------------------------------------------------------------------
939
940	# --------------------------------------------------------------------
941	#D- Other specifications
942	# --------------------------------------------------------------------
943
944	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
945	}
946
947	############################################################
948	# Update job headers to be used by the scheduler
949
950	#D-#==================================================
951	#D-function IGCM_sys_updateHeaders
952	#D-* Purpose: Update job headers to be used by the scheduler
953	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
954	#D-
955	function IGCM_sys_updateHeaders {
956	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
957	if ( $DEBUG_sys ) ; then
958	echo "IGCM_sys_updateHeaders"
959	fi
960	typeset file
961	file=$1
962
963	if [ ${executionType} -eq 1 ] ; then
964	# MPMD + MPI
965	if [ X${MPIEnvironment} = XIBM ] ; then
966	sed -e "/::openMPthreads::/d" \
967	-e "s/::JobNumProcTot::/${coreNumber}/" \
968	${file} > ${file}.tmp
969	else
970	sed -e "/::openMPthreads::/d" \
971	-e "s/@ job_type = parallel/@ job_type = mpich/" \
972	-e "s/::JobNumProcTot::/${coreNumber}/" \
973	${file} > ${file}.tmp
974	fi
975
976	elif [ ${executionType} -eq 2 ] ; then
977	# MPMD + MPI + OMP
978	if [ X${MPIEnvironment} = XIBM ] ; then
979	sed -e "s/::openMPthreads::/${openMPthreads}/" \
980	-e "s/::JobNumProcTot::/${mpiTasks}/" \
981	${file} > ${file}.tmp
982	else
983	(( nodeNumber = coreNumber / core_per_node ))
984	[ $(( ${coreNumber} % ${core_per_node} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
985	sed -e "/::openMPthreads::/d" \
986	-e "s/@ job_type = parallel/@ job_type = mpich/" \
987	-e "s/@ total_tasks = ::JobNumProcTot::/@ node = ${nodeNumber} /" \
988	-e "/@ as_limit = 3.5gb/d" \
989	-e "s/::JobNumProcTot::/${mpiTasks}/" \
990	${file} > ${file}.tmp
991	fi
992
993	elif [ ${executionType} -eq 3 ] ; then
994	# SPMD + MPI/OMP
995	if [ X${MPIEnvironment} = XIBM ] ; then
996	sed -e "s/::openMPthreads::/${openMPthreads}/" \
997	-e "s/::JobNumProcTot::/${mpiTasks}/" \
998	${file} > ${file}.tmp
999	else
1000	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1001	-e "s/@ job_type = parallel/@ job_type = mpich/" \
1002	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1003	${file} > ${file}.tmp
1004	fi
1005
1006	elif [ ${executionType} -eq 4 ] ; then
1007	# SPMD + MPI only
1008	if [ X${MPIEnvironment} = XIBM ] ; then
1009	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
1010	-e "/::openMPthreads::/d" \
1011	${file} > ${file}.tmp
1012	else
1013	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
1014	-e "s/@ job_type = parallel/@ job_type = mpich/" \
1015	-e "/::openMPthreads::/d" \
1016	${file} > ${file}.tmp
1017	fi
1018
1019	elif [ ${executionType} -eq 5 ] ; then
1020	# SPMD + OMP only
1021	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1022	-e "s/@ job_type = parallel/@ job_type = serial/" \
1023	-e "/::JobNumProcTot::/d" \
1024	${file} > ${file}.tmp
1025
1026	elif [ ${executionType} -eq 6 ] ; then
1027	# SEQUENTIAL THEN
1028	sed -e "s/::JobNumProcTot::/1/" \
1029	-e "s/@ job_type = parallel/@ job_type = serial/" \
1030	-e "/::openMPthreads::/d" \
1031	${file} > ${file}.tmp
1032
1033	fi
1034
1035	IGCM_sys_Mv ${file}.tmp ${file}
1036
1037	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1038	}
1039
1040	############################################################
1041	# Build MPI/OMP scripts run file (dummy function)
1042
1043	#D-#==================================================
1044	#D-function IGCM_sys_build_run_file
1045	#D-* Purpose: build run file (deprecated)
1046	#D-* Examples:
1047	#D-
1048	function IGCM_sys_build_run_file {
1049
1050	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1051
1052	}
1053
1054	############################################################
1055	# Build MPI/OMP scripts
1056
1057	#D-#==================================================
1058	#D-function IGCM_sys_build_execution_scripts
1059	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1060	#D-* Examples:
1061	#D-
1062	function IGCM_sys_build_execution_scripts
1063	{
1064	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1065	if ( $DEBUG_sys ) ; then
1066	echo "IGCM_sys_build_execution_scripts " $@
1067	fi
1068
1069	EXECUTION=${HOST_MPIRUN_COMMAND}
1070
1071	# MPMD mode
1072	if ( ${OK_PARA_MPMD} ) ; then
1073
1074	# MPI IBM Environment
1075	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1076	IGCM_debug_Print 1 "You use IBM MPI environment"
1077	if [ -f run_file ] ; then
1078	IGCM_sys_Rm -f run_file
1079	fi
1080	touch run_file
1081
1082	# Build run_file
1083
1084	# First loop on the components for the coupler ie oasis (only if oasis3)
1085	# the coupler ie oasis3 must be the first one
1086	for comp in ${config_ListOfComponents[*]} ; do
1087
1088	eval ExeNameIn=\${config_Executable_${comp}[0]}
1089	eval ExeNameOut=\${config_Executable_${comp}[1]}
1090
1091	# for CPL component only
1092	if [ "X${comp}" = "XCPL" ] ; then
1093
1094	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1095	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1096
1097	if ( ${OK_PARA_MPI} ) ; then
1098	(( mpi_count = 1 ))
1099	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1100	echo "./${ExeNameOut}" >> run_file
1101	(( mpi_count = mpi_count + 1 ))
1102	done
1103	else
1104	echo "./${ExeNameOut} " >> run_file
1105	fi
1106	fi
1107
1108	done
1109
1110	# Then second loop on the components
1111	for comp in ${config_ListOfComponents[*]} ; do
1112
1113	eval ExeNameIn=\${config_Executable_${comp}[0]}
1114	eval ExeNameOut=\${config_Executable_${comp}[1]}
1115
1116	# Only if we really have an executable for the component and not the coupler ie oasis:
1117	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1118
1119	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1120	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1121
1122	if ( ${OK_PARA_OMP} ) ; then
1123	# Check if the number of threads is correct
1124	case ${comp_proc_omp_loc} in
1125	1\|2\|4)
1126	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads on IBM MPI Environment"
1127	IGCM_debug_Print 1 "Beware : it may you use more CPU than needed."
1128	IGCM_debug_Print 1 "Think to the Intel MPI Environment to do what you want to do !"
1129	;;
1130	8\|16)
1131	IGCM_debug_Exit "ERROR with OMP parameters !"
1132	IGCM_debug_Print 2 "Beware : ${comp_proc_omp_loc} is too much for MPMD mode"
1133	IGCM_debug_Print 2 "You will use more CPU than needed : try to use Intel-MPI environment to do such a thing !"
1134	IGCM_debug_Verif_Exit
1135	;;
1136	*)
1137	IGCM_debug_Exit "ERROR with OMP parameters !"
1138	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1139	IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1140	IGCM_debug_Verif_Exit
1141	;;
1142	esac
1143
1144	fi
1145
1146	if ( ${OK_PARA_MPI} ) ; then
1147	(( mpi_count = 1 ))
1148	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1149	if ( ${OK_PARA_OMP} ) ; then
1150	echo "env OMP_NUM_THREADS=$comp_proc_omp_loc ./${ExeNameOut}" >> run_file
1151	else
1152	echo "./${ExeNameOut}" >> run_file
1153	fi
1154	(( mpi_count = mpi_count + 1 ))
1155	done
1156	else
1157	echo "./${ExeNameOut} " >> run_file
1158	fi
1159	fi
1160	done
1161	if ( ${OK_PARA_OMP} ) ; then
1162	export KMP_STACKSIZE=200m
1163	fi
1164
1165	EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
1166
1167	IGCM_sys_Chmod u+x run_file
1168	if ( $DEBUG_sys ) ; then
1169	echo "run_file contains : "
1170	cat run_file
1171	fi
1172
1173
1174	# MPI Intel Environment
1175	else
1176	IGCM_debug_Print 1 "You use Intel MPI environment"
1177
1178	# Only MPI (MPMD)
1179	if ( ! ${OK_PARA_OMP} ) ; then
1180	init_exec=n
1181
1182	# First loop on the components for the coupler ie oasis (only if oasis3)
1183	# the coupler ie oasis3 must be the first one
1184	for comp in ${config_ListOfComponents[*]} ; do
1185
1186	eval ExeNameIn=\${config_Executable_${comp}[0]}
1187	eval ExeNameOut=\${config_Executable_${comp}[1]}
1188
1189	# for CPL component only
1190	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1191
1192	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1193	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1194
1195	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1196	echo "" >> script_${ExeNameOut}.ksh
1197	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1198	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1199	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1200	init_exec=y
1201	fi
1202	done
1203	# Then second loop on the components
1204	for comp in ${config_ListOfComponents[*]} ; do
1205
1206	eval ExeNameIn=\${config_Executable_${comp}[0]}
1207	eval ExeNameOut=\${config_Executable_${comp}[1]}
1208
1209	# Only if we really have an executable for the component and not the coupler ie oasis:
1210	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1211
1212	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1213	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1214
1215	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1216	echo "" >> script_${ExeNameOut}.ksh
1217	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1218	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1219
1220	if [ ${init_exec} = y ] ; then
1221	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1222	else
1223	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1224	init_exec=y
1225	fi
1226	fi
1227	done
1228	# MPI-OpenMP (MPMD)
1229	else
1230
1231	# Execution specifications
1232	EXECUTION="${EXECUTION} -configfile run_file"
1233	export KMP_STACKSIZE=200m
1234	if [ -f run_file ] ; then
1235	IGCM_sys_Rm -f run_file
1236	fi
1237	touch run_file
1238
1239	# Initialisation of variables
1240	string_final=""
1241	string_final_hexa=""
1242	current_core=0
1243	executable_nb=1
1244
1245	# Hosts treatment
1246	for nodes in `echo $LOADL_PROCESSOR_LIST`
1247	do
1248	host[$i]=$nodes
1249	i=$((i+1))
1250	done
1251
1252	# Loop on the components
1253	for comp in ${config_ListOfComponents[*]} ; do
1254
1255	eval ExeNameIn=\${config_Executable_${comp}[0]}
1256	eval ExeNameOut=\${config_Executable_${comp}[1]}
1257
1258	# Not possible if oasis has an executable (i.e old version of oasis3)
1259	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1260	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1261	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1262	IGCM_debug_Verif_Exit
1263	fi
1264
1265	# Only if we really have an executable for the component :
1266	if [ "X${ExeNameOut}" != X\"\" ] ; then
1267
1268	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1269	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1270
1271	# Check if the number of threads is correct
1272	case ${comp_proc_omp_loc} in
1273	1\|2\|4\|8\|16)
1274	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1275	;;
1276	*)
1277	IGCM_debug_Exit "ERROR with OMP parameters !"
1278	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1279	IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1280	IGCM_debug_Verif_Exit
1281	;;
1282	esac
1283
1284
1285	# Build run_file for Ada Intel-MPI environment : method used to assign cores and nodes for the MPI process by using hexadecimal mask
1286	# Example of run_file :
1287	#-host host0 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1288	#-host host1 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1289	#-host host2 -n 10 -env I_MPI_PIN_DOMAIN=[3,c,30,c0,300,c00,3000,c000,30000,c0000,100000,200000,400000,800000,1000000,2000000,4000000,8000000,10000000,20000000] ./b.out
1290	#-host host2 -n 10 ./c.out
1291	# Example of final command :
1292	# mpirun -configfile run_file
1293
1294	rank=0
1295	# For one specific executable, loop on mpi process
1296	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1297	(( index_host = current_core / core_per_node ))
1298	host_value=${host[${index_host}]}
1299	(( slot = current_core % core_per_node ))
1300	# loop on omp threads for each mpi process (set the appropriate bit to 1 and append it to previous one)
1301	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1302	string_final="1"$string_final
1303	done
1304	# convert binary mask to hexadecimal mask
1305	if [ $rank -ne 0 ] ; then
1306	string_final_hexa=$string_final_hexa","$( printf '%x\n' "$((2#$string_final))" )
1307	else
1308	string_final_hexa=$( printf '%x\n' "$((2#$string_final))" )
1309	fi
1310	# replace bit 1 by bit 0 in order to append next one (next one wil be 1)
1311	string_final=$( echo $string_final \| sed "s/1/0/g" )
1312	# mpi rank = mpi_rank + 1
1313	(( rank = rank + 1 ))
1314	# current core takes into account the number of omp threads which was previously append
1315	(( current_core = current_core + comp_proc_omp_loc ))
1316	# We write to the configuration file either we switch to the next node or we switch to the next executable
1317	if ( [ $(( current_core / core_per_node )) -ne $index_host ] \|\| [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ) ; then
1318	# I_MPI_PIN_DOMAIN variable must be given once per node
1319	if [ $executable_nb -eq 1 ] ; then
1320	echo "-host $host_value -n $rank -env I_MPI_PIN_DOMAIN=[$string_final_hexa] ./$ExeNameOut" >> run_file
1321	else
1322	sed -i "/$host_value/s/\]/\,$string_final_hexa\]/g" run_file
1323	echo "-host $host_value -n $rank ./$ExeNameOut" >> run_file
1324	fi
1325	# +1 for the number of executbale on the same node
1326	if [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ; then
1327	(( executable_nb = executable_nb + 1 ))
1328	fi
1329	# Some initializations if we switch to the next node
1330	if [ $(( current_core / core_per_node )) -ne $index_host ] ; then
1331	string_final=""
1332	string_final_hexa=""
1333	rank=0
1334	executable_nb=1
1335	fi
1336	fi
1337	done
1338
1339	fi
1340	done
1341	IGCM_sys_Chmod u+x run_file
1342	if ( $DEBUG_sys ) ; then
1343	echo "run_file contains : "
1344	cat run_file
1345	fi
1346	fi
1347	fi
1348
1349	# Only one executable (SPMD mode).
1350	else
1351
1352	for comp in ${config_ListOfComponents[*]} ; do
1353
1354	# Only if we really have an executable for the component :
1355	eval ExeNameOut=\${config_Executable_${comp}[1]}
1356	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1357
1358	if ( ${OK_PARA_OMP} ) ; then
1359	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1360	export KMP_STACKSIZE=200m
1361	#export KMP_LIBRARY=turnaround
1362	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1363	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1364	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1365	fi
1366	if ( ${OK_PARA_MPI} ) ; then
1367	if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] \|\| [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1368	EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
1369	else
1370	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1371	EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
1372	fi
1373	else
1374	EXECUTION="/usr/bin/time ./${ExeNameOut}"
1375	fi
1376	fi
1377
1378	done
1379
1380	fi
1381
1382	IGCM_debug_Print 1 "sys ada : execution command is "
1383	IGCM_debug_Print 1 "$EXECUTION"
1384
1385	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1386	}
1387
1388	#D-#==================================================
1389	#D-function IGCM_sys_check_path
1390	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1391	#D-* do not point to an important use directory. Stop immediately in that case.
1392	#D-* Examples:
1393	#D-
1394	function IGCM_sys_check_path {
1395	IGCM_debug_PushStack "IGCM_sys_check_path"
1396	if ( $DEBUG_sys ) ; then
1397	echo "IGCM_sys_check_path"
1398	fi
1399
1400	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1401	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1402	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1403	IGCM_debug_Exit "This will stop the job"
1404	fi
1405	IGCM_debug_PopStack "IGCM_sys_check_path"
1406	}
1407
1408	#D-#==================================================
1409	#D-function IGCM_sys_check_quota
1410	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1411	#D-* Examples:
1412	#D-
1413	function IGCM_sys_check_quota {
1414	IGCM_debug_PushStack "IGCM_sys_check_quota"
1415	if ( $DEBUG_sys ) ; then
1416	echo "IGCM_sys_check_quota"
1417	fi
1418	# Limit of quota (in %)
1419	limit_quota=90
1420
1421	# Check of the volume
1422	volume_quota=$(quota_u -w 2>/dev/null\| grep 'Quota soft' \| gawk '{print $5}')
1423	if [ ! X${volume_quota} = X ] ; then
1424	quota_volume=${volume_quota%%\%}
1425	# echo $quota_volume
1426	if [ $(echo "${quota_volume} > ${limit_quota}" \| bc) -eq 1 ] ; then
1427	IGCM_debug_Print 1 "Please, check your quota of volume on workgpfs"
1428	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1429	IGCM_debug_Print 1 "Use the quota_u -w command to check"
1430	IGCM_debug_Print 1 "You must have more than 10% available to run"
1431	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1432	IGCM_debug_Verif_Exit
1433	fi
1434
1435	fi
1436	IGCM_debug_PopStack "IGCM_sys_check_quota"
1437	}
1438
1439	#D-#==================================================
1440	#D-function IGCM_sys_projectAccounting
1441	#D-* Purpose: store project accounting information in a file
1442	#D-* Examples:
1443	#D-
1444	function IGCM_sys_projectAccounting {
1445	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1446	if ( $DEBUG_sys ) ; then
1447	echo "IGCM_sys_check_quota"
1448	fi
1449
1450	cpt > $1
1451
1452	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1453	}
1454
1455	#D-#==================================================
1456	#D-function IGCM_sys_getJobSchedulerID
1457	#D-* Purpose: Get the job ID during execution
1458	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1459	#D-
1460	function IGCM_sys_getJobSchedulerID {
1461	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1462	if ( $DEBUG_sys ) ; then
1463	echo "IGCM_sys_getJobSchedulerID"
1464	fi
1465
1466	eval ${1}=$( echo $LOADL_STEP_ID \| awk -F. '{print $4}' )
1467
1468	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1469	}
1470
1471	#D-#==================================================
1472	#D-function IGCM_sys_GetJobID
1473	#D-* Purpose: Get the job ID from the JobName
1474	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1475	#D-
1476	function IGCM_sys_GetJobID {
1477	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1478	if ( $DEBUG_sys ) ; then
1479	echo "IGCM_sys_GetJobID"
1480	fi
1481
1482	# Print only the full (-W) JobID (%id) and JobName (%jn)
1483	ID=$( llq -u $2 -W -f %id %jn \| \
1484	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1485
1486	eval ${3}=${ID}
1487	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1488	}
1489
1490	#D-#==================================================
1491	#D-function IGCM_sys_CountJobInQueue
1492	#D-* Purpose: Count number of users job
1493	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1494	#D-
1495	function IGCM_sys_CountJobInQueue {
1496	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1497	if ( $DEBUG_sys ) ; then
1498	echo "IGCM_sys_CountJobInQueue"
1499	fi
1500
1501	# Print only the full (-W) JobName (%jn)
1502	NbRun=$( llq -W -f %jn \| grep -c "$1" )
1503
1504	eval ${2}=${NbRun}
1505
1506	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1507	}
1508
1509	#D-#==================================================
1510	#D-function IGCM_sys_ListJobInQueue
1511	#D-* Purpose: Produce a list of users jobs
1512	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1513	#D-
1514	function IGCM_sys_ListJobInQueue {
1515	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1516	if ( $DEBUG_sys ) ; then
1517	echo "IGCM_sys_ListJobInQueue"
1518	fi
1519
1520	# With -W option, column width is as large as necessary
1521	set -A JobList $( llq -u $1 -W -f %jn \| head -n -2 \| tail -n +3 \| \
1522	gawk '( $1 != /TS/ && \
1523	$1 !~ /PACK/ && \
1524	$1 !~ /REBUILD/ && \
1525	$1 !~ /pack/ ) \
1526	{ print $1 }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1527
1528	eval set -A ${2} ${JobList[*]}
1529
1530	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1531	}
1532
1533	#D-#==================================================
1534	#D-function IGCM_sys_atlas
1535	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1536	#D-* Examples:
1537	#D-
1538	function IGCM_sys_atlas {
1539	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1540	if ( $DEBUG_sys ) ; then
1541	echo "IGCM_sys_atlas :" $@
1542	fi
1543
1544	typeset status
1545
1546	\atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1547	status=$?
1548	if [ ${status} -gt 0 ] ; then
1549	echo "IGCM_sys_atlas : error code ${status}"
1550	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1551	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1552	IGCM_debug_PopStack "IGCM_sys_atlas"
1553	return 1
1554	else
1555	IGCM_debug_PopStack "IGCM_sys_atlas"
1556	return 0
1557	fi
1558
1559	IGCM_debug_PopStack "IGCM_sys_atlas"
1560	}
1561

Note: See TracBrowser for help on using the repository browser.

Download in other formats: