Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_obelix.ksh @ 933

Last change on this file since 933 was 933, checked in by sdipsl, 11 years ago
properly remove RUN_DIR_PATH used by AA_job on curie. see #141 but add some check to be sure RUN_DIR_PATH do not point to important directory. Indeed within AA_Job you can overwrite RUN_DIR_PATH and define something like this: RUN_DIR_PATH=$SCRATCHDIR You don't want to erase your $SCRATCHDIR don't you
Property svn:keywords set to `Revision Author Date`
File size: 70.0 KB

Rev	Line
[657]	1	#!/bin/ksh
[16]	2
	3	#**************************************************************
	4	# Author: Sebastien Denvil, Martial Mancip, Christian Laguerre
[373]	5	# Contact: Martial.Mancip__at__ipsl.jussieu.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
[16]	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
[657]	11	#
[16]	12	#**************************************************************
	13
	14	#=========================================================
	15	# The documentation of this file can be automatically generated
[657]	16	# if you use the prefix #D- for comments to be extracted.
[16]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#=========================================================
	19
	20	#D-#==================================================
	21	#D-LibIGCM_sys for obelix
	22	#D-#==================================================
	23	#D-
[657]	24	#D- This ksh library if a layer under some usefull
[16]	25	#D-environment variables and shell commands.
	26	#D-All those definitions depend on host particularities.
	27	#D-It manages a stack mechanism and test validity of operations.
	28	#D-All function described bellow must be prefixed by IGCM_sys.
	29
	30	#====================================================
	31	# libIGCM_sys PARAMETERS
	32	#====================================================
	33
[343]	34	#====================================================
[16]	35	# set DEBUG_sys to true to output calls of function
	36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
	37
[343]	38	#====================================================
[16]	39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
	40	typeset -r DRYRUN=${DRYRUN:=0}
	41
	42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
	43	# -------------------------------------------------------------------------------------
	44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
[809]	45	# \| \| Cp/Exe/param/files \| \| \| \|
	46	# \| \| Chmod Qsub \| \| \| \|
[16]	47	# -------------------------------------------------------------------------------------
	48	# \| 0 \| yes \| yes \| yes \| yes \|
	49	# -------------------------------------------------------------------------------------
	50	# \| 1 \| yes \| yes \| yes \| no \|
	51	# -------------------------------------------------------------------------------------
	52	# \| 2 \| yes \| yes \| no \| no \|
	53	# -------------------------------------------------------------------------------------
	54	# \| 3 \| yes \| no \| no \| no \|
	55	# -------------------------------------------------------------------------------------
	56
	57	#=====================================================
	58	# Global Variables :
	59	#=====================================================
	60	# Language : "fr" or "en"
	61	typeset -r MYLANG="fr"
	62
	63	#=====================================================
[657]	64	# Host and user names
[217]	65	# $hostname ou hostname
[800]	66	typeset HOST=${HOST:=$( hostname )}
[16]	67	# $username ou whoami
[800]	68	typeset LOGIN=${LOGIN:=$( whoami )}
[657]	69	# $hostname of the MASTER job
[742]	70	typeset MASTER=${MASTER:=$( hostname )}
[16]	71
	72	#D-
	73	#D-#==================================================
	74	#D-Program used in libIGCM
	75	#D-#==================================================
	76
[742]	77	# Submit command
	78	typeset SUBMIT=${SUBMIT:=qsub}
[16]	79	# rsync with path
	80	typeset -r RSYNC=/usr/bin/rsync
	81	# RSYNC_opt args to rsync
[282]	82	typeset -r RSYNC_opt="-va"
[657]	83	# ie storage filesystem
	84	typeset -r STOREHOST=${MASTER}
[16]	85
	86	#====================================================
[305]	87	# Set environment tools (ferret, nco, cdo)
	88	#====================================================
[16]	89	. /home/users/brock/.atlas_env_asterix_ksh
	90
[343]	91	#====================================================
[421]	92	# Set lf95 environment
	93	#====================================================
	94	. /usr/local/install/lf6481/bash_laheyfort_setup
	95
	96	#====================================================
[16]	97	# Host specific DIRECTORIES
	98	#====================================================
	99
	100	#====================================================
[657]	101	#- MirrorlibIGCM for frontend
	102	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	103
	104	#====================================================
	105	#- libIGCM_POST for frontend
	106	typeset -r libIGCM_POST=${libIGCM}
	107
	108	#====================================================
[16]	109	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
	110	typeset -r R_EXE="${MODIPSL}/bin"
	111
	112	#====================================================
	113	#- SUBMIT_DIR : submission dir
[256]	114	if [ X${PBS_O_WORKDIR} != X ] ; then
[657]	115	typeset -r SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
[256]	116	else
[657]	117	typeset -r SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
[256]	118	fi
[657]	119
[16]	120	#====================================================
[725]	121	#- IN
	122	typeset -r R_IN=${R_IN:=/home/orchidee01/mmancip/IGCM}
	123
	124	#====================================================
[811]	125	#- ARCHIVE
[800]	126	typeset ARCHIVE=${ARCHIVE:=/home/scratch01/${LOGIN}}
[16]	127
[725]	128	#====================================================
	129	#- STORAGE (dedicated to small/medium files)
[800]	130	typeset STORAGE=${ARCHIVE}
[282]	131
	132	#====================================================
[717]	133	#- R_OUT
[800]	134	typeset R_OUT=${ARCHIVE}/IGCM_OUT
[16]	135
	136	#====================================================
[657]	137	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[800]	138	typeset R_FIG=${STORAGE}/IGCM_OUT
[657]	139
	140	#====================================================
	141	#- R_BUF (ONLY FOR double copy an scratch)
[800]	142	typeset -r R_BUF=${STORAGE}/IGCM_OUT
[657]	143
	144	#====================================================
[16]	145	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[838]	146	if [ X${PBS_JOBID} != X ] ; then
	147	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/scratch/$PBS_O_LOGNAME.$PBS_JOBID}
	148	else
[839]	149	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/tmp/tmp$$}
[838]	150	fi
[16]	151
	152	#====================================================
	153	#- HOST_MPIRUN_COMMAND
	154	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
	155
[61]	156	#====================================================
	157	#- Max number of arguments passed to nco operator or demigration command
	158	UNIX_MAX_LIMIT=360
	159
[657]	160	#====================================================
	161	#- Number of core per node (max number of OpenMP task)
	162	NUM_COREPERNODE=1
	163
	164	#====================================================
	165	#- Default number of MPI task for IPSL coupled model
	166	#- required for backward compatibility
	167	#-
	168	DEFAULT_NUM_PROC_OCE=1
	169	DEFAULT_NUM_PROC_CPL=1
	170	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
	171	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
	172
[16]	173	#D-#==================================================
[725]	174	#D-function IGCM_sys_ChangeArchive
	175	#D-* Purpose: Just a dummy call on this machine
	176	#D-* Examples:
	177	#D-
	178	function IGCM_sys_ChangeArchive {
	179	IGCM_debug_Print 1 " dummy function : IGCM_sys_ChangeArchive "
	180	}
	181
	182	#D-#==================================================
[16]	183	#D-function IGCM_sys_RshMaster
[800]	184	#D-* Purpose: Connection to frontend machine.
[16]	185	#D-* Examples:
	186	#D-
	187	function IGCM_sys_RshMaster {
[657]	188	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
[862]	189	/bin/ksh <<-EOF
[16]	190	export libIGCM=${libIGCM}
[98]	191	export DEBUG_debug=${DEBUG_debug}
[16]	192	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
	193	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
	194	${@}
	195	EOF
[657]	196	if [ $? -gt 0 ] ; then
	197	echo "IGCM_sys_RshMaster : erreur."
	198	IGCM_debug_Exit "IGCM_sys_RshMaster"
	199	fi
	200	IGCM_debug_PopStack "IGCM_sys_RshMaster"
[16]	201	}
	202
	203	#D-#==================================================
[37]	204	#D-function IGCM_sys_RshArchive
	205	#D-* Purpose: Archive rsh command
	206	#D-* Examples:
	207	#D-
	208	function IGCM_sys_RshArchive {
[657]	209	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
	210	/bin/ksh <<-EOF
[40]	211	${@}
	212	EOF
[811]	213	status=$?
	214	if [ ${status} -gt 0 ] ; then
	215	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
[657]	216	IGCM_debug_Exit "IGCM_sys_RshArchive"
	217	fi
	218	IGCM_debug_PopStack "IGCM_sys_RshArchive"
[37]	219	}
	220
	221	#D-#==================================================
[16]	222	#D-function IGCM_sys_RshPost
[282]	223	#D-* Purpose: Post-process rsh command
[16]	224	#D-* Examples:
	225	#D-
	226	function IGCM_sys_RshPost {
[657]	227	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
	228	if ( $DEBUG_sys ) ; then
	229	echo "IGCM_sys_RshPost :" $@
	230	fi
[800]	231
	232	# keep standard input (stdin) for the loop onto temporary file
	233	cat >tmp_IGCM_sys_RshPost_$$
	234
	235	# ============ FRONTEND START ============ #
	236
[862]	237	/bin/ksh <tmp_IGCM_sys_RshPost_$$
[657]	238	if [ $? -gt 0 ] ; then
	239	echo "IGCM_sys_RshPost : erreur."
	240	IGCM_debug_Exit "IGCM_sys_RshPost"
	241	fi
[800]	242	# delete temporary file
	243	\rm tmp_IGCM_sys_RshPost_$$
[657]	244	IGCM_debug_PopStack "IGCM_sys_RshPost"
[16]	245	}
	246
	247	#D-#==================================================
[153]	248	#D-function IGCM_sys_SendMail
	249	#D-* Purpose: Send mail when simulation is over
	250	#D-* Examples:
	251	#D-
	252	function IGCM_sys_SendMail {
[657]	253	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
	254	if ( $DEBUG_sys ) ; then
	255	echo "IGCM_sys_SendMail :" $@
	256	fi
[153]	257
[922]	258	if [ X${1} = XAccounting ] ; then
	259	status=Accounting
	260	mailText=jobAccounting.mail
	261	elif ( ${ExitFlag} ) ; then
[657]	262	status=failed
[922]	263	mailText=jobEnd.mail
[657]	264	else
	265	status=completed
[922]	266	mailText=jobEnd.mail
[657]	267	fi
[717]	268
[922]	269	# Update selected mail template
	270	while read -r line; do
	271	eval echo $line >> mail.txt ;
	272	done < ${libIGCM}/libIGCM_sys/${mailText}
[153]	273
[657]	274	if [ ! -z ${config_UserChoices_MailName} ] ; then
[922]	275	mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < mail.txt
[657]	276	elif [ -f ~/.forward ] ; then
[922]	277	mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < mail.txt
[657]	278	fi
[153]	279
[811]	280	sleep 10
[922]	281	rm -f mail.txt
[811]	282
[657]	283	IGCM_debug_PopStack "IGCM_sys_SendMail"
[153]	284	}
	285
	286	#D-#==================================================
[16]	287	#D-function IGCM_sys_Mkdir
	288	#D-* Purpose: Master locale mkdir command
	289	#D-* Examples:
	290	#D-
	291	function IGCM_sys_Mkdir {
[657]	292	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
	293	if ( $DEBUG_sys ) ; then
	294	echo "IGCM_sys_Mkdir :" $@
	295	fi
	296	if [ ! -d ${1} ]; then
	297	\mkdir -p $1
	298	if [ $? -gt 0 ] ; then
	299	echo "IGCM_sys_Mkdir : erreur."
	300	IGCM_debug_Exit "IGCM_sys_Mkdir"
[16]	301	fi
[657]	302	fi
[717]	303	# vérification :
[657]	304	if [ ! -d ${1} ] ; then
	305	echo "IGCM_sys_Mkdir : erreur."
	306	IGCM_debug_Exit "IGCM_sys_Mkdir"
	307	fi
	308	IGCM_debug_PopStack "IGCM_sys_Mkdir"
[16]	309	}
	310
	311	#D-#==================================================
	312	#D-function IGCM_sys_MkdirArchive
	313	#D-* Purpose: Mkdir on Archive
	314	#D-* Examples:
	315	#D-
	316	function IGCM_sys_MkdirArchive {
[657]	317	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
	318	if ( $DEBUG_sys ) ; then
	319	echo "IGCM_sys_MkdirArchive :" $@
	320	fi
[717]	321	#- creation de repertoire sur le serveur fichier
[657]	322	if [ ! -d ${1} ]; then
[811]	323	mkdir -p $1
	324	status=$?
	325
	326	if [ ${status} -gt 0 ] ; then
	327	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
[657]	328	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
[16]	329	fi
[657]	330	fi
	331	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[16]	332	}
	333
	334	#D-#==================================================
	335	#D-function IGCM_sys_MkdirWork
	336	#D-* Purpose: Mkdir on Work
	337	#D-* Examples:
	338	#D-
	339	function IGCM_sys_MkdirWork {
[657]	340	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
	341	if ( $DEBUG_sys ) ; then
	342	echo "IGCM_sys_MkdirWork :" $@
	343	fi
[717]	344	#- creation de repertoire sur le serveur fichier
[657]	345	if [ ! -d ${1} ]; then
	346	\mkdir -p $1
	347	if [ $? -gt 0 ] ; then
	348	echo "IGCM_sys_MkdirWork : erreur."
	349	IGCM_debug_Exit "IGCM_sys_MkdirWork"
[16]	350	fi
[657]	351	fi
	352	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
[16]	353	}
	354
	355	#D-#==================================================
	356	#D-function IGCM_sys_Cd
	357	#D-* Purpose: master cd command
	358	#D-* Examples:
	359	#D-
	360	function IGCM_sys_Cd {
[657]	361	IGCM_debug_PushStack "IGCM_sys_Cd" $@
	362	if ( $DEBUG_sys ) ; then
	363	echo "IGCM_sys_Cd :" $@
	364	fi
	365	\cd $1
	366	if [ $? -gt 0 ] ; then
	367	echo "IGCM_sys_Cd : erreur."
	368	IGCM_debug_Exit "IGCM_sys_Cd"
	369	fi
	370	IGCM_debug_PopStack "IGCM_sys_Cd"
[16]	371	}
	372
	373	#D-#==================================================
	374	#D-function IGCM_sys_Chmod
	375	#D-* Purpose: Chmod
	376	#D-* Examples:
	377	#D-
	378	function IGCM_sys_Chmod {
[657]	379	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
	380	if ( $DEBUG_sys ) ; then
	381	echo "IGCM_sys_Chmod :" $@
	382	fi
[809]	383	\chmod $@
	384	if [ $? -gt 0 ] ; then
	385	echo "IGCM_sys_Chmod : erreur."
	386	IGCM_debug_Exit "IGCM_sys_Chmod"
[657]	387	fi
	388	IGCM_debug_PopStack "IGCM_sys_Chmod"
[16]	389	}
	390
	391	#D-#==================================================
	392	#D-function IGCM_sys_FileSize
	393	#D-* Purpose: Filesize
	394	#D-* Examples:
	395	#D-
	396	function IGCM_sys_FileSize {
[657]	397	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
[16]	398
[657]	399	typeset sizeF
	400	set +A sizeF -- $( ls -la ${1} )
	401	if [ $? -gt 0 ] ; then
	402	IGCM_debug_Exit "IGCM_sys_FileSize"
	403	fi
	404	eval ${2}=${sizeF[4]}
[16]	405
[657]	406	IGCM_debug_PopStack "IGCM_sys_FileSize"
[16]	407	}
	408
	409	#D-#==================================================
	410	#D-function IGCM_sys_TestDir
	411	#D-* Purpose: Test Directory that must exists
	412	#D-* Examples:
	413	#D-
	414	function IGCM_sys_TestDir {
[657]	415	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
	416	if ( $DEBUG_sys ) ; then
	417	echo "IGCM_sys_TestDir :" $@
	418	fi
	419	typeset ExistFlag
	420	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	421	IGCM_debug_PopStack "IGCM_sys_TestDir"
[67]	422
[657]	423	return ${ExistFlag}
[16]	424	}
	425
	426	#D-#==================================================
	427	#D-function IGCM_sys_TestDirArchive
	428	#D-* Purpose: Test Directory that must exists on Archive
	429	#D-* Examples:
	430	#D-
	431	function IGCM_sys_TestDirArchive {
[657]	432	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
	433	if ( $DEBUG_sys ) ; then
	434	echo "IGCM_sys_TestDirArchive :" $@
	435	fi
	436	typeset ExistFlag
	437	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	438	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
	439	return ${ExistFlag}
[16]	440	}
	441
	442	#D-#==================================================
[800]	443	#D-function IGCM_sys_IsFileArchived
	444	#D-* Purpose: Test file that must NOT EXISTS on Archive
	445	#D-* Examples:
	446	#D-
	447	function IGCM_sys_IsFileArchived {
	448	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
	449	if ( $DEBUG_sys ) ; then
	450	echo "IGCM_sys_IsFileArchived :" $@
	451	fi
	452	typeset IsArchivedFlag
	453	# Never archived for this system
	454	IsArchivedFlag=1
	455	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
	456
	457	return ${IsArchivedFlag}
	458	}
	459
	460	#D-#==================================================
[16]	461	#D-function IGCM_sys_TestFileArchive
	462	#D-* Purpose: Test file that must NOT EXISTS on Archive
	463	#D-* Examples:
	464	#D-
	465	function IGCM_sys_TestFileArchive {
[657]	466	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
	467	typeset ExistFlag
	468	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	469	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[67]	470
[657]	471	return ${ExistFlag}
[67]	472	}
	473
	474	#D-#==================================================
[657]	475	#D-function IGCM_sys_TestFileBuffer
	476	#D-* Purpose: Test file that must NOT EXISTS on Buffer
	477	#D-* Examples:
	478	#D-
	479	function IGCM_sys_TestFileBuffer {
	480	IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
	481	typeset ExistFlag
	482	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	483	IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
	484
	485	return ${ExistFlag}
	486	}
	487
	488	#D-#==================================================
[67]	489	#D-function IGCM_sys_CountFileArchive
	490	#D-* Purpose: Count files on Archive filesystem
	491	#D-* Examples:
	492	#D-
	493	function IGCM_sys_CountFileArchive {
[657]	494	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
	495	ls ${@} 2>/dev/null \| wc -l
	496	if [ $? -gt 0 ] ; then
	497	echo "IGCM_sys_CountFileArchive : erreur."
	498	fi
	499	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[16]	500	}
	501
	502	#D-#==================================================
[657]	503	#D-function IGCM_sys_CountFileBuffer
	504	#D-* Purpose: Count files on Scratch filesystem
	505	#D-* Examples:
	506	#D-
	507	function IGCM_sys_CountFileBuffer {
	508	IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
	509	ls ${@} 2>/dev/null \| wc -l
	510	if [ $? -gt 0 ] ; then
	511	echo "IGCM_sys_CountFileBuffer : erreur."
	512	fi
	513	IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
	514	}
	515
	516	#D-#==================================================
[16]	517	#D-function IGCM_sys_Tree
	518	#D-* Purpose: Tree directories with files on ${ARCHIVE}
	519	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
	520	#D-
	521	function IGCM_sys_Tree {
[657]	522	IGCM_debug_PushStack "IGCM_sys_Tree" $@
	523	if ( $DEBUG_sys ) ; then
	524	echo "IGCM_sys_Tree :" $@
	525	fi
[16]	526
[657]	527	\tree -f $@
[16]	528
[657]	529	IGCM_debug_PopStack "IGCM_sys_Tree"
[16]	530	}
	531
	532	#D-#==================================================
	533	#D-function IGCM_sys_Tar
[657]	534	#D-* Purpose: master tar command
[16]	535	#D-* Examples:
	536	#D-
	537	function IGCM_sys_Tar {
[657]	538	IGCM_debug_PushStack "IGCM_sys_Tar" $@
	539	if ( $DEBUG_sys ) ; then
	540	echo "IGCM_sys_Tar :" $@
	541	fi
	542	\tar cf $@
	543	if [ $? -gt 0 ] ; then
	544	echo "IGCM_sys_Tar : erreur."
	545	IGCM_debug_Exit "IGCM_sys_Tar"
	546	fi
	547	IGCM_debug_PopStack "IGCM_sys_Tar"
[16]	548	}
	549
	550	#D-#==================================================
	551	#D-function IGCM_sys_UnTar
	552	#D-* Purpose: master un-tar command
	553	#D-* Examples:
	554	#D-
	555	function IGCM_sys_UnTar {
[657]	556	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
	557	if ( $DEBUG_sys ) ; then
	558	echo "IGCM_sys_UnTar :" $@
	559	fi
	560	\tar xvf $1
	561	if [ $? -gt 0 ] ; then
	562	echo "IGCM_sys_UnTar : erreur."
	563	IGCM_debug_Exit "IGCM_sys_UnTar"
	564	fi
	565	IGCM_debug_PopStack "IGCM_sys_UnTar"
[16]	566	}
	567
	568	#D-#==================================================
	569	#D-function IGCM_sys_Qsub
	570	#D-* Purpose: Qsub new job
	571	#D-* Examples:
	572	#D-
	573	function IGCM_sys_Qsub {
[657]	574	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
	575	if ( $DEBUG_sys ) ; then
	576	echo "IGCM_sys_Qsub :" $@
	577	fi
[923]	578	typeset options status
	579	options="-o ${SUBMIT_DIR}/${Script_Output}"
	580	/usr/local/bin/qsub ${options} < $1
[811]	581	status=$?
	582	if [ ${status} -gt 0 ] ; then
	583	IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
[657]	584	IGCM_debug_Exit "IGCM_sys_Qsub"
	585	fi
	586	IGCM_debug_PopStack "IGCM_sys_Qsub"
[16]	587	}
	588
	589	#D-#==================================================
	590	#D-function IGCM_sys_QsubPost
	591	#D-* Purpose: Qsub new job on scalaire
	592	#D-* Examples:
	593	#D-
	594	function IGCM_sys_QsubPost {
[657]	595	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
	596	if ( $DEBUG_sys ) ; then
	597	echo "IGCM_sys_QsubPost :" $@
	598	fi
[923]	599	typeset options status
	600	options="-o ${POST_DIR}/${Script_Post_Output}.out -v ${listVarEnv}"
	601	/usr/local/bin/qsub ${options} < ${libIGCM_POST}/$1.job
[811]	602	status=$?
	603	if [ ${status} -gt 0 ] ; then
[923]	604	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
[657]	605	IGCM_debug_Exit "IGCM_sys_QsubPost"
	606	fi
	607	IGCM_debug_PopStack "IGCM_sys_QsubPost"
[16]	608	}
	609
	610	#D-*************************
[657]	611	#D- File transfer functions
[16]	612	#D-*************************
	613	#D-
	614
	615	#D-#==================================================
	616	#D-function IGCM_sys_Rsync_out
	617	#D-* Purpose: treat return val of rsync
	618	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
	619	#D- Error values and explanations can depend on your system version.
	620	function IGCM_sys_Rsync_out {
[717]	621	status=$1
	622	if [ ! $status ] ; then
[657]	623	echo "rsync error !"
	624	fi
[16]	625
[657]	626	if [ $MYLANG = "fr" ]; then
[717]	627	case $status in
[657]	628	0) return ;;
	629	1) echo "Erreur de rsync ; RERR_SYNTAX : "
	630	echo "Erreur de syntaxe ou d'utilisation."
	631	return;;
	632	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
	633	echo "Incompatibilité de protocole."
	634	return;;
	635	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
	636	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
	637	echo "répertoires"
	638	return;;
	639	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
	640	echo "Action demandée non supportée : une tentative de manipulation de"
	641	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
	642	echo "été faite ; ou une option qui est supportée par le client mais"
	643	echo "pas par le serveur a été spécifiée."
	644	return;;
	645	10) echo "Erreur de rsync ; RERR_SOCKETIO"
	646	echo "Erreur dans le socket d'entrée sortie"
	647	return;;
	648	11) echo "Erreur de rsync ; RERR_FILEIO"
	649	echo "Erreur d'entrée sortie fichier"
	650	return;;
	651	12) echo "Erreur de rsync ; RERR_STREAMIO"
	652	echo "Erreur dans flux de donnée du protocole rsync"
	653	return;;
	654	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
	655	echo "Erreur avec les diagnostics du programme"
	656	return;;
	657	14) echo "Erreur de rsync ; RERR_IPC"
	658	echo "Erreur dans le code IPC"
	659	return;;
	660	20) echo "Erreur de rsync ; RERR_SIGNAL"
	661	echo "SIGUSR1 ou SIGINT reçu"
	662	return;;
	663	21) echo "Erreur de rsync ; RERR_WAITCHILD"
	664	echo "Une erreur retournée par waitpid()"
	665	return;;
	666	22) echo "Erreur de rsync ; RERR_MALLOC"
	667	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
	668	return;;
	669	23) echo ""
	670	echo "Erreur fichier inexistant"
	671	return;;
	672	30) echo "Erreur de rsync ; RERR_TIMEOUT"
	673	echo "Temps d'attente écoulé dans l'envoi/réception de données"
	674	return;;
[717]	675	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $status
[657]	676	return;;
	677	esac
	678	elif [ $MYLANG = "en" ] ; then
[717]	679	case $status in
[657]	680	0) return;;
	681	1) echo "rsync error : Syntax or usage error "
	682	return;;
	683	2) echo "rsync error : Protocol incompatibility "
	684	return;;
	685	3) echo "rsync error : Errors selecting input/output files, dirs"
	686	return;;
	687	4) echo "rsync error : Requested action not supported: an attempt"
	688	echo "was made to manipulate 64-bit files on a platform that cannot support"
	689	echo "them; or an option was specified that is supported by the client and"
	690	echo "not by the server."
	691	return;;
	692	5) echo "rsync error : Error starting client-server protocol"
	693	return;;
	694	10) echo "rsync error : Error in socket I/O "
	695	return;;
	696	11) echo "rsync error : Error in file I/O "
	697	return;;
	698	12) echo "rsync error : Error in rsync protocol data stream "
	699	return;;
	700	13) echo "rsync error : Errors with program diagnostics "
	701	return;;
	702	14) echo "rsync error : Error in IPC code "
	703	return;;
	704	20) echo "rsync error : Received SIGUSR1 or SIGINT "
	705	return;;
	706	21) echo "rsync error : Some error returned by waitpid() "
	707	return;;
	708	22) echo "rsync error : Error allocating core memory buffers "
	709	return;;
	710	23) echo "rsync error : Partial transfer due to error"
	711	return;;
	712	24) echo "rsync error : Partial transfer due to vanished source files"
	713	return;;
	714	30) echo "rsync error : Timeout in data send/receive "
	715	return;;
[717]	716	*) echo "rsync error : return code of rsync unknown :" $status
[657]	717	return;;
	718	esac
	719	else
	720	echo "unknown language $MYLANG."
	721	return
	722	fi
[16]	723	}
[657]	724
[16]	725	#D-#==================================================
[811]	726	#D-function IGCM_sys_Miror_libIGCM
	727	#D-* Purpose: Mirror libIGCM PATH and lib to frontend
	728	#D-* Examples:
	729	#D-
	730	function IGCM_sys_Mirror_libIGCM {
	731	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
	732	if ( $DEBUG_sys ) ; then
	733	echo "IGCM_sys_Mirror_libIGCM"
	734	fi
	735
	736	typeset status
	737
	738	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
	739
	740	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
	741	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
	742	status=$?
	743
	744	if [ ${status} -gt 0 ] ; then
	745	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
	746	cat out_rsync
	747	fi
	748	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
	749	}
	750
	751	#D-#==================================================
[16]	752	#D-function IGCM_sys_Cp
	753	#D-* Purpose: generic cp
	754	#D-* Examples:
	755	#D-
	756	function IGCM_sys_Cp {
[657]	757	IGCM_debug_PushStack "IGCM_sys_Cp" $@
	758	if ( $DEBUG_sys ) ; then
	759	echo "IGCM_sys_Cp :" $@
	760	fi
[16]	761
[717]	762	typeset status
[16]	763
[717]	764	echo cp $@ > out_rsync 2>&1
	765	\cp $@ >> out_rsync 2>&1
	766	status=$?
[657]	767
[717]	768	if [ ${status} -gt 0 ] ; then
	769	echo "IGCM_sys_Cp : error code ${status}"
[657]	770	cat out_rsync
	771	IGCM_debug_Exit "IGCM_sys_Cp"
	772	else
[717]	773	\rm out_rsync
[657]	774	fi
	775	IGCM_debug_PopStack "IGCM_sys_Cp"
[16]	776	}
	777
	778	#D-#==================================================
	779	#D-function IGCM_sys_Rm
	780	#D-* Purpose: generic rm
	781	#D-* Examples:
	782	#D-
	783	function IGCM_sys_Rm {
[657]	784	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
	785	if ( $DEBUG_sys ) ; then
	786	echo "IGCM_sys_Rm :" $@
	787	fi
[16]	788
[717]	789	typeset status
[16]	790
[657]	791	echo rm $@ > out_rsync 2>&1
	792	\rm $@ >> out_rsync 2>&1
[717]	793	status=$?
[657]	794
[717]	795	if [ ${status} -gt 0 ] ; then
	796	echo "IGCM_sys_Rm : error code ${status}"
[657]	797	cat out_rsync
	798	IGCM_debug_Exit "IGCM_sys_Rm"
	799	else
[717]	800	\rm out_rsync
[657]	801	fi
	802	IGCM_debug_PopStack "IGCM_sys_Rm"
[16]	803	}
	804
	805	#D-#==================================================
[203]	806	#D-function IGCM_sys_RmRunDir
	807	#D-* Purpose: rm tmpdir (dummy function most of the time batch
	808	#D- scheduler will do the job)
	809	#D-* Examples:
	810	#D-
	811	function IGCM_sys_RmRunDir {
[657]	812	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
	813	if ( $DEBUG_sys ) ; then
	814	echo "IGCM_sys_RmRunDir :" $@
	815	fi
[282]	816
[717]	817	typeset status
[282]	818
[657]	819	echo rm $@ > out_rsync 2>&1
	820	\rm $@ >> out_rsync 2>&1
[717]	821	status=$?
[282]	822
[717]	823	if [ ${status} -gt 0 ] ; then
[811]	824	echo "IGCM_sys_RmRunDir : rm error code is ${status}."
[657]	825	cat out_rsync
	826	IGCM_debug_Exit "IGCM_sys_RmRunDir"
[717]	827	else
	828	\rm out_rsync
[657]	829	fi
	830	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[203]	831	}
	832
	833	#D-#==================================================
[16]	834	#D-function IGCM_sys_Mv
	835	#D-* Purpose: generic move
	836	#D-* Examples:
	837	#D-
	838	function IGCM_sys_Mv {
[657]	839	IGCM_debug_PushStack "IGCM_sys_Mv" $@
	840	if ( $DEBUG_sys ) ; then
	841	echo "IGCM_sys_Mv :" $@
	842	fi
[16]	843
[657]	844	if [ $DRYRUN = 0 ]; then
[16]	845
[717]	846	typeset status
[657]	847
	848	echo mv $@ > out_rsync 2>&1
	849	\mv $@ >> out_rsync 2>&1
[717]	850	status=$?
[657]	851
[717]	852	if [ ${status} -gt 0 ] ; then
	853	echo "IGCM_sys_Mv : error code ${status}"
[657]	854	cat out_rsync
	855	IGCM_debug_Exit "IGCM_sys_Mv"
[173]	856	else
[717]	857	\rm out_rsync
[16]	858	fi
[657]	859	fi
	860	IGCM_debug_PopStack "IGCM_sys_Mv"
[16]	861	}
	862
	863	#D-#==================================================
	864	#D-function IGCM_sys_Put_Dir
	865	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
	866	#D-* Examples:
	867	#D-
	868	function IGCM_sys_Put_Dir {
[657]	869	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
	870	if ( $DEBUG_sys ) ; then
	871	echo "IGCM_sys_Put_Dir :" $@
	872	fi
	873	if [ $DRYRUN = 0 ]; then
	874	if [ ! -d ${1} ] ; then
	875	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
	876	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
	877	return
[16]	878	fi
	879
[717]	880	typeset status
[16]	881
[717]	882	# Only if we use rsync
	883	#IGCM_sys_TestDirArchive $( dirname $2 )
	884	#
	885	#USUAL WAY
	886	\cp -r $1 $2 > out_rsync 2>&1
	887	status=$?
[16]	888
[717]	889	if [ ${status} -gt 0 ] ; then
[811]	890	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
[657]	891	cat out_rsync
	892	IGCM_debug_Exit "IGCM_sys_Put_Dir"
[16]	893	else
[717]	894	\rm out_rsync
[16]	895	fi
[657]	896	fi
	897	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[16]	898	}
	899
	900	#D-#==================================================
	901	#D-function IGCM_sys_Get_Dir
[717]	902	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[16]	903	#D-* Examples:
	904	#D-
	905	function IGCM_sys_Get_Dir {
[657]	906	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
	907	if ( $DEBUG_sys ) ; then
	908	echo "IGCM_sys_Get_Dir :" $@
	909	fi
	910	if [ $DRYRUN = 0 ]; then
	911	if [ ! -d ${1} ] ; then
	912	echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
	913	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
	914	return
[16]	915	fi
	916
[717]	917	typeset status
[16]	918
[719]	919	#USUAL WAY
[811]	920	\cp -ur $1 $2 > out_rsync 2>&1
[717]	921	status=$?
[16]	922
[717]	923	if [ ${status} -gt 0 ] ; then
[811]	924	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
[657]	925	cat out_rsync
	926	IGCM_debug_Exit "IGCM_sys_Get_Dir"
[16]	927	else
[717]	928	\rm out_rsync
[16]	929	fi
[657]	930	fi
	931	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[16]	932	}
	933
	934	#D-#==================================================
	935	#D-function IGCM_sys_Get_Master
	936	#D-* Purpose: Copy a complete directory from MASTER filesystem
	937	#D-* Examples:
	938	#D-
	939	function IGCM_sys_Get_Master {
[657]	940	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
	941	if ( $DEBUG_sys ) ; then
	942	echo "IGCM_sys_Get_Master :" $@
	943	fi
	944	if [ $DRYRUN = 0 ]; then
	945	if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
	946	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
	947	IGCM_debug_PopStack "IGCM_sys_Get_Master"
	948	return
[16]	949	fi
	950
[811]	951	typeset NB_ESSAI DELAI status i
	952	# number of tentative
	953	NB_ESSAI=3
	954	# time delay between tentative
	955	DELAI=2
[16]	956
[811]	957	i=0
	958	while [ $i -lt $NB_ESSAI ] ; do
	959	\cp -urL $1 $2 > out_rsync 2>&1
	960	status=$?
	961	if [ ${status} -gt 0 ]; then
	962	IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
	963	IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
	964	sleep $DELAI
	965	else
	966	break
	967	fi
	968	(( i = i + 1 ))
	969	done
[16]	970
[717]	971	if [ ${status} -gt 0 ] ; then
[657]	972	echo "IGCM_sys_Get_Master : error."
	973	cat out_rsync
	974	IGCM_debug_Exit "IGCM_sys_Get_Master"
[16]	975	else
[811]	976	\rm out_rsync
[16]	977	fi
[657]	978	fi
	979	IGCM_debug_PopStack "IGCM_sys_Get_Master"
[16]	980	}
	981
[811]	982	#====================================================
	983	#- Call IGCM_sys_Mirror_libIGCM now !
	984	if ( $MirrorlibIGCM ) ; then
	985	IGCM_sys_Mirror_libIGCM
	986	fi
	987
[16]	988	#D-#==================================================
	989	#D-function IGCM_sys_Put_Rest
[717]	990	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[282]	991	#D- File and target directory must exist.
[16]	992	#D-* Examples:
	993	#D-
	994	function IGCM_sys_Put_Rest {
[657]	995	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
	996	if ( $DEBUG_sys ) ; then
	997	echo "IGCM_sys_Put_Rest :" $@
	998	fi
	999	if [ $DRYRUN = 0 ]; then
	1000	if [ ! -f ${1} ] ; then
	1001	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
	1002	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[16]	1003	fi
[717]	1004
	1005	typeset status
	1006	#
[657]	1007	if [ X${JobType} = XRUN ] ; then
	1008	IGCM_sys_Chmod 444 ${1}
	1009	fi
[717]	1010	#
[811]	1011	# Only if we use rsync
	1012	#IGCM_sys_MkdirArchive $( dirname $2 )
	1013	#
	1014	#USUAL WAY
[717]	1015	\cp $1 $2 > out_rsync 2>&1
	1016	status=$?
[16]	1017
[717]	1018	# #RSYNC WITH NETWORK SSH CALL
	1019	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
	1020	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
[16]	1021
[717]	1022	# #RSYNC WITH NFS USE
	1023	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
	1024	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
[16]	1025
[717]	1026	# status=$?
	1027	# IGCM_sys_Rsync_out $status
	1028
	1029	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
	1030	# (( status=status+$? ))
	1031
	1032	if [ ${status} -gt 0 ] ; then
	1033	echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
[657]	1034	cat out_rsync
	1035	IGCM_debug_Exit "IGCM_sys_Put_Rest"
	1036	else
[717]	1037	\rm out_rsync
[657]	1038	fi
	1039	fi
	1040	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
	1041	}
[16]	1042
[657]	1043	#D-#==================================================
	1044	#D-function IGCM_sys_PutBuffer_Rest
	1045	#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
	1046	#D- File and target directory must exist.
	1047	#D-* Examples:
	1048	#D-
	1049	function IGCM_sys_PutBuffer_Rest {
	1050	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
	1051	if ( $DEBUG_sys ) ; then
	1052	echo "IGCM_sys_PutBuffer_Rest :" $@
	1053	fi
	1054	if [ $DRYRUN = 0 ]; then
	1055	if [ ! -f ${1} ] ; then
	1056	echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
	1057	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
	1058	fi
[16]	1059
[717]	1060	typeset status
[657]	1061	#
	1062	if [ X${JobType} = XRUN ] ; then
	1063	IGCM_sys_Chmod 444 ${1}
	1064	fi
	1065
	1066	#
	1067	# USUAL WAY
	1068	\cp $1 $2 > out_rsync 2>&1
[717]	1069	status=$?
[657]	1070
[717]	1071	if [ ${status} -gt 0 ] ; then
	1072	echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
[811]	1073	[ -f ${2} ] && ls -l ${2}
	1074	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[657]	1075	cat out_rsync
	1076	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
[16]	1077	else
[657]	1078	\rm out_rsync
[16]	1079	fi
[657]	1080	fi
	1081	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
[16]	1082	}
	1083
	1084	#D-#==================================================
[801]	1085	#D-function IGCM_sys_PrepareTaredRestart
	1086	#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
	1087	#D-* Examples:
	1088	#D-
	1089	function IGCM_sys_PrepareTaredRestart {
	1090	IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
[918]	1091
[801]	1092	IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
	1093	}
	1094
	1095	#D-#==================================================
[16]	1096	#D-function IGCM_sys_Put_Out
[717]	1097	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[16]	1098	#D-* Examples:
	1099	#D-
	1100	function IGCM_sys_Put_Out {
[657]	1101	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
	1102	if ( $DEBUG_sys ) ; then
	1103	echo "IGCM_sys_Put_Out :" $@
	1104	fi
[811]	1105
	1106	typeset status
	1107
[657]	1108	if [ $DRYRUN = 0 ]; then
[811]	1109	if [ ! -f ${1} ] ; then
[657]	1110	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
	1111	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	1112	return 1
[16]	1113	fi
[719]	1114	#
[811]	1115	IGCM_sys_MkdirArchive $( dirname $2 )
	1116	#
[657]	1117	if [ X${JobType} = XRUN ] ; then
	1118	if [ X${3} = X ] ; then
	1119	IGCM_sys_Chmod 444 ${1}
	1120	fi
	1121	fi
[717]	1122	#
[16]	1123
[657]	1124	echo ${RSYNC} ${RSYNC_opt} $1 $2 > out_rsync 2>&1
	1125	${RSYNC} ${RSYNC_opt} $1 $2 >> out_rsync 2>&1
[717]	1126	status=$?
	1127	IGCM_sys_Rsync_out $status
[16]	1128
[657]	1129	${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
[717]	1130	(( status=status+$? ))
[16]	1131
[717]	1132	if [ ${status} -gt 0 ] ; then
[811]	1133	IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
[657]	1134	cat out_rsync
	1135	IGCM_debug_Exit "IGCM_sys_Put_Out"
[16]	1136	else
[717]	1137	\rm out_rsync
[16]	1138	fi
[657]	1139	fi
	1140	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	1141	return 0
[16]	1142	}
	1143
	1144	#D-#==================================================
[657]	1145	#D-function IGCM_sys_PutBuffer_Out
	1146	#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
	1147	#D-* Examples:
	1148	#D-
	1149	function IGCM_sys_PutBuffer_Out {
	1150	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
	1151	if ( $DEBUG_sys ) ; then
	1152	echo "IGCM_sys_PutBuffer_Out :" $@
	1153	fi
[719]	1154
[811]	1155	typeset NB_ESSAI DELAI status i exist skip
[719]	1156
[811]	1157	# number of tentative
	1158	NB_ESSAI=3
	1159	# time delay between tentative
	1160	DELAI=2
	1161
[657]	1162	if [ $DRYRUN = 0 ]; then
	1163	if [ ! -f ${1} ] ; then
	1164	echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
	1165	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
	1166	return 1
	1167	fi
	1168	#
	1169	IGCM_sys_Mkdir $( dirname $2 )
	1170	#
	1171
[811]	1172	exist=false
	1173	skip=false
	1174	if [ -f $2 ] ; then
	1175	IGCM_debug_Print 1 "$2 already exist"
	1176	exist=true
	1177	if [ "X$( diff $1 $2 )" = X ] ; then
	1178	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
	1179	status=0
	1180	skip=true
	1181	else
	1182	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	1183	skip=false
[657]	1184	fi
	1185	fi
	1186	#
[811]	1187	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	1188	IGCM_sys_Chmod u+w $2
	1189	fi
[657]	1190
[811]	1191	if [ X${skip} = Xfalse ] ; then
	1192	i=0
	1193	while [ $i -lt $NB_ESSAI ] ; do
	1194	# USUAL WAY
	1195	\cp $1 $2 > out_rsync 2>&1
	1196	status=$?
	1197	if [ ${status} -gt 0 ]; then
	1198	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1199	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
	1200	[ -f ${2} ] && ls -l ${2}
	1201	[ -f ${2}/${1} ] && ls -l ${2}/${1}
	1202	sleep $DELAI
	1203	else
	1204	break
	1205	fi
	1206	(( i = i + 1 ))
	1207	done
	1208	fi
	1209
[717]	1210	if [ ${status} -gt 0 ] ; then
[657]	1211	echo "IGCM_sys_PutBuffer_Out : error."
[811]	1212	[ -f ${2} ] && ls -l ${2}
	1213	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[657]	1214	cat out_rsync
	1215	IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
	1216	else
[811]	1217
	1218	if [ X${JobType} = XRUN ] ; then
	1219	if [ X${3} = X ] ; then
	1220	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	1221	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	1222	fi
	1223	fi
	1224
[657]	1225	\rm out_rsync
	1226	fi
	1227	fi
	1228	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
	1229	return 0
	1230	}
	1231
	1232	#D-#==================================================
[16]	1233	#D-function IGCM_sys_Get
	1234	#D-* Purpose: Get a file from ${ARCHIVE}
	1235	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
	1236	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
	1237	function IGCM_sys_Get {
[657]	1238	IGCM_debug_PushStack "IGCM_sys_Get" $@
[16]	1239
[811]	1240	typeset DEST status dm_liste
[16]	1241
[657]	1242	if ( $DEBUG_sys ) ; then
	1243	echo "IGCM_sys_Get :" $@
	1244	fi
	1245	if [ $DRYRUN -le 2 ]; then
	1246	if [ X${1} = X'/l' ] ; then
	1247	eval set +A dm_liste \${${2}}
	1248	else
	1249	dm_liste=${1}
[16]	1250	fi
[657]	1251	eval DEST=\${${#}}
[16]	1252
[717]	1253	# test if the (first) file is present in the old computation :
[657]	1254	IGCM_sys_TestFileArchive ${dm_liste[0]}
[717]	1255	status=$?
	1256	if [ ${status} -gt 0 ] ; then
[657]	1257	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
	1258	IGCM_debug_Exit "IGCM_sys_Get"
[811]	1259	return
[657]	1260	fi
[16]	1261
[811]	1262	#USUAL WAY
	1263	\cp ${dm_liste[*]} ${DEST} > out_rsync 2>&1
	1264	status=$?
[16]	1265
[811]	1266	if [ ${status} -gt 0 ] ; then
	1267	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
[657]	1268	cat out_rsync
	1269	IGCM_debug_Exit "IGCM_sys_Get"
[16]	1270	else
[717]	1271	\rm out_rsync
[16]	1272	fi
[657]	1273	fi
	1274	IGCM_debug_PopStack "IGCM_sys_Get"
[16]	1275	}
	1276
	1277	#D-#==================================================
[657]	1278	#D-function IGCM_sys_GetBuffer
	1279	#D-* Purpose: Get a file from ${SCRATCHDIR}
	1280	#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
	1281	#D- IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
	1282	function IGCM_sys_GetBuffer {
	1283	IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
	1284
[811]	1285	typeset DEST buf_liste target file_work
	1286	typeset NB_ESSAI DELAI status i
[657]	1287
	1288	if ( $DEBUG_sys ) ; then
	1289	echo "IGCM_sys_GetBuffer :" $@
	1290	fi
[811]	1291
	1292	# number of tentative
	1293	NB_ESSAI=3
	1294	# time delay between tentative
	1295	DELAI=2
	1296
[657]	1297	if [ $DRYRUN -le 2 ]; then
	1298	if [ X${1} = X'/l' ] ; then
	1299	eval set +A buf_liste \${${2}}
	1300	else
	1301	eval set +A buf_liste ${1}
	1302	fi
	1303	eval DEST=\${${#}}
	1304
	1305	#USUAL WAY
	1306	if [ X${1} = X'/l' ] ; then
	1307	for target in ${buf_liste[*]} ; do
	1308	local_file=$( basename ${target} )
[811]	1309	i=0
	1310	while [ $i -lt $NB_ESSAI ] ; do
	1311	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
	1312	status=$?
	1313	if [ ${status} -gt 0 ]; then
	1314	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1315	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
	1316	sleep $DELAI
	1317	else
	1318	break
	1319	fi
	1320	(( i = i + 1 ))
	1321	done
	1322	if [ ${status} -gt 0 ] ; then
	1323	echo "IGCM_sys_Get : error"
	1324	cat out_rsync
	1325	\rm out_rsync
	1326	IGCM_debug_Exit "IGCM_sys_GetBuffer"
	1327	else
	1328	\rm out_rsync
	1329	fi
[657]	1330	done
	1331	else
[811]	1332	i=0
	1333	while [ $i -lt $NB_ESSAI ] ; do
	1334	\cp ${buf_liste} ${DEST} >> out_rsync 2>&1
	1335	status=$?
	1336	if [ ${status} -gt 0 ]; then
	1337	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1338	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
	1339	sleep $DELAI
	1340	else
	1341	break
	1342	fi
	1343	(( i = i + 1 ))
	1344	done
	1345	if [ ${status} -gt 0 ] ; then
	1346	echo "IGCM_sys_Get : error"
	1347	cat out_rsync
	1348	\rm out_rsync
	1349	IGCM_debug_Exit "IGCM_sys_GetBuffer"
	1350	else
	1351	\rm out_rsync
	1352	fi
[657]	1353	fi
	1354	fi
	1355	IGCM_debug_PopStack "IGCM_sys_GetBuffer"
	1356	}
	1357
	1358	#D-#==================================================
	1359	#D-function IGCM_sys_GetDate_FichWork
	1360	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
	1361	#D-* Examples:
	1362	#D-
	1363	function IGCM_sys_GetDate_FichWork {
	1364	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
	1365	if ( $DEBUG_sys ) ; then
	1366	echo "IGCM_sys_GetDate_FichWork :" $@
	1367	fi
	1368	typeset dateF
	1369	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
	1370	eval ${2}=${dateF[5]}
	1371
[800]	1372	# donne la date filesys d'un fichier sur la machine work
[657]	1373	IGCM_debug_PopStack "IGCM_sys_FichWork"
	1374	}
	1375
	1376	#D-#==================================================
	1377	#D-function IGCM_sys_GetDate_FichArchive
	1378	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
	1379	#D-* Examples:
	1380	#D-
	1381	function IGCM_sys_GetDate_FichArchive {
	1382	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
	1383	if ( $DEBUG_sys ) ; then
	1384	echo "IGCM_sys_GetDate_FichArchive :" $@
	1385	fi
	1386	typeset dateF
	1387	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
	1388	eval ${2}=${dateF[5]}
	1389
	1390	IGCM_debug_PopStack "IGCM_sys_FichArchive"
	1391	}
	1392
	1393	#D-#==================================================
[343]	1394	#D-function IGCM_sys_Dods_Rm
[717]	1395	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
[343]	1396	#D-* Examples:
	1397	#D-
	1398	function IGCM_sys_Dods_Rm {
[657]	1399	if ( $DEBUG_sys ) ; then
	1400	echo "IGCM_sys_Dods_Rm :" $@
	1401	fi
	1402	return 0
[343]	1403	}
	1404
	1405	#D-#==================================================
	1406	#D-function IGCM_sys_Dods_Cp
[717]	1407	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
[343]	1408	#D-* Examples:
	1409	#D-
	1410	function IGCM_sys_Dods_Cp {
[657]	1411	if ( $DEBUG_sys ) ; then
	1412	echo "IGCM_sys_Dods_Cp :" $@
	1413	fi
	1414	return 0
[343]	1415	}
	1416
	1417	#D-#==================================================
[16]	1418	#D-function IGCM_sys_Put_Dods
[811]	1419	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
[16]	1420	#D-* Examples:
	1421	#D-
	1422	function IGCM_sys_Put_Dods {
[657]	1423	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
	1424	if ( $DEBUG_sys ) ; then
	1425	echo "IGCM_sys_Put_Dods :" $@
	1426	fi
	1427	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
[16]	1428	}
	1429
[657]	1430	##############################################################
	1431	# REBUILD OPERATOR
[536]	1432
[811]	1433	#D-#==================================================
	1434	#D-function IGCM_sys_rebuild
	1435	#D-* Purpose: rebuild parallel files
	1436	#D-* Examples:
	1437	#D-
[657]	1438	function IGCM_sys_rebuild {
	1439	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
	1440	if ( $DEBUG_sys ) ; then
	1441	echo "IGCM_sys_rebuild :" $@
	1442	fi
[717]	1443
[811]	1444	typeset NB_ESSAI DELAI status i firstArg
	1445	# number of tentative
	1446	NB_ESSAI=3
	1447	# time delay between tentative
	1448	DELAI=2
[717]	1449
[811]	1450	i=0
	1451	while [ $i -lt $NB_ESSAI ] ; do
	1452	/home/users/igcmg/rebuild/bin/rebuild -f -o $@ > out_rsync 2>&1
	1453	status=$?
	1454	if [ ${status} -gt 0 ] ; then
	1455	IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
	1456	cat out_rsync
	1457	\rm out_rsync
	1458	IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
	1459	firstArg=${1}
	1460	\rm ${firstArg}
	1461	sleep $DELAI
	1462	else
	1463	\rm out_rsync
	1464	break
	1465	fi
	1466	(( i = i + 1 ))
	1467	done
	1468
[717]	1469	if [ ${status} -gt 0 ] ; then
[811]	1470	echo "IGCM_sys_rebuild : rebuild error code is ${status}"
[657]	1471	IGCM_debug_Exit "rebuild"
	1472	fi
	1473
	1474	IGCM_debug_PopStack "IGCM_sys_rebuild"
[16]	1475	}
	1476
[811]	1477	#D-#==================================================
	1478	#D-function IGCM_sys_rebuild_station
[922]	1479	#D-* Purpose: rebuild parallel files describing station
[811]	1480	#D-* Examples:
	1481	#D-
[697]	1482	function IGCM_sys_rebuild_station {
	1483	IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
	1484	typeset i list_opt file_in file_out prefix_invert list_invert
	1485	if ( $DEBUG_sys ) ; then
	1486	echo "IGCM_sys_rebuild_station :" $@
	1487	fi
	1488	list_opt=$@
	1489
[800]	1490	# Invert Axis : t,x -> x,t
[697]	1491	# t,pres,x -> x,t,pres
	1492	# So that we can concatenate along x
	1493	i=0
	1494	for file_in in ${list_opt} ; do
	1495	(( i = i + 1))
	1496	[ ${i} = 1 ] && file_out=${file_in} && continue
	1497	prefix_invert=$( basename ${file_in} .nc )
	1498	IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
	1499	list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
	1500	done
	1501
	1502	# Concatenate
	1503	IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
	1504
	1505	# Re-ivert file
	1506	IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
	1507
[922]	1508	# Station re-ordering is too expansive to be run within libIGCM
[697]	1509	# This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
	1510	# This re-ordering must be done "in memory" by the cmorization process
	1511	# Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
	1512	# BEGIN reordering
	1513
	1514	# Only LMDZ text output contains the exact ordering of the station.
	1515	# We isolate this in the code below:
	1516	# 0 38 -157.5000000000000 70.98591549295774
	1517	# 0 54 27.49999999999999 67.18309859154928
	1518	# 0 56 -62.50000000000001 82.39436619718309
	1519	# 0 79 12.49999999999999 78.59154929577466
	1520	# 0 116 -165.0000000000000 76.05633802816901
	1521	# 0 117 130.0000000000000 70.98591549295774
	1522	# 0 118 110.0000000000000 87.46478873239437
	1523	# 1 40 4.999999999999995 51.97183098591550
	1524	# typeset iStation iProc list_opt file_in file_out prefix_invert
	1525	# typeset -Z4 j4
	1526	# typeset -Z3 j3
	1527
	1528	# unset list_opt
	1529	# set +A list_opt $@
	1530
	1531	# Filename after rebuild
	1532	# file_out=${list_opt[0]}
	1533	# Prefix of output files
	1534	# prefix_invert=$( basename ${file_out} .nc )
	1535	# Number of procs
	1536	# num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} \| wc -l )
	1537
	1538	# iProc=0
	1539	# while [ ${iProc} -lt ${num_proc} ] ; do
	1540	# Array containing Station as a number
	1541	# unset proc_stn
[912]	1542	# set +A proc_stn $( grep "iophy_mpi rank ip lon lat $iProc" ${PREFIX}_${Exe_Output} \| sed -e "s/iophy_mpi rank ip lon lat //g" \| gawk ' {print $2}' )
[697]	1543	# Number of stations produced by processor proc
	1544	# stationLast=${#proc_stn[*]}
	1545	# Proc number on 4 digits
	1546	# j4=${iProc}
	1547	# Init
	1548	# iStation=0
	1549	# while [ ${iStation} -lt ${stationLast} ] ; do
	1550	# Station number on 3 digits
	1551	# j3=${proc_stn[${iStation}]}
	1552	# Extract station
[800]	1553	# Invert Axis : t,x -> x,t
[697]	1554	# t,pres,x -> x,t,pres
	1555	# So that we can concatenate along x
	1556	# IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
	1557	# (( iStation = iStation + 1 ))
	1558	# done
	1559	# (( iProc = iProc + 1 ))
	1560	# done
	1561
	1562	# Concatenate all station along x
	1563	# IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
	1564
	1565	# Re-invert file
	1566	# IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
	1567
	1568	# END reordering
	1569
	1570	IGCM_debug_PopStack "IGCM_sys_rebuild_station"
	1571	}
	1572
[657]	1573	############################################################
	1574	# Activate Running Environnment Variables
[536]	1575
[811]	1576	#D-#==================================================
	1577	#D-function IGCM_sys_desactiv_variables
	1578	#D-* Purpose: set environement variables prior to execution
	1579	#D-* Examples:
	1580	#D-
[657]	1581	function IGCM_sys_activ_variables {
	1582	IGCM_debug_PushStack "IGCM_sys_activ_variables"
	1583	if ( $DEBUG_sys ) ; then
	1584	echo "IGCM_sys_activ_variables"
	1585	fi
[811]	1586
	1587	# --------------------------------------------------------------------
	1588	#D- MPI specifications
	1589	# --------------------------------------------------------------------
	1590
	1591	# --------------------------------------------------------------------
	1592	#D- Other specifications
	1593	# --------------------------------------------------------------------
	1594
[657]	1595	IGCM_debug_PopStack "IGCM_sys_activ_variables"
[16]	1596	}
	1597
[657]	1598	############################################################
	1599	# Desactivate Running Environnment Variables
[16]	1600
[811]	1601	#D-#==================================================
	1602	#D-function IGCM_sys_desactiv_variables
	1603	#D-* Purpose: unset environement variables after execution
	1604	#D-* Examples:
	1605	#D-
[657]	1606	function IGCM_sys_desactiv_variables {
	1607	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
	1608	if ( $DEBUG_sys ) ; then
	1609	echo "IGCM_sys_desactiv_variables"
	1610	fi
[811]	1611	# --------------------------------------------------------------------
	1612	#D- MPI specifications
	1613	# --------------------------------------------------------------------
	1614
	1615	# --------------------------------------------------------------------
	1616	#D- Other specifications
	1617	# --------------------------------------------------------------------
	1618
[657]	1619	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
	1620	}
	1621
	1622	############################################################
[800]	1623	# Build MPI/OMP scripts run file (dummy function)
[657]	1624
[811]	1625	#D-#==================================================
	1626	#D-function IGCM_sys_build_run_file
	1627	#D-* Purpose: build run file (deprecated)
	1628	#D-* Examples:
	1629	#D-
[657]	1630	function IGCM_sys_build_run_file {
	1631
[664]	1632	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[657]	1633
	1634	}
	1635
	1636	############################################################
	1637	# Build MPI/OMP scripts
[811]	1638
	1639	#D-#==================================================
	1640	#D-function IGCM_sys_build_execution_scripts
	1641	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
	1642	#D-* Examples:
	1643	#D-
[657]	1644	function IGCM_sys_build_execution_scripts
	1645	{
	1646	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
	1647	if ( $DEBUG_sys ) ; then
	1648	echo "IGCM_sys_build_execution_scripts " $@
	1649	fi
	1650	typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
	1651	typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
	1652	typeset num_corempi nombre_restant_node nombre_restant_comp
	1653
	1654	if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ] ; then
[811]	1655	IGCM_debug_Exit "IGCM_sys_obelix build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
[657]	1656	fi
	1657
	1658	if ( ${OK_PARA_MPMD} ) ; then
	1659
	1660	if [ -f run_file ] ; then
	1661	IGCM_sys_Rm -f run_file
[16]	1662	fi
[657]	1663	touch run_file
	1664
	1665	if ( ${OK_PARA_OMP} ) ; then
	1666
	1667	# Hosts treatment
	1668
	1669	${HOST_MPIRUN_COMMAND} hostname \| sort \| uniq > hosts.tmp
	1670
	1671	i=0
	1672	rm -f hosts
	1673	IGCM_debug_Print 1 "sys Obelix, Hosts avaible :"
	1674	for nodes in `cat hosts.tmp` ; do
	1675	host[$i]=$nodes
	1676	echo "${host[$i]} slots=1 max_slots=1" >> hosts
	1677	IGCM_debug_Print 1 ${host[$i]}
	1678	i=$((i+1))
	1679	done
	1680	rm -f hosts.tmp
	1681
	1682	listnodes=${host[*]}
	1683
	1684	EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
	1685
	1686	# Initialisation
	1687
	1688	init_node=y
	1689	node_num_current=0
	1690	start_num=0
	1691	init_exec=n
	1692
	1693	# Test : if oasis is there, we put it at the first position
	1694
	1695	for comp in ${config_ListOfComponents[*]} ; do
	1696
	1697	if [ "X${comp}" = "XCPL" ] ; then
	1698
	1699	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1700	eval ExeNameOut=\${config_Executable_${comp}[1]}
	1701
	1702	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1703	echo "" >> script_${ExeNameOut}.ksh
	1704	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1705	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1706	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1707	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
	1708	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1709
	1710	init_node=n
	1711
	1712	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
	1713	node_num_current=0
	1714	node_current=${host[${node_num_current}]}
	1715
	1716	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
	1717
	1718	init_exec=y
	1719	start_num=1
	1720
	1721	fi
	1722
	1723	done
	1724
	1725	# Then loop on the components (except for oasis)
	1726
	1727	for comp in ${config_ListOfComponents[*]} ; do
	1728
	1729	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1730	eval ExeNameOut=\${config_Executable_${comp}[1]}
	1731
	1732	# Only if we really have an executable for the component :
	1733	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
	1734
	1735	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1736	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1737
	1738	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1739	# echo "set -vx" >> script_${ExeNameOut}.ksh
	1740	echo "" >> script_${ExeNameOut}.ksh
	1741	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1742	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1743	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1744	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1745	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
	1746	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
	1747	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1748
	1749	node_num=0
	1750
	1751	# We define the number of MPI process to be assigned for the component
	1752
	1753	nombre_restant_comp=${comp_proc_mpi_loc}
	1754
	1755	# Loop on the allocated nodes
	1756
	1757	for node in ${listnodes} ; do
	1758
	1759	# We go to the current node
	1760	if [ ${node_num} = ${node_num_current} ] ; then
	1761
	1762	node_current=${host[${node_num_current}]}
	1763
	1764	# If first time on the node : initialisation
	1765
	1766	if [ ${init_node} = y ] ; then
	1767	nombre_restant_node=${NUM_COREPERNODE}
	1768	fi
	1769
	1770	# Test on the number of OMP threads
	1771
	1772	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
	1773	(( node_num = node_num + 1 ))
	1774	node_num_current=${node_num}
	1775	init_node=y
	1776	continue
	1777	fi
	1778
	1779	# Number of MPI process to assign
	1780
	1781	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
	1782
	1783	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
	1784	num_corempi=${nombre_restant_comp}
	1785	fi
	1786
	1787	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
	1788	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
	1789
	1790	if [ ${init_exec} = y ] ; then
	1791	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
	1792	else
	1793	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
	1794	init_exec=y
	1795	fi
	1796
	1797	(( start_num = num_corempi + start_num ))
	1798
	1799	else
	1800
	1801	(( node_num = node_num + 1 ))
	1802	continue
	1803	fi
	1804
	1805	# Test on the number of core/process remaining on the node/component
	1806
	1807	if [ ${nombre_restant_node} = 0 ] ; then
	1808	(( node_num = node_num + 1 ))
	1809	node_num_current=${node_num}
	1810	init_node=y
	1811
	1812	if [ ${nombre_restant_comp} = 0 ] ; then
	1813	break 1
	1814	fi
	1815	else
	1816
	1817	node_num_current=${node_num}
	1818	init_node=n
	1819
	1820	if [ ${nombre_restant_comp} = 0 ] ; then
	1821	break 1
	1822	fi
	1823	fi
	1824	done
	1825	fi
	1826	done
	1827
	1828	else
	1829
[664]	1830	# Then first loop on the components for the coupler ie oasis
[657]	1831
[664]	1832	## the coupler ie oasis must be the first one
	1833	for comp in ${config_ListOfComponents[*]} ; do
[657]	1834
[664]	1835	eval ExeNameOut=\${config_Executable_${comp}[1]}
[657]	1836
	1837	# for CPL component only
[664]	1838	if [ "X${comp}" = "XCPL" ] ; then
	1839	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1840	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
	1841	fi
	1842	done
[657]	1843
[811]	1844	# Then second loop on the components
[657]	1845
[664]	1846	for comp in ${config_ListOfComponents[*]} ; do
[657]	1847
[664]	1848	eval ExeNameOut=\${config_Executable_${comp}[1]}
[657]	1849
[664]	1850	# Only if we really have an executable for the component and not the coupler ie oasis:
	1851	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
	1852	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1853	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1854	fi
	1855	done
	1856	IGCM_sys_Chmod u+x run_file
[657]	1857
[664]	1858	EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
[657]	1859
[16]	1860	fi
	1861
[657]	1862	else # Only one executable. launch it.
	1863
[664]	1864	for comp in ${config_ListOfComponents[*]} ; do
[657]	1865
[664]	1866	# Only if we really have an executable for the component :
	1867	eval ExeNameOut=\${config_Executable_${comp}[1]}
	1868	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
[657]	1869
[664]	1870	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1871	echo "" >> script_${ExeNameOut}.ksh
	1872	if ( ${OK_PARA_OMP} ) ; then
	1873	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1874	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1875	fi
	1876	if ( ${OK_PARA_MPI} ) ; then
	1877	# Default : mpirun used if nb_proc gt 1
	1878	# pour sortie out/err par process
	1879	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}" >> script_${ExeNameOut}.ksh
	1880	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
	1881	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1882	EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
	1883	else
	1884	# Default : mpirun is NOT used if nb_proc eq 1
	1885	# pour sortie out/err par process
	1886	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
	1887	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
	1888	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1889	EXECUTION="time ./script_${ExeNameOut}.ksh"
	1890	fi
	1891	fi
	1892	done
[657]	1893
	1894	fi
	1895
[811]	1896	IGCM_debug_Print 1 "sys Obelix : execution command is"
	1897	IGCM_debug_Print 1 "$EXECUTION"
[657]	1898
	1899	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[16]	1900	}
	1901
[933]	1902	#D-#==================================================
	1903	#D-function IGCM_sys_check_path
	1904	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
	1905	#D-* do not point to an important use directory. Stop immediately in that case.
	1906	#D-* Examples:
	1907	#D-
	1908	function IGCM_sys_check_path {
	1909	IGCM_debug_PushStack "IGCM_sys_check_path"
	1910	if ( $DEBUG_sys ) ; then
	1911	echo "IGCM_sys_check_path"
	1912	fi
[811]	1913
[933]	1914	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
	1915	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
	1916	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
	1917	IGCM_debug_Exit "This will stop the job" ;;
	1918	fi
	1919	IGCM_debug_PopStack "IGCM_sys_check_path"
	1920	}
	1921
[811]	1922	#D-#==================================================
	1923	#D-function IGCM_sys_check_quota. Dummy call here
	1924	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
	1925	#D-* Examples:
	1926	#D-
[657]	1927	function IGCM_sys_check_quota {
	1928	IGCM_debug_PushStack "IGCM_sys_check_quota"
	1929	if ( $DEBUG_sys ) ; then
	1930	echo "IGCM_sys_check_quota"
	1931	fi
	1932	IGCM_debug_PopStack "IGCM_sys_check_quota"
	1933	}
	1934
[811]	1935	#D-#==================================================
	1936	#D-function IGCM_sys_CountJobInQueue
	1937	#D-* Purpose: Check if job_name is currently
	1938	#D- running or in queue
	1939	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
	1940	#D-
	1941	function IGCM_sys_CountJobInQueue {
	1942	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
	1943	if ( $DEBUG_sys ) ; then
	1944	echo "IGCM_sys_CountJobInQueue"
	1945	fi
	1946	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
	1947	}
	1948
[16]	1949	##############################################################
	1950	# NCO OPERATOR
	1951
[811]	1952	#D-#==================================================
	1953	#D-function IGCM_sys_ncap2
	1954	#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
	1955	#D-* Examples:
	1956	#D-
[377]	1957	function IGCM_sys_ncap2 {
[657]	1958	IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
	1959	if ( $DEBUG_sys ) ; then
	1960	echo "IGCM_sys_ncap2 :" $@
	1961	fi
[714]	1962
	1963	typeset NB_ESSAI DELAI status i
	1964	# number of tentative
	1965	NB_ESSAI=3
	1966	# time delay between tentative
	1967	DELAI=2
	1968
	1969	i=0
	1970	while [ $i -lt $NB_ESSAI ] ; do
	1971	ncap2 "$@" > out_rsync 2>&1
	1972	status=$?
	1973	if [ ${status} -gt 0 ] ; then
	1974	IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
	1975	cat out_rsync
	1976	\rm out_rsync
	1977	IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	1978	sleep $DELAI
[714]	1979	else
	1980	\rm out_rsync
	1981	break
	1982	fi
	1983	(( i = i + 1 ))
	1984	done
	1985
	1986	if [ ${status} -gt 0 ] ; then
[811]	1987	echo "IGCM_sys_ncap2 : ncap2 error"
	1988	IGCM_debug_Exit "ncap2"
[657]	1989	fi
[16]	1990
[657]	1991	IGCM_debug_PopStack "IGCM_sys_ncap2"
[16]	1992	}
	1993
[811]	1994	#D-#==================================================
	1995	#D-function IGCM_sys_ncatted
	1996	#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
	1997	#D-* Examples:
	1998	#D-
[375]	1999	function IGCM_sys_ncatted {
[657]	2000	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
	2001	if ( $DEBUG_sys ) ; then
	2002	echo "IGCM_sys_ncatted :" $@
	2003	fi
[714]	2004
	2005	typeset NB_ESSAI DELAI status i
	2006	# number of tentative
	2007	NB_ESSAI=3
	2008	# time delay between tentative
	2009	DELAI=2
	2010
	2011	i=0
	2012	while [ $i -lt $NB_ESSAI ] ; do
	2013	ncatted "$@" > out_rsync 2>&1
	2014	status=$?
	2015	if [ ${status} -gt 0 ] ; then
	2016	IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
	2017	cat out_rsync
	2018	\rm out_rsync
	2019	IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2020	sleep $DELAI
[714]	2021	else
	2022	\rm out_rsync
	2023	break
	2024	fi
	2025	(( i = i + 1 ))
	2026	done
	2027
	2028	if [ ${status} -gt 0 ] ; then
[811]	2029	echo "IGCM_sys_ncatted : ncatted error"
	2030	IGCM_debug_Exit "ncatted"
[657]	2031	fi
[16]	2032
[657]	2033	IGCM_debug_PopStack "IGCM_sys_ncatted"
[375]	2034	}
[16]	2035
[811]	2036	#D-#==================================================
	2037	#D-function IGCM_sys_ncbo
	2038	#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
	2039	#D-* Examples:
	2040	#D-
[16]	2041	function IGCM_sys_ncbo {
[657]	2042	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
	2043	if ( $DEBUG_sys ) ; then
	2044	echo "IGCM_sys_ncbo :" $@
	2045	fi
[714]	2046
	2047	typeset NB_ESSAI DELAI status i
	2048	# number of tentative
	2049	NB_ESSAI=3
	2050	# time delay between tentative
	2051	DELAI=2
	2052
	2053	i=0
	2054	while [ $i -lt $NB_ESSAI ] ; do
	2055	ncbo $@ > out_rsync 2>&1
	2056	status=$?
	2057	if [ ${status} -gt 0 ] ; then
	2058	IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
	2059	cat out_rsync
	2060	\rm out_rsync
	2061	IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2062	sleep $DELAI
[714]	2063	else
	2064	\rm out_rsync
	2065	break
	2066	fi
	2067	(( i = i + 1 ))
	2068	done
	2069
	2070	if [ ${status} -gt 0 ] ; then
[811]	2071	echo "IGCM_sys_ncbo : ncbo error"
	2072	IGCM_debug_Exit "ncbo"
[657]	2073	fi
[16]	2074
[657]	2075	IGCM_debug_PopStack "IGCM_sys_ncbo"
[16]	2076	}
	2077
[811]	2078	#D-#==================================================
	2079	#D-function IGCM_sys_ncdif
	2080	#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
	2081	#D-* Examples:
	2082	#D-
[16]	2083	function IGCM_sys_ncdiff {
[657]	2084	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
	2085	if ( $DEBUG_sys ) ; then
	2086	echo "IGCM_sys_ncdiff :" $@
	2087	fi
[714]	2088
	2089	typeset NB_ESSAI DELAI status i
	2090	# number of tentative
	2091	NB_ESSAI=3
	2092	# time delay between tentative
	2093	DELAI=2
	2094
	2095	i=0
	2096	while [ $i -lt $NB_ESSAI ] ; do
	2097	ncdiff $@ > out_rsync 2>&1
	2098	status=$?
	2099	if [ ${status} -gt 0 ] ; then
	2100	IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
	2101	cat out_rsync
	2102	\rm out_rsync
	2103	IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2104	sleep $DELAI
[714]	2105	else
	2106	\rm out_rsync
	2107	break
	2108	fi
	2109	(( i = i + 1 ))
	2110	done
	2111
	2112	if [ ${status} -gt 0 ] ; then
[811]	2113	echo "IGCM_sys_ncdiff : ncdiff error"
	2114	IGCM_debug_Exit "ncdiff"
[657]	2115	fi
[16]	2116
[657]	2117	IGCM_debug_PopStack "IGCM_sys_ncdiff"
[16]	2118	}
	2119
[811]	2120	#D-#==================================================
	2121	#D-function IGCM_sys_ncea
	2122	#D-* Purpose: encapsulate ncea call so as to manage error code and retry
	2123	#D-* Examples:
	2124	#D-
[16]	2125	function IGCM_sys_ncea {
[657]	2126	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
	2127	if ( $DEBUG_sys ) ; then
	2128	echo "IGCM_sys_ncea :" $@
	2129	fi
[714]	2130
	2131	typeset NB_ESSAI DELAI status i
	2132	# number of tentative
	2133	NB_ESSAI=3
	2134	# time delay between tentative
	2135	DELAI=2
	2136
	2137	i=0
	2138	while [ $i -lt $NB_ESSAI ] ; do
	2139	ncea $@ > out_rsync 2>&1
	2140	status=$?
	2141	if [ ${status} -gt 0 ] ; then
	2142	IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
	2143	cat out_rsync
	2144	\rm out_rsync
	2145	IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2146	sleep $DELAI
[714]	2147	else
	2148	\rm out_rsync
	2149	break
	2150	fi
	2151	(( i = i + 1 ))
	2152	done
	2153
	2154	if [ ${status} -gt 0 ] ; then
[811]	2155	echo "IGCM_sys_ncea : ncea error"
	2156	IGCM_debug_Exit "ncea"
[657]	2157	fi
[16]	2158
[657]	2159	IGCM_debug_PopStack "IGCM_sys_ncea"
[16]	2160	}
	2161
[811]	2162	#D-#==================================================
	2163	#D-function IGCM_sys_ncecat
	2164	#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
	2165	#D-* Examples:
	2166	#D-
[16]	2167	function IGCM_sys_ncecat {
[657]	2168	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
	2169	if ( $DEBUG_sys ) ; then
	2170	echo "IGCM_sys_ncecat :" $@
	2171	fi
[714]	2172
	2173	typeset NB_ESSAI DELAI status i
	2174	# number of tentative
	2175	NB_ESSAI=3
	2176	# time delay between tentative
	2177	DELAI=2
	2178
	2179	i=0
	2180	while [ $i -lt $NB_ESSAI ] ; do
	2181	ncecat $@ > out_rsync 2>&1
	2182	status=$?
	2183	if [ ${status} -gt 0 ] ; then
	2184	IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
	2185	cat out_rsync
	2186	\rm out_rsync
	2187	IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2188	sleep $DELAI
[714]	2189	else
	2190	\rm out_rsync
	2191	break
	2192	fi
	2193	(( i = i + 1 ))
	2194	done
	2195
	2196	if [ ${status} -gt 0 ] ; then
[811]	2197	echo "IGCM_sys_ncecat : ncecat error"
	2198	IGCM_debug_Exit "ncecat"
[657]	2199	fi
[16]	2200
[657]	2201	IGCM_debug_PopStack "IGCM_sys_ncecat"
[16]	2202	}
	2203
[811]	2204	#D-#==================================================
	2205	#D-function IGCM_sys_ncflint
	2206	#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
	2207	#D-* Examples:
	2208	#D-
[16]	2209	function IGCM_sys_ncflint {
[657]	2210	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
	2211	if ( $DEBUG_sys ) ; then
	2212	echo "IGCM_sys_ncflint :" $@
	2213	fi
[714]	2214
	2215	typeset NB_ESSAI DELAI status i
	2216	# number of tentative
	2217	NB_ESSAI=3
	2218	# time delay between tentative
	2219	DELAI=2
	2220
	2221	i=0
	2222	while [ $i -lt $NB_ESSAI ] ; do
	2223	ncflint $@ > out_rsync 2>&1
	2224	status=$?
	2225	if [ ${status} -gt 0 ] ; then
	2226	IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
	2227	cat out_rsync
	2228	\rm out_rsync
	2229	IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2230	sleep $DELAI
[714]	2231	else
	2232	\rm out_rsync
	2233	break
	2234	fi
	2235	(( i = i + 1 ))
	2236	done
	2237
	2238	if [ ${status} -gt 0 ] ; then
[811]	2239	echo "IGCM_sys_ncflint : ncflint error"
	2240	IGCM_debug_Exit "ncflint"
[657]	2241	fi
[16]	2242
[657]	2243	IGCM_debug_PopStack "IGCM_sys_ncflint"
[16]	2244	}
	2245
[811]	2246	#D-#==================================================
	2247	#D-function IGCM_sys_ncks
	2248	#D-* Purpose: encapsulate ncks call so as to manage error code and retry
	2249	#D-* Examples:
	2250	#D-
[16]	2251	function IGCM_sys_ncks {
[657]	2252	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
	2253	if ( $DEBUG_sys ) ; then
	2254	echo "IGCM_sys_ncks :" $@
	2255	fi
[714]	2256
	2257	typeset NB_ESSAI DELAI status i
	2258	# number of tentative
	2259	NB_ESSAI=3
	2260	# time delay between tentative
	2261	DELAI=2
	2262
	2263	i=0
	2264	while [ $i -lt $NB_ESSAI ] ; do
	2265	ncks $@ > out_rsync 2>&1
	2266	status=$?
	2267	if [ ${status} -gt 0 ] ; then
	2268	IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
	2269	cat out_rsync
	2270	\rm out_rsync
	2271	IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2272	sleep $DELAI
[714]	2273	else
	2274	\rm out_rsync
	2275	break
	2276	fi
	2277	(( i = i + 1 ))
	2278	done
	2279
	2280	if [ ${status} -gt 0 ] ; then
[811]	2281	echo "IGCM_sys_ncks : ncks error"
	2282	IGCM_debug_Exit "ncks"
[657]	2283	fi
[16]	2284
[657]	2285	IGCM_debug_PopStack "IGCM_sys_ncks"
[16]	2286	}
	2287
[811]	2288	#D-#==================================================
	2289	#D-function IGCM_sys_ncpdq
	2290	#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
	2291	#D-* Examples:
	2292	#D-
[16]	2293	function IGCM_sys_ncpdq {
[657]	2294	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
	2295	if ( $DEBUG_sys ) ; then
	2296	echo "IGCM_sys_ncpdq :" $@
	2297	fi
[714]	2298
	2299	typeset NB_ESSAI DELAI status i
	2300	# number of tentative
	2301	NB_ESSAI=3
	2302	# time delay between tentative
	2303	DELAI=2
	2304
	2305	i=0
	2306	while [ $i -lt $NB_ESSAI ] ; do
	2307	ncpdq $@ > out_rsync 2>&1
	2308	status=$?
	2309	if [ ${status} -gt 0 ] ; then
	2310	IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
	2311	cat out_rsync
	2312	\rm out_rsync
	2313	IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2314	sleep $DELAI
[714]	2315	else
	2316	\rm out_rsync
	2317	break
	2318	fi
	2319	(( i = i + 1 ))
	2320	done
	2321
	2322	if [ ${status} -gt 0 ] ; then
[811]	2323	echo "IGCM_sys_ncpdq : ncpdq error"
	2324	IGCM_debug_Exit "ncpdq"
[657]	2325	fi
[16]	2326
[657]	2327	IGCM_debug_PopStack "IGCM_sys_ncpdq"
[16]	2328	}
	2329
[811]	2330	#D-#==================================================
	2331	#D-function IGCM_sys_ncra
	2332	#D-* Purpose: encapsulate ncra call so as to manage error code and retry
	2333	#D-* Examples:
	2334	#D-
[16]	2335	function IGCM_sys_ncra {
[657]	2336	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
	2337	if ( $DEBUG_sys ) ; then
	2338	echo "IGCM_sys_ncra :" $@
	2339	fi
[714]	2340
	2341	typeset NB_ESSAI DELAI status i
	2342	# number of tentative
	2343	NB_ESSAI=3
	2344	# time delay between tentative
	2345	DELAI=2
	2346
	2347	i=0
	2348	while [ $i -lt $NB_ESSAI ] ; do
	2349	ncra $@ > out_rsync 2>&1
	2350	status=$?
	2351	if [ ${status} -gt 0 ] ; then
	2352	IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
	2353	cat out_rsync
	2354	\rm out_rsync
	2355	IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2356	sleep $DELAI
[714]	2357	else
	2358	\rm out_rsync
	2359	break
	2360	fi
	2361	(( i = i + 1 ))
	2362	done
	2363
	2364	if [ ${status} -gt 0 ] ; then
[811]	2365	echo "IGCM_sys_ncra : ncra error"
	2366	IGCM_debug_Exit "ncra"
[657]	2367	fi
[16]	2368
[657]	2369	IGCM_debug_PopStack "IGCM_sys_ncra"
[16]	2370	}
	2371
[811]	2372	#D-#==================================================
	2373	#D-function IGCM_sys_ncrcat
	2374	#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
	2375	#D-* Examples:
	2376	#D-
[16]	2377	function IGCM_sys_ncrcat {
[657]	2378	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
	2379	if ( $DEBUG_sys ) ; then
	2380	echo "IGCM_sys_ncrcat :" $@
	2381	fi
[714]	2382
[811]	2383	typeset NB_ESSAI DELAI status i lastArg
[714]	2384	# number of tentative
	2385	NB_ESSAI=3
	2386	# time delay between tentative
	2387	DELAI=2
	2388
	2389	i=0
	2390	while [ $i -lt $NB_ESSAI ] ; do
	2391	ncrcat $@ > out_rsync 2>&1
	2392	status=$?
	2393	if [ ${status} -gt 0 ] ; then
	2394	IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
	2395	cat out_rsync
	2396	\rm out_rsync
	2397	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2398	sleep $DELAI
[811]	2399	elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" out_rsync )" = "X" ] ; then
	2400	IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
	2401	cat out_rsync
	2402	# remove files having corrupted time axis
	2403	eval lastArg=\${$#}
	2404	IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
	2405	\rm ${lastArg}
	2406	\rm out_rsync
	2407	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
	2408	sleep $DELAI
[714]	2409	else
	2410	\rm out_rsync
	2411	break
	2412	fi
	2413	(( i = i + 1 ))
	2414	done
	2415
	2416	if [ ${status} -gt 0 ] ; then
[811]	2417	echo "IGCM_sys_ncrcat : ncrcat error"
	2418	#IGCM_debug_Exit "ncrcat"
[657]	2419	fi
[16]	2420
[657]	2421	IGCM_debug_PopStack "IGCM_sys_ncrcat"
[16]	2422	}
	2423
[811]	2424	#D-#==================================================
	2425	#D-function IGCM_sys_ncrename
	2426	#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
	2427	#D-* Examples:
	2428	#D-
[16]	2429	function IGCM_sys_ncrename {
[657]	2430	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
	2431	if ( $DEBUG_sys ) ; then
	2432	echo "IGCM_sys_ncrename :" $@
	2433	fi
[714]	2434
	2435	typeset NB_ESSAI DELAI status i
	2436	# number of tentative
	2437	NB_ESSAI=3
	2438	# time delay between tentative
	2439	DELAI=2
	2440
	2441	i=0
	2442	while [ $i -lt $NB_ESSAI ] ; do
	2443	ncrename $@ > out_rsync 2>&1
	2444	status=$?
	2445	if [ ${status} -gt 0 ] ; then
	2446	IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
	2447	cat out_rsync
	2448	\rm out_rsync
	2449	IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2450	sleep $DELAI
[714]	2451	else
	2452	\rm out_rsync
	2453	break
	2454	fi
	2455	(( i = i + 1 ))
	2456	done
	2457
	2458	if [ ${status} -gt 0 ] ; then
[811]	2459	echo "IGCM_sys_ncrename : ncrename error"
	2460	IGCM_debug_Exit "ncrename"
[657]	2461	fi
[16]	2462
[657]	2463	IGCM_debug_PopStack "IGCM_sys_ncrename"
[16]	2464	}
	2465
[811]	2466	#D-#==================================================
	2467	#D-function IGCM_sys_ncwa
	2468	#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
	2469	#D-* Examples:
	2470	#D-
[16]	2471	function IGCM_sys_ncwa {
[657]	2472	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
	2473	if ( $DEBUG_sys ) ; then
	2474	echo "IGCM_sys_ncwa :" $@
	2475	fi
[714]	2476
	2477	typeset NB_ESSAI DELAI status i
	2478	# number of tentative
	2479	NB_ESSAI=3
	2480	# time delay between tentative
	2481	DELAI=2
	2482
	2483	i=0
	2484	while [ $i -lt $NB_ESSAI ] ; do
	2485	ncwa $@ > out_rsync 2>&1
	2486	status=$?
	2487	if [ ${status} -gt 0 ] ; then
	2488	IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
	2489	cat out_rsync
	2490	\rm out_rsync
	2491	IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[800]	2492	sleep $DELAI
[714]	2493	else
	2494	\rm out_rsync
	2495	break
	2496	fi
	2497	(( i = i + 1 ))
	2498	done
	2499
	2500	if [ ${status} -gt 0 ] ; then
[811]	2501	echo "IGCM_sys_ncwa : ncwa error"
	2502	IGCM_debug_Exit "ncwa"
[657]	2503	fi
[16]	2504
[657]	2505	IGCM_debug_PopStack "IGCM_sys_ncwa"
[16]	2506	}
	2507
[284]	2508	##############################################################
	2509	# CDO OPERATOR
	2510
[811]	2511	#D-#==================================================
	2512	#D-function IGCM_sys_cdo
	2513	#D-* Purpose: encapsulate cdo call so as to manage error code and retry
	2514	#D-* Examples:
	2515	#D-
[284]	2516	function IGCM_sys_cdo {
[657]	2517	IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
[714]	2518	if ( $DEBUG_sys ) ; then
	2519	echo "IGCM_sys_cdo :" $@
	2520	fi
[284]	2521
[714]	2522	typeset status
	2523
	2524	\cdo $@ > out_rsync 2>&1
	2525	status=$?
	2526	if [ ${status} -gt 0 ] ; then
	2527	echo "IGCM_sys_cdo : error code ${status}"
	2528	cat out_rsync
	2529	\rm out_rsync
[284]	2530	IGCM_debug_PopStack "IGCM_sys_cdo"
[657]	2531	return 1
	2532	else
	2533	IGCM_debug_PopStack "IGCM_sys_cdo"
	2534	return 0
	2535	fi
[284]	2536
[657]	2537	IGCM_debug_PopStack "IGCM_sys_cdo"
[16]	2538	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: