Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1369

Last change on this file since 1369 was 1364, checked in by sdipsl, 8 years ago
fix some environment problem with the PCMDI_MP
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 55.9 KB

Rev	Line
[622]	1	#!/bin/ksh
	2
	3	#**************************************************************
	4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
	5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#=========================================================
	15	# The documentation of this file can be automatically generated
[623]	16	# if you use the prefix #D- for comments to be extracted.
[622]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#=========================================================
	19
	20	#D-#==================================================
[623]	21	#D-LibIGCM_sys for Curie
[622]	22	#D-#==================================================
	23	#D-
[623]	24	#D- This ksh library if a layer under some usefull
[622]	25	#D-environment variables and shell commands.
	26	#D-All those definitions depend on host particularities.
	27	#D-It manages a stack mechanism and test validity of operations.
	28	#D-All function described bellow must be prefixed by IGCM_sys.
	29
	30	#====================================================
	31	# libIGCM_sys PARAMETERS
	32	#====================================================
	33
	34	#====================================================
	35	# set DEBUG_sys to true to output calls of function
	36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
	37
	38	#====================================================
	39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
	40	typeset -r DRYRUN=${DRYRUN:=0}
	41
	42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
	43	# -------------------------------------------------------------------------------------
	44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
[809]	45	# \| \| Cp/Exe/param/files \| \| \| \|
	46	# \| \| Chmod Qsub \| \| \| \|
[622]	47	# -------------------------------------------------------------------------------------
	48	# \| 0 \| yes \| yes \| yes \| yes \|
	49	# -------------------------------------------------------------------------------------
	50	# \| 1 \| yes \| yes \| yes \| no \|
	51	# -------------------------------------------------------------------------------------
	52	# \| 2 \| yes \| yes \| no \| no \|
	53	# -------------------------------------------------------------------------------------
	54	# \| 3 \| yes \| no \| no \| no \|
	55	# -------------------------------------------------------------------------------------
	56
	57	#=====================================================
	58	# Global Variables :
	59	#=====================================================
	60	# Language : "fr" or "en"
	61	typeset -r MYLANG="fr"
	62
	63	#=====================================================
[1201]	64	# Host user names project maxCpuTime
[622]	65	# $hostname ou hostname
[1158]	66	typeset HOST=${HOST:=$( hostname )}
[622]	67	# $username ou whoami
[1158]	68	typeset LOGIN=${LOGIN:=$( whoami )}
[622]	69	# $hostname of the MASTER job
	70	typeset MASTER=curie
[880]	71	# add default project on curie
[1201]	72	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} \| cut -d@ -f1 )
	73	# jobWarningDelay in seconds
	74	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
[622]	75
	76	#D-
	77	#D-#==================================================
	78	#D-Program used in libIGCM
	79	#D-#==================================================
	80
[742]	81	# Submit command
[747]	82	typeset SUBMIT=${SUBMIT:=ccc_msub}
[622]	83	# rsync with path
	84	typeset -r RSYNC=/usr/bin/rsync
	85	# RSYNC_opt args to rsync
	86	typeset -r RSYNC_opt="-va"
	87	# ie storage filesystem
[623]	88	typeset -r STOREHOST=${MASTER}
[1180]	89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
[622]	90
	91	#====================================================
[1180]	92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
[622]	93	#====================================================
[996]	94	if [ X${TaskType} = Xcomputing ] ; then
[1358]	95	module load python/2.7.8 > /dev/null 2>&1
[1193]	96	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
[996]	97	# to run with netcdf 3.6.3 ie compilation done before 17/2/2014
	98	# uncomment 2 lines :
	99	# module unload netcdf
	100	# module load netcdf/3.6.3
	101	else
[1196]	102	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
[1358]	103	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
[1364]	104	export UVCDAT_ANONYMOUS_LOG=FALSE
[996]	105	fi
[1315]	106
	107	# Use CMIP6 storage space when using CMIP6 cpu ressources
[1330]	108	[[ "X${PROJECT}" = "Xdevcmip6" && ! $(module list --terse 2>&1 \| grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
[1315]	109
	110	# FYI
[1210]	111	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
	112	[ ! X${TaskType} = Xchecking ] && module list
[1208]	113
[1179]	114	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
	115	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
	116	export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
[622]	117
	118	#====================================================
	119	# Specific for ocean additionnal diagnostic
	120	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
	121	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
	122
	123	#====================================================
	124	# Host specific DIRECTORIES
	125	#====================================================
	126
[623]	127	# ============ CESIUM START ============ #
	128
[622]	129	#====================================================
[623]	130	#- Mirror libIGCM from titane to cesium if needed
	131	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
	132	#if [ ! ${ROOTSYS} = "home" ] ; then
	133	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
	134	#else
	135	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	136	#fi
	137
	138	#====================================================
	139	#- libIGCM_POST
	140	#if ( ${MirrorlibIGCM} ) ; then
	141	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
	142	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
	143	#else
	144	# typeset -r libIGCM_POST=${libIGCM}
	145	#fi
	146
	147	# ============ CESIUM END ============ #
	148
	149	#====================================================
	150	#- MirrorlibIGCM for frontend
	151	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	152
	153	#====================================================
	154	#- libIGCM_POST for frontend
	155	typeset -r libIGCM_POST=${libIGCM}
	156
	157	#====================================================
[622]	158	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
	159	typeset -r R_EXE="${MODIPSL}/bin"
	160
	161	#====================================================
	162	#- SUBMIT_DIR : submission dir
[1010]	163	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
[622]	164
	165	#====================================================
	166	#- IN
[1276]	167	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
[622]	168	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
	169
	170	#====================================================
	171	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[850]	172	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
[622]	173
	174	#====================================================
[1179]	175	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
[1186]	176	typeset -r OUTCOMMAND_PATH=/tmp
[1179]	177
	178	#====================================================
[622]	179	#- HOST_MPIRUN_COMMAND
[808]	180	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}
[622]	181
	182	#====================================================
	183	#- Max number of arguments passed to nco operator or demigration command
	184	UNIX_MAX_LIMIT=360
	185
[623]	186	#====================================================
[1180]	187	#- set PackDefault to true on curie
[623]	188	PackDefault=true
	189
[664]	190	#====================================================
[623]	191	#- Default number of MPI task for IPSL coupled model
	192	#- required for backward compatibility
	193	#-
	194	DEFAULT_NUM_PROC_OCE=5
	195	DEFAULT_NUM_PROC_CPL=1
[637]	196	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
[623]	197
[725]	198	#D-#==================================================
[1057]	199	#D-function IGCM_sys_defineArchives
	200	#D-* Purpose:
	201	#D-* Define ARCHIVE : Dedicated to large files
	202	#D-* Define STORAGE : Dedicated to small/medium files
	203	#D-* Define R_OUT : Output tree located on ARCHIVE
	204	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
	205	#D-* Define R_BUF : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
	206	#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
[725]	207	#D-* Examples:
	208	#D-
[1057]	209	function IGCM_sys_defineArchives {
	210	IGCM_debug_PushStack "IGCM_sys_defineArchives"
[676]	211
[1060]	212	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
[1057]	213	#====================================================
	214	#- ARCHIVE (dedicated to large files)
[1063]	215	ARCHIVE=${config_UserChoices_ARCHIVE}
[1057]	216	else
	217	#====================================================
	218	#- ARCHIVE (dedicated to large files)
[1063]	219	ARCHIVE=${CCCSTOREDIR}
[1057]	220	fi
[725]	221
[1060]	222	if [ ! X${config_UserChoices_STORAGE} = X ]; then
[1057]	223	#====================================================
	224	#- STORAGE (dedicated to small/medium files)
[1063]	225	STORAGE=${config_UserChoices_STORAGE}
[1057]	226	else
	227	#====================================================
	228	#- STORAGE (dedicated to small/medium files)
[1063]	229	STORAGE=${CCCWORKDIR}
[1057]	230	fi
[725]	231
[1057]	232	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
	233	#====================================================
	234	#- R_OUT
[1063]	235	R_OUT=${SCRATCHDIR}/IGCM_OUT
[1057]	236
	237	#====================================================
	238	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]	239	R_FIG=${SCRATCHDIR}/IGCM_OUT
[1057]	240
	241	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
	242
	243	else
	244	#====================================================
	245	#- R_OUT
[1063]	246	R_OUT=${ARCHIVE}/IGCM_OUT
[1057]	247
	248	#====================================================
	249	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]	250	R_FIG=${STORAGE}/IGCM_OUT
[1057]	251	fi
	252
	253	#====================================================
	254	#- R_BUF (ONLY FOR double copy an scratch)
[1063]	255	R_BUF=${SCRATCHDIR}/IGCM_OUT
[1057]	256
	257	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
	258	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
	259	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
	260
	261	IGCM_debug_PopStack "IGCM_sys_defineArchives"
[725]	262	}
	263
[622]	264	#D-#==================================================
	265	#D-function IGCM_sys_RshArchive
	266	#D-* Purpose: Archive rsh command
	267	#D-* Examples:
	268	#D-
	269	function IGCM_sys_RshArchive {
[623]	270	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
	271	/bin/ksh <<-EOF
[622]	272	${@}
	273	EOF
[811]	274	status=$?
	275	if [ ${status} -gt 0 ] ; then
	276	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
[623]	277	IGCM_debug_Exit "IGCM_sys_RshArchive"
	278	fi
	279	IGCM_debug_PopStack "IGCM_sys_RshArchive"
[622]	280	}
	281
	282	#D-#==================================================
[845]	283	#D-function IGCM_sys_RshArchive_NoError
	284	#D-* Purpose: Archive rsh command, without error
	285	#D-* used only in monitoring.job
	286	#D-* Examples:
	287	#D-
	288	function IGCM_sys_RshArchive_NoError {
	289	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
	290	/bin/ksh <<-EOF
[1181]	291	${@} 2> /dev/null
[845]	292	EOF
	293	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
	294	}
	295
	296	#D-#==================================================
[622]	297	#D-function IGCM_sys_MkdirArchive
	298	#D-* Purpose: Mkdir on Archive
	299	#D-* Examples:
	300	#D-
	301	function IGCM_sys_MkdirArchive {
[623]	302	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
	303	if ( $DEBUG_sys ) ; then
	304	echo "IGCM_sys_MkdirArchive :" $@
	305	fi
	306	#- creation de repertoire sur le serveur fichier
	307	if [ ! -d ${1} ]; then
	308	\mkdir -p $1
[811]	309	status=$?
	310
	311	if [ ${status} -gt 0 ] ; then
	312	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
[623]	313	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
[622]	314	fi
[623]	315	fi
	316	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[622]	317	}
	318
	319	#D-#==================================================
	320	#D-function IGCM_sys_TestDirArchive
	321	#D-* Purpose: Test Directory that must exists on Archive
	322	#D-* Examples:
	323	#D-
	324	function IGCM_sys_TestDirArchive {
[623]	325	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
	326	if ( $DEBUG_sys ) ; then
	327	echo "IGCM_sys_TestDirArchive :" $@
	328	fi
	329	typeset ExistFlag
	330	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	331	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
	332	return ${ExistFlag}
[622]	333	}
	334
	335	#D-#==================================================
[623]	336	#D-function IGCM_sys_IsFileArchived
[1179]	337	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
[623]	338	#D-* Examples:
	339	#D-
	340	function IGCM_sys_IsFileArchived {
	341	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
	342	if ( $DEBUG_sys ) ; then
	343	echo "IGCM_sys_IsFileArchived :" $@
	344	fi
	345	typeset IsArchivedFlag
[818]	346	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
[623]	347	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
	348
	349	return ${IsArchivedFlag}
	350	}
	351
	352	#D-#==================================================
[622]	353	#D-function IGCM_sys_TestFileArchive
	354	#D-* Purpose: Test file that must NOT EXISTS on Archive
	355	#D-* Examples:
	356	#D-
	357	function IGCM_sys_TestFileArchive {
[623]	358	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
	359	typeset ExistFlag
	360	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	361	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[622]	362
[623]	363	return ${ExistFlag}
[622]	364	}
	365
	366	#D-#==================================================
	367	#D-function IGCM_sys_CountFileArchive
	368	#D-* Purpose: Count files on Archive filesystem
	369	#D-* Examples:
	370	#D-
	371	function IGCM_sys_CountFileArchive {
[623]	372	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
	373	ls ${@} 2>/dev/null \| wc -l
	374	if [ $? -gt 0 ] ; then
	375	echo "IGCM_sys_CountFileArchive : erreur."
	376	fi
	377	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[622]	378	}
	379
	380	#D-#==================================================
	381	#D-function IGCM_sys_Tree
	382	#D-* Purpose: Tree directories with files on ${ARCHIVE}
	383	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
	384	#D-
	385	function IGCM_sys_Tree {
[623]	386	IGCM_debug_PushStack "IGCM_sys_Tree" $@
	387	if ( $DEBUG_sys ) ; then
	388	echo "IGCM_sys_Tree :" $@
	389	fi
[622]	390
[623]	391	\ls -lR ${@}
[622]	392
[623]	393	IGCM_debug_PopStack "IGCM_sys_Tree"
[622]	394	}
	395
	396	#D-#==================================================
	397	#D-function IGCM_sys_Qsub
	398	#D-* Purpose: Qsub new job
	399	#D-* Examples:
	400	#D-
	401	function IGCM_sys_Qsub {
[623]	402	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
	403	if ( $DEBUG_sys ) ; then
	404	echo "IGCM_sys_Qsub :" $@
	405	fi
[811]	406	typeset options status
[923]	407	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
[622]	408
[1179]	409	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	410	status=$?
[948]	411
[1179]	412	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	413	if [ ${status} -gt 0 ] ; then
	414	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
[623]	415	IGCM_debug_Exit "IGCM_sys_Qsub"
[948]	416	else
[1179]	417	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	418	fi
	419	IGCM_debug_PopStack "IGCM_sys_Qsub"
[622]	420	}
	421
	422	#D-#==================================================
	423	#D-function IGCM_sys_QsubPost
	424	#D-* Purpose: Qsub new job on scalaire
	425	#D-* Examples:
	426	#D-
	427	function IGCM_sys_QsubPost {
[623]	428	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
	429	if ( $DEBUG_sys ) ; then
	430	echo "IGCM_sys_QsubPost :" $@
	431	fi
[923]	432	typeset options status
	433
[1062]	434	# EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
	435	case $( hostname -s ) in
	436	curie*)
[1201]	437	options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
[1062]	438	airain*)
	439	options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
	440	esac
	441
[1179]	442	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	443	status=$?
[948]	444
[1179]	445	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	446	if [ ${status} -gt 0 ] ; then
[923]	447	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
[623]	448	IGCM_debug_Exit "IGCM_sys_QsubPost"
[948]	449	else
[1179]	450	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	451	fi
	452	IGCM_debug_PopStack "IGCM_sys_QsubPost"
[622]	453	}
	454
	455	#D-*************************
[623]	456	#D- File transfer functions
[622]	457	#D-*************************
	458	#D-
	459
	460	#D-#==================================================
	461	#D-function IGCM_sys_RmRunDir
	462	#D-* Purpose: rm tmpdir (dummy function most of the time batch
	463	#D- scheduler will do the job)
	464	#D-* Examples:
	465	#D-
	466	function IGCM_sys_RmRunDir {
[1057]	467	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
[623]	468	if ( $DEBUG_sys ) ; then
	469	echo "IGCM_sys_RmRunDir :" $@
	470	fi
[622]	471
[715]	472	typeset status
[622]	473
[1179]	474	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	475	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	476	status=$?
[622]	477
[715]	478	if [ ${status} -gt 0 ] ; then
[1183]	479	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
[1179]	480	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	481	IGCM_debug_Exit "IGCM_sys_RmRunDir"
	482	else
[1179]	483	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	484	fi
	485	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[622]	486	}
	487
	488	#D-#==================================================
	489	#D-function IGCM_sys_Put_Dir
	490	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
	491	#D-* Examples:
	492	#D-
	493	function IGCM_sys_Put_Dir {
[623]	494	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
	495	if ( $DEBUG_sys ) ; then
	496	echo "IGCM_sys_Put_Dir :" $@
	497	fi
	498	if [ $DRYRUN = 0 ]; then
	499	if [ ! -d ${1} ] ; then
	500	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
	501	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
	502	return
[622]	503	fi
	504
[715]	505	typeset status
[622]	506
[623]	507	# Only if we use rsync
	508	#IGCM_sys_TestDirArchive $( dirname $2 )
	509	#
	510	#USUAL WAY
[1179]	511	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	512	status=$?
[622]	513
[715]	514	if [ ${status} -gt 0 ] ; then
[811]	515	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
[1179]	516	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	517	IGCM_debug_Exit "IGCM_sys_Put_Dir"
[622]	518	else
[1179]	519	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	520	fi
[623]	521	fi
	522	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[622]	523	}
	524
	525	#D-#==================================================
	526	#D-function IGCM_sys_Get_Dir
[623]	527	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[622]	528	#D-* Examples:
	529	#D-
	530	function IGCM_sys_Get_Dir {
[623]	531	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
	532	if ( $DEBUG_sys ) ; then
	533	echo "IGCM_sys_Get_Dir :" $@
	534	fi
	535	if [ $DRYRUN = 0 ]; then
[708]	536	typeset NB_ESSAI DELAI status i
	537	# number of tentative
	538	NB_ESSAI=3
	539	# time delay between tentative
	540	DELAI=2
[622]	541
[676]	542	#
[708]	543	# USUAL WAY
	544	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
[713]	545	ccc_hsm get -r $1
[622]	546
[709]	547	i=0
[708]	548	while [ $i -lt $NB_ESSAI ] ; do
[1179]	549	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[708]	550	status=$?
[811]	551	if [ ${status} -gt 0 ] ; then
[713]	552	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
[708]	553	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
	554	sleep $DELAI
	555	else
	556	break
	557	fi
	558	(( i = i + 1 ))
	559	done
	560
	561	if [ ${status} -gt 0 ] ; then
[1178]	562	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
[1179]	563	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	564	IGCM_debug_Exit "IGCM_sys_Get_Dir"
[622]	565	else
[1179]	566	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	567	fi
[623]	568	fi
	569	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[622]	570	}
	571
	572	#D-#==================================================
	573	#D-function IGCM_sys_Put_Rest
[623]	574	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[622]	575	#D- File and target directory must exist.
	576	#D-* Examples:
	577	#D-
	578	function IGCM_sys_Put_Rest {
[623]	579	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
	580	if ( $DEBUG_sys ) ; then
	581	echo "IGCM_sys_Put_Rest :" $@
	582	fi
	583	if [ $DRYRUN = 0 ]; then
	584	if [ ! -f ${1} ] ; then
	585	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
	586	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	587	fi
	588
[715]	589	typeset status
[623]	590	#
	591	# USUAL WAY
[1179]	592	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	593	status=$?
[622]	594
[623]	595	# #RSYNC WITH NETWORK SSH CALL
[1179]	596	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	597	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	598
[623]	599	# #RSYNC WITH NFS USE
[1179]	600	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	601	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	602
[715]	603	# status=$?
	604	# IGCM_sys_Rsync_out $status
[622]	605
[1179]	606	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[715]	607	# (( status=status+$? ))
[623]	608
[715]	609	if [ ${status} -gt 0 ] ; then
[1179]	610	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
[765]	611	[ -f ${1} ] && ls -l ${1}
[759]	612	[ -f ${2} ] && ls -l ${2}
	613	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	614	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	615	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	616	else
[759]	617
	618	if [ X${JobType} = XRUN ] ; then
	619	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	620	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	621	fi
	622
[1179]	623	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	624	fi
[623]	625	fi
	626	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[622]	627	}
	628
	629	#D-#==================================================
	630	#D-function IGCM_sys_Put_Out
[623]	631	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[622]	632	#D-* Examples:
	633	#D-
	634	function IGCM_sys_Put_Out {
[623]	635	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
	636	if ( $DEBUG_sys ) ; then
	637	echo "IGCM_sys_Put_Out :" $@
	638	fi
[667]	639
[711]	640	typeset NB_ESSAI DELAI status i exist skip
[1234]	641	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
[667]	642
	643	# number of tentative
[668]	644	NB_ESSAI=3
[667]	645	# time delay between tentative
[668]	646	DELAI=2
[667]	647
[623]	648	if [ $DRYRUN = 0 ]; then
	649	if [ ! -f ${1} ] ; then
	650	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
	651	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	652	return 1
[622]	653	fi
[623]	654	#
	655	IGCM_sys_MkdirArchive $( dirname $2 )
	656	#
[711]	657	exist=false
	658	skip=false
	659	if [ -f $2 ] ; then
	660	IGCM_debug_Print 1 "$2 already exist"
	661	ccc_hsm get $2
	662	exist=true
	663	if [ "X$( diff $1 $2 )" = X ] ; then
	664	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
	665	skip=true
[667]	666	else
[711]	667	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	668	skip=false
[667]	669	fi
[711]	670	fi
	671	#
	672	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	673	IGCM_sys_Chmod u+w $2
	674	fi
[622]	675
[711]	676	if [ X${skip} = Xfalse ] ; then
[719]	677	i=0
[711]	678	while [ $i -lt $NB_ESSAI ] ; do
[1234]	679	# Identify file system
	680	fileDeviceNumberInHex=$( stat -c %d $1 )
	681	status=$?
	682	if [ ${status} -gt 0 ] ; then
	683	IGCM_debug_Exit "IGCM_sys_Put_Out"
	684	fi
	685	# Identify file system
	686	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
	687	status=$?
	688	if [ ${status} -gt 0 ] ; then
	689	IGCM_debug_Exit "IGCM_sys_Put_Out"
	690	fi
	691
	692	if [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] ; then
	693	# They are not on the same device. USUAL WAY
[1179]	694	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	695	status=$?
	696	else
[1234]	697	# They are on the same device. NOT SO USUAL WAY
[1179]	698	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	699	status=$?
	700	fi
[711]	701	if [ ${status} -gt 0 ]; then
[713]	702	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
[711]	703	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
[765]	704	[ -f ${1} ] && ls -l ${1}
[759]	705	[ -f ${2} ] && ls -l ${2}
	706	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[711]	707	sleep $DELAI
	708	else
	709	break
	710	fi
	711	(( i = i + 1 ))
	712	done
	713	fi
	714
[623]	715	# #RSYNC WITH NETWORK SSH CALL
[1179]	716	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	717	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	718
[623]	719	# #RSYNC WITH NFS USE
[1179]	720	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	721	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	722
[711]	723	# status=$?
	724	# IGCM_sys_Rsync_out $status
[622]	725
[1179]	726	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[711]	727	# (( status=status+$? ))
[622]	728
[711]	729	if [ ${status} -gt 0 ] ; then
[1179]	730	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
[765]	731	[ -f ${1} ] && ls -l ${1}
[759]	732	[ -f ${2} ] && ls -l ${2}
	733	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	734	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	735	IGCM_debug_Exit "IGCM_sys_Put_Out"
[622]	736	else
[759]	737
	738	if [ X${JobType} = XRUN ] ; then
	739	if [ X${3} = X ] ; then
	740	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	741	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	742	fi
	743	fi
	744
[1179]	745	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	746	fi
[623]	747	fi
	748	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	749	return 0
[622]	750	}
	751
	752	#D-#==================================================
	753	#D-function IGCM_sys_Get
	754	#D-* Purpose: Get a file from ${ARCHIVE}
	755	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
	756	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
	757	function IGCM_sys_Get {
[623]	758	IGCM_debug_PushStack "IGCM_sys_Get" $@
[622]	759
[773]	760	typeset DEST dm_liste target file_work
[667]	761	typeset NB_ESSAI DELAI status i
[622]	762
[623]	763	if ( $DEBUG_sys ) ; then
	764	echo "IGCM_sys_Get :" $@
	765	fi
[667]	766
	767	# number of tentative
[668]	768	NB_ESSAI=3
[667]	769	# time delay between tentative
[668]	770	DELAI=2
[667]	771
[623]	772	if [ $DRYRUN -le 2 ]; then
	773	if [ X${1} = X'/l' ] ; then
	774	eval set +A dm_liste \${${2}}
	775	else
	776	eval set +A dm_liste ${1}
[622]	777	fi
[623]	778	eval DEST=\${${#}}
[1179]	779	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	780	status=$?
	781	if [ ${status} -gt 0 ] ; then
	782	echo "WARNING IGCM_sys_Get : error code ${status}"
[1179]	783	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	784	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
	785	fi
[622]	786
[715]	787	#if [ ${status} -gt 0 ] ; then
[1179]	788	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
	789	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	790	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
	791	# sleep 30
	792	# echo "We try another time"
[1179]	793	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	794	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[785]	795	# status=$?
	796	# if [ ${status} -gt 0 ] ; then
	797	# echo "ERROR IGCM_sys_Get : again demigration error :"
[1179]	798	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	799	# IGCM_debug_Exit "IGCM_sys_Get"
[623]	800	# fi
[785]	801	# else
	802	# echo "ERROR IGCM_sys_Get : demigration error :"
[1179]	803	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	804	# IGCM_debug_Exit "IGCM_sys_Get"
	805	# fi
[623]	806	#fi
[622]	807
[719]	808	# #RSYNC WITH NETWORK SSH CALL
[1179]	809	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	810	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	811
	812	# #RSYNC WITH NFS USE
[1179]	813	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	814	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	815
	816	# status=$?
	817	# IGCM_sys_Rsync_out $status
	818
[1179]	819	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	820	# (( status=status+$? ))
	821
[623]	822	#USUAL WAY
	823	if [ X${1} = X'/l' ] ; then
	824	for target in ${dm_liste[*]} ; do
	825	local_file=$( basename ${target} )
[668]	826	# test if the target file is present before the loop
	827	IGCM_sys_TestFileArchive ${target}
[715]	828	status=$?
	829	if [ ${status} -gt 0 ] ; then
[701]	830	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
[668]	831	IGCM_debug_Exit "IGCM_sys_Get"
	832	else
[719]	833	i=0
[668]	834	while [ $i -lt $NB_ESSAI ] ; do
[1037]	835	#if [ X${DoLink} = Xtrue ] ; then
[1179]	836	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	837	# status=$?
	838	# else
[1179]	839	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	840	# status=$?
	841	#fi
[1179]	842	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[668]	843	status=$?
	844	if [ ${status} -gt 0 ]; then
[713]	845	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
[676]	846	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	847	sleep $DELAI
[668]	848	else
[676]	849	break
[668]	850	fi
	851	(( i = i + 1 ))
	852	done
[719]	853	if [ ${status} -gt 0 ] ; then
	854	echo "IGCM_sys_Get : error"
[1179]	855	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	856	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	857	IGCM_debug_Exit "IGCM_sys_Get"
	858	else
[1179]	859	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	860	fi
[668]	861	fi
[623]	862	done
	863	else
[719]	864	i=0
	865	while [ $i -lt $NB_ESSAI ] ; do
[1179]	866	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	867	status=$?
	868	if [ ${status} -gt 0 ]; then
	869	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
	870	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	871	sleep $DELAI
	872	else
	873	break
	874	fi
	875	(( i = i + 1 ))
	876	done
	877	if [ ${status} -gt 0 ] ; then
	878	echo "IGCM_sys_Get : error"
[1179]	879	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	880	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	881	IGCM_debug_Exit "IGCM_sys_Get"
	882	else
[1179]	883	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	884	fi
[623]	885	fi
	886	fi
	887	IGCM_debug_PopStack "IGCM_sys_Get"
	888	}
[622]	889
[623]	890	#D-#==================================================
[835]	891	#D-function IGCM_sys_GetDate_Monitoring
	892	#D-* Purpose: get the last year for which the monitoring has been computed
	893	#D-* Examples:
	894	#D-
	895	function IGCM_sys_GetDate_Monitoring {
	896	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
	897	if ( $DEBUG_sys ) ; then
	898	echo "IGCM_sys_GetDate_Monitoring :" $@
	899	fi
	900
	901	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
	902
	903	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
	904	}
	905
	906	#D-#==================================================
[623]	907	#D-function IGCM_sys_Dods_Rm
	908	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
	909	#D-* Examples:
	910	#D-
	911	function IGCM_sys_Dods_Rm {
	912	if ( $DEBUG_sys ) ; then
	913	echo "IGCM_sys_Dods_Rm :" $@
	914	fi
[715]	915	typeset status
[623]	916	if [ $DRYRUN = 0 ]; then
	917
	918	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
	919	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
	920	# echo "Nothing has been done."
	921	# return
	922	# fi
	923
[649]	924	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
[715]	925	status=$?
[623]	926
[785]	927	# if [ ${status} -gt 0 ] ; then
	928	# echo "IGCM_sys_Dods_Rm : error."
	929	# cat out_dods_rm
	930	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
	931	# else
	932	# rm out_dods_rm
	933	# fi
[623]	934
	935	fi
[715]	936	return $status
[623]	937	}
	938
	939	#D-#==================================================
	940	#D-function IGCM_sys_Dods_Cp
	941	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
	942	#D-* Examples:
	943	#D-
	944	function IGCM_sys_Dods_Cp {
	945	if ( $DEBUG_sys ) ; then
	946	echo "IGCM_sys_Dods_Cp :" $@
	947	fi
[715]	948	typeset status
[623]	949	if [ $DRYRUN = 0 ]; then
	950
	951	# if [ ! -d ${R_SAVE}/${1} ] ; then
	952	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
	953	# echo "Nothing has been done."
	954	# return
	955	# fi
	956
[649]	957	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
[715]	958	status=$?
[623]	959
[715]	960	# if [ ${status} -gt 0 ] ; then
[623]	961	# echo "IGCM_sys_Dods_Cp : error."
	962	# cat out_dods_cp
	963	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
	964	# else
	965	# rm out_dods_cp
	966	# fi
	967
	968	fi
[715]	969	return $status
[623]	970	}
	971
	972	#D-#==================================================
	973	#D-function IGCM_sys_Put_Dods
[1178]	974	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
[623]	975	#D-* Examples:
	976	#D-
	977	function IGCM_sys_Put_Dods {
	978	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
	979	if ( $DEBUG_sys ) ; then
	980	echo "IGCM_sys_Put_Dods :" $@
	981	fi
[715]	982	typeset status
[623]	983	if [ $DRYRUN = 0 ]; then
	984	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
	985	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
	986	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
	987	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
	988	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	989	return
[622]	990	fi
[623]	991
	992	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
	993	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
	994	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
	995	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
	996	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	997	return
	998	fi
	999	#
	1000	if [ -d ${R_SAVE}/${1} ] ; then
	1001	cd ${R_SAVE}
	1002	elif [ -d ${R_FIGR}/${1} ] ; then
	1003	cd ${R_FIGR}
	1004	fi
	1005
	1006	IGCM_sys_Dods_Rm ${1}
	1007	IGCM_sys_Dods_Cp ${1}
[715]	1008	status=0
[623]	1009
[715]	1010	if [ ${status} -gt 0 ] ; then
[623]	1011	echo "IGCM_sys_Put_Dods : error."
	1012	IGCM_debug_Exit "IGCM_sys_Put_Dods"
	1013	fi
	1014	fi
	1015	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
[622]	1016	}
	1017
	1018	##############################################################
	1019	# REBUILD OPERATOR
	1020
[811]	1021	#D-#==================================================
[1163]	1022	#D-function IGCM_sys_sync
	1023	#D-* Purpose: flush buffer on disk
	1024	#D-* Examples:
	1025	#D-
	1026	function IGCM_sys_sync {
	1027	IGCM_debug_PushStack "IGCM_sys_sync" $@
	1028	if ( $DEBUG_sys ) ; then
	1029	echo "IGCM_sys_sync :" $@
	1030	fi
	1031
	1032	/bin/sync
	1033
	1034	IGCM_debug_PopStack "IGCM_sys_sync"
	1035	}
	1036
[622]	1037	############################################################
	1038	# Activate Running Environnment Variables
	1039
[811]	1040	#D-#==================================================
[1239]	1041	#D-function IGCM_sys_activ_variables
[811]	1042	#D-* Purpose: set environement variables prior to execution
	1043	#D-* Examples:
	1044	#D-
[622]	1045	function IGCM_sys_activ_variables {
[623]	1046	IGCM_debug_PushStack "IGCM_sys_activ_variables"
	1047	if ( $DEBUG_sys ) ; then
	1048	echo "IGCM_sys_activ_variables"
	1049	fi
[622]	1050
[811]	1051	# --------------------------------------------------------------------
	1052	#D- MPI specifications
	1053	# --------------------------------------------------------------------
[1191]	1054
[811]	1055	# --------------------------------------------------------------------
	1056	#D- Other specifications
	1057	# --------------------------------------------------------------------
	1058
[623]	1059	ulimit -s unlimited
[622]	1060
[623]	1061	IGCM_debug_PopStack "IGCM_sys_activ_variables"
[622]	1062	}
	1063
	1064	############################################################
	1065	# Desactivate Running Environnment Variables
	1066
[811]	1067	#D-#==================================================
	1068	#D-function IGCM_sys_desactiv_variables
	1069	#D-* Purpose: unset environement variables after execution
	1070	#D-* Examples:
	1071	#D-
[622]	1072	function IGCM_sys_desactiv_variables {
[623]	1073	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
	1074	if ( $DEBUG_sys ) ; then
	1075	echo "IGCM_sys_desactiv_variables"
	1076	fi
[811]	1077	# --------------------------------------------------------------------
	1078	#D- MPI specifications
	1079	# --------------------------------------------------------------------
	1080
	1081	# --------------------------------------------------------------------
	1082	#D- Other specifications
	1083	# --------------------------------------------------------------------
	1084
[623]	1085	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
[622]	1086	}
	1087
[623]	1088	############################################################
[1238]	1089	# Update job headers to be used by the scheduler
	1090
	1091	#D-#==================================================
	1092	#D-function IGCM_sys_updateHeaders
	1093	#D-* Purpose: Update job headers to be used by the scheduler
	1094	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
	1095	#D-
	1096	function IGCM_sys_updateHeaders {
	1097	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
	1098	if ( $DEBUG_sys ) ; then
	1099	echo "IGCM_sys_updateHeaders"
	1100	fi
	1101	typeset file
	1102	file=$1
	1103
	1104	if [ ${executionType} -eq 1 ] ; then
	1105	# MPMD + MPI
	1106	sed -e "/::openMPthreads::/d" \
	1107	-e "s/::JobNumProcTot::/${coreNumber}/" \
[1256]	1108	-e "/#MSUB -x/d" \
	1109	-e "/--cpu_bind=none/d" \
[1238]	1110	${file} > ${file}.tmp
	1111
	1112	elif [ ${executionType} -eq 2 ] ; then
	1113	# MPMD + MPI + OMP
[1245]	1114	sed -e "/::openMPthreads::/d" \
[1238]	1115	-e "s/::JobNumProcTot::/${coreNumber}/" \
	1116	${file} > ${file}.tmp
	1117
	1118	elif [ ${executionType} -eq 3 ] ; then
	1119	# SPMD + MPI/OMP
	1120	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1121	-e "s/::JobNumProcTot::/${mpiTasks}/" \
[1240]	1122	-e "/#MSUB -x/d" \
[1238]	1123	-e "/--cpu_bind=none/d" \
	1124	${file} > ${file}.tmp
	1125
	1126	elif [ ${executionType} -eq 4 ] ; then
	1127	# SPMD + MPI only
	1128	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
	1129	-e "/::openMPthreads::/d" \
[1240]	1130	-e "/#MSUB -x/d" \
[1238]	1131	-e "/--cpu_bind=none/d" \
	1132	${file} > ${file}.tmp
	1133
	1134	elif [ ${executionType} -eq 5 ] ; then
	1135	# SPMD + OMP only
	1136	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1137	-e "/::JobNumProcTot::/d" \
[1240]	1138	-e "/#MSUB -x/d" \
[1238]	1139	-e "/--cpu_bind=none/d" \
	1140	${file} > ${file}.tmp
	1141
	1142	elif [ ${executionType} -eq 6 ] ; then
	1143	# SEQUENTIAL THEN
	1144	sed -e "s/::JobNumProcTot::/1/" \
	1145	-e "/::openMPthreads::/d" \
[1240]	1146	-e "/#MSUB -x/d" \
[1238]	1147	-e "/--cpu_bind=none/d" \
	1148	${file} > ${file}.tmp
	1149
	1150	fi
	1151
	1152	IGCM_sys_Mv ${file}.tmp ${file}
	1153
	1154	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
	1155	}
	1156
	1157	############################################################
[623]	1158	# Build MPI/OMP scripts run file (dummy function)
[622]	1159
[811]	1160	#D-#==================================================
	1161	#D-function IGCM_sys_build_run_file
	1162	#D-* Purpose: build run file (deprecated)
	1163	#D-* Examples:
	1164	#D-
[622]	1165	function IGCM_sys_build_run_file {
[811]	1166	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[622]	1167	}
	1168
	1169	############################################################
	1170	# Build MPI/OMP scripts
[811]	1171
	1172	#D-#==================================================
	1173	#D-function IGCM_sys_build_execution_scripts
	1174	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
	1175	#D-* Examples:
	1176	#D-
[623]	1177	function IGCM_sys_build_execution_scripts
[622]	1178	{
[623]	1179	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
	1180	if ( $DEBUG_sys ) ; then
	1181	echo "IGCM_sys_build_execution_scripts " $@
	1182	fi
[622]	1183
[623]	1184	EXECUTION=${HOST_MPIRUN_COMMAND}
[622]	1185
[1078]	1186	# MPMD mode
[623]	1187	if ( ${OK_PARA_MPMD} ) ; then
[622]	1188
[1078]	1189	# Only MPI (MPMD)
	1190	if ( ! ${OK_PARA_OMP} ) ; then
[622]	1191
[1078]	1192	if [ -f run_file ] ; then
	1193	IGCM_sys_Rm -f run_file
	1194	fi
	1195	touch run_file
[622]	1196
[1078]	1197	# Build run_file
[622]	1198
[1078]	1199	# First loop on the components for the coupler ie oasis (only if oasis3)
	1200	# the coupler ie oasis3 must be the first one
	1201	for comp in ${config_ListOfComponents[*]} ; do
[622]	1202
[1078]	1203	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1204	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1205
[1078]	1206	# for CPL component only
	1207	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1208
[1078]	1209	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1210	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1211	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1212	fi
	1213	done
[622]	1214
[1078]	1215	# Then second loop on the components
	1216	for comp in ${config_ListOfComponents[*]} ; do
[622]	1217
[1078]	1218	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1219	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1220
[1078]	1221	# Only if we really have an executable for the component and not the coupler ie oasis:
	1222	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
[622]	1223
[1078]	1224	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1225	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1226	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1227	fi
	1228	done
[622]	1229
[1078]	1230	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
[622]	1231
[1078]	1232	IGCM_sys_Chmod u+x run_file
	1233	if ( $DEBUG_sys ) ; then
	1234	echo "run_file contains : "
	1235	cat run_file
	1236	fi
[622]	1237
[1078]	1238	# MPI-OpenMP (MPMD)
	1239	else
[622]	1240
[1078]	1241	# Use of mpirun instead of ccc_mprun
	1242	EXECUTION="time mpirun"
[622]	1243
[1078]	1244	# Hosts treatment
	1245	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
[622]	1246
[1078]	1247	i=0
	1248	rm -f hosts rankfile
	1249	IGCM_debug_Print 1 "sys Curie, Hosts available :"
	1250	for nodes in `cat hosts.tmp`
	1251	do
	1252	host[$i]=$nodes
	1253	echo "${host[$i]}" >> hosts
	1254	IGCM_debug_Print 1 ${host[$i]}
	1255	i=$((i+1))
	1256	done
	1257	rm -f hosts.tmp
[1000]	1258
[1078]	1259	listnodes=${host[*]}
[1000]	1260
[1078]	1261	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
[1000]	1262
[1078]	1263	# Initialisation
	1264	rank=0
	1265	current_core=0
	1266	core_per_node=16
	1267	init_exec=n
[1000]	1268
[1078]	1269	# Loop on the components
	1270	for comp in ${config_ListOfComponents[*]} ; do
[1000]	1271
[1078]	1272	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1273	eval ExeNameOut=\${config_Executable_${comp}[1]}
[1000]	1274
[1078]	1275	# Not possible if oasis has an executable (i.e old version of oasis3)
	1276	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
	1277	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
	1278	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
	1279	IGCM_debug_Verif_Exit
	1280	fi
[1000]	1281
[1078]	1282	# Only if we really have an executable for the component :
	1283	if [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1284
[1078]	1285	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1286	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[1000]	1287
[1078]	1288	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1289	echo "" >> script_${ExeNameOut}.ksh
	1290	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
	1291
	1292	# Check if the number of threads is correct
	1293	case ${comp_proc_omp_loc} in
	1294	2\|4\|8\|16)
	1295	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
	1296	;;
	1297	*)
	1298	IGCM_debug_Exit "ERROR with OMP parameters !"
	1299	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
	1300	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
	1301	IGCM_debug_Verif_Exit
	1302	;;
	1303	esac
	1304	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1305	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1306	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1307	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1308	fi
[1318]	1309
[1317]	1310	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[1318]	1311	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
	1312	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
	1313	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
	1314	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
[1078]	1315
	1316	if [ ${init_exec} = y ] ; then
	1317	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
	1318	else
	1319	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
	1320	init_exec=y
	1321	fi
	1322
[1296]	1323	# Build rankfile : method used to assign cores and nodes for the MPI process
	1324	# Ex :
	1325	#rank 0=curie5296 slot=0,1,2,3
	1326	#rank 1=curie5296 slot=4,5,6,7
	1327	# Example of final command :
	1328	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
	1329	# with script_lmdz.x.ksh :
	1330	# #!/bin/ksh
	1331	#export KMP_STACKSIZE=3g
	1332	#export KMP_LIBRARY=turnaround
	1333	#export MKL_SERIAL=YES
	1334	#OMP_NUM_THREADS=4
	1335	#./lmdz.x
[1000]	1336
[1078]	1337	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
	1338	(( index_host = current_core / core_per_node ))
	1339	host_value=${host[${index_host}]}
	1340	(( slot = current_core % core_per_node ))
	1341	virg=","
	1342	string_final=""
	1343	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
[1191]	1344	string=$index$virg
[1078]	1345	string_final=$string_final$string
	1346	done
	1347	string_final=$( echo $string_final \| sed "s/.$//" )
	1348	echo "rank $rank=$host_value slot=$string_final" >> rankfile
	1349	(( rank = rank + 1 ))
	1350	(( current_core = current_core + comp_proc_omp_loc ))
	1351	done
	1352	fi
	1353	done
	1354	fi
[1000]	1355
[1078]	1356	# Only one executable (SPMD mode).
	1357	else
[1000]	1358
[622]	1359	for comp in ${config_ListOfComponents[*]} ; do
	1360
[685]	1361	# Only if we really have an executable for the component :
[623]	1362	eval ExeNameOut=\${config_Executable_${comp}[1]}
[685]	1363	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
[622]	1364
[685]	1365	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1366	echo "" >> script_${ExeNameOut}.ksh
	1367	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1368
	1369	if ( ${OK_PARA_OMP} ) ; then
	1370	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1371	echo "" >> script_${ExeNameOut}.ksh
	1372	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1373	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1374	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1375	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1376	fi
	1377
	1378	if ( ${OK_PARA_MPI} ) ; then
[623]	1379	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
[685]	1380	# Default : ccc_mprun used if nb_proc gt 1
	1381	# to have out/err per process on different files
[1317]	1382	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
	1383	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1384	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
[623]	1385	else
[685]	1386	# Default : ccc_mprun is NOT used if nb_proc eq 1
	1387	# to have out/err per process on different files
[1317]	1388	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
	1389	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1390	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
[622]	1391	fi
[685]	1392
	1393	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
	1394	cat script_${ExeNameOut}.ksh
	1395
[623]	1396	fi
[622]	1397
	1398	done
	1399
[623]	1400	fi
[622]	1401
[623]	1402	IGCM_debug_Print 1 "sys Curie : execution command is "
	1403	IGCM_debug_Print 1 "$EXECUTION"
[622]	1404
[623]	1405	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[622]	1406	}
	1407
[933]	1408	#D-#==================================================
	1409	#D-function IGCM_sys_check_path
	1410	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
	1411	#D-* do not point to an important use directory. Stop immediately in that case.
	1412	#D-* Examples:
	1413	#D-
	1414	function IGCM_sys_check_path {
	1415	IGCM_debug_PushStack "IGCM_sys_check_path"
	1416	if ( $DEBUG_sys ) ; then
	1417	echo "IGCM_sys_check_path"
	1418	fi
[811]	1419
[933]	1420	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
	1421	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
	1422	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
[936]	1423	IGCM_debug_Exit "This will stop the job"
[933]	1424	fi
	1425	IGCM_debug_PopStack "IGCM_sys_check_path"
	1426	}
	1427
[811]	1428	#D-#==================================================
	1429	#D-function IGCM_sys_check_quota
	1430	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
	1431	#D-* Examples:
	1432	#D-
[714]	1433	function IGCM_sys_check_quota {
[776]	1434	IGCM_debug_PushStack "IGCM_sys_check_quota"
	1435	if ( $DEBUG_sys ) ; then
	1436	echo "IGCM_sys_check_quota"
	1437	fi
	1438	# Limit of quota (in %)
	1439	limit_quota=90
	1440
	1441	# Check of the volume
[912]	1442	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
	1443	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
[776]	1444
	1445	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
	1446
	1447	unit_avail=${volume_avail: -1}
	1448	unit_quota=${volume_quota: -1}
	1449
	1450	if [ "${unit_quota}" = "*" ] ; then
[811]	1451	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1452	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1453	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1454	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1455	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1456	IGCM_debug_Verif_Exit
[714]	1457	fi
[776]	1458
	1459	temp_avail=${volume_avail%%${volume_avail: -1}*}
	1460	temp_quota=${volume_quota%%${volume_quota: -1}*}
	1461
	1462	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1463
	1464	# Convertion
	1465	if [ ${volume_avail: -1} = "T" ] ; then
	1466	(( temp_avail = temp_avail * 1000000000000 ))
	1467	elif [ ${volume_avail: -1} = "G" ] ; then
	1468	(( temp_avail = temp_avail * 1000000000 ))
	1469	elif [ ${volume_avail: -1} = "M" ] ; then
	1470	(( temp_avail = temp_avail * 1000000 ))
	1471	elif [ ${volume_avail: -1} = "k" ] ; then
	1472	(( temp_avail = temp_avail * 1000 ))
	1473	else
	1474	(( temp_avail = volume_avail ))
	1475	fi
	1476	if [ ${volume_quota: -1} = "T" ] ; then
	1477	(( temp_quota = temp_quota * 1000000000000 ))
	1478	elif [ ${volume_quota: -1} = "G" ] ; then
	1479	(( temp_quota = temp_quota * 1000000000 ))
	1480	elif [ ${volume_quota: -1} = "M" ] ; then
	1481	(( temp_quota = temp_quota * 1000000 ))
	1482	elif [ ${volume_quota: -1} = "k" ] ; then
	1483	(( temp_quota = temp_quota * 1000 ))
	1484	else
	1485	(( temp_quota = volume_quota ))
	1486	fi
	1487	fi
	1488
	1489	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1490	# echo "volume ratio is " $quota_volume
	1491
	1492	if [ ${quota_volume} -ge ${limit_quota} ] ; then
	1493	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1494	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
	1495	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1496	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1497	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1498	IGCM_debug_Verif_Exit
	1499	fi
	1500
	1501	fi
	1502
	1503	# Check of the number of inodes
	1504
[912]	1505	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
	1506	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
[776]	1507
	1508	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
	1509
	1510	unit_avail=${inode_avail: -1}
	1511	unit_quota=${inode_quota: -1}
	1512
	1513	if [ "${unit_quota}" = "*" ] ; then
[811]	1514	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1515	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1516	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1517	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1518	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1519	IGCM_debug_Verif_Exit
[776]	1520	fi
	1521
	1522	temp_avail=${inode_avail%%${inode_avail: -1}*}
	1523	temp_quota=${inode_quota%%${inode_quota: -1}*}
	1524
	1525	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1526
	1527	# Convertion
	1528	if [ ${inode_avail: -1} = "T" ] ; then
	1529	(( temp_avail = temp_avail * 1000000000000 ))
	1530	elif [ ${inode_avail: -1} = "G" ] ; then
	1531	(( temp_avail = temp_avail * 1000000000 ))
	1532	elif [ ${inode_avail: -1} = "M" ] ; then
	1533	(( temp_avail = temp_avail * 1000000 ))
	1534	elif [ ${inode_avail: -1} = "k" ] ; then
	1535	(( temp_avail = temp_avail * 1000 ))
	1536	else
	1537	(( temp_avail = inode_avail ))
	1538	fi
	1539
	1540	if [ ${inode_quota: -1} = "T" ] ; then
	1541	(( temp_quota = temp_quota * 1000000000000 ))
	1542	elif [ ${inode_quota: -1} = "G" ] ; then
	1543	(( temp_quota = temp_quota * 1000000000 ))
	1544	elif [ ${inode_quota: -1} = "M" ] ; then
	1545	(( temp_quota = temp_quota * 1000000 ))
	1546	elif [ ${inode_quota: -1} = "k" ] ; then
	1547	(( temp_quota = temp_quota * 1000 ))
	1548	else
	1549	(( temp_quota = inode_quota ))
	1550	fi
	1551	fi
	1552	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1553	# echo "inode ratio is " $quota_inode
	1554
	1555	if [ ${quota_inode} -ge ${limit_quota} ] ; then
	1556	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1557	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
	1558	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1559	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1560	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1561	IGCM_debug_Verif_Exit
	1562	fi
	1563	fi
	1564	IGCM_debug_PopStack "IGCM_sys_check_quota"
[714]	1565	}
	1566
[773]	1567	#D-#==================================================
[1286]	1568	#D-function IGCM_sys_projectAccounting
	1569	#D-* Purpose: store project accounting information in a file
	1570	#D-* Examples:
	1571	#D-
	1572	function IGCM_sys_projectAccounting {
	1573	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
	1574	if ( $DEBUG_sys ) ; then
	1575	echo "IGCM_sys_check_quota"
	1576	fi
	1577
[1288]	1578	ssh curie91 /usr/bin/ccc_myproject > $1
[1286]	1579
	1580	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
	1581	}
	1582
	1583	#D-#==================================================
[1244]	1584	#D-function IGCM_sys_getJobSchedulerID
	1585	#D-* Purpose: Get the job ID during execution
	1586	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
	1587	#D-
	1588	function IGCM_sys_getJobSchedulerID {
	1589	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
	1590	if ( $DEBUG_sys ) ; then
	1591	echo "IGCM_sys_getJobSchedulerID"
	1592	fi
	1593
	1594	eval ${1}=${BRIDGE_MSUB_JOBID}
	1595
	1596	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
	1597	}
	1598
	1599	#D-#==================================================
[958]	1600	#D-function IGCM_sys_GetJobID
[1244]	1601	#D-* Purpose: Get the job ID from the JobName
[958]	1602	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
	1603	#D-
	1604	function IGCM_sys_GetJobID {
	1605	IGCM_debug_PushStack "IGCM_sys_GetJobID"
	1606	if ( $DEBUG_sys ) ; then
	1607	echo "IGCM_sys_GetJobID"
	1608	fi
	1609
	1610	# With -f option, the full job name is given in the last column
	1611	ID=$( ccc_mstat -f -u $2 \| \
	1612	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
	1613
	1614	eval ${3}=${ID}
	1615	IGCM_debug_PopStack "IGCM_sys_GetJobID"
	1616	}
	1617
	1618	#D-#==================================================
[773]	1619	#D-function IGCM_sys_CountJobInQueue
[1244]	1620	#D-* Purpose: Count number of users job
[773]	1621	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
	1622	#D-
	1623	function IGCM_sys_CountJobInQueue {
	1624	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
	1625	if ( $DEBUG_sys ) ; then
	1626	echo "IGCM_sys_CountJobInQueue"
	1627	fi
	1628
	1629	# With -f option, the full job name is given in the last column
[834]	1630	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
[773]	1631
	1632	eval ${2}=${NbRun}
	1633
	1634	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
	1635	}
	1636
[955]	1637	#D-#==================================================
	1638	#D-function IGCM_sys_ListJobInQueue
[1244]	1639	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
[955]	1640	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
	1641	#D-
	1642	function IGCM_sys_ListJobInQueue {
	1643	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
	1644	if ( $DEBUG_sys ) ; then
	1645	echo "IGCM_sys_ListJobInQueue"
	1646	fi
	1647
	1648	# With -f option, the full job name is given in the last column
	1649	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
	1650	'( $2 == User && \
	1651	$NF != /TS/ && \
	1652	$NF !~ /PACK/ && \
	1653	$NF !~ /REBUILD/ && \
	1654	$NF !~ /pack/ ) \
	1655	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
	1656
	1657	eval set -A ${2} ${JobList[*]}
	1658
	1659	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
	1660	}
[1272]	1661
	1662	#D-#==================================================
	1663	#D-function IGCM_sys_atlas
	1664	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
	1665	#D-* Examples:
	1666	#D-
	1667	function IGCM_sys_atlas {
	1668	IGCM_debug_PushStack "IGCM_sys_atlas" $@
	1669	if ( $DEBUG_sys ) ; then
	1670	echo "IGCM_sys_atlas :" $@
	1671	fi
	1672
	1673	typeset status
	1674
	1675	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	1676	status=$?
	1677	if [ ${status} -gt 0 ] ; then
	1678	echo "IGCM_sys_atlas : error code ${status}"
	1679	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1680	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1681	IGCM_debug_PopStack "IGCM_sys_atlas"
	1682	return 1
	1683	else
	1684	IGCM_debug_PopStack "IGCM_sys_atlas"
	1685	return 0
	1686	fi
	1687
	1688	IGCM_debug_PopStack "IGCM_sys_atlas"
	1689	}
	1690

Note: See TracBrowser for help on using the repository browser.

Download in other formats: