Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1392

Last change on this file since 1392 was 1392, checked in by sdipsl, 7 years ago
fix #326 #322 #318 and #330
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 60.8 KB

Rev	Line
[622]	1	#!/bin/ksh
	2
	3	#**************************************************************
	4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
	5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#=========================================================
	15	# The documentation of this file can be automatically generated
[623]	16	# if you use the prefix #D- for comments to be extracted.
[622]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#=========================================================
	19
	20	#D-#==================================================
[623]	21	#D-LibIGCM_sys for Curie
[622]	22	#D-#==================================================
	23	#D-
[623]	24	#D- This ksh library if a layer under some usefull
[622]	25	#D-environment variables and shell commands.
	26	#D-All those definitions depend on host particularities.
	27	#D-It manages a stack mechanism and test validity of operations.
	28	#D-All function described bellow must be prefixed by IGCM_sys.
	29
	30	#====================================================
	31	# libIGCM_sys PARAMETERS
	32	#====================================================
	33
	34	#====================================================
	35	# set DEBUG_sys to true to output calls of function
	36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
	37
	38	#====================================================
	39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
	40	typeset -r DRYRUN=${DRYRUN:=0}
	41
	42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
	43	# -------------------------------------------------------------------------------------
	44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
[809]	45	# \| \| Cp/Exe/param/files \| \| \| \|
	46	# \| \| Chmod Qsub \| \| \| \|
[622]	47	# -------------------------------------------------------------------------------------
	48	# \| 0 \| yes \| yes \| yes \| yes \|
	49	# -------------------------------------------------------------------------------------
	50	# \| 1 \| yes \| yes \| yes \| no \|
	51	# -------------------------------------------------------------------------------------
	52	# \| 2 \| yes \| yes \| no \| no \|
	53	# -------------------------------------------------------------------------------------
	54	# \| 3 \| yes \| no \| no \| no \|
	55	# -------------------------------------------------------------------------------------
	56
	57	#=====================================================
	58	# Global Variables :
	59	#=====================================================
	60	# Language : "fr" or "en"
	61	typeset -r MYLANG="fr"
	62
	63	#=====================================================
[1201]	64	# Host user names project maxCpuTime
[622]	65	# $hostname ou hostname
[1158]	66	typeset HOST=${HOST:=$( hostname )}
[622]	67	# $username ou whoami
[1158]	68	typeset LOGIN=${LOGIN:=$( whoami )}
[622]	69	# $hostname of the MASTER job
	70	typeset MASTER=curie
[880]	71	# add default project on curie
[1201]	72	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} \| cut -d@ -f1 )
	73	# jobWarningDelay in seconds
	74	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
[622]	75
	76	#D-
	77	#D-#==================================================
	78	#D-Program used in libIGCM
	79	#D-#==================================================
	80
[742]	81	# Submit command
[747]	82	typeset SUBMIT=${SUBMIT:=ccc_msub}
[622]	83	# rsync with path
	84	typeset -r RSYNC=/usr/bin/rsync
	85	# RSYNC_opt args to rsync
	86	typeset -r RSYNC_opt="-va"
	87	# ie storage filesystem
[623]	88	typeset -r STOREHOST=${MASTER}
[1180]	89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
[622]	90
	91	#====================================================
[1180]	92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
[622]	93	#====================================================
[996]	94	if [ X${TaskType} = Xcomputing ] ; then
[1358]	95	module load python/2.7.8 > /dev/null 2>&1
[1193]	96	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
[996]	97	# to run with netcdf 3.6.3 ie compilation done before 17/2/2014
	98	# uncomment 2 lines :
	99	# module unload netcdf
	100	# module load netcdf/3.6.3
	101	else
[1196]	102	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
[1358]	103	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
[1364]	104	export UVCDAT_ANONYMOUS_LOG=FALSE
[996]	105	fi
[1315]	106
	107	# Use CMIP6 storage space when using CMIP6 cpu ressources
[1330]	108	[[ "X${PROJECT}" = "Xdevcmip6" && ! $(module list --terse 2>&1 \| grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
[1315]	109
	110	# FYI
[1210]	111	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
	112	[ ! X${TaskType} = Xchecking ] && module list
[1208]	113
[1179]	114	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
	115	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
	116	export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
[622]	117
	118	#====================================================
	119	# Specific for ocean additionnal diagnostic
	120	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
	121	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
	122
	123	#====================================================
	124	# Host specific DIRECTORIES
	125	#====================================================
	126
[623]	127	# ============ CESIUM START ============ #
	128
[622]	129	#====================================================
[623]	130	#- Mirror libIGCM from titane to cesium if needed
	131	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
	132	#if [ ! ${ROOTSYS} = "home" ] ; then
	133	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
	134	#else
	135	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	136	#fi
	137
	138	#====================================================
	139	#- libIGCM_POST
	140	#if ( ${MirrorlibIGCM} ) ; then
	141	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
	142	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
	143	#else
	144	# typeset -r libIGCM_POST=${libIGCM}
	145	#fi
	146
	147	# ============ CESIUM END ============ #
	148
	149	#====================================================
	150	#- MirrorlibIGCM for frontend
	151	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	152
	153	#====================================================
	154	#- libIGCM_POST for frontend
	155	typeset -r libIGCM_POST=${libIGCM}
	156
	157	#====================================================
[622]	158	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
	159	typeset -r R_EXE="${MODIPSL}/bin"
	160
	161	#====================================================
	162	#- SUBMIT_DIR : submission dir
[1010]	163	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
[622]	164
	165	#====================================================
	166	#- IN
[1276]	167	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
[622]	168	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
	169
	170	#====================================================
	171	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[850]	172	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
[622]	173
	174	#====================================================
[1179]	175	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
[1186]	176	typeset -r OUTCOMMAND_PATH=/tmp
[1179]	177
	178	#====================================================
[622]	179	#- HOST_MPIRUN_COMMAND
[1382]	180	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
[622]	181
	182	#====================================================
	183	#- Max number of arguments passed to nco operator or demigration command
	184	UNIX_MAX_LIMIT=360
	185
[623]	186	#====================================================
[1180]	187	#- set PackDefault to true on curie
[623]	188	PackDefault=true
	189
[664]	190	#====================================================
[623]	191	#- Default number of MPI task for IPSL coupled model
	192	#- required for backward compatibility
	193	#-
	194	DEFAULT_NUM_PROC_OCE=5
	195	DEFAULT_NUM_PROC_CPL=1
[637]	196	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
[623]	197
[725]	198	#D-#==================================================
[1057]	199	#D-function IGCM_sys_defineArchives
	200	#D-* Purpose:
	201	#D-* Define ARCHIVE : Dedicated to large files
	202	#D-* Define STORAGE : Dedicated to small/medium files
	203	#D-* Define R_OUT : Output tree located on ARCHIVE
	204	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
	205	#D-* Define R_BUF : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
	206	#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
[725]	207	#D-* Examples:
	208	#D-
[1057]	209	function IGCM_sys_defineArchives {
	210	IGCM_debug_PushStack "IGCM_sys_defineArchives"
[676]	211
[1060]	212	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
[1057]	213	#====================================================
	214	#- ARCHIVE (dedicated to large files)
[1063]	215	ARCHIVE=${config_UserChoices_ARCHIVE}
[1057]	216	else
	217	#====================================================
	218	#- ARCHIVE (dedicated to large files)
[1063]	219	ARCHIVE=${CCCSTOREDIR}
[1057]	220	fi
[725]	221
[1060]	222	if [ ! X${config_UserChoices_STORAGE} = X ]; then
[1057]	223	#====================================================
	224	#- STORAGE (dedicated to small/medium files)
[1063]	225	STORAGE=${config_UserChoices_STORAGE}
[1057]	226	else
	227	#====================================================
	228	#- STORAGE (dedicated to small/medium files)
[1063]	229	STORAGE=${CCCWORKDIR}
[1057]	230	fi
[725]	231
[1057]	232	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
	233	#====================================================
	234	#- R_OUT
[1063]	235	R_OUT=${SCRATCHDIR}/IGCM_OUT
[1057]	236
	237	#====================================================
	238	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]	239	R_FIG=${SCRATCHDIR}/IGCM_OUT
[1057]	240
	241	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
	242
	243	else
	244	#====================================================
	245	#- R_OUT
[1063]	246	R_OUT=${ARCHIVE}/IGCM_OUT
[1057]	247
	248	#====================================================
	249	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]	250	R_FIG=${STORAGE}/IGCM_OUT
[1057]	251	fi
	252
	253	#====================================================
	254	#- R_BUF (ONLY FOR double copy an scratch)
[1063]	255	R_BUF=${SCRATCHDIR}/IGCM_OUT
[1057]	256
	257	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
	258	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
	259	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
	260
	261	IGCM_debug_PopStack "IGCM_sys_defineArchives"
[725]	262	}
	263
[622]	264	#D-#==================================================
	265	#D-function IGCM_sys_RshArchive
	266	#D-* Purpose: Archive rsh command
	267	#D-* Examples:
	268	#D-
	269	function IGCM_sys_RshArchive {
[623]	270	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
	271	/bin/ksh <<-EOF
[622]	272	${@}
	273	EOF
[811]	274	status=$?
	275	if [ ${status} -gt 0 ] ; then
	276	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
[623]	277	IGCM_debug_Exit "IGCM_sys_RshArchive"
	278	fi
	279	IGCM_debug_PopStack "IGCM_sys_RshArchive"
[622]	280	}
	281
	282	#D-#==================================================
[845]	283	#D-function IGCM_sys_RshArchive_NoError
	284	#D-* Purpose: Archive rsh command, without error
	285	#D-* used only in monitoring.job
	286	#D-* Examples:
	287	#D-
	288	function IGCM_sys_RshArchive_NoError {
	289	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
	290	/bin/ksh <<-EOF
[1181]	291	${@} 2> /dev/null
[845]	292	EOF
	293	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
	294	}
	295
	296	#D-#==================================================
[622]	297	#D-function IGCM_sys_MkdirArchive
	298	#D-* Purpose: Mkdir on Archive
	299	#D-* Examples:
	300	#D-
	301	function IGCM_sys_MkdirArchive {
[623]	302	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
	303	if ( $DEBUG_sys ) ; then
	304	echo "IGCM_sys_MkdirArchive :" $@
	305	fi
	306	#- creation de repertoire sur le serveur fichier
	307	if [ ! -d ${1} ]; then
	308	\mkdir -p $1
[811]	309	status=$?
	310
	311	if [ ${status} -gt 0 ] ; then
	312	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
[623]	313	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
[622]	314	fi
[623]	315	fi
	316	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[622]	317	}
	318
	319	#D-#==================================================
	320	#D-function IGCM_sys_TestDirArchive
	321	#D-* Purpose: Test Directory that must exists on Archive
	322	#D-* Examples:
	323	#D-
	324	function IGCM_sys_TestDirArchive {
[623]	325	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
	326	if ( $DEBUG_sys ) ; then
	327	echo "IGCM_sys_TestDirArchive :" $@
	328	fi
	329	typeset ExistFlag
	330	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	331	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
	332	return ${ExistFlag}
[622]	333	}
	334
	335	#D-#==================================================
[623]	336	#D-function IGCM_sys_IsFileArchived
[1179]	337	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
[623]	338	#D-* Examples:
	339	#D-
	340	function IGCM_sys_IsFileArchived {
	341	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
	342	if ( $DEBUG_sys ) ; then
	343	echo "IGCM_sys_IsFileArchived :" $@
	344	fi
	345	typeset IsArchivedFlag
[818]	346	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
[623]	347	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
	348
	349	return ${IsArchivedFlag}
	350	}
	351
	352	#D-#==================================================
[622]	353	#D-function IGCM_sys_TestFileArchive
	354	#D-* Purpose: Test file that must NOT EXISTS on Archive
	355	#D-* Examples:
	356	#D-
	357	function IGCM_sys_TestFileArchive {
[623]	358	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
	359	typeset ExistFlag
	360	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	361	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[622]	362
[623]	363	return ${ExistFlag}
[622]	364	}
	365
	366	#D-#==================================================
	367	#D-function IGCM_sys_CountFileArchive
	368	#D-* Purpose: Count files on Archive filesystem
	369	#D-* Examples:
	370	#D-
	371	function IGCM_sys_CountFileArchive {
[623]	372	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
	373	ls ${@} 2>/dev/null \| wc -l
	374	if [ $? -gt 0 ] ; then
	375	echo "IGCM_sys_CountFileArchive : erreur."
	376	fi
	377	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[622]	378	}
	379
	380	#D-#==================================================
	381	#D-function IGCM_sys_Tree
	382	#D-* Purpose: Tree directories with files on ${ARCHIVE}
	383	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
	384	#D-
	385	function IGCM_sys_Tree {
[623]	386	IGCM_debug_PushStack "IGCM_sys_Tree" $@
	387	if ( $DEBUG_sys ) ; then
	388	echo "IGCM_sys_Tree :" $@
	389	fi
[622]	390
[623]	391	\ls -lR ${@}
[622]	392
[623]	393	IGCM_debug_PopStack "IGCM_sys_Tree"
[622]	394	}
	395
	396	#D-#==================================================
	397	#D-function IGCM_sys_Qsub
	398	#D-* Purpose: Qsub new job
	399	#D-* Examples:
	400	#D-
	401	function IGCM_sys_Qsub {
[623]	402	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
	403	if ( $DEBUG_sys ) ; then
	404	echo "IGCM_sys_Qsub :" $@
	405	fi
[811]	406	typeset options status
[923]	407	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
[622]	408
[1179]	409	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	410	status=$?
[948]	411
[1179]	412	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	413	if [ ${status} -gt 0 ] ; then
	414	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
[623]	415	IGCM_debug_Exit "IGCM_sys_Qsub"
[948]	416	else
[1179]	417	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	418	fi
	419	IGCM_debug_PopStack "IGCM_sys_Qsub"
[622]	420	}
	421
	422	#D-#==================================================
	423	#D-function IGCM_sys_QsubPost
	424	#D-* Purpose: Qsub new job on scalaire
	425	#D-* Examples:
	426	#D-
	427	function IGCM_sys_QsubPost {
[623]	428	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
	429	if ( $DEBUG_sys ) ; then
	430	echo "IGCM_sys_QsubPost :" $@
	431	fi
[923]	432	typeset options status
	433
[1062]	434	# EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
	435	case $( hostname -s ) in
	436	curie*)
[1201]	437	options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
[1062]	438	airain*)
	439	options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
	440	esac
	441
[1179]	442	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	443	status=$?
[948]	444
[1179]	445	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	446	if [ ${status} -gt 0 ] ; then
[923]	447	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
[623]	448	IGCM_debug_Exit "IGCM_sys_QsubPost"
[948]	449	else
[1179]	450	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	451	fi
	452	IGCM_debug_PopStack "IGCM_sys_QsubPost"
[622]	453	}
	454
	455	#D-*************************
[623]	456	#D- File transfer functions
[622]	457	#D-*************************
	458	#D-
	459
	460	#D-#==================================================
	461	#D-function IGCM_sys_RmRunDir
	462	#D-* Purpose: rm tmpdir (dummy function most of the time batch
	463	#D- scheduler will do the job)
	464	#D-* Examples:
	465	#D-
	466	function IGCM_sys_RmRunDir {
[1057]	467	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
[623]	468	if ( $DEBUG_sys ) ; then
	469	echo "IGCM_sys_RmRunDir :" $@
	470	fi
[622]	471
[715]	472	typeset status
[622]	473
[1179]	474	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	475	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	476	status=$?
[622]	477
[715]	478	if [ ${status} -gt 0 ] ; then
[1183]	479	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
[1179]	480	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	481	IGCM_debug_Exit "IGCM_sys_RmRunDir"
	482	else
[1179]	483	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	484	fi
	485	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[622]	486	}
	487
	488	#D-#==================================================
	489	#D-function IGCM_sys_Put_Dir
	490	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
	491	#D-* Examples:
	492	#D-
	493	function IGCM_sys_Put_Dir {
[623]	494	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
	495	if ( $DEBUG_sys ) ; then
	496	echo "IGCM_sys_Put_Dir :" $@
	497	fi
	498	if [ $DRYRUN = 0 ]; then
	499	if [ ! -d ${1} ] ; then
	500	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
	501	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
	502	return
[622]	503	fi
	504
[715]	505	typeset status
[622]	506
[623]	507	# Only if we use rsync
	508	#IGCM_sys_TestDirArchive $( dirname $2 )
	509	#
	510	#USUAL WAY
[1179]	511	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	512	status=$?
[622]	513
[715]	514	if [ ${status} -gt 0 ] ; then
[811]	515	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
[1179]	516	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	517	IGCM_debug_Exit "IGCM_sys_Put_Dir"
[622]	518	else
[1179]	519	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	520	fi
[623]	521	fi
	522	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[622]	523	}
	524
	525	#D-#==================================================
	526	#D-function IGCM_sys_Get_Dir
[623]	527	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[622]	528	#D-* Examples:
	529	#D-
	530	function IGCM_sys_Get_Dir {
[623]	531	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
	532	if ( $DEBUG_sys ) ; then
	533	echo "IGCM_sys_Get_Dir :" $@
	534	fi
	535	if [ $DRYRUN = 0 ]; then
[708]	536	typeset NB_ESSAI DELAI status i
	537	# number of tentative
	538	NB_ESSAI=3
	539	# time delay between tentative
	540	DELAI=2
[622]	541
[676]	542	#
[708]	543	# USUAL WAY
	544	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
[713]	545	ccc_hsm get -r $1
[622]	546
[709]	547	i=0
[708]	548	while [ $i -lt $NB_ESSAI ] ; do
[1179]	549	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[708]	550	status=$?
[811]	551	if [ ${status} -gt 0 ] ; then
[713]	552	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
[708]	553	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
	554	sleep $DELAI
	555	else
	556	break
	557	fi
	558	(( i = i + 1 ))
	559	done
	560
	561	if [ ${status} -gt 0 ] ; then
[1178]	562	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
[1179]	563	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	564	IGCM_debug_Exit "IGCM_sys_Get_Dir"
[622]	565	else
[1179]	566	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	567	fi
[623]	568	fi
	569	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[622]	570	}
	571
	572	#D-#==================================================
	573	#D-function IGCM_sys_Put_Rest
[623]	574	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[622]	575	#D- File and target directory must exist.
	576	#D-* Examples:
	577	#D-
	578	function IGCM_sys_Put_Rest {
[623]	579	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
	580	if ( $DEBUG_sys ) ; then
	581	echo "IGCM_sys_Put_Rest :" $@
	582	fi
	583	if [ $DRYRUN = 0 ]; then
	584	if [ ! -f ${1} ] ; then
	585	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
	586	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	587	fi
	588
[715]	589	typeset status
[623]	590	#
	591	# USUAL WAY
[1179]	592	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	593	status=$?
[622]	594
[623]	595	# #RSYNC WITH NETWORK SSH CALL
[1179]	596	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	597	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	598
[623]	599	# #RSYNC WITH NFS USE
[1179]	600	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	601	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	602
[715]	603	# status=$?
	604	# IGCM_sys_Rsync_out $status
[622]	605
[1179]	606	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[715]	607	# (( status=status+$? ))
[623]	608
[715]	609	if [ ${status} -gt 0 ] ; then
[1179]	610	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
[765]	611	[ -f ${1} ] && ls -l ${1}
[759]	612	[ -f ${2} ] && ls -l ${2}
	613	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	614	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	615	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	616	else
[759]	617
	618	if [ X${JobType} = XRUN ] ; then
	619	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	620	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	621	fi
	622
[1179]	623	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	624	fi
[623]	625	fi
	626	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[622]	627	}
	628
	629	#D-#==================================================
	630	#D-function IGCM_sys_Put_Out
[623]	631	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[622]	632	#D-* Examples:
	633	#D-
	634	function IGCM_sys_Put_Out {
[623]	635	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
	636	if ( $DEBUG_sys ) ; then
	637	echo "IGCM_sys_Put_Out :" $@
	638	fi
[667]	639
[711]	640	typeset NB_ESSAI DELAI status i exist skip
[1234]	641	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
[667]	642
	643	# number of tentative
[668]	644	NB_ESSAI=3
[667]	645	# time delay between tentative
[668]	646	DELAI=2
[667]	647
[623]	648	if [ $DRYRUN = 0 ]; then
	649	if [ ! -f ${1} ] ; then
	650	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
	651	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	652	return 1
[622]	653	fi
[623]	654	#
	655	IGCM_sys_MkdirArchive $( dirname $2 )
	656	#
[711]	657	exist=false
	658	skip=false
	659	if [ -f $2 ] ; then
	660	IGCM_debug_Print 1 "$2 already exist"
	661	ccc_hsm get $2
	662	exist=true
	663	if [ "X$( diff $1 $2 )" = X ] ; then
	664	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
	665	skip=true
[667]	666	else
[711]	667	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	668	skip=false
[667]	669	fi
[711]	670	fi
	671	#
	672	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	673	IGCM_sys_Chmod u+w $2
	674	fi
[622]	675
[711]	676	if [ X${skip} = Xfalse ] ; then
[719]	677	i=0
[711]	678	while [ $i -lt $NB_ESSAI ] ; do
[1234]	679	# Identify file system
	680	fileDeviceNumberInHex=$( stat -c %d $1 )
	681	status=$?
	682	if [ ${status} -gt 0 ] ; then
	683	IGCM_debug_Exit "IGCM_sys_Put_Out"
	684	fi
	685	# Identify file system
	686	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
	687	status=$?
	688	if [ ${status} -gt 0 ] ; then
	689	IGCM_debug_Exit "IGCM_sys_Put_Out"
	690	fi
	691
	692	if [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] ; then
	693	# They are not on the same device. USUAL WAY
[1179]	694	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	695	status=$?
	696	else
[1234]	697	# They are on the same device. NOT SO USUAL WAY
[1179]	698	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	699	status=$?
	700	fi
[711]	701	if [ ${status} -gt 0 ]; then
[713]	702	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
[711]	703	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
[765]	704	[ -f ${1} ] && ls -l ${1}
[759]	705	[ -f ${2} ] && ls -l ${2}
	706	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[711]	707	sleep $DELAI
	708	else
	709	break
	710	fi
	711	(( i = i + 1 ))
	712	done
	713	fi
	714
[623]	715	# #RSYNC WITH NETWORK SSH CALL
[1179]	716	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	717	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	718
[623]	719	# #RSYNC WITH NFS USE
[1179]	720	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	721	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	722
[711]	723	# status=$?
	724	# IGCM_sys_Rsync_out $status
[622]	725
[1179]	726	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[711]	727	# (( status=status+$? ))
[622]	728
[711]	729	if [ ${status} -gt 0 ] ; then
[1179]	730	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
[765]	731	[ -f ${1} ] && ls -l ${1}
[759]	732	[ -f ${2} ] && ls -l ${2}
	733	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	734	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	735	IGCM_debug_Exit "IGCM_sys_Put_Out"
[622]	736	else
[759]	737
	738	if [ X${JobType} = XRUN ] ; then
	739	if [ X${3} = X ] ; then
	740	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	741	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	742	fi
	743	fi
	744
[1179]	745	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	746	fi
[623]	747	fi
	748	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	749	return 0
[622]	750	}
	751
	752	#D-#==================================================
	753	#D-function IGCM_sys_Get
	754	#D-* Purpose: Get a file from ${ARCHIVE}
	755	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
	756	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
	757	function IGCM_sys_Get {
[623]	758	IGCM_debug_PushStack "IGCM_sys_Get" $@
[622]	759
[773]	760	typeset DEST dm_liste target file_work
[667]	761	typeset NB_ESSAI DELAI status i
[622]	762
[623]	763	if ( $DEBUG_sys ) ; then
	764	echo "IGCM_sys_Get :" $@
	765	fi
[667]	766
	767	# number of tentative
[668]	768	NB_ESSAI=3
[667]	769	# time delay between tentative
[668]	770	DELAI=2
[667]	771
[623]	772	if [ $DRYRUN -le 2 ]; then
	773	if [ X${1} = X'/l' ] ; then
	774	eval set +A dm_liste \${${2}}
	775	else
	776	eval set +A dm_liste ${1}
[622]	777	fi
[623]	778	eval DEST=\${${#}}
[1179]	779	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	780	status=$?
	781	if [ ${status} -gt 0 ] ; then
	782	echo "WARNING IGCM_sys_Get : error code ${status}"
[1179]	783	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	784	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
	785	fi
[622]	786
[715]	787	#if [ ${status} -gt 0 ] ; then
[1179]	788	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
	789	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	790	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
	791	# sleep 30
	792	# echo "We try another time"
[1179]	793	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	794	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[785]	795	# status=$?
	796	# if [ ${status} -gt 0 ] ; then
	797	# echo "ERROR IGCM_sys_Get : again demigration error :"
[1179]	798	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	799	# IGCM_debug_Exit "IGCM_sys_Get"
[623]	800	# fi
[785]	801	# else
	802	# echo "ERROR IGCM_sys_Get : demigration error :"
[1179]	803	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	804	# IGCM_debug_Exit "IGCM_sys_Get"
	805	# fi
[623]	806	#fi
[622]	807
[719]	808	# #RSYNC WITH NETWORK SSH CALL
[1179]	809	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	810	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	811
	812	# #RSYNC WITH NFS USE
[1179]	813	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	814	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	815
	816	# status=$?
	817	# IGCM_sys_Rsync_out $status
	818
[1179]	819	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	820	# (( status=status+$? ))
	821
[623]	822	#USUAL WAY
	823	if [ X${1} = X'/l' ] ; then
	824	for target in ${dm_liste[*]} ; do
	825	local_file=$( basename ${target} )
[668]	826	# test if the target file is present before the loop
	827	IGCM_sys_TestFileArchive ${target}
[715]	828	status=$?
	829	if [ ${status} -gt 0 ] ; then
[701]	830	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
[668]	831	IGCM_debug_Exit "IGCM_sys_Get"
	832	else
[719]	833	i=0
[668]	834	while [ $i -lt $NB_ESSAI ] ; do
[1037]	835	#if [ X${DoLink} = Xtrue ] ; then
[1179]	836	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	837	# status=$?
	838	# else
[1179]	839	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	840	# status=$?
	841	#fi
[1179]	842	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[668]	843	status=$?
	844	if [ ${status} -gt 0 ]; then
[713]	845	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
[676]	846	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	847	sleep $DELAI
[668]	848	else
[676]	849	break
[668]	850	fi
	851	(( i = i + 1 ))
	852	done
[719]	853	if [ ${status} -gt 0 ] ; then
	854	echo "IGCM_sys_Get : error"
[1179]	855	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	856	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	857	IGCM_debug_Exit "IGCM_sys_Get"
	858	else
[1179]	859	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	860	fi
[668]	861	fi
[623]	862	done
	863	else
[719]	864	i=0
	865	while [ $i -lt $NB_ESSAI ] ; do
[1179]	866	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	867	status=$?
	868	if [ ${status} -gt 0 ]; then
	869	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
	870	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	871	sleep $DELAI
	872	else
	873	break
	874	fi
	875	(( i = i + 1 ))
	876	done
	877	if [ ${status} -gt 0 ] ; then
	878	echo "IGCM_sys_Get : error"
[1179]	879	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	880	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	881	IGCM_debug_Exit "IGCM_sys_Get"
	882	else
[1179]	883	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	884	fi
[623]	885	fi
	886	fi
	887	IGCM_debug_PopStack "IGCM_sys_Get"
	888	}
[622]	889
[623]	890	#D-#==================================================
[835]	891	#D-function IGCM_sys_GetDate_Monitoring
	892	#D-* Purpose: get the last year for which the monitoring has been computed
	893	#D-* Examples:
	894	#D-
	895	function IGCM_sys_GetDate_Monitoring {
	896	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
	897	if ( $DEBUG_sys ) ; then
	898	echo "IGCM_sys_GetDate_Monitoring :" $@
	899	fi
	900
	901	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
	902
	903	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
	904	}
	905
	906	#D-#==================================================
[623]	907	#D-function IGCM_sys_Dods_Rm
	908	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
	909	#D-* Examples:
	910	#D-
	911	function IGCM_sys_Dods_Rm {
	912	if ( $DEBUG_sys ) ; then
	913	echo "IGCM_sys_Dods_Rm :" $@
	914	fi
[715]	915	typeset status
[623]	916	if [ $DRYRUN = 0 ]; then
	917
	918	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
	919	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
	920	# echo "Nothing has been done."
	921	# return
	922	# fi
	923
[1392]	924	if [ "$#" -eq 1 ]; then
	925	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
	926	status=$?
	927	else
	928	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
	929	status=$?
	930	fi
[623]	931
[785]	932	# if [ ${status} -gt 0 ] ; then
	933	# echo "IGCM_sys_Dods_Rm : error."
	934	# cat out_dods_rm
	935	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
	936	# else
	937	# rm out_dods_rm
	938	# fi
[623]	939
	940	fi
[715]	941	return $status
[623]	942	}
	943
	944	#D-#==================================================
	945	#D-function IGCM_sys_Dods_Cp
	946	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
	947	#D-* Examples:
	948	#D-
	949	function IGCM_sys_Dods_Cp {
	950	if ( $DEBUG_sys ) ; then
	951	echo "IGCM_sys_Dods_Cp :" $@
	952	fi
[715]	953	typeset status
[623]	954	if [ $DRYRUN = 0 ]; then
	955
	956	# if [ ! -d ${R_SAVE}/${1} ] ; then
	957	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
	958	# echo "Nothing has been done."
	959	# return
	960	# fi
	961
[649]	962	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
[715]	963	status=$?
[623]	964
[715]	965	# if [ ${status} -gt 0 ] ; then
[623]	966	# echo "IGCM_sys_Dods_Cp : error."
	967	# cat out_dods_cp
	968	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
	969	# else
	970	# rm out_dods_cp
	971	# fi
	972
	973	fi
[715]	974	return $status
[623]	975	}
	976
	977	#D-#==================================================
	978	#D-function IGCM_sys_Put_Dods
[1178]	979	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
[623]	980	#D-* Examples:
	981	#D-
	982	function IGCM_sys_Put_Dods {
	983	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
	984	if ( $DEBUG_sys ) ; then
	985	echo "IGCM_sys_Put_Dods :" $@
	986	fi
[715]	987	typeset status
[623]	988	if [ $DRYRUN = 0 ]; then
	989	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
	990	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
	991	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
	992	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
	993	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	994	return
[622]	995	fi
[623]	996
	997	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
	998	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
	999	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
	1000	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
	1001	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	1002	return
	1003	fi
	1004	#
	1005	if [ -d ${R_SAVE}/${1} ] ; then
	1006	cd ${R_SAVE}
	1007	elif [ -d ${R_FIGR}/${1} ] ; then
	1008	cd ${R_FIGR}
	1009	fi
	1010
	1011	IGCM_sys_Dods_Rm ${1}
	1012	IGCM_sys_Dods_Cp ${1}
[715]	1013	status=0
[623]	1014
[715]	1015	if [ ${status} -gt 0 ] ; then
[623]	1016	echo "IGCM_sys_Put_Dods : error."
	1017	IGCM_debug_Exit "IGCM_sys_Put_Dods"
	1018	fi
	1019	fi
	1020	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
[622]	1021	}
	1022
	1023	##############################################################
	1024	# REBUILD OPERATOR
	1025
[811]	1026	#D-#==================================================
[1163]	1027	#D-function IGCM_sys_sync
	1028	#D-* Purpose: flush buffer on disk
	1029	#D-* Examples:
	1030	#D-
	1031	function IGCM_sys_sync {
	1032	IGCM_debug_PushStack "IGCM_sys_sync" $@
	1033	if ( $DEBUG_sys ) ; then
	1034	echo "IGCM_sys_sync :" $@
	1035	fi
	1036
	1037	/bin/sync
	1038
	1039	IGCM_debug_PopStack "IGCM_sys_sync"
	1040	}
	1041
[622]	1042	############################################################
	1043	# Activate Running Environnment Variables
	1044
[811]	1045	#D-#==================================================
[1239]	1046	#D-function IGCM_sys_activ_variables
[811]	1047	#D-* Purpose: set environement variables prior to execution
	1048	#D-* Examples:
	1049	#D-
[622]	1050	function IGCM_sys_activ_variables {
[623]	1051	IGCM_debug_PushStack "IGCM_sys_activ_variables"
	1052	if ( $DEBUG_sys ) ; then
	1053	echo "IGCM_sys_activ_variables"
	1054	fi
[622]	1055
[811]	1056	# --------------------------------------------------------------------
	1057	#D- MPI specifications
	1058	# --------------------------------------------------------------------
[1191]	1059
[811]	1060	# --------------------------------------------------------------------
	1061	#D- Other specifications
	1062	# --------------------------------------------------------------------
	1063
[623]	1064	ulimit -s unlimited
[622]	1065
[623]	1066	IGCM_debug_PopStack "IGCM_sys_activ_variables"
[622]	1067	}
	1068
	1069	############################################################
	1070	# Desactivate Running Environnment Variables
	1071
[811]	1072	#D-#==================================================
	1073	#D-function IGCM_sys_desactiv_variables
	1074	#D-* Purpose: unset environement variables after execution
	1075	#D-* Examples:
	1076	#D-
[622]	1077	function IGCM_sys_desactiv_variables {
[623]	1078	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
	1079	if ( $DEBUG_sys ) ; then
	1080	echo "IGCM_sys_desactiv_variables"
	1081	fi
[811]	1082	# --------------------------------------------------------------------
	1083	#D- MPI specifications
	1084	# --------------------------------------------------------------------
	1085
	1086	# --------------------------------------------------------------------
	1087	#D- Other specifications
	1088	# --------------------------------------------------------------------
	1089
[623]	1090	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
[622]	1091	}
	1092
[623]	1093	############################################################
[1238]	1094	# Update job headers to be used by the scheduler
	1095
	1096	#D-#==================================================
	1097	#D-function IGCM_sys_updateHeaders
	1098	#D-* Purpose: Update job headers to be used by the scheduler
	1099	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
	1100	#D-
	1101	function IGCM_sys_updateHeaders {
	1102	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
	1103	if ( $DEBUG_sys ) ; then
	1104	echo "IGCM_sys_updateHeaders"
	1105	fi
	1106	typeset file
	1107	file=$1
	1108
	1109	if [ ${executionType} -eq 1 ] ; then
	1110	# MPMD + MPI
	1111	sed -e "/::openMPthreads::/d" \
	1112	-e "s/::JobNumProcTot::/${coreNumber}/" \
[1256]	1113	-e "/#MSUB -x/d" \
	1114	-e "/--cpu_bind=none/d" \
[1238]	1115	${file} > ${file}.tmp
	1116
	1117	elif [ ${executionType} -eq 2 ] ; then
[1390]	1118	# MPMD + MPI + OMP : mpirun/ccc_mprun/error
	1119	if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
	1120	sed -e "/::openMPthreads::/d" \
	1121	-e "s/::JobNumProcTot::/${coreNumber}/" \
	1122	${file} > ${file}.tmp
	1123	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
	1124	sed -e "/::openMPthreads::/d" \
	1125	-e "s/::JobNumProcTot::/${coreNumber}/" \
	1126	-e "/--cpu_bind=none/d" \
	1127	${file} > ${file}.tmp
	1128	else
	1129	IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
	1130	IGCM_debug_Exit "IGCM_sys_updateHeaders"
	1131	fi
[1238]	1132	elif [ ${executionType} -eq 3 ] ; then
	1133	# SPMD + MPI/OMP
	1134	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1135	-e "s/::JobNumProcTot::/${mpiTasks}/" \
[1240]	1136	-e "/#MSUB -x/d" \
[1238]	1137	-e "/--cpu_bind=none/d" \
	1138	${file} > ${file}.tmp
	1139
	1140	elif [ ${executionType} -eq 4 ] ; then
	1141	# SPMD + MPI only
	1142	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
	1143	-e "/::openMPthreads::/d" \
[1240]	1144	-e "/#MSUB -x/d" \
[1238]	1145	-e "/--cpu_bind=none/d" \
	1146	${file} > ${file}.tmp
	1147
	1148	elif [ ${executionType} -eq 5 ] ; then
	1149	# SPMD + OMP only
	1150	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1151	-e "/::JobNumProcTot::/d" \
[1240]	1152	-e "/#MSUB -x/d" \
[1238]	1153	-e "/--cpu_bind=none/d" \
	1154	${file} > ${file}.tmp
	1155
	1156	elif [ ${executionType} -eq 6 ] ; then
	1157	# SEQUENTIAL THEN
	1158	sed -e "s/::JobNumProcTot::/1/" \
	1159	-e "/::openMPthreads::/d" \
[1240]	1160	-e "/#MSUB -x/d" \
[1238]	1161	-e "/--cpu_bind=none/d" \
	1162	${file} > ${file}.tmp
	1163
	1164	fi
	1165
	1166	IGCM_sys_Mv ${file}.tmp ${file}
	1167
	1168	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
	1169	}
	1170
	1171	############################################################
[623]	1172	# Build MPI/OMP scripts run file (dummy function)
[622]	1173
[811]	1174	#D-#==================================================
	1175	#D-function IGCM_sys_build_run_file
	1176	#D-* Purpose: build run file (deprecated)
	1177	#D-* Examples:
	1178	#D-
[622]	1179	function IGCM_sys_build_run_file {
[811]	1180	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[622]	1181	}
	1182
	1183	############################################################
	1184	# Build MPI/OMP scripts
[811]	1185
	1186	#D-#==================================================
	1187	#D-function IGCM_sys_build_execution_scripts
	1188	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
	1189	#D-* Examples:
	1190	#D-
[623]	1191	function IGCM_sys_build_execution_scripts
[622]	1192	{
[623]	1193	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
	1194	if ( $DEBUG_sys ) ; then
	1195	echo "IGCM_sys_build_execution_scripts " $@
	1196	fi
[622]	1197
[623]	1198	EXECUTION=${HOST_MPIRUN_COMMAND}
[622]	1199
[623]	1200	if ( ${OK_PARA_MPMD} ) ; then
[622]	1201
[1390]	1202	# MPMD mode
	1203	# 1 MPI only
	1204	# 2 MPI/OpenMP mpirun method
	1205	# 3 MPI/OpenMP ccc_mprun method
	1206
[1381]	1207	if [ -f run_file ] ; then
	1208	IGCM_sys_Rm -f run_file
	1209	fi
	1210	touch run_file
	1211
[1390]	1212	# case 1 : Only MPI (MPMD)
[1078]	1213	if ( ! ${OK_PARA_OMP} ) ; then
[622]	1214
[1078]	1215	# Build run_file
[622]	1216
[1078]	1217	# First loop on the components for the coupler ie oasis (only if oasis3)
	1218	# the coupler ie oasis3 must be the first one
	1219	for comp in ${config_ListOfComponents[*]} ; do
[622]	1220
[1078]	1221	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1222	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1223
[1078]	1224	# for CPL component only
	1225	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1226
[1078]	1227	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1228	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1229	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1230	fi
	1231	done
[622]	1232
[1078]	1233	# Then second loop on the components
	1234	for comp in ${config_ListOfComponents[*]} ; do
[622]	1235
[1078]	1236	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1237	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1238
[1078]	1239	# Only if we really have an executable for the component and not the coupler ie oasis:
	1240	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
[622]	1241
[1078]	1242	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1243	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1244	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1245	fi
	1246	done
[622]	1247
[1390]	1248	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
	1249
	1250	IGCM_sys_Chmod u+x run_file
	1251	if ( $DEBUG_sys ) ; then
	1252	echo "run_file contains : "
	1253	cat run_file
	1254	fi
	1255
	1256	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
	1257
	1258	# 2 MPI/OpenMP mpirun method
	1259
	1260	# Use of mpirun instead of ccc_mprun
	1261	EXECUTION="time mpirun"
	1262
	1263	# Hosts treatment
	1264	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
	1265
	1266	i=0
	1267	rm -f hosts rankfile
	1268	IGCM_debug_Print 1 "sys Curie, Hosts available :"
	1269	for nodes in `cat hosts.tmp`
	1270	do
	1271	host[$i]=$nodes
	1272	echo "${host[$i]}" >> hosts
	1273	IGCM_debug_Print 1 ${host[$i]}
	1274	i=$((i+1))
	1275	done
	1276	rm -f hosts.tmp
	1277
	1278	listnodes=${host[*]}
	1279
	1280	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
	1281
	1282	# Initialisation
	1283	rank=0
	1284	current_core=0
	1285	core_per_node=16
	1286	init_exec=n
	1287
	1288	# Loop on the components
	1289	for comp in ${config_ListOfComponents[*]} ; do
	1290
	1291	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1292	eval ExeNameOut=\${config_Executable_${comp}[1]}
	1293
	1294	# Not possible if oasis has an executable (i.e old version of oasis3)
	1295	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
	1296	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
	1297	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
	1298	IGCM_debug_Verif_Exit
	1299	fi
	1300
	1301	# Only if we really have an executable for the component :
	1302	if [ "X${ExeNameOut}" != X\"\" ] ; then
	1303
	1304	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1305	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1306
	1307	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1308	echo "" >> script_${ExeNameOut}.ksh
	1309	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
	1310
	1311	# Check if the number of threads is correct
	1312	case ${comp_proc_omp_loc} in
	1313	2\|4\|8\|16)
	1314	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
	1315	;;
	1316	*)
	1317	IGCM_debug_Exit "ERROR with OMP parameters !"
	1318	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
	1319	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
	1320	IGCM_debug_Verif_Exit
	1321	;;
	1322	esac
	1323	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1324	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1325	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1326	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1327	fi
	1328
	1329	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
	1330	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
	1331	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
	1332	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
	1333	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1334
	1335	if [ ${init_exec} = y ] ; then
	1336	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
	1337	else
	1338	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
	1339	init_exec=y
	1340	fi
	1341
	1342	# Build rankfile : method used to assign cores and nodes for the MPI process
	1343	# Ex :
	1344	#rank 0=curie5296 slot=0,1,2,3
	1345	#rank 1=curie5296 slot=4,5,6,7
	1346	# Example of final command :
	1347	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
	1348	# with script_lmdz.x.ksh :
	1349	# #!/bin/ksh
	1350	#export KMP_STACKSIZE=3g
	1351	#export KMP_LIBRARY=turnaround
	1352	#export MKL_SERIAL=YES
	1353	#OMP_NUM_THREADS=4
	1354	#./lmdz.x
	1355
	1356	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
	1357	(( index_host = current_core / core_per_node ))
	1358	host_value=${host[${index_host}]}
	1359	(( slot = current_core % core_per_node ))
	1360	virg=","
	1361	string_final=""
	1362	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
	1363	string=$index$virg
	1364	string_final=$string_final$string
	1365	done
	1366	string_final=$( echo $string_final \| sed "s/.$//" )
	1367	echo "rank $rank=$host_value slot=$string_final" >> rankfile
	1368	(( rank = rank + 1 ))
	1369	(( current_core = current_core + comp_proc_omp_loc ))
	1370	done
	1371	fi
	1372	done
	1373
	1374	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
	1375
	1376	# 3 MPI/OpenMP ccc_mprun method
	1377
[1381]	1378	# MPI-OpenMP (MPMD)
	1379	# example of run_file
	1380	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
	1381	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
	1382	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
[622]	1383
[1078]	1384	# Loop on the components
	1385	for comp in ${config_ListOfComponents[*]} ; do
[1000]	1386
[1078]	1387	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1388	eval ExeNameOut=\${config_Executable_${comp}[1]}
[1000]	1389
[1078]	1390	# Not possible if oasis has an executable (i.e old version of oasis3)
	1391	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
	1392	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
	1393	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
	1394	IGCM_debug_Verif_Exit
	1395	fi
[1000]	1396
[1078]	1397	# Only if we really have an executable for the component :
	1398	if [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1399
[1078]	1400	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1401	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[1000]	1402
[1381]	1403	# Build script files
	1404
[1078]	1405	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1406	echo "" >> script_${ExeNameOut}.ksh
	1407	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
	1408
	1409	# Check if the number of threads is correct
	1410	case ${comp_proc_omp_loc} in
	1411	2\|4\|8\|16)
	1412	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
	1413	;;
	1414	*)
	1415	IGCM_debug_Exit "ERROR with OMP parameters !"
	1416	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
	1417	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
	1418	IGCM_debug_Verif_Exit
	1419	;;
	1420	esac
[1381]	1421	### only for LMDZ?
[1078]	1422	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1423	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1424	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1425	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
[1381]	1426
[1078]	1427	fi
[1318]	1428
[1381]	1429	# to have out/err per process on different files
	1430	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
[1317]	1431	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[1078]	1432
[1381]	1433	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
[1078]	1434
[1381]	1435	# Complete run_file
[1000]	1436
[1381]	1437	echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
	1438
[1078]	1439	fi
	1440	done
[1381]	1441
[1385]	1442	## variable added to stop after 60s instead of 600s by default.
	1443	## This is used when no error comes from executables and when something stopped an executable without notice.
	1444	export SLURM_WAIT=60
[1381]	1445
[1385]	1446	## module advised by TGCC (instead of 2 variables)
	1447	module load feature/bridge/heterogenous_mpmd
[1381]	1448
[1390]	1449	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
[1000]	1450
[1390]	1451	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
[1381]	1452
[1390]	1453	IGCM_sys_Chmod u+x run_file
	1454	if ( $DEBUG_sys ) ; then
	1455	echo "run_file contains : "
	1456	cat run_file
	1457	fi
[1381]	1458
[1390]	1459	else
[1381]	1460
[1390]	1461	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
	1462
	1463	fi # 1 2 or 3 if ${OK_PARA_MPMD}
	1464
	1465	else
[1078]	1466	# Only one executable (SPMD mode).
[1000]	1467
[622]	1468	for comp in ${config_ListOfComponents[*]} ; do
	1469
[685]	1470	# Only if we really have an executable for the component :
[623]	1471	eval ExeNameOut=\${config_Executable_${comp}[1]}
[685]	1472	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
[622]	1473
[1381]	1474	# Build script files
	1475
[685]	1476	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1477	echo "" >> script_${ExeNameOut}.ksh
	1478	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1479
	1480	if ( ${OK_PARA_OMP} ) ; then
	1481	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1482	echo "" >> script_${ExeNameOut}.ksh
	1483	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1484	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1485	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1486	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1487	fi
	1488
	1489	if ( ${OK_PARA_MPI} ) ; then
[623]	1490	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
[685]	1491	# Default : ccc_mprun used if nb_proc gt 1
	1492	# to have out/err per process on different files
[1317]	1493	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
	1494	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1495	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
[623]	1496	else
[685]	1497	# Default : ccc_mprun is NOT used if nb_proc eq 1
	1498	# to have out/err per process on different files
[1317]	1499	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
	1500	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1501	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
[622]	1502	fi
[685]	1503
	1504	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
	1505	cat script_${ExeNameOut}.ksh
	1506
[623]	1507	fi
[622]	1508
	1509	done
	1510
[1390]	1511	fi # ${OK_PARA_MPMD}
[622]	1512
[623]	1513	IGCM_debug_Print 1 "sys Curie : execution command is "
	1514	IGCM_debug_Print 1 "$EXECUTION"
[622]	1515
[623]	1516	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[622]	1517	}
	1518
[933]	1519	#D-#==================================================
	1520	#D-function IGCM_sys_check_path
	1521	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
	1522	#D-* do not point to an important use directory. Stop immediately in that case.
	1523	#D-* Examples:
	1524	#D-
	1525	function IGCM_sys_check_path {
	1526	IGCM_debug_PushStack "IGCM_sys_check_path"
	1527	if ( $DEBUG_sys ) ; then
	1528	echo "IGCM_sys_check_path"
	1529	fi
[811]	1530
[933]	1531	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
	1532	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
	1533	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
[936]	1534	IGCM_debug_Exit "This will stop the job"
[933]	1535	fi
	1536	IGCM_debug_PopStack "IGCM_sys_check_path"
	1537	}
	1538
[811]	1539	#D-#==================================================
	1540	#D-function IGCM_sys_check_quota
	1541	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
	1542	#D-* Examples:
	1543	#D-
[714]	1544	function IGCM_sys_check_quota {
[776]	1545	IGCM_debug_PushStack "IGCM_sys_check_quota"
	1546	if ( $DEBUG_sys ) ; then
	1547	echo "IGCM_sys_check_quota"
	1548	fi
	1549	# Limit of quota (in %)
	1550	limit_quota=90
	1551
	1552	# Check of the volume
[912]	1553	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
	1554	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
[776]	1555
	1556	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
	1557
	1558	unit_avail=${volume_avail: -1}
	1559	unit_quota=${volume_quota: -1}
	1560
	1561	if [ "${unit_quota}" = "*" ] ; then
[811]	1562	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1563	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1564	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1565	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1566	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1567	IGCM_debug_Verif_Exit
[714]	1568	fi
[776]	1569
	1570	temp_avail=${volume_avail%%${volume_avail: -1}*}
	1571	temp_quota=${volume_quota%%${volume_quota: -1}*}
	1572
	1573	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1574
	1575	# Convertion
	1576	if [ ${volume_avail: -1} = "T" ] ; then
	1577	(( temp_avail = temp_avail * 1000000000000 ))
	1578	elif [ ${volume_avail: -1} = "G" ] ; then
	1579	(( temp_avail = temp_avail * 1000000000 ))
	1580	elif [ ${volume_avail: -1} = "M" ] ; then
	1581	(( temp_avail = temp_avail * 1000000 ))
	1582	elif [ ${volume_avail: -1} = "k" ] ; then
	1583	(( temp_avail = temp_avail * 1000 ))
	1584	else
	1585	(( temp_avail = volume_avail ))
	1586	fi
	1587	if [ ${volume_quota: -1} = "T" ] ; then
	1588	(( temp_quota = temp_quota * 1000000000000 ))
	1589	elif [ ${volume_quota: -1} = "G" ] ; then
	1590	(( temp_quota = temp_quota * 1000000000 ))
	1591	elif [ ${volume_quota: -1} = "M" ] ; then
	1592	(( temp_quota = temp_quota * 1000000 ))
	1593	elif [ ${volume_quota: -1} = "k" ] ; then
	1594	(( temp_quota = temp_quota * 1000 ))
	1595	else
	1596	(( temp_quota = volume_quota ))
	1597	fi
	1598	fi
	1599
	1600	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1601	# echo "volume ratio is " $quota_volume
	1602
	1603	if [ ${quota_volume} -ge ${limit_quota} ] ; then
	1604	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1605	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
	1606	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1607	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1608	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1609	IGCM_debug_Verif_Exit
	1610	fi
	1611
	1612	fi
	1613
	1614	# Check of the number of inodes
	1615
[912]	1616	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
	1617	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
[776]	1618
	1619	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
	1620
	1621	unit_avail=${inode_avail: -1}
	1622	unit_quota=${inode_quota: -1}
	1623
	1624	if [ "${unit_quota}" = "*" ] ; then
[811]	1625	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1626	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1627	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1628	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1629	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1630	IGCM_debug_Verif_Exit
[776]	1631	fi
	1632
	1633	temp_avail=${inode_avail%%${inode_avail: -1}*}
	1634	temp_quota=${inode_quota%%${inode_quota: -1}*}
	1635
	1636	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1637
	1638	# Convertion
	1639	if [ ${inode_avail: -1} = "T" ] ; then
	1640	(( temp_avail = temp_avail * 1000000000000 ))
	1641	elif [ ${inode_avail: -1} = "G" ] ; then
	1642	(( temp_avail = temp_avail * 1000000000 ))
	1643	elif [ ${inode_avail: -1} = "M" ] ; then
	1644	(( temp_avail = temp_avail * 1000000 ))
	1645	elif [ ${inode_avail: -1} = "k" ] ; then
	1646	(( temp_avail = temp_avail * 1000 ))
	1647	else
	1648	(( temp_avail = inode_avail ))
	1649	fi
	1650
	1651	if [ ${inode_quota: -1} = "T" ] ; then
	1652	(( temp_quota = temp_quota * 1000000000000 ))
	1653	elif [ ${inode_quota: -1} = "G" ] ; then
	1654	(( temp_quota = temp_quota * 1000000000 ))
	1655	elif [ ${inode_quota: -1} = "M" ] ; then
	1656	(( temp_quota = temp_quota * 1000000 ))
	1657	elif [ ${inode_quota: -1} = "k" ] ; then
	1658	(( temp_quota = temp_quota * 1000 ))
	1659	else
	1660	(( temp_quota = inode_quota ))
	1661	fi
	1662	fi
	1663	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1664	# echo "inode ratio is " $quota_inode
	1665
	1666	if [ ${quota_inode} -ge ${limit_quota} ] ; then
	1667	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1668	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
	1669	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1670	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1671	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1672	IGCM_debug_Verif_Exit
	1673	fi
	1674	fi
	1675	IGCM_debug_PopStack "IGCM_sys_check_quota"
[714]	1676	}
	1677
[773]	1678	#D-#==================================================
[1286]	1679	#D-function IGCM_sys_projectAccounting
	1680	#D-* Purpose: store project accounting information in a file
	1681	#D-* Examples:
	1682	#D-
	1683	function IGCM_sys_projectAccounting {
	1684	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
	1685	if ( $DEBUG_sys ) ; then
	1686	echo "IGCM_sys_check_quota"
	1687	fi
	1688
[1288]	1689	ssh curie91 /usr/bin/ccc_myproject > $1
[1286]	1690
	1691	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
	1692	}
	1693
	1694	#D-#==================================================
[1244]	1695	#D-function IGCM_sys_getJobSchedulerID
	1696	#D-* Purpose: Get the job ID during execution
	1697	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
	1698	#D-
	1699	function IGCM_sys_getJobSchedulerID {
	1700	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
	1701	if ( $DEBUG_sys ) ; then
	1702	echo "IGCM_sys_getJobSchedulerID"
	1703	fi
	1704
	1705	eval ${1}=${BRIDGE_MSUB_JOBID}
	1706
	1707	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
	1708	}
	1709
	1710	#D-#==================================================
[958]	1711	#D-function IGCM_sys_GetJobID
[1244]	1712	#D-* Purpose: Get the job ID from the JobName
[958]	1713	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
	1714	#D-
	1715	function IGCM_sys_GetJobID {
	1716	IGCM_debug_PushStack "IGCM_sys_GetJobID"
	1717	if ( $DEBUG_sys ) ; then
	1718	echo "IGCM_sys_GetJobID"
	1719	fi
	1720
	1721	# With -f option, the full job name is given in the last column
	1722	ID=$( ccc_mstat -f -u $2 \| \
	1723	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
	1724
	1725	eval ${3}=${ID}
	1726	IGCM_debug_PopStack "IGCM_sys_GetJobID"
	1727	}
	1728
	1729	#D-#==================================================
[773]	1730	#D-function IGCM_sys_CountJobInQueue
[1244]	1731	#D-* Purpose: Count number of users job
[773]	1732	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
	1733	#D-
	1734	function IGCM_sys_CountJobInQueue {
	1735	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
	1736	if ( $DEBUG_sys ) ; then
	1737	echo "IGCM_sys_CountJobInQueue"
	1738	fi
	1739
	1740	# With -f option, the full job name is given in the last column
[834]	1741	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
[773]	1742
	1743	eval ${2}=${NbRun}
	1744
	1745	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
	1746	}
	1747
[955]	1748	#D-#==================================================
	1749	#D-function IGCM_sys_ListJobInQueue
[1244]	1750	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
[955]	1751	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
	1752	#D-
	1753	function IGCM_sys_ListJobInQueue {
	1754	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
	1755	if ( $DEBUG_sys ) ; then
	1756	echo "IGCM_sys_ListJobInQueue"
	1757	fi
	1758
	1759	# With -f option, the full job name is given in the last column
	1760	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
	1761	'( $2 == User && \
	1762	$NF != /TS/ && \
	1763	$NF !~ /PACK/ && \
	1764	$NF !~ /REBUILD/ && \
	1765	$NF !~ /pack/ ) \
	1766	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
	1767
	1768	eval set -A ${2} ${JobList[*]}
	1769
	1770	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
	1771	}
[1272]	1772
	1773	#D-#==================================================
	1774	#D-function IGCM_sys_atlas
	1775	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
	1776	#D-* Examples:
	1777	#D-
	1778	function IGCM_sys_atlas {
	1779	IGCM_debug_PushStack "IGCM_sys_atlas" $@
	1780	if ( $DEBUG_sys ) ; then
	1781	echo "IGCM_sys_atlas :" $@
	1782	fi
	1783
	1784	typeset status
	1785
	1786	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	1787	status=$?
	1788	if [ ${status} -gt 0 ] ; then
	1789	echo "IGCM_sys_atlas : error code ${status}"
	1790	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1791	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1792	IGCM_debug_PopStack "IGCM_sys_atlas"
	1793	return 1
	1794	else
	1795	IGCM_debug_PopStack "IGCM_sys_atlas"
	1796	return 0
	1797	fi
	1798
	1799	IGCM_debug_PopStack "IGCM_sys_atlas"
	1800	}
	1801
[1380]	1802	#D-#==================================================
	1803	#D-function IGCM_sys_rebuild_nemo
	1804	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
	1805	#D-* Examples:
	1806	#D-
	1807
	1808	function IGCM_sys_rebuild_nemo {
	1809	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
	1810	if ( $DEBUG_sys ) ; then
	1811	echo "IGCM_sys_rebuild_nemo :" $@
	1812	fi
	1813
	1814	$( ccc_home -u p86ipsl )/REBUILD_NEMO/rebuild_nemo ${1} ${2}
	1815
	1816	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
	1817	}
	1818

Note: See TracBrowser for help on using the repository browser.

Download in other formats: