Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene.ksh @ 1440

Last change on this file since 1440 was 1440, checked in by jgipsl, 6 years ago
Added more possibilities at irene
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 62.1 KB

Rev	Line
[622]	1	#!/bin/ksh
	2
	3	#**************************************************************
	4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
	5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#=========================================================
	15	# The documentation of this file can be automatically generated
[623]	16	# if you use the prefix #D- for comments to be extracted.
[622]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#=========================================================
	19
	20	#D-#==================================================
[623]	21	#D-LibIGCM_sys for Curie
[622]	22	#D-#==================================================
	23	#D-
[623]	24	#D- This ksh library if a layer under some usefull
[622]	25	#D-environment variables and shell commands.
	26	#D-All those definitions depend on host particularities.
	27	#D-It manages a stack mechanism and test validity of operations.
	28	#D-All function described bellow must be prefixed by IGCM_sys.
	29
	30	#====================================================
	31	# libIGCM_sys PARAMETERS
	32	#====================================================
	33
	34	#====================================================
	35	# set DEBUG_sys to true to output calls of function
	36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
	37
	38	#====================================================
	39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
	40	typeset -r DRYRUN=${DRYRUN:=0}
	41
	42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
	43	# -------------------------------------------------------------------------------------
	44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
[809]	45	# \| \| Cp/Exe/param/files \| \| \| \|
	46	# \| \| Chmod Qsub \| \| \| \|
[622]	47	# -------------------------------------------------------------------------------------
	48	# \| 0 \| yes \| yes \| yes \| yes \|
	49	# -------------------------------------------------------------------------------------
	50	# \| 1 \| yes \| yes \| yes \| no \|
	51	# -------------------------------------------------------------------------------------
	52	# \| 2 \| yes \| yes \| no \| no \|
	53	# -------------------------------------------------------------------------------------
	54	# \| 3 \| yes \| no \| no \| no \|
	55	# -------------------------------------------------------------------------------------
	56
	57	#=====================================================
	58	# Global Variables :
	59	#=====================================================
	60	# Language : "fr" or "en"
	61	typeset -r MYLANG="fr"
	62
	63	#=====================================================
[1201]	64	# Host user names project maxCpuTime
[622]	65	# $hostname ou hostname
[1158]	66	typeset HOST=${HOST:=$( hostname )}
[622]	67	# $username ou whoami
[1158]	68	typeset LOGIN=${LOGIN:=$( whoami )}
[622]	69	# $hostname of the MASTER job
	70	typeset MASTER=curie
[880]	71	# add default project on curie
[1201]	72	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} \| cut -d@ -f1 )
	73	# jobWarningDelay in seconds
	74	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
[622]	75
	76	#D-
	77	#D-#==================================================
	78	#D-Program used in libIGCM
	79	#D-#==================================================
	80
[742]	81	# Submit command
[747]	82	typeset SUBMIT=${SUBMIT:=ccc_msub}
[622]	83	# rsync with path
	84	typeset -r RSYNC=/usr/bin/rsync
	85	# RSYNC_opt args to rsync
	86	typeset -r RSYNC_opt="-va"
	87	# ie storage filesystem
[623]	88	typeset -r STOREHOST=${MASTER}
[1180]	89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
[622]	90
	91	#====================================================
[1180]	92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
[622]	93	#====================================================
[996]	94	if [ X${TaskType} = Xcomputing ] ; then
[1409]	95	IGCM_debug_Print 1 "Module will be loaded once we know the compiler version"
[996]	96	else
[1413]	97	module purge > /dev/null 2>&1
[1433]	98	. /ccc/cont003/home/dsm/p86ipsl/.env_irene > /dev/null 2>&1
[1438]	99	. /ccc/cont003/home/dsm/p86ipsl/.env_atlas_irene > /dev/null 2>&1
[1433]	100
	101	# . /ccc/cont003/home/dsm/p86ipsl/.env_intel17.0.2_curie > /dev/null 2>&1
	102	# . /ccc/cont003/home/dsm/p86ipsl/.env_netcdf4.3.3.1_curie > /dev/null 2>&1
	103	# . /ccc/cont003/home/dsm/p86ipsl/.env_atlas_curie > /dev/null 2>&1
[1358]	104	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
[1364]	105	export UVCDAT_ANONYMOUS_LOG=FALSE
[996]	106	fi
[1315]	107
[1437]	108	# Use CMIP6 storage space when using project id gch0316
	109	[[ "X${PROJECT}" = "Xgch0316" && ! $(module list --terse 2>&1 \| grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
[1315]	110
	111	# FYI
[1210]	112	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
	113	[ ! X${TaskType} = Xchecking ] && module list
[1208]	114
[1179]	115	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
	116	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
	117	export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
[622]	118
	119	#====================================================
	120	# Specific for ocean additionnal diagnostic
	121	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
	122	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
	123
	124	#====================================================
	125	# Host specific DIRECTORIES
	126	#====================================================
	127
[623]	128	# ============ CESIUM START ============ #
	129
[622]	130	#====================================================
[623]	131	#- Mirror libIGCM from titane to cesium if needed
	132	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
	133	#if [ ! ${ROOTSYS} = "home" ] ; then
	134	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
	135	#else
	136	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	137	#fi
	138
	139	#====================================================
	140	#- libIGCM_POST
	141	#if ( ${MirrorlibIGCM} ) ; then
	142	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
	143	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
	144	#else
	145	# typeset -r libIGCM_POST=${libIGCM}
	146	#fi
	147
	148	# ============ CESIUM END ============ #
	149
	150	#====================================================
	151	#- MirrorlibIGCM for frontend
	152	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	153
	154	#====================================================
	155	#- libIGCM_POST for frontend
	156	typeset -r libIGCM_POST=${libIGCM}
	157
	158	#====================================================
[622]	159	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
	160	typeset -r R_EXE="${MODIPSL}/bin"
	161
	162	#====================================================
	163	#- SUBMIT_DIR : submission dir
[1010]	164	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
[622]	165
	166	#====================================================
	167	#- IN
[1276]	168	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
[622]	169	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
	170
	171	#====================================================
	172	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[1432]	173	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
[622]	174
	175	#====================================================
[1179]	176	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
[1186]	177	typeset -r OUTCOMMAND_PATH=/tmp
[1179]	178
	179	#====================================================
[622]	180	#- HOST_MPIRUN_COMMAND
[1382]	181	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
[622]	182
	183	#====================================================
	184	#- Max number of arguments passed to nco operator or demigration command
	185	UNIX_MAX_LIMIT=360
	186
[623]	187	#====================================================
[1180]	188	#- set PackDefault to true on curie
[623]	189	PackDefault=true
	190
[664]	191	#====================================================
[623]	192	#- Default number of MPI task for IPSL coupled model
	193	#- required for backward compatibility
	194	#-
	195	DEFAULT_NUM_PROC_OCE=5
	196	DEFAULT_NUM_PROC_CPL=1
[637]	197	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
[623]	198
[1426]	199	#====================================================
	200	#- Number of computing cores per node
	201	#-
	202	NB_CORE_PER_NODE=16
	203
[725]	204	#D-#==================================================
[1057]	205	#D-function IGCM_sys_defineArchives
	206	#D-* Purpose:
	207	#D-* Define ARCHIVE : Dedicated to large files
	208	#D-* Define STORAGE : Dedicated to small/medium files
	209	#D-* Define R_OUT : Output tree located on ARCHIVE
	210	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
[1432]	211	#D-* Define R_BUF : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
	212	#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
[725]	213	#D-* Examples:
	214	#D-
[1057]	215	function IGCM_sys_defineArchives {
	216	IGCM_debug_PushStack "IGCM_sys_defineArchives"
[676]	217
[1060]	218	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
[1057]	219	#====================================================
	220	#- ARCHIVE (dedicated to large files)
[1063]	221	ARCHIVE=${config_UserChoices_ARCHIVE}
[1057]	222	else
	223	#====================================================
	224	#- ARCHIVE (dedicated to large files)
[1063]	225	ARCHIVE=${CCCSTOREDIR}
[1057]	226	fi
[725]	227
[1060]	228	if [ ! X${config_UserChoices_STORAGE} = X ]; then
[1057]	229	#====================================================
	230	#- STORAGE (dedicated to small/medium files)
[1063]	231	STORAGE=${config_UserChoices_STORAGE}
[1057]	232	else
	233	#====================================================
	234	#- STORAGE (dedicated to small/medium files)
[1063]	235	STORAGE=${CCCWORKDIR}
[1057]	236	fi
[725]	237
[1057]	238	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
	239	#====================================================
	240	#- R_OUT
[1432]	241	R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
[1057]	242
	243	#====================================================
	244	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1432]	245	R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
[1057]	246
	247	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
	248
	249	else
	250	#====================================================
	251	#- R_OUT
[1063]	252	R_OUT=${ARCHIVE}/IGCM_OUT
[1057]	253
	254	#====================================================
	255	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]	256	R_FIG=${STORAGE}/IGCM_OUT
[1057]	257	fi
	258
	259	#====================================================
[1431]	260	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
	261	CMIP6_BUF=${STORAGE}/IGCM_OUT
	262
	263	#====================================================
[1057]	264	#- R_BUF (ONLY FOR double copy an scratch)
[1432]	265	R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
[1057]	266
	267	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
	268	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
	269	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
	270
	271	IGCM_debug_PopStack "IGCM_sys_defineArchives"
[725]	272	}
	273
[622]	274	#D-#==================================================
	275	#D-function IGCM_sys_RshArchive
	276	#D-* Purpose: Archive rsh command
	277	#D-* Examples:
	278	#D-
	279	function IGCM_sys_RshArchive {
[623]	280	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
	281	/bin/ksh <<-EOF
[622]	282	${@}
	283	EOF
[811]	284	status=$?
	285	if [ ${status} -gt 0 ] ; then
	286	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
[623]	287	IGCM_debug_Exit "IGCM_sys_RshArchive"
	288	fi
	289	IGCM_debug_PopStack "IGCM_sys_RshArchive"
[622]	290	}
	291
	292	#D-#==================================================
[845]	293	#D-function IGCM_sys_RshArchive_NoError
	294	#D-* Purpose: Archive rsh command, without error
	295	#D-* used only in monitoring.job
	296	#D-* Examples:
	297	#D-
	298	function IGCM_sys_RshArchive_NoError {
	299	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
	300	/bin/ksh <<-EOF
[1181]	301	${@} 2> /dev/null
[845]	302	EOF
	303	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
	304	}
	305
	306	#D-#==================================================
[622]	307	#D-function IGCM_sys_MkdirArchive
	308	#D-* Purpose: Mkdir on Archive
	309	#D-* Examples:
	310	#D-
	311	function IGCM_sys_MkdirArchive {
[623]	312	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
	313	if ( $DEBUG_sys ) ; then
	314	echo "IGCM_sys_MkdirArchive :" $@
	315	fi
	316	#- creation de repertoire sur le serveur fichier
	317	if [ ! -d ${1} ]; then
	318	\mkdir -p $1
[811]	319	status=$?
	320
	321	if [ ${status} -gt 0 ] ; then
	322	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
[623]	323	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
[622]	324	fi
[623]	325	fi
	326	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[622]	327	}
	328
	329	#D-#==================================================
	330	#D-function IGCM_sys_TestDirArchive
	331	#D-* Purpose: Test Directory that must exists on Archive
	332	#D-* Examples:
	333	#D-
	334	function IGCM_sys_TestDirArchive {
[623]	335	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
	336	if ( $DEBUG_sys ) ; then
	337	echo "IGCM_sys_TestDirArchive :" $@
	338	fi
	339	typeset ExistFlag
	340	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	341	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
	342	return ${ExistFlag}
[622]	343	}
	344
	345	#D-#==================================================
[623]	346	#D-function IGCM_sys_IsFileArchived
[1179]	347	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
[623]	348	#D-* Examples:
	349	#D-
	350	function IGCM_sys_IsFileArchived {
	351	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
	352	if ( $DEBUG_sys ) ; then
	353	echo "IGCM_sys_IsFileArchived :" $@
	354	fi
	355	typeset IsArchivedFlag
[818]	356	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
[623]	357	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
	358
	359	return ${IsArchivedFlag}
	360	}
	361
	362	#D-#==================================================
[622]	363	#D-function IGCM_sys_TestFileArchive
	364	#D-* Purpose: Test file that must NOT EXISTS on Archive
	365	#D-* Examples:
	366	#D-
	367	function IGCM_sys_TestFileArchive {
[623]	368	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
	369	typeset ExistFlag
	370	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	371	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[622]	372
[623]	373	return ${ExistFlag}
[622]	374	}
	375
	376	#D-#==================================================
	377	#D-function IGCM_sys_CountFileArchive
	378	#D-* Purpose: Count files on Archive filesystem
	379	#D-* Examples:
	380	#D-
	381	function IGCM_sys_CountFileArchive {
[623]	382	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
	383	ls ${@} 2>/dev/null \| wc -l
	384	if [ $? -gt 0 ] ; then
	385	echo "IGCM_sys_CountFileArchive : erreur."
	386	fi
	387	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[622]	388	}
	389
	390	#D-#==================================================
	391	#D-function IGCM_sys_Tree
	392	#D-* Purpose: Tree directories with files on ${ARCHIVE}
	393	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
	394	#D-
	395	function IGCM_sys_Tree {
[623]	396	IGCM_debug_PushStack "IGCM_sys_Tree" $@
	397	if ( $DEBUG_sys ) ; then
	398	echo "IGCM_sys_Tree :" $@
	399	fi
[622]	400
[623]	401	\ls -lR ${@}
[622]	402
[623]	403	IGCM_debug_PopStack "IGCM_sys_Tree"
[622]	404	}
	405
	406	#D-#==================================================
	407	#D-function IGCM_sys_Qsub
	408	#D-* Purpose: Qsub new job
	409	#D-* Examples:
	410	#D-
	411	function IGCM_sys_Qsub {
[623]	412	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
	413	if ( $DEBUG_sys ) ; then
	414	echo "IGCM_sys_Qsub :" $@
	415	fi
[811]	416	typeset options status
[923]	417	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
[622]	418
[1179]	419	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	420	status=$?
[948]	421
[1179]	422	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	423	if [ ${status} -gt 0 ] ; then
	424	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
[623]	425	IGCM_debug_Exit "IGCM_sys_Qsub"
[948]	426	else
[1179]	427	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	428	fi
	429	IGCM_debug_PopStack "IGCM_sys_Qsub"
[622]	430	}
	431
	432	#D-#==================================================
	433	#D-function IGCM_sys_QsubPost
	434	#D-* Purpose: Qsub new job on scalaire
	435	#D-* Examples:
	436	#D-
	437	function IGCM_sys_QsubPost {
[623]	438	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
	439	if ( $DEBUG_sys ) ; then
	440	echo "IGCM_sys_QsubPost :" $@
	441	fi
[923]	442	typeset options status
	443
[1062]	444	# EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
	445	case $( hostname -s ) in
	446	curie*)
[1201]	447	options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
[1062]	448	airain*)
	449	options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
	450	esac
	451
[1179]	452	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	453	status=$?
[948]	454
[1179]	455	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	456	if [ ${status} -gt 0 ] ; then
[923]	457	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
[623]	458	IGCM_debug_Exit "IGCM_sys_QsubPost"
[948]	459	else
[1179]	460	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	461	fi
	462	IGCM_debug_PopStack "IGCM_sys_QsubPost"
[622]	463	}
	464
	465	#D-*************************
[623]	466	#D- File transfer functions
[622]	467	#D-*************************
	468	#D-
	469
	470	#D-#==================================================
	471	#D-function IGCM_sys_RmRunDir
	472	#D-* Purpose: rm tmpdir (dummy function most of the time batch
	473	#D- scheduler will do the job)
	474	#D-* Examples:
	475	#D-
	476	function IGCM_sys_RmRunDir {
[1057]	477	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
[623]	478	if ( $DEBUG_sys ) ; then
	479	echo "IGCM_sys_RmRunDir :" $@
	480	fi
[622]	481
[715]	482	typeset status
[622]	483
[1179]	484	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	485	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	486	status=$?
[622]	487
[715]	488	if [ ${status} -gt 0 ] ; then
[1183]	489	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
[1179]	490	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	491	IGCM_debug_Exit "IGCM_sys_RmRunDir"
	492	else
[1179]	493	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	494	fi
	495	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[622]	496	}
	497
	498	#D-#==================================================
	499	#D-function IGCM_sys_Put_Dir
	500	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
	501	#D-* Examples:
	502	#D-
	503	function IGCM_sys_Put_Dir {
[623]	504	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
	505	if ( $DEBUG_sys ) ; then
	506	echo "IGCM_sys_Put_Dir :" $@
	507	fi
	508	if [ $DRYRUN = 0 ]; then
	509	if [ ! -d ${1} ] ; then
	510	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
	511	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
	512	return
[622]	513	fi
	514
[715]	515	typeset status
[622]	516
[623]	517	# Only if we use rsync
	518	#IGCM_sys_TestDirArchive $( dirname $2 )
	519	#
	520	#USUAL WAY
[1179]	521	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	522	status=$?
[622]	523
[715]	524	if [ ${status} -gt 0 ] ; then
[811]	525	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
[1179]	526	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	527	IGCM_debug_Exit "IGCM_sys_Put_Dir"
[622]	528	else
[1179]	529	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	530	fi
[623]	531	fi
	532	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[622]	533	}
	534
	535	#D-#==================================================
	536	#D-function IGCM_sys_Get_Dir
[623]	537	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[622]	538	#D-* Examples:
	539	#D-
	540	function IGCM_sys_Get_Dir {
[623]	541	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
	542	if ( $DEBUG_sys ) ; then
	543	echo "IGCM_sys_Get_Dir :" $@
	544	fi
	545	if [ $DRYRUN = 0 ]; then
[708]	546	typeset NB_ESSAI DELAI status i
	547	# number of tentative
	548	NB_ESSAI=3
	549	# time delay between tentative
	550	DELAI=2
[622]	551
[676]	552	#
[708]	553	# USUAL WAY
	554	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
[713]	555	ccc_hsm get -r $1
[622]	556
[709]	557	i=0
[708]	558	while [ $i -lt $NB_ESSAI ] ; do
[1179]	559	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[708]	560	status=$?
[811]	561	if [ ${status} -gt 0 ] ; then
[713]	562	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
[708]	563	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
	564	sleep $DELAI
	565	else
	566	break
	567	fi
	568	(( i = i + 1 ))
	569	done
	570
	571	if [ ${status} -gt 0 ] ; then
[1178]	572	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
[1179]	573	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	574	IGCM_debug_Exit "IGCM_sys_Get_Dir"
[622]	575	else
[1179]	576	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	577	fi
[623]	578	fi
	579	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[622]	580	}
	581
	582	#D-#==================================================
	583	#D-function IGCM_sys_Put_Rest
[623]	584	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[622]	585	#D- File and target directory must exist.
	586	#D-* Examples:
	587	#D-
	588	function IGCM_sys_Put_Rest {
[623]	589	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
	590	if ( $DEBUG_sys ) ; then
	591	echo "IGCM_sys_Put_Rest :" $@
	592	fi
	593	if [ $DRYRUN = 0 ]; then
	594	if [ ! -f ${1} ] ; then
	595	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
	596	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	597	fi
	598
[715]	599	typeset status
[623]	600	#
	601	# USUAL WAY
[1179]	602	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	603	status=$?
[622]	604
[623]	605	# #RSYNC WITH NETWORK SSH CALL
[1179]	606	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	607	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	608
[623]	609	# #RSYNC WITH NFS USE
[1179]	610	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	611	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	612
[715]	613	# status=$?
	614	# IGCM_sys_Rsync_out $status
[622]	615
[1179]	616	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[715]	617	# (( status=status+$? ))
[623]	618
[715]	619	if [ ${status} -gt 0 ] ; then
[1179]	620	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
[765]	621	[ -f ${1} ] && ls -l ${1}
[759]	622	[ -f ${2} ] && ls -l ${2}
	623	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	624	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	625	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	626	else
[759]	627
	628	if [ X${JobType} = XRUN ] ; then
	629	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	630	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	631	fi
	632
[1179]	633	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	634	fi
[623]	635	fi
	636	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[622]	637	}
	638
	639	#D-#==================================================
	640	#D-function IGCM_sys_Put_Out
[623]	641	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[622]	642	#D-* Examples:
	643	#D-
	644	function IGCM_sys_Put_Out {
[623]	645	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
	646	if ( $DEBUG_sys ) ; then
	647	echo "IGCM_sys_Put_Out :" $@
	648	fi
[667]	649
[711]	650	typeset NB_ESSAI DELAI status i exist skip
[1234]	651	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
[667]	652
	653	# number of tentative
[668]	654	NB_ESSAI=3
[667]	655	# time delay between tentative
[668]	656	DELAI=2
[667]	657
[623]	658	if [ $DRYRUN = 0 ]; then
	659	if [ ! -f ${1} ] ; then
	660	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
	661	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	662	return 1
[622]	663	fi
[623]	664	#
	665	IGCM_sys_MkdirArchive $( dirname $2 )
	666	#
[711]	667	exist=false
	668	skip=false
	669	if [ -f $2 ] ; then
	670	IGCM_debug_Print 1 "$2 already exist"
	671	ccc_hsm get $2
	672	exist=true
	673	if [ "X$( diff $1 $2 )" = X ] ; then
	674	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
	675	skip=true
[667]	676	else
[711]	677	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	678	skip=false
[667]	679	fi
[711]	680	fi
	681	#
	682	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	683	IGCM_sys_Chmod u+w $2
	684	fi
[622]	685
[711]	686	if [ X${skip} = Xfalse ] ; then
[719]	687	i=0
[711]	688	while [ $i -lt $NB_ESSAI ] ; do
[1234]	689	# Identify file system
	690	fileDeviceNumberInHex=$( stat -c %d $1 )
	691	status=$?
	692	if [ ${status} -gt 0 ] ; then
	693	IGCM_debug_Exit "IGCM_sys_Put_Out"
	694	fi
	695	# Identify file system
	696	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
	697	status=$?
	698	if [ ${status} -gt 0 ] ; then
	699	IGCM_debug_Exit "IGCM_sys_Put_Out"
	700	fi
	701
[1412]	702	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
[1234]	703	# They are not on the same device. USUAL WAY
[1179]	704	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	705	status=$?
	706	else
[1234]	707	# They are on the same device. NOT SO USUAL WAY
[1179]	708	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	709	status=$?
	710	fi
[711]	711	if [ ${status} -gt 0 ]; then
[713]	712	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
[711]	713	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
[765]	714	[ -f ${1} ] && ls -l ${1}
[759]	715	[ -f ${2} ] && ls -l ${2}
	716	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[711]	717	sleep $DELAI
	718	else
	719	break
	720	fi
	721	(( i = i + 1 ))
	722	done
	723	fi
	724
[623]	725	# #RSYNC WITH NETWORK SSH CALL
[1179]	726	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	727	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	728
[623]	729	# #RSYNC WITH NFS USE
[1179]	730	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	731	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	732
[711]	733	# status=$?
	734	# IGCM_sys_Rsync_out $status
[622]	735
[1179]	736	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[711]	737	# (( status=status+$? ))
[622]	738
[711]	739	if [ ${status} -gt 0 ] ; then
[1179]	740	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
[765]	741	[ -f ${1} ] && ls -l ${1}
[759]	742	[ -f ${2} ] && ls -l ${2}
	743	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	744	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	745	IGCM_debug_Exit "IGCM_sys_Put_Out"
[622]	746	else
[759]	747
	748	if [ X${JobType} = XRUN ] ; then
	749	if [ X${3} = X ] ; then
	750	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	751	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	752	fi
	753	fi
	754
[1179]	755	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	756	fi
[623]	757	fi
	758	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	759	return 0
[622]	760	}
	761
	762	#D-#==================================================
	763	#D-function IGCM_sys_Get
	764	#D-* Purpose: Get a file from ${ARCHIVE}
	765	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
	766	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
	767	function IGCM_sys_Get {
[623]	768	IGCM_debug_PushStack "IGCM_sys_Get" $@
[622]	769
[773]	770	typeset DEST dm_liste target file_work
[667]	771	typeset NB_ESSAI DELAI status i
[622]	772
[623]	773	if ( $DEBUG_sys ) ; then
	774	echo "IGCM_sys_Get :" $@
	775	fi
[667]	776
	777	# number of tentative
[668]	778	NB_ESSAI=3
[667]	779	# time delay between tentative
[668]	780	DELAI=2
[667]	781
[623]	782	if [ $DRYRUN -le 2 ]; then
	783	if [ X${1} = X'/l' ] ; then
	784	eval set +A dm_liste \${${2}}
	785	else
	786	eval set +A dm_liste ${1}
[622]	787	fi
[623]	788	eval DEST=\${${#}}
[1179]	789	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	790	status=$?
	791	if [ ${status} -gt 0 ] ; then
	792	echo "WARNING IGCM_sys_Get : error code ${status}"
[1179]	793	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	794	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
	795	fi
[622]	796
[715]	797	#if [ ${status} -gt 0 ] ; then
[1179]	798	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
	799	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	800	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
	801	# sleep 30
	802	# echo "We try another time"
[1179]	803	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	804	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[785]	805	# status=$?
	806	# if [ ${status} -gt 0 ] ; then
	807	# echo "ERROR IGCM_sys_Get : again demigration error :"
[1179]	808	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	809	# IGCM_debug_Exit "IGCM_sys_Get"
[623]	810	# fi
[785]	811	# else
	812	# echo "ERROR IGCM_sys_Get : demigration error :"
[1179]	813	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	814	# IGCM_debug_Exit "IGCM_sys_Get"
	815	# fi
[623]	816	#fi
[622]	817
[719]	818	# #RSYNC WITH NETWORK SSH CALL
[1179]	819	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	820	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	821
	822	# #RSYNC WITH NFS USE
[1179]	823	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	824	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	825
	826	# status=$?
	827	# IGCM_sys_Rsync_out $status
	828
[1179]	829	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	830	# (( status=status+$? ))
	831
[623]	832	#USUAL WAY
	833	if [ X${1} = X'/l' ] ; then
	834	for target in ${dm_liste[*]} ; do
	835	local_file=$( basename ${target} )
[668]	836	# test if the target file is present before the loop
	837	IGCM_sys_TestFileArchive ${target}
[715]	838	status=$?
	839	if [ ${status} -gt 0 ] ; then
[701]	840	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
[668]	841	IGCM_debug_Exit "IGCM_sys_Get"
	842	else
[719]	843	i=0
[668]	844	while [ $i -lt $NB_ESSAI ] ; do
[1037]	845	#if [ X${DoLink} = Xtrue ] ; then
[1179]	846	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	847	# status=$?
	848	# else
[1179]	849	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	850	# status=$?
	851	#fi
[1179]	852	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[668]	853	status=$?
	854	if [ ${status} -gt 0 ]; then
[713]	855	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
[676]	856	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	857	sleep $DELAI
[668]	858	else
[676]	859	break
[668]	860	fi
	861	(( i = i + 1 ))
	862	done
[719]	863	if [ ${status} -gt 0 ] ; then
	864	echo "IGCM_sys_Get : error"
[1179]	865	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	866	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	867	IGCM_debug_Exit "IGCM_sys_Get"
	868	else
[1179]	869	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	870	fi
[668]	871	fi
[623]	872	done
	873	else
[719]	874	i=0
	875	while [ $i -lt $NB_ESSAI ] ; do
[1179]	876	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	877	status=$?
	878	if [ ${status} -gt 0 ]; then
	879	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
	880	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	881	sleep $DELAI
	882	else
	883	break
	884	fi
	885	(( i = i + 1 ))
	886	done
	887	if [ ${status} -gt 0 ] ; then
	888	echo "IGCM_sys_Get : error"
[1179]	889	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	890	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	891	IGCM_debug_Exit "IGCM_sys_Get"
	892	else
[1179]	893	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	894	fi
[623]	895	fi
	896	fi
	897	IGCM_debug_PopStack "IGCM_sys_Get"
	898	}
[622]	899
[623]	900	#D-#==================================================
[835]	901	#D-function IGCM_sys_GetDate_Monitoring
	902	#D-* Purpose: get the last year for which the monitoring has been computed
	903	#D-* Examples:
	904	#D-
	905	function IGCM_sys_GetDate_Monitoring {
	906	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
	907	if ( $DEBUG_sys ) ; then
	908	echo "IGCM_sys_GetDate_Monitoring :" $@
	909	fi
	910
	911	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
	912
	913	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
	914	}
	915
	916	#D-#==================================================
[623]	917	#D-function IGCM_sys_Dods_Rm
	918	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
	919	#D-* Examples:
	920	#D-
	921	function IGCM_sys_Dods_Rm {
	922	if ( $DEBUG_sys ) ; then
	923	echo "IGCM_sys_Dods_Rm :" $@
	924	fi
[715]	925	typeset status
[623]	926	if [ $DRYRUN = 0 ]; then
	927
	928	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
	929	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
	930	# echo "Nothing has been done."
	931	# return
	932	# fi
	933
[1392]	934	if [ "$#" -eq 1 ]; then
	935	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
	936	status=$?
	937	else
	938	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
	939	status=$?
	940	fi
[623]	941
[785]	942	# if [ ${status} -gt 0 ] ; then
	943	# echo "IGCM_sys_Dods_Rm : error."
	944	# cat out_dods_rm
	945	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
	946	# else
	947	# rm out_dods_rm
	948	# fi
[623]	949
	950	fi
[715]	951	return $status
[623]	952	}
	953
	954	#D-#==================================================
	955	#D-function IGCM_sys_Dods_Cp
	956	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
	957	#D-* Examples:
	958	#D-
	959	function IGCM_sys_Dods_Cp {
	960	if ( $DEBUG_sys ) ; then
	961	echo "IGCM_sys_Dods_Cp :" $@
	962	fi
[715]	963	typeset status
[623]	964	if [ $DRYRUN = 0 ]; then
	965
	966	# if [ ! -d ${R_SAVE}/${1} ] ; then
	967	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
	968	# echo "Nothing has been done."
	969	# return
	970	# fi
	971
[649]	972	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
[715]	973	status=$?
[623]	974
[715]	975	# if [ ${status} -gt 0 ] ; then
[623]	976	# echo "IGCM_sys_Dods_Cp : error."
	977	# cat out_dods_cp
	978	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
	979	# else
	980	# rm out_dods_cp
	981	# fi
	982
	983	fi
[715]	984	return $status
[623]	985	}
	986
	987	#D-#==================================================
	988	#D-function IGCM_sys_Put_Dods
[1178]	989	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
[623]	990	#D-* Examples:
	991	#D-
	992	function IGCM_sys_Put_Dods {
	993	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
	994	if ( $DEBUG_sys ) ; then
	995	echo "IGCM_sys_Put_Dods :" $@
	996	fi
[715]	997	typeset status
[623]	998	if [ $DRYRUN = 0 ]; then
	999	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
	1000	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
	1001	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
	1002	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
	1003	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	1004	return
[622]	1005	fi
[623]	1006
	1007	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
	1008	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
	1009	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
	1010	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
	1011	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	1012	return
	1013	fi
	1014	#
	1015	if [ -d ${R_SAVE}/${1} ] ; then
	1016	cd ${R_SAVE}
	1017	elif [ -d ${R_FIGR}/${1} ] ; then
	1018	cd ${R_FIGR}
	1019	fi
	1020
	1021	IGCM_sys_Dods_Rm ${1}
	1022	IGCM_sys_Dods_Cp ${1}
[715]	1023	status=0
[623]	1024
[715]	1025	if [ ${status} -gt 0 ] ; then
[623]	1026	echo "IGCM_sys_Put_Dods : error."
	1027	IGCM_debug_Exit "IGCM_sys_Put_Dods"
	1028	fi
	1029	fi
	1030	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
[622]	1031	}
	1032
	1033	##############################################################
	1034	# REBUILD OPERATOR
	1035
[811]	1036	#D-#==================================================
[1163]	1037	#D-function IGCM_sys_sync
	1038	#D-* Purpose: flush buffer on disk
	1039	#D-* Examples:
	1040	#D-
	1041	function IGCM_sys_sync {
	1042	IGCM_debug_PushStack "IGCM_sys_sync" $@
	1043	if ( $DEBUG_sys ) ; then
	1044	echo "IGCM_sys_sync :" $@
	1045	fi
	1046
	1047	/bin/sync
	1048
	1049	IGCM_debug_PopStack "IGCM_sys_sync"
	1050	}
	1051
[622]	1052	############################################################
	1053	# Activate Running Environnment Variables
	1054
[811]	1055	#D-#==================================================
[1239]	1056	#D-function IGCM_sys_activ_variables
[811]	1057	#D-* Purpose: set environement variables prior to execution
	1058	#D-* Examples:
	1059	#D-
[622]	1060	function IGCM_sys_activ_variables {
[623]	1061	IGCM_debug_PushStack "IGCM_sys_activ_variables"
	1062	if ( $DEBUG_sys ) ; then
	1063	echo "IGCM_sys_activ_variables"
	1064	fi
[622]	1065
[811]	1066	# --------------------------------------------------------------------
[1409]	1067	#D- MODULE specifications
	1068	# --------------------------------------------------------------------
[1413]	1069	module purge > /dev/null 2>&1
[1410]	1070	if [ X${compilerVersion} = XV17 ] ; then
[1433]	1071	IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17"
[1409]	1072	else
	1073	IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
	1074	fi
[1433]	1075	. /ccc/cont003/home/dsm/p86ipsl/.env_irene > /dev/null 2>&1
	1076	IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded module for Irene. "
	1077	module list
[1409]	1078
	1079	# Keep track of this
	1080	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
	1081	[ ! X${TaskType} = Xchecking ] && module list
	1082
	1083	# --------------------------------------------------------------------
[811]	1084	#D- MPI specifications
	1085	# --------------------------------------------------------------------
[1191]	1086
[811]	1087	# --------------------------------------------------------------------
	1088	#D- Other specifications
	1089	# --------------------------------------------------------------------
	1090
[623]	1091	ulimit -s unlimited
[622]	1092
[623]	1093	IGCM_debug_PopStack "IGCM_sys_activ_variables"
[622]	1094	}
	1095
	1096	############################################################
	1097	# Desactivate Running Environnment Variables
	1098
[811]	1099	#D-#==================================================
	1100	#D-function IGCM_sys_desactiv_variables
	1101	#D-* Purpose: unset environement variables after execution
	1102	#D-* Examples:
	1103	#D-
[622]	1104	function IGCM_sys_desactiv_variables {
[623]	1105	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
	1106	if ( $DEBUG_sys ) ; then
	1107	echo "IGCM_sys_desactiv_variables"
	1108	fi
[811]	1109	# --------------------------------------------------------------------
	1110	#D- MPI specifications
	1111	# --------------------------------------------------------------------
	1112
	1113	# --------------------------------------------------------------------
	1114	#D- Other specifications
	1115	# --------------------------------------------------------------------
	1116
[623]	1117	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
[622]	1118	}
	1119
[623]	1120	############################################################
[1238]	1121	# Update job headers to be used by the scheduler
	1122
	1123	#D-#==================================================
	1124	#D-function IGCM_sys_updateHeaders
	1125	#D-* Purpose: Update job headers to be used by the scheduler
	1126	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
	1127	#D-
	1128	function IGCM_sys_updateHeaders {
	1129	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
	1130	if ( $DEBUG_sys ) ; then
	1131	echo "IGCM_sys_updateHeaders"
	1132	fi
	1133	typeset file
	1134	file=$1
	1135
	1136	if [ ${executionType} -eq 1 ] ; then
	1137	# MPMD + MPI
	1138	sed -e "/::openMPthreads::/d" \
	1139	-e "s/::JobNumProcTot::/${coreNumber}/" \
[1256]	1140	-e "/#MSUB -x/d" \
	1141	-e "/--cpu_bind=none/d" \
[1238]	1142	${file} > ${file}.tmp
	1143
	1144	elif [ ${executionType} -eq 2 ] ; then
[1390]	1145	# MPMD + MPI + OMP : mpirun/ccc_mprun/error
[1432]	1146	if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
[1390]	1147	sed -e "/::openMPthreads::/d" \
	1148	-e "s/::JobNumProcTot::/${coreNumber}/" \
	1149	${file} > ${file}.tmp
[1432]	1150	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
[1390]	1151	sed -e "/::openMPthreads::/d" \
	1152	-e "s/::JobNumProcTot::/${coreNumber}/" \
	1153	-e "/--cpu_bind=none/d" \
	1154	${file} > ${file}.tmp
	1155	else
	1156	IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
	1157	IGCM_debug_Exit "IGCM_sys_updateHeaders"
	1158	fi
[1238]	1159	elif [ ${executionType} -eq 3 ] ; then
	1160	# SPMD + MPI/OMP
	1161	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1162	-e "s/::JobNumProcTot::/${mpiTasks}/" \
[1240]	1163	-e "/#MSUB -x/d" \
[1238]	1164	-e "/--cpu_bind=none/d" \
	1165	${file} > ${file}.tmp
	1166
	1167	elif [ ${executionType} -eq 4 ] ; then
	1168	# SPMD + MPI only
	1169	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
	1170	-e "/::openMPthreads::/d" \
[1240]	1171	-e "/#MSUB -x/d" \
[1238]	1172	-e "/--cpu_bind=none/d" \
	1173	${file} > ${file}.tmp
	1174
	1175	elif [ ${executionType} -eq 5 ] ; then
	1176	# SPMD + OMP only
	1177	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1178	-e "/::JobNumProcTot::/d" \
[1240]	1179	-e "/#MSUB -x/d" \
[1238]	1180	-e "/--cpu_bind=none/d" \
	1181	${file} > ${file}.tmp
	1182
	1183	elif [ ${executionType} -eq 6 ] ; then
	1184	# SEQUENTIAL THEN
	1185	sed -e "s/::JobNumProcTot::/1/" \
	1186	-e "/::openMPthreads::/d" \
[1240]	1187	-e "/#MSUB -x/d" \
[1238]	1188	-e "/--cpu_bind=none/d" \
	1189	${file} > ${file}.tmp
	1190
	1191	fi
	1192
	1193	IGCM_sys_Mv ${file}.tmp ${file}
	1194
	1195	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
	1196	}
	1197
	1198	############################################################
[623]	1199	# Build MPI/OMP scripts run file (dummy function)
[622]	1200
[811]	1201	#D-#==================================================
	1202	#D-function IGCM_sys_build_run_file
	1203	#D-* Purpose: build run file (deprecated)
	1204	#D-* Examples:
	1205	#D-
[622]	1206	function IGCM_sys_build_run_file {
[811]	1207	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[622]	1208	}
	1209
	1210	############################################################
	1211	# Build MPI/OMP scripts
[811]	1212
	1213	#D-#==================================================
	1214	#D-function IGCM_sys_build_execution_scripts
	1215	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
	1216	#D-* Examples:
	1217	#D-
[623]	1218	function IGCM_sys_build_execution_scripts
[622]	1219	{
[623]	1220	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
	1221	if ( $DEBUG_sys ) ; then
	1222	echo "IGCM_sys_build_execution_scripts " $@
	1223	fi
[622]	1224
[623]	1225	EXECUTION=${HOST_MPIRUN_COMMAND}
[622]	1226
[623]	1227	if ( ${OK_PARA_MPMD} ) ; then
[622]	1228
[1390]	1229	# MPMD mode
	1230	# 1 MPI only
	1231	# 2 MPI/OpenMP mpirun method
	1232	# 3 MPI/OpenMP ccc_mprun method
	1233
[1381]	1234	if [ -f run_file ] ; then
	1235	IGCM_sys_Rm -f run_file
	1236	fi
	1237	touch run_file
	1238
[1390]	1239	# case 1 : Only MPI (MPMD)
[1078]	1240	if ( ! ${OK_PARA_OMP} ) ; then
[622]	1241
[1078]	1242	# Build run_file
[622]	1243
[1078]	1244	# First loop on the components for the coupler ie oasis (only if oasis3)
	1245	# the coupler ie oasis3 must be the first one
	1246	for comp in ${config_ListOfComponents[*]} ; do
[622]	1247
[1078]	1248	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1249	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1250
[1078]	1251	# for CPL component only
	1252	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1253
[1078]	1254	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1255	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1256	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1257	fi
	1258	done
[622]	1259
[1078]	1260	# Then second loop on the components
	1261	for comp in ${config_ListOfComponents[*]} ; do
[622]	1262
[1078]	1263	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1264	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1265
[1078]	1266	# Only if we really have an executable for the component and not the coupler ie oasis:
	1267	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
[622]	1268
[1078]	1269	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1270	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1271	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1272	fi
	1273	done
[622]	1274
[1390]	1275	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
	1276
	1277	IGCM_sys_Chmod u+x run_file
	1278	if ( $DEBUG_sys ) ; then
	1279	echo "run_file contains : "
	1280	cat run_file
	1281	fi
	1282
[1432]	1283	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
[1390]	1284
	1285	# 2 MPI/OpenMP mpirun method
	1286
	1287	# Use of mpirun instead of ccc_mprun
	1288	EXECUTION="time mpirun"
	1289
	1290	# Hosts treatment
	1291	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
	1292
	1293	i=0
	1294	rm -f hosts rankfile
	1295	IGCM_debug_Print 1 "sys Curie, Hosts available :"
	1296	for nodes in `cat hosts.tmp`
	1297	do
	1298	host[$i]=$nodes
	1299	echo "${host[$i]}" >> hosts
	1300	IGCM_debug_Print 1 ${host[$i]}
	1301	i=$((i+1))
	1302	done
	1303	rm -f hosts.tmp
	1304
	1305	listnodes=${host[*]}
	1306
	1307	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
	1308
	1309	# Initialisation
	1310	rank=0
	1311	current_core=0
	1312	init_exec=n
	1313
	1314	# Loop on the components
	1315	for comp in ${config_ListOfComponents[*]} ; do
	1316
	1317	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1318	eval ExeNameOut=\${config_Executable_${comp}[1]}
	1319
	1320	# Not possible if oasis has an executable (i.e old version of oasis3)
	1321	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
	1322	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
	1323	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
	1324	IGCM_debug_Verif_Exit
	1325	fi
	1326
	1327	# Only if we really have an executable for the component :
	1328	if [ "X${ExeNameOut}" != X\"\" ] ; then
	1329
	1330	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1331	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[1426]	1332	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
[1390]	1333
	1334	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1335	echo "" >> script_${ExeNameOut}.ksh
	1336	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
	1337
	1338	# Check if the number of threads is correct
	1339	case ${comp_proc_omp_loc} in
	1340	2\|4\|8\|16)
	1341	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
	1342	;;
	1343	*)
	1344	IGCM_debug_Exit "ERROR with OMP parameters !"
	1345	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
	1346	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
	1347	IGCM_debug_Verif_Exit
	1348	;;
	1349	esac
	1350	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1351	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1352	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1353	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1354	fi
	1355
	1356	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
	1357	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
	1358	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
	1359	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
	1360	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1361
	1362	if [ ${init_exec} = y ] ; then
	1363	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
	1364	else
	1365	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
	1366	init_exec=y
	1367	fi
	1368
	1369	# Build rankfile : method used to assign cores and nodes for the MPI process
	1370	# Ex :
	1371	#rank 0=curie5296 slot=0,1,2,3
	1372	#rank 1=curie5296 slot=4,5,6,7
	1373	# Example of final command :
	1374	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
	1375	# with script_lmdz.x.ksh :
	1376	# #!/bin/ksh
	1377	#export KMP_STACKSIZE=3g
	1378	#export KMP_LIBRARY=turnaround
	1379	#export MKL_SERIAL=YES
	1380	#OMP_NUM_THREADS=4
	1381	#./lmdz.x
[1426]	1382	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
[1430]	1383	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
[1426]	1384	else
	1385	(( offset_comp_proc_loc = comp_proc_omp_loc ))
	1386	fi
[1390]	1387	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
[1426]	1388	(( index_host = current_core / NB_CORE_PER_NODE ))
	1389	host_value=${host[${index_host}]}
	1390	(( slot = current_core % NB_CORE_PER_NODE ))
	1391	virg=","
	1392	string_final=""
	1393	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
	1394	string=$index$virg
	1395	string_final=$string_final$string
	1396	done
	1397	string_final=$( echo $string_final \| sed "s/.$//" )
	1398	echo "rank $rank=$host_value slot=$string_final" >> rankfile
	1399	(( rank = rank + 1 ))
	1400	(( current_core = current_core + offset_comp_proc_loc ))
[1390]	1401	done
	1402	fi
	1403	done
	1404
[1432]	1405	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
[1390]	1406
	1407	# 3 MPI/OpenMP ccc_mprun method
	1408
[1381]	1409	# MPI-OpenMP (MPMD)
	1410	# example of run_file
	1411	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
	1412	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
	1413	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
[622]	1414
[1078]	1415	# Loop on the components
	1416	for comp in ${config_ListOfComponents[*]} ; do
[1000]	1417
[1078]	1418	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1419	eval ExeNameOut=\${config_Executable_${comp}[1]}
[1000]	1420
[1078]	1421	# Not possible if oasis has an executable (i.e old version of oasis3)
	1422	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
	1423	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
	1424	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
	1425	IGCM_debug_Verif_Exit
	1426	fi
[1000]	1427
[1078]	1428	# Only if we really have an executable for the component :
	1429	if [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1430
[1078]	1431	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1432	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[1000]	1433
[1381]	1434	# Build script files
	1435
[1078]	1436	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1437	echo "" >> script_${ExeNameOut}.ksh
	1438	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
	1439
	1440	# Check if the number of threads is correct
	1441	case ${comp_proc_omp_loc} in
[1440]	1442	2\|4\|8\|12\|16\|24)
[1078]	1443	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
	1444	;;
	1445	*)
	1446	IGCM_debug_Exit "ERROR with OMP parameters !"
	1447	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
[1440]	1448	IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible "
[1078]	1449	IGCM_debug_Verif_Exit
	1450	;;
	1451	esac
[1381]	1452	### only for LMDZ?
[1078]	1453	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1454	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1455	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1456	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
[1381]	1457
[1078]	1458	fi
[1318]	1459
[1381]	1460	# to have out/err per process on different files
	1461	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
[1317]	1462	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[1078]	1463
[1381]	1464	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
[1078]	1465
[1381]	1466	# Complete run_file
[1000]	1467
[1381]	1468	echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
	1469
[1078]	1470	fi
	1471	done
[1381]	1472
[1385]	1473	## variable added to stop after 60s instead of 600s by default.
	1474	## This is used when no error comes from executables and when something stopped an executable without notice.
	1475	export SLURM_WAIT=60
[1381]	1476
[1385]	1477	## module advised by TGCC (instead of 2 variables)
	1478	module load feature/bridge/heterogenous_mpmd
[1381]	1479
[1390]	1480	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
[1000]	1481
[1390]	1482	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
[1381]	1483
[1390]	1484	IGCM_sys_Chmod u+x run_file
	1485	if ( $DEBUG_sys ) ; then
	1486	echo "run_file contains : "
	1487	cat run_file
	1488	fi
[1381]	1489
[1390]	1490	else
[1381]	1491
[1390]	1492	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
	1493
	1494	fi # 1 2 or 3 if ${OK_PARA_MPMD}
	1495
	1496	else
[1078]	1497	# Only one executable (SPMD mode).
[1000]	1498
[622]	1499	for comp in ${config_ListOfComponents[*]} ; do
	1500
[685]	1501	# Only if we really have an executable for the component :
[623]	1502	eval ExeNameOut=\${config_Executable_${comp}[1]}
[685]	1503	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
[622]	1504
[1381]	1505	# Build script files
	1506
[685]	1507	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1508	echo "" >> script_${ExeNameOut}.ksh
	1509	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1510
	1511	if ( ${OK_PARA_OMP} ) ; then
	1512	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1513	echo "" >> script_${ExeNameOut}.ksh
	1514	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1515	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1516	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1517	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1518	fi
	1519
	1520	if ( ${OK_PARA_MPI} ) ; then
[623]	1521	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
[685]	1522	# Default : ccc_mprun used if nb_proc gt 1
	1523	# to have out/err per process on different files
[1317]	1524	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
	1525	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1526	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
[623]	1527	else
[685]	1528	# Default : ccc_mprun is NOT used if nb_proc eq 1
	1529	# to have out/err per process on different files
[1317]	1530	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
	1531	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1532	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
[622]	1533	fi
[685]	1534
	1535	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
	1536	cat script_${ExeNameOut}.ksh
	1537
[623]	1538	fi
[622]	1539
	1540	done
	1541
[1390]	1542	fi # ${OK_PARA_MPMD}
[622]	1543
[623]	1544	IGCM_debug_Print 1 "sys Curie : execution command is "
	1545	IGCM_debug_Print 1 "$EXECUTION"
[622]	1546
[623]	1547	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[622]	1548	}
	1549
[933]	1550	#D-#==================================================
	1551	#D-function IGCM_sys_check_path
	1552	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
	1553	#D-* do not point to an important use directory. Stop immediately in that case.
	1554	#D-* Examples:
	1555	#D-
	1556	function IGCM_sys_check_path {
	1557	IGCM_debug_PushStack "IGCM_sys_check_path"
	1558	if ( $DEBUG_sys ) ; then
	1559	echo "IGCM_sys_check_path"
	1560	fi
[811]	1561
[1432]	1562	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
[933]	1563	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
	1564	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
[936]	1565	IGCM_debug_Exit "This will stop the job"
[933]	1566	fi
	1567	IGCM_debug_PopStack "IGCM_sys_check_path"
	1568	}
	1569
[811]	1570	#D-#==================================================
	1571	#D-function IGCM_sys_check_quota
	1572	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
	1573	#D-* Examples:
	1574	#D-
[714]	1575	function IGCM_sys_check_quota {
[776]	1576	IGCM_debug_PushStack "IGCM_sys_check_quota"
	1577	if ( $DEBUG_sys ) ; then
	1578	echo "IGCM_sys_check_quota"
	1579	fi
	1580	# Limit of quota (in %)
	1581	limit_quota=90
	1582
	1583	# Check of the volume
[912]	1584	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
	1585	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
[776]	1586
	1587	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
	1588
	1589	unit_avail=${volume_avail: -1}
	1590	unit_quota=${volume_quota: -1}
	1591
	1592	if [ "${unit_quota}" = "*" ] ; then
[811]	1593	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1594	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1595	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1596	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1597	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1598	IGCM_debug_Verif_Exit
[714]	1599	fi
[776]	1600
	1601	temp_avail=${volume_avail%%${volume_avail: -1}*}
	1602	temp_quota=${volume_quota%%${volume_quota: -1}*}
	1603
	1604	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1605
	1606	# Convertion
	1607	if [ ${volume_avail: -1} = "T" ] ; then
	1608	(( temp_avail = temp_avail * 1000000000000 ))
	1609	elif [ ${volume_avail: -1} = "G" ] ; then
	1610	(( temp_avail = temp_avail * 1000000000 ))
	1611	elif [ ${volume_avail: -1} = "M" ] ; then
	1612	(( temp_avail = temp_avail * 1000000 ))
	1613	elif [ ${volume_avail: -1} = "k" ] ; then
	1614	(( temp_avail = temp_avail * 1000 ))
	1615	else
	1616	(( temp_avail = volume_avail ))
	1617	fi
	1618	if [ ${volume_quota: -1} = "T" ] ; then
	1619	(( temp_quota = temp_quota * 1000000000000 ))
	1620	elif [ ${volume_quota: -1} = "G" ] ; then
	1621	(( temp_quota = temp_quota * 1000000000 ))
	1622	elif [ ${volume_quota: -1} = "M" ] ; then
	1623	(( temp_quota = temp_quota * 1000000 ))
	1624	elif [ ${volume_quota: -1} = "k" ] ; then
	1625	(( temp_quota = temp_quota * 1000 ))
	1626	else
	1627	(( temp_quota = volume_quota ))
	1628	fi
	1629	fi
	1630
	1631	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1632	# echo "volume ratio is " $quota_volume
	1633
	1634	if [ ${quota_volume} -ge ${limit_quota} ] ; then
	1635	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1636	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
	1637	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1638	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1639	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1640	IGCM_debug_Verif_Exit
	1641	fi
	1642
	1643	fi
	1644
	1645	# Check of the number of inodes
	1646
[912]	1647	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
	1648	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
[776]	1649
	1650	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
	1651
	1652	unit_avail=${inode_avail: -1}
	1653	unit_quota=${inode_quota: -1}
	1654
	1655	if [ "${unit_quota}" = "*" ] ; then
[811]	1656	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1657	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1658	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1659	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1660	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1661	IGCM_debug_Verif_Exit
[776]	1662	fi
	1663
	1664	temp_avail=${inode_avail%%${inode_avail: -1}*}
	1665	temp_quota=${inode_quota%%${inode_quota: -1}*}
	1666
	1667	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1668
	1669	# Convertion
	1670	if [ ${inode_avail: -1} = "T" ] ; then
	1671	(( temp_avail = temp_avail * 1000000000000 ))
	1672	elif [ ${inode_avail: -1} = "G" ] ; then
	1673	(( temp_avail = temp_avail * 1000000000 ))
	1674	elif [ ${inode_avail: -1} = "M" ] ; then
	1675	(( temp_avail = temp_avail * 1000000 ))
	1676	elif [ ${inode_avail: -1} = "k" ] ; then
	1677	(( temp_avail = temp_avail * 1000 ))
	1678	else
	1679	(( temp_avail = inode_avail ))
	1680	fi
	1681
	1682	if [ ${inode_quota: -1} = "T" ] ; then
	1683	(( temp_quota = temp_quota * 1000000000000 ))
	1684	elif [ ${inode_quota: -1} = "G" ] ; then
	1685	(( temp_quota = temp_quota * 1000000000 ))
	1686	elif [ ${inode_quota: -1} = "M" ] ; then
	1687	(( temp_quota = temp_quota * 1000000 ))
	1688	elif [ ${inode_quota: -1} = "k" ] ; then
	1689	(( temp_quota = temp_quota * 1000 ))
	1690	else
	1691	(( temp_quota = inode_quota ))
	1692	fi
	1693	fi
	1694	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1695	# echo "inode ratio is " $quota_inode
	1696
	1697	if [ ${quota_inode} -ge ${limit_quota} ] ; then
	1698	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1699	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
	1700	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1701	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1702	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1703	IGCM_debug_Verif_Exit
	1704	fi
	1705	fi
	1706	IGCM_debug_PopStack "IGCM_sys_check_quota"
[714]	1707	}
	1708
[773]	1709	#D-#==================================================
[1286]	1710	#D-function IGCM_sys_projectAccounting
	1711	#D-* Purpose: store project accounting information in a file
	1712	#D-* Examples:
	1713	#D-
	1714	function IGCM_sys_projectAccounting {
	1715	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
	1716	if ( $DEBUG_sys ) ; then
	1717	echo "IGCM_sys_check_quota"
	1718	fi
	1719
[1288]	1720	ssh curie91 /usr/bin/ccc_myproject > $1
[1286]	1721
	1722	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
	1723	}
	1724
	1725	#D-#==================================================
[1244]	1726	#D-function IGCM_sys_getJobSchedulerID
	1727	#D-* Purpose: Get the job ID during execution
	1728	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
	1729	#D-
	1730	function IGCM_sys_getJobSchedulerID {
	1731	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
	1732	if ( $DEBUG_sys ) ; then
	1733	echo "IGCM_sys_getJobSchedulerID"
	1734	fi
	1735
	1736	eval ${1}=${BRIDGE_MSUB_JOBID}
	1737
	1738	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
	1739	}
	1740
	1741	#D-#==================================================
[958]	1742	#D-function IGCM_sys_GetJobID
[1244]	1743	#D-* Purpose: Get the job ID from the JobName
[958]	1744	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
	1745	#D-
	1746	function IGCM_sys_GetJobID {
	1747	IGCM_debug_PushStack "IGCM_sys_GetJobID"
	1748	if ( $DEBUG_sys ) ; then
	1749	echo "IGCM_sys_GetJobID"
	1750	fi
	1751
	1752	# With -f option, the full job name is given in the last column
	1753	ID=$( ccc_mstat -f -u $2 \| \
	1754	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
	1755
	1756	eval ${3}=${ID}
	1757	IGCM_debug_PopStack "IGCM_sys_GetJobID"
	1758	}
	1759
	1760	#D-#==================================================
[773]	1761	#D-function IGCM_sys_CountJobInQueue
[1244]	1762	#D-* Purpose: Count number of users job
[773]	1763	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
	1764	#D-
	1765	function IGCM_sys_CountJobInQueue {
	1766	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
	1767	if ( $DEBUG_sys ) ; then
	1768	echo "IGCM_sys_CountJobInQueue"
	1769	fi
	1770
	1771	# With -f option, the full job name is given in the last column
[834]	1772	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
[773]	1773
	1774	eval ${2}=${NbRun}
	1775
	1776	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
	1777	}
	1778
[955]	1779	#D-#==================================================
	1780	#D-function IGCM_sys_ListJobInQueue
[1244]	1781	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
[955]	1782	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
	1783	#D-
	1784	function IGCM_sys_ListJobInQueue {
	1785	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
	1786	if ( $DEBUG_sys ) ; then
	1787	echo "IGCM_sys_ListJobInQueue"
	1788	fi
	1789
	1790	# With -f option, the full job name is given in the last column
	1791	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
	1792	'( $2 == User && \
	1793	$NF != /TS/ && \
	1794	$NF !~ /PACK/ && \
	1795	$NF !~ /REBUILD/ && \
	1796	$NF !~ /pack/ ) \
	1797	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
	1798
	1799	eval set -A ${2} ${JobList[*]}
	1800
	1801	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
	1802	}
[1272]	1803
	1804	#D-#==================================================
	1805	#D-function IGCM_sys_atlas
	1806	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
	1807	#D-* Examples:
	1808	#D-
	1809	function IGCM_sys_atlas {
	1810	IGCM_debug_PushStack "IGCM_sys_atlas" $@
	1811	if ( $DEBUG_sys ) ; then
	1812	echo "IGCM_sys_atlas :" $@
	1813	fi
	1814
	1815	typeset status
	1816
	1817	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	1818	status=$?
	1819	if [ ${status} -gt 0 ] ; then
	1820	echo "IGCM_sys_atlas : error code ${status}"
	1821	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1822	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1823	IGCM_debug_PopStack "IGCM_sys_atlas"
	1824	return 1
	1825	else
	1826	IGCM_debug_PopStack "IGCM_sys_atlas"
	1827	return 0
	1828	fi
	1829
	1830	IGCM_debug_PopStack "IGCM_sys_atlas"
	1831	}
	1832
[1380]	1833	#D-#==================================================
	1834	#D-function IGCM_sys_rebuild_nemo
	1835	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
	1836	#D-* Examples:
	1837	#D-
	1838
	1839	function IGCM_sys_rebuild_nemo {
	1840	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
	1841	if ( $DEBUG_sys ) ; then
	1842	echo "IGCM_sys_rebuild_nemo :" $@
	1843	fi
	1844
	1845	$( ccc_home -u p86ipsl )/REBUILD_NEMO/rebuild_nemo ${1} ${2}
	1846
	1847	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
	1848	}
	1849

Note: See TracBrowser for help on using the repository browser.

Download in other formats: