Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1439

Last change on this file since 1439 was 1431, checked in by sdipsl, 6 years ago
CMIP6 files go in CCCWORKDIR and alike.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 62.0 KB

Rev	Line
[622]	1	#!/bin/ksh
	2
	3	#**************************************************************
	4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
	5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#=========================================================
	15	# The documentation of this file can be automatically generated
[623]	16	# if you use the prefix #D- for comments to be extracted.
[622]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#=========================================================
	19
	20	#D-#==================================================
[623]	21	#D-LibIGCM_sys for Curie
[622]	22	#D-#==================================================
	23	#D-
[623]	24	#D- This ksh library if a layer under some usefull
[622]	25	#D-environment variables and shell commands.
	26	#D-All those definitions depend on host particularities.
	27	#D-It manages a stack mechanism and test validity of operations.
	28	#D-All function described bellow must be prefixed by IGCM_sys.
	29
	30	#====================================================
	31	# libIGCM_sys PARAMETERS
	32	#====================================================
	33
	34	#====================================================
	35	# set DEBUG_sys to true to output calls of function
	36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
	37
	38	#====================================================
	39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
	40	typeset -r DRYRUN=${DRYRUN:=0}
	41
	42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
	43	# -------------------------------------------------------------------------------------
	44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
[809]	45	# \| \| Cp/Exe/param/files \| \| \| \|
	46	# \| \| Chmod Qsub \| \| \| \|
[622]	47	# -------------------------------------------------------------------------------------
	48	# \| 0 \| yes \| yes \| yes \| yes \|
	49	# -------------------------------------------------------------------------------------
	50	# \| 1 \| yes \| yes \| yes \| no \|
	51	# -------------------------------------------------------------------------------------
	52	# \| 2 \| yes \| yes \| no \| no \|
	53	# -------------------------------------------------------------------------------------
	54	# \| 3 \| yes \| no \| no \| no \|
	55	# -------------------------------------------------------------------------------------
	56
	57	#=====================================================
	58	# Global Variables :
	59	#=====================================================
	60	# Language : "fr" or "en"
	61	typeset -r MYLANG="fr"
	62
	63	#=====================================================
[1201]	64	# Host user names project maxCpuTime
[622]	65	# $hostname ou hostname
[1158]	66	typeset HOST=${HOST:=$( hostname )}
[622]	67	# $username ou whoami
[1158]	68	typeset LOGIN=${LOGIN:=$( whoami )}
[622]	69	# $hostname of the MASTER job
	70	typeset MASTER=curie
[880]	71	# add default project on curie
[1201]	72	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} \| cut -d@ -f1 )
	73	# jobWarningDelay in seconds
	74	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
[622]	75
	76	#D-
	77	#D-#==================================================
	78	#D-Program used in libIGCM
	79	#D-#==================================================
	80
[742]	81	# Submit command
[747]	82	typeset SUBMIT=${SUBMIT:=ccc_msub}
[622]	83	# rsync with path
	84	typeset -r RSYNC=/usr/bin/rsync
	85	# RSYNC_opt args to rsync
	86	typeset -r RSYNC_opt="-va"
	87	# ie storage filesystem
[623]	88	typeset -r STOREHOST=${MASTER}
[1180]	89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
[622]	90
	91	#====================================================
[1180]	92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
[622]	93	#====================================================
[996]	94	if [ X${TaskType} = Xcomputing ] ; then
[1409]	95	IGCM_debug_Print 1 "Module will be loaded once we know the compiler version"
[996]	96	else
[1413]	97	module purge > /dev/null 2>&1
[1409]	98	. /ccc/cont003/home/dsm/p86ipsl/.env_intel17.0.2_curie > /dev/null 2>&1
	99	. /ccc/cont003/home/dsm/p86ipsl/.env_netcdf4.3.3.1_curie > /dev/null 2>&1
	100	. /ccc/cont003/home/dsm/p86ipsl/.env_atlas_curie > /dev/null 2>&1
[1358]	101	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
[1364]	102	export UVCDAT_ANONYMOUS_LOG=FALSE
[996]	103	fi
[1315]	104
	105	# Use CMIP6 storage space when using CMIP6 cpu ressources
[1330]	106	[[ "X${PROJECT}" = "Xdevcmip6" && ! $(module list --terse 2>&1 \| grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
[1315]	107
	108	# FYI
[1210]	109	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
	110	[ ! X${TaskType} = Xchecking ] && module list
[1208]	111
[1179]	112	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
	113	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
	114	export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
[622]	115
	116	#====================================================
	117	# Specific for ocean additionnal diagnostic
	118	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
	119	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
	120
	121	#====================================================
	122	# Host specific DIRECTORIES
	123	#====================================================
	124
[623]	125	# ============ CESIUM START ============ #
	126
[622]	127	#====================================================
[623]	128	#- Mirror libIGCM from titane to cesium if needed
	129	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
	130	#if [ ! ${ROOTSYS} = "home" ] ; then
	131	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
	132	#else
	133	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	134	#fi
	135
	136	#====================================================
	137	#- libIGCM_POST
	138	#if ( ${MirrorlibIGCM} ) ; then
	139	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
	140	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
	141	#else
	142	# typeset -r libIGCM_POST=${libIGCM}
	143	#fi
	144
	145	# ============ CESIUM END ============ #
	146
	147	#====================================================
	148	#- MirrorlibIGCM for frontend
	149	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	150
	151	#====================================================
	152	#- libIGCM_POST for frontend
	153	typeset -r libIGCM_POST=${libIGCM}
	154
	155	#====================================================
[622]	156	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
	157	typeset -r R_EXE="${MODIPSL}/bin"
	158
	159	#====================================================
	160	#- SUBMIT_DIR : submission dir
[1010]	161	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
[622]	162
	163	#====================================================
	164	#- IN
[1276]	165	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
[622]	166	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
	167
	168	#====================================================
	169	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[850]	170	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
[622]	171
	172	#====================================================
[1179]	173	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
[1186]	174	typeset -r OUTCOMMAND_PATH=/tmp
[1179]	175
	176	#====================================================
[622]	177	#- HOST_MPIRUN_COMMAND
[1382]	178	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
[622]	179
	180	#====================================================
	181	#- Max number of arguments passed to nco operator or demigration command
	182	UNIX_MAX_LIMIT=360
	183
[623]	184	#====================================================
[1180]	185	#- set PackDefault to true on curie
[623]	186	PackDefault=true
	187
[664]	188	#====================================================
[623]	189	#- Default number of MPI task for IPSL coupled model
	190	#- required for backward compatibility
	191	#-
	192	DEFAULT_NUM_PROC_OCE=5
	193	DEFAULT_NUM_PROC_CPL=1
[637]	194	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
[623]	195
[1426]	196	#====================================================
	197	#- Number of computing cores per node
	198	#-
	199	NB_CORE_PER_NODE=16
	200
[725]	201	#D-#==================================================
[1057]	202	#D-function IGCM_sys_defineArchives
	203	#D-* Purpose:
	204	#D-* Define ARCHIVE : Dedicated to large files
	205	#D-* Define STORAGE : Dedicated to small/medium files
	206	#D-* Define R_OUT : Output tree located on ARCHIVE
	207	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
	208	#D-* Define R_BUF : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
	209	#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
[725]	210	#D-* Examples:
	211	#D-
[1057]	212	function IGCM_sys_defineArchives {
	213	IGCM_debug_PushStack "IGCM_sys_defineArchives"
[676]	214
[1060]	215	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
[1057]	216	#====================================================
	217	#- ARCHIVE (dedicated to large files)
[1063]	218	ARCHIVE=${config_UserChoices_ARCHIVE}
[1057]	219	else
	220	#====================================================
	221	#- ARCHIVE (dedicated to large files)
[1063]	222	ARCHIVE=${CCCSTOREDIR}
[1057]	223	fi
[725]	224
[1060]	225	if [ ! X${config_UserChoices_STORAGE} = X ]; then
[1057]	226	#====================================================
	227	#- STORAGE (dedicated to small/medium files)
[1063]	228	STORAGE=${config_UserChoices_STORAGE}
[1057]	229	else
	230	#====================================================
	231	#- STORAGE (dedicated to small/medium files)
[1063]	232	STORAGE=${CCCWORKDIR}
[1057]	233	fi
[725]	234
[1057]	235	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
	236	#====================================================
	237	#- R_OUT
[1063]	238	R_OUT=${SCRATCHDIR}/IGCM_OUT
[1057]	239
	240	#====================================================
	241	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]	242	R_FIG=${SCRATCHDIR}/IGCM_OUT
[1057]	243
	244	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
	245
	246	else
	247	#====================================================
	248	#- R_OUT
[1063]	249	R_OUT=${ARCHIVE}/IGCM_OUT
[1057]	250
	251	#====================================================
	252	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]	253	R_FIG=${STORAGE}/IGCM_OUT
[1057]	254	fi
	255
	256	#====================================================
[1431]	257	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
	258	CMIP6_BUF=${STORAGE}/IGCM_OUT
	259
	260	#====================================================
[1057]	261	#- R_BUF (ONLY FOR double copy an scratch)
[1063]	262	R_BUF=${SCRATCHDIR}/IGCM_OUT
[1057]	263
	264	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
	265	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
	266	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
	267
	268	IGCM_debug_PopStack "IGCM_sys_defineArchives"
[725]	269	}
	270
[622]	271	#D-#==================================================
	272	#D-function IGCM_sys_RshArchive
	273	#D-* Purpose: Archive rsh command
	274	#D-* Examples:
	275	#D-
	276	function IGCM_sys_RshArchive {
[623]	277	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
	278	/bin/ksh <<-EOF
[622]	279	${@}
	280	EOF
[811]	281	status=$?
	282	if [ ${status} -gt 0 ] ; then
	283	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
[623]	284	IGCM_debug_Exit "IGCM_sys_RshArchive"
	285	fi
	286	IGCM_debug_PopStack "IGCM_sys_RshArchive"
[622]	287	}
	288
	289	#D-#==================================================
[845]	290	#D-function IGCM_sys_RshArchive_NoError
	291	#D-* Purpose: Archive rsh command, without error
	292	#D-* used only in monitoring.job
	293	#D-* Examples:
	294	#D-
	295	function IGCM_sys_RshArchive_NoError {
	296	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
	297	/bin/ksh <<-EOF
[1181]	298	${@} 2> /dev/null
[845]	299	EOF
	300	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
	301	}
	302
	303	#D-#==================================================
[622]	304	#D-function IGCM_sys_MkdirArchive
	305	#D-* Purpose: Mkdir on Archive
	306	#D-* Examples:
	307	#D-
	308	function IGCM_sys_MkdirArchive {
[623]	309	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
	310	if ( $DEBUG_sys ) ; then
	311	echo "IGCM_sys_MkdirArchive :" $@
	312	fi
	313	#- creation de repertoire sur le serveur fichier
	314	if [ ! -d ${1} ]; then
	315	\mkdir -p $1
[811]	316	status=$?
	317
	318	if [ ${status} -gt 0 ] ; then
	319	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
[623]	320	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
[622]	321	fi
[623]	322	fi
	323	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[622]	324	}
	325
	326	#D-#==================================================
	327	#D-function IGCM_sys_TestDirArchive
	328	#D-* Purpose: Test Directory that must exists on Archive
	329	#D-* Examples:
	330	#D-
	331	function IGCM_sys_TestDirArchive {
[623]	332	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
	333	if ( $DEBUG_sys ) ; then
	334	echo "IGCM_sys_TestDirArchive :" $@
	335	fi
	336	typeset ExistFlag
	337	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	338	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
	339	return ${ExistFlag}
[622]	340	}
	341
	342	#D-#==================================================
[623]	343	#D-function IGCM_sys_IsFileArchived
[1179]	344	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
[623]	345	#D-* Examples:
	346	#D-
	347	function IGCM_sys_IsFileArchived {
	348	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
	349	if ( $DEBUG_sys ) ; then
	350	echo "IGCM_sys_IsFileArchived :" $@
	351	fi
	352	typeset IsArchivedFlag
[818]	353	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
[623]	354	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
	355
	356	return ${IsArchivedFlag}
	357	}
	358
	359	#D-#==================================================
[622]	360	#D-function IGCM_sys_TestFileArchive
	361	#D-* Purpose: Test file that must NOT EXISTS on Archive
	362	#D-* Examples:
	363	#D-
	364	function IGCM_sys_TestFileArchive {
[623]	365	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
	366	typeset ExistFlag
	367	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	368	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[622]	369
[623]	370	return ${ExistFlag}
[622]	371	}
	372
	373	#D-#==================================================
	374	#D-function IGCM_sys_CountFileArchive
	375	#D-* Purpose: Count files on Archive filesystem
	376	#D-* Examples:
	377	#D-
	378	function IGCM_sys_CountFileArchive {
[623]	379	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
	380	ls ${@} 2>/dev/null \| wc -l
	381	if [ $? -gt 0 ] ; then
	382	echo "IGCM_sys_CountFileArchive : erreur."
	383	fi
	384	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[622]	385	}
	386
	387	#D-#==================================================
	388	#D-function IGCM_sys_Tree
	389	#D-* Purpose: Tree directories with files on ${ARCHIVE}
	390	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
	391	#D-
	392	function IGCM_sys_Tree {
[623]	393	IGCM_debug_PushStack "IGCM_sys_Tree" $@
	394	if ( $DEBUG_sys ) ; then
	395	echo "IGCM_sys_Tree :" $@
	396	fi
[622]	397
[623]	398	\ls -lR ${@}
[622]	399
[623]	400	IGCM_debug_PopStack "IGCM_sys_Tree"
[622]	401	}
	402
	403	#D-#==================================================
	404	#D-function IGCM_sys_Qsub
	405	#D-* Purpose: Qsub new job
	406	#D-* Examples:
	407	#D-
	408	function IGCM_sys_Qsub {
[623]	409	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
	410	if ( $DEBUG_sys ) ; then
	411	echo "IGCM_sys_Qsub :" $@
	412	fi
[811]	413	typeset options status
[923]	414	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
[622]	415
[1179]	416	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	417	status=$?
[948]	418
[1179]	419	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	420	if [ ${status} -gt 0 ] ; then
	421	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
[623]	422	IGCM_debug_Exit "IGCM_sys_Qsub"
[948]	423	else
[1179]	424	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	425	fi
	426	IGCM_debug_PopStack "IGCM_sys_Qsub"
[622]	427	}
	428
	429	#D-#==================================================
	430	#D-function IGCM_sys_QsubPost
	431	#D-* Purpose: Qsub new job on scalaire
	432	#D-* Examples:
	433	#D-
	434	function IGCM_sys_QsubPost {
[623]	435	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
	436	if ( $DEBUG_sys ) ; then
	437	echo "IGCM_sys_QsubPost :" $@
	438	fi
[923]	439	typeset options status
	440
[1062]	441	# EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
	442	case $( hostname -s ) in
	443	curie*)
[1201]	444	options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
[1062]	445	airain*)
	446	options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
	447	esac
	448
[1179]	449	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	450	status=$?
[948]	451
[1179]	452	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	453	if [ ${status} -gt 0 ] ; then
[923]	454	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
[623]	455	IGCM_debug_Exit "IGCM_sys_QsubPost"
[948]	456	else
[1179]	457	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	458	fi
	459	IGCM_debug_PopStack "IGCM_sys_QsubPost"
[622]	460	}
	461
	462	#D-*************************
[623]	463	#D- File transfer functions
[622]	464	#D-*************************
	465	#D-
	466
	467	#D-#==================================================
	468	#D-function IGCM_sys_RmRunDir
	469	#D-* Purpose: rm tmpdir (dummy function most of the time batch
	470	#D- scheduler will do the job)
	471	#D-* Examples:
	472	#D-
	473	function IGCM_sys_RmRunDir {
[1057]	474	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
[623]	475	if ( $DEBUG_sys ) ; then
	476	echo "IGCM_sys_RmRunDir :" $@
	477	fi
[622]	478
[715]	479	typeset status
[622]	480
[1179]	481	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	482	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	483	status=$?
[622]	484
[715]	485	if [ ${status} -gt 0 ] ; then
[1183]	486	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
[1179]	487	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	488	IGCM_debug_Exit "IGCM_sys_RmRunDir"
	489	else
[1179]	490	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	491	fi
	492	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[622]	493	}
	494
	495	#D-#==================================================
	496	#D-function IGCM_sys_Put_Dir
	497	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
	498	#D-* Examples:
	499	#D-
	500	function IGCM_sys_Put_Dir {
[623]	501	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
	502	if ( $DEBUG_sys ) ; then
	503	echo "IGCM_sys_Put_Dir :" $@
	504	fi
	505	if [ $DRYRUN = 0 ]; then
	506	if [ ! -d ${1} ] ; then
	507	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
	508	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
	509	return
[622]	510	fi
	511
[715]	512	typeset status
[622]	513
[623]	514	# Only if we use rsync
	515	#IGCM_sys_TestDirArchive $( dirname $2 )
	516	#
	517	#USUAL WAY
[1179]	518	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	519	status=$?
[622]	520
[715]	521	if [ ${status} -gt 0 ] ; then
[811]	522	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
[1179]	523	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	524	IGCM_debug_Exit "IGCM_sys_Put_Dir"
[622]	525	else
[1179]	526	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	527	fi
[623]	528	fi
	529	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[622]	530	}
	531
	532	#D-#==================================================
	533	#D-function IGCM_sys_Get_Dir
[623]	534	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[622]	535	#D-* Examples:
	536	#D-
	537	function IGCM_sys_Get_Dir {
[623]	538	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
	539	if ( $DEBUG_sys ) ; then
	540	echo "IGCM_sys_Get_Dir :" $@
	541	fi
	542	if [ $DRYRUN = 0 ]; then
[708]	543	typeset NB_ESSAI DELAI status i
	544	# number of tentative
	545	NB_ESSAI=3
	546	# time delay between tentative
	547	DELAI=2
[622]	548
[676]	549	#
[708]	550	# USUAL WAY
	551	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
[713]	552	ccc_hsm get -r $1
[622]	553
[709]	554	i=0
[708]	555	while [ $i -lt $NB_ESSAI ] ; do
[1179]	556	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[708]	557	status=$?
[811]	558	if [ ${status} -gt 0 ] ; then
[713]	559	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
[708]	560	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
	561	sleep $DELAI
	562	else
	563	break
	564	fi
	565	(( i = i + 1 ))
	566	done
	567
	568	if [ ${status} -gt 0 ] ; then
[1178]	569	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
[1179]	570	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	571	IGCM_debug_Exit "IGCM_sys_Get_Dir"
[622]	572	else
[1179]	573	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	574	fi
[623]	575	fi
	576	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[622]	577	}
	578
	579	#D-#==================================================
	580	#D-function IGCM_sys_Put_Rest
[623]	581	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[622]	582	#D- File and target directory must exist.
	583	#D-* Examples:
	584	#D-
	585	function IGCM_sys_Put_Rest {
[623]	586	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
	587	if ( $DEBUG_sys ) ; then
	588	echo "IGCM_sys_Put_Rest :" $@
	589	fi
	590	if [ $DRYRUN = 0 ]; then
	591	if [ ! -f ${1} ] ; then
	592	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
	593	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	594	fi
	595
[715]	596	typeset status
[623]	597	#
	598	# USUAL WAY
[1179]	599	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	600	status=$?
[622]	601
[623]	602	# #RSYNC WITH NETWORK SSH CALL
[1179]	603	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	604	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	605
[623]	606	# #RSYNC WITH NFS USE
[1179]	607	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	608	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	609
[715]	610	# status=$?
	611	# IGCM_sys_Rsync_out $status
[622]	612
[1179]	613	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[715]	614	# (( status=status+$? ))
[623]	615
[715]	616	if [ ${status} -gt 0 ] ; then
[1179]	617	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
[765]	618	[ -f ${1} ] && ls -l ${1}
[759]	619	[ -f ${2} ] && ls -l ${2}
	620	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	621	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	622	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	623	else
[759]	624
	625	if [ X${JobType} = XRUN ] ; then
	626	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	627	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	628	fi
	629
[1179]	630	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	631	fi
[623]	632	fi
	633	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[622]	634	}
	635
	636	#D-#==================================================
	637	#D-function IGCM_sys_Put_Out
[623]	638	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[622]	639	#D-* Examples:
	640	#D-
	641	function IGCM_sys_Put_Out {
[623]	642	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
	643	if ( $DEBUG_sys ) ; then
	644	echo "IGCM_sys_Put_Out :" $@
	645	fi
[667]	646
[711]	647	typeset NB_ESSAI DELAI status i exist skip
[1234]	648	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
[667]	649
	650	# number of tentative
[668]	651	NB_ESSAI=3
[667]	652	# time delay between tentative
[668]	653	DELAI=2
[667]	654
[623]	655	if [ $DRYRUN = 0 ]; then
	656	if [ ! -f ${1} ] ; then
	657	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
	658	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	659	return 1
[622]	660	fi
[623]	661	#
	662	IGCM_sys_MkdirArchive $( dirname $2 )
	663	#
[711]	664	exist=false
	665	skip=false
	666	if [ -f $2 ] ; then
	667	IGCM_debug_Print 1 "$2 already exist"
	668	ccc_hsm get $2
	669	exist=true
	670	if [ "X$( diff $1 $2 )" = X ] ; then
	671	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
	672	skip=true
[667]	673	else
[711]	674	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	675	skip=false
[667]	676	fi
[711]	677	fi
	678	#
	679	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	680	IGCM_sys_Chmod u+w $2
	681	fi
[622]	682
[711]	683	if [ X${skip} = Xfalse ] ; then
[719]	684	i=0
[711]	685	while [ $i -lt $NB_ESSAI ] ; do
[1234]	686	# Identify file system
	687	fileDeviceNumberInHex=$( stat -c %d $1 )
	688	status=$?
	689	if [ ${status} -gt 0 ] ; then
	690	IGCM_debug_Exit "IGCM_sys_Put_Out"
	691	fi
	692	# Identify file system
	693	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
	694	status=$?
	695	if [ ${status} -gt 0 ] ; then
	696	IGCM_debug_Exit "IGCM_sys_Put_Out"
	697	fi
	698
[1412]	699	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
[1234]	700	# They are not on the same device. USUAL WAY
[1179]	701	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	702	status=$?
	703	else
[1234]	704	# They are on the same device. NOT SO USUAL WAY
[1179]	705	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	706	status=$?
	707	fi
[711]	708	if [ ${status} -gt 0 ]; then
[713]	709	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
[711]	710	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
[765]	711	[ -f ${1} ] && ls -l ${1}
[759]	712	[ -f ${2} ] && ls -l ${2}
	713	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[711]	714	sleep $DELAI
	715	else
	716	break
	717	fi
	718	(( i = i + 1 ))
	719	done
	720	fi
	721
[623]	722	# #RSYNC WITH NETWORK SSH CALL
[1179]	723	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	724	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	725
[623]	726	# #RSYNC WITH NFS USE
[1179]	727	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	728	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	729
[711]	730	# status=$?
	731	# IGCM_sys_Rsync_out $status
[622]	732
[1179]	733	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[711]	734	# (( status=status+$? ))
[622]	735
[711]	736	if [ ${status} -gt 0 ] ; then
[1179]	737	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
[765]	738	[ -f ${1} ] && ls -l ${1}
[759]	739	[ -f ${2} ] && ls -l ${2}
	740	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	741	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	742	IGCM_debug_Exit "IGCM_sys_Put_Out"
[622]	743	else
[759]	744
	745	if [ X${JobType} = XRUN ] ; then
	746	if [ X${3} = X ] ; then
	747	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	748	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	749	fi
	750	fi
	751
[1179]	752	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	753	fi
[623]	754	fi
	755	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	756	return 0
[622]	757	}
	758
	759	#D-#==================================================
	760	#D-function IGCM_sys_Get
	761	#D-* Purpose: Get a file from ${ARCHIVE}
	762	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
	763	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
	764	function IGCM_sys_Get {
[623]	765	IGCM_debug_PushStack "IGCM_sys_Get" $@
[622]	766
[773]	767	typeset DEST dm_liste target file_work
[667]	768	typeset NB_ESSAI DELAI status i
[622]	769
[623]	770	if ( $DEBUG_sys ) ; then
	771	echo "IGCM_sys_Get :" $@
	772	fi
[667]	773
	774	# number of tentative
[668]	775	NB_ESSAI=3
[667]	776	# time delay between tentative
[668]	777	DELAI=2
[667]	778
[623]	779	if [ $DRYRUN -le 2 ]; then
	780	if [ X${1} = X'/l' ] ; then
	781	eval set +A dm_liste \${${2}}
	782	else
	783	eval set +A dm_liste ${1}
[622]	784	fi
[623]	785	eval DEST=\${${#}}
[1179]	786	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	787	status=$?
	788	if [ ${status} -gt 0 ] ; then
	789	echo "WARNING IGCM_sys_Get : error code ${status}"
[1179]	790	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	791	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
	792	fi
[622]	793
[715]	794	#if [ ${status} -gt 0 ] ; then
[1179]	795	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
	796	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	797	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
	798	# sleep 30
	799	# echo "We try another time"
[1179]	800	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	801	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[785]	802	# status=$?
	803	# if [ ${status} -gt 0 ] ; then
	804	# echo "ERROR IGCM_sys_Get : again demigration error :"
[1179]	805	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	806	# IGCM_debug_Exit "IGCM_sys_Get"
[623]	807	# fi
[785]	808	# else
	809	# echo "ERROR IGCM_sys_Get : demigration error :"
[1179]	810	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	811	# IGCM_debug_Exit "IGCM_sys_Get"
	812	# fi
[623]	813	#fi
[622]	814
[719]	815	# #RSYNC WITH NETWORK SSH CALL
[1179]	816	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	817	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	818
	819	# #RSYNC WITH NFS USE
[1179]	820	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	821	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	822
	823	# status=$?
	824	# IGCM_sys_Rsync_out $status
	825
[1179]	826	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	827	# (( status=status+$? ))
	828
[623]	829	#USUAL WAY
	830	if [ X${1} = X'/l' ] ; then
	831	for target in ${dm_liste[*]} ; do
	832	local_file=$( basename ${target} )
[668]	833	# test if the target file is present before the loop
	834	IGCM_sys_TestFileArchive ${target}
[715]	835	status=$?
	836	if [ ${status} -gt 0 ] ; then
[701]	837	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
[668]	838	IGCM_debug_Exit "IGCM_sys_Get"
	839	else
[719]	840	i=0
[668]	841	while [ $i -lt $NB_ESSAI ] ; do
[1037]	842	#if [ X${DoLink} = Xtrue ] ; then
[1179]	843	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	844	# status=$?
	845	# else
[1179]	846	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	847	# status=$?
	848	#fi
[1179]	849	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[668]	850	status=$?
	851	if [ ${status} -gt 0 ]; then
[713]	852	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
[676]	853	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	854	sleep $DELAI
[668]	855	else
[676]	856	break
[668]	857	fi
	858	(( i = i + 1 ))
	859	done
[719]	860	if [ ${status} -gt 0 ] ; then
	861	echo "IGCM_sys_Get : error"
[1179]	862	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	863	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	864	IGCM_debug_Exit "IGCM_sys_Get"
	865	else
[1179]	866	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	867	fi
[668]	868	fi
[623]	869	done
	870	else
[719]	871	i=0
	872	while [ $i -lt $NB_ESSAI ] ; do
[1179]	873	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	874	status=$?
	875	if [ ${status} -gt 0 ]; then
	876	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
	877	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	878	sleep $DELAI
	879	else
	880	break
	881	fi
	882	(( i = i + 1 ))
	883	done
	884	if [ ${status} -gt 0 ] ; then
	885	echo "IGCM_sys_Get : error"
[1179]	886	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	887	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	888	IGCM_debug_Exit "IGCM_sys_Get"
	889	else
[1179]	890	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	891	fi
[623]	892	fi
	893	fi
	894	IGCM_debug_PopStack "IGCM_sys_Get"
	895	}
[622]	896
[623]	897	#D-#==================================================
[835]	898	#D-function IGCM_sys_GetDate_Monitoring
	899	#D-* Purpose: get the last year for which the monitoring has been computed
	900	#D-* Examples:
	901	#D-
	902	function IGCM_sys_GetDate_Monitoring {
	903	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
	904	if ( $DEBUG_sys ) ; then
	905	echo "IGCM_sys_GetDate_Monitoring :" $@
	906	fi
	907
	908	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
	909
	910	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
	911	}
	912
	913	#D-#==================================================
[623]	914	#D-function IGCM_sys_Dods_Rm
	915	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
	916	#D-* Examples:
	917	#D-
	918	function IGCM_sys_Dods_Rm {
	919	if ( $DEBUG_sys ) ; then
	920	echo "IGCM_sys_Dods_Rm :" $@
	921	fi
[715]	922	typeset status
[623]	923	if [ $DRYRUN = 0 ]; then
	924
	925	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
	926	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
	927	# echo "Nothing has been done."
	928	# return
	929	# fi
	930
[1392]	931	if [ "$#" -eq 1 ]; then
	932	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
	933	status=$?
	934	else
	935	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
	936	status=$?
	937	fi
[623]	938
[785]	939	# if [ ${status} -gt 0 ] ; then
	940	# echo "IGCM_sys_Dods_Rm : error."
	941	# cat out_dods_rm
	942	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
	943	# else
	944	# rm out_dods_rm
	945	# fi
[623]	946
	947	fi
[715]	948	return $status
[623]	949	}
	950
	951	#D-#==================================================
	952	#D-function IGCM_sys_Dods_Cp
	953	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
	954	#D-* Examples:
	955	#D-
	956	function IGCM_sys_Dods_Cp {
	957	if ( $DEBUG_sys ) ; then
	958	echo "IGCM_sys_Dods_Cp :" $@
	959	fi
[715]	960	typeset status
[623]	961	if [ $DRYRUN = 0 ]; then
	962
	963	# if [ ! -d ${R_SAVE}/${1} ] ; then
	964	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
	965	# echo "Nothing has been done."
	966	# return
	967	# fi
	968
[649]	969	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
[715]	970	status=$?
[623]	971
[715]	972	# if [ ${status} -gt 0 ] ; then
[623]	973	# echo "IGCM_sys_Dods_Cp : error."
	974	# cat out_dods_cp
	975	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
	976	# else
	977	# rm out_dods_cp
	978	# fi
	979
	980	fi
[715]	981	return $status
[623]	982	}
	983
	984	#D-#==================================================
	985	#D-function IGCM_sys_Put_Dods
[1178]	986	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
[623]	987	#D-* Examples:
	988	#D-
	989	function IGCM_sys_Put_Dods {
	990	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
	991	if ( $DEBUG_sys ) ; then
	992	echo "IGCM_sys_Put_Dods :" $@
	993	fi
[715]	994	typeset status
[623]	995	if [ $DRYRUN = 0 ]; then
	996	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
	997	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
	998	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
	999	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
	1000	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	1001	return
[622]	1002	fi
[623]	1003
	1004	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
	1005	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
	1006	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
	1007	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
	1008	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	1009	return
	1010	fi
	1011	#
	1012	if [ -d ${R_SAVE}/${1} ] ; then
	1013	cd ${R_SAVE}
	1014	elif [ -d ${R_FIGR}/${1} ] ; then
	1015	cd ${R_FIGR}
	1016	fi
	1017
	1018	IGCM_sys_Dods_Rm ${1}
	1019	IGCM_sys_Dods_Cp ${1}
[715]	1020	status=0
[623]	1021
[715]	1022	if [ ${status} -gt 0 ] ; then
[623]	1023	echo "IGCM_sys_Put_Dods : error."
	1024	IGCM_debug_Exit "IGCM_sys_Put_Dods"
	1025	fi
	1026	fi
	1027	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
[622]	1028	}
	1029
	1030	##############################################################
	1031	# REBUILD OPERATOR
	1032
[811]	1033	#D-#==================================================
[1163]	1034	#D-function IGCM_sys_sync
	1035	#D-* Purpose: flush buffer on disk
	1036	#D-* Examples:
	1037	#D-
	1038	function IGCM_sys_sync {
	1039	IGCM_debug_PushStack "IGCM_sys_sync" $@
	1040	if ( $DEBUG_sys ) ; then
	1041	echo "IGCM_sys_sync :" $@
	1042	fi
	1043
	1044	/bin/sync
	1045
	1046	IGCM_debug_PopStack "IGCM_sys_sync"
	1047	}
	1048
[622]	1049	############################################################
	1050	# Activate Running Environnment Variables
	1051
[811]	1052	#D-#==================================================
[1239]	1053	#D-function IGCM_sys_activ_variables
[811]	1054	#D-* Purpose: set environement variables prior to execution
	1055	#D-* Examples:
	1056	#D-
[622]	1057	function IGCM_sys_activ_variables {
[623]	1058	IGCM_debug_PushStack "IGCM_sys_activ_variables"
	1059	if ( $DEBUG_sys ) ; then
	1060	echo "IGCM_sys_activ_variables"
	1061	fi
[622]	1062
[811]	1063	# --------------------------------------------------------------------
[1409]	1064	#D- MODULE specifications
	1065	# --------------------------------------------------------------------
[1413]	1066	module purge > /dev/null 2>&1
[1410]	1067	if [ X${compilerVersion} = XV17 ] ; then
[1409]	1068	. /ccc/cont003/home/dsm/p86ipsl/.env_intel17.0.2_curie > /dev/null 2>&1
[1410]	1069	elif [ X${compilerVersion} = XV14 ] ; then
[1409]	1070	. /ccc/cont003/home/dsm/p86ipsl/.env_intel14.0.3_curie > /dev/null 2>&1
	1071	else
	1072	IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
	1073	fi
	1074	. /ccc/cont003/home/dsm/p86ipsl/.env_netcdf4.3.3.1_curie > /dev/null 2>&1
	1075
	1076	# Keep track of this
	1077	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
	1078	[ ! X${TaskType} = Xchecking ] && module list
	1079
	1080	# --------------------------------------------------------------------
[811]	1081	#D- MPI specifications
	1082	# --------------------------------------------------------------------
[1191]	1083
[811]	1084	# --------------------------------------------------------------------
	1085	#D- Other specifications
	1086	# --------------------------------------------------------------------
	1087
[623]	1088	ulimit -s unlimited
[622]	1089
[623]	1090	IGCM_debug_PopStack "IGCM_sys_activ_variables"
[622]	1091	}
	1092
	1093	############################################################
	1094	# Desactivate Running Environnment Variables
	1095
[811]	1096	#D-#==================================================
	1097	#D-function IGCM_sys_desactiv_variables
	1098	#D-* Purpose: unset environement variables after execution
	1099	#D-* Examples:
	1100	#D-
[622]	1101	function IGCM_sys_desactiv_variables {
[623]	1102	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
	1103	if ( $DEBUG_sys ) ; then
	1104	echo "IGCM_sys_desactiv_variables"
	1105	fi
[811]	1106	# --------------------------------------------------------------------
	1107	#D- MPI specifications
	1108	# --------------------------------------------------------------------
	1109
	1110	# --------------------------------------------------------------------
	1111	#D- Other specifications
	1112	# --------------------------------------------------------------------
	1113
[623]	1114	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
[622]	1115	}
	1116
[623]	1117	############################################################
[1238]	1118	# Update job headers to be used by the scheduler
	1119
	1120	#D-#==================================================
	1121	#D-function IGCM_sys_updateHeaders
	1122	#D-* Purpose: Update job headers to be used by the scheduler
	1123	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
	1124	#D-
	1125	function IGCM_sys_updateHeaders {
	1126	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
	1127	if ( $DEBUG_sys ) ; then
	1128	echo "IGCM_sys_updateHeaders"
	1129	fi
	1130	typeset file
	1131	file=$1
	1132
	1133	if [ ${executionType} -eq 1 ] ; then
	1134	# MPMD + MPI
	1135	sed -e "/::openMPthreads::/d" \
	1136	-e "s/::JobNumProcTot::/${coreNumber}/" \
[1256]	1137	-e "/#MSUB -x/d" \
	1138	-e "/--cpu_bind=none/d" \
[1238]	1139	${file} > ${file}.tmp
	1140
	1141	elif [ ${executionType} -eq 2 ] ; then
[1390]	1142	# MPMD + MPI + OMP : mpirun/ccc_mprun/error
	1143	if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
	1144	sed -e "/::openMPthreads::/d" \
	1145	-e "s/::JobNumProcTot::/${coreNumber}/" \
	1146	${file} > ${file}.tmp
	1147	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
	1148	sed -e "/::openMPthreads::/d" \
	1149	-e "s/::JobNumProcTot::/${coreNumber}/" \
	1150	-e "/--cpu_bind=none/d" \
	1151	${file} > ${file}.tmp
	1152	else
	1153	IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
	1154	IGCM_debug_Exit "IGCM_sys_updateHeaders"
	1155	fi
[1238]	1156	elif [ ${executionType} -eq 3 ] ; then
	1157	# SPMD + MPI/OMP
	1158	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1159	-e "s/::JobNumProcTot::/${mpiTasks}/" \
[1240]	1160	-e "/#MSUB -x/d" \
[1238]	1161	-e "/--cpu_bind=none/d" \
	1162	${file} > ${file}.tmp
	1163
	1164	elif [ ${executionType} -eq 4 ] ; then
	1165	# SPMD + MPI only
	1166	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
	1167	-e "/::openMPthreads::/d" \
[1240]	1168	-e "/#MSUB -x/d" \
[1238]	1169	-e "/--cpu_bind=none/d" \
	1170	${file} > ${file}.tmp
	1171
	1172	elif [ ${executionType} -eq 5 ] ; then
	1173	# SPMD + OMP only
	1174	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1175	-e "/::JobNumProcTot::/d" \
[1240]	1176	-e "/#MSUB -x/d" \
[1238]	1177	-e "/--cpu_bind=none/d" \
	1178	${file} > ${file}.tmp
	1179
	1180	elif [ ${executionType} -eq 6 ] ; then
	1181	# SEQUENTIAL THEN
	1182	sed -e "s/::JobNumProcTot::/1/" \
	1183	-e "/::openMPthreads::/d" \
[1240]	1184	-e "/#MSUB -x/d" \
[1238]	1185	-e "/--cpu_bind=none/d" \
	1186	${file} > ${file}.tmp
	1187
	1188	fi
	1189
	1190	IGCM_sys_Mv ${file}.tmp ${file}
	1191
	1192	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
	1193	}
	1194
	1195	############################################################
[623]	1196	# Build MPI/OMP scripts run file (dummy function)
[622]	1197
[811]	1198	#D-#==================================================
	1199	#D-function IGCM_sys_build_run_file
	1200	#D-* Purpose: build run file (deprecated)
	1201	#D-* Examples:
	1202	#D-
[622]	1203	function IGCM_sys_build_run_file {
[811]	1204	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[622]	1205	}
	1206
	1207	############################################################
	1208	# Build MPI/OMP scripts
[811]	1209
	1210	#D-#==================================================
	1211	#D-function IGCM_sys_build_execution_scripts
	1212	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
	1213	#D-* Examples:
	1214	#D-
[623]	1215	function IGCM_sys_build_execution_scripts
[622]	1216	{
[623]	1217	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
	1218	if ( $DEBUG_sys ) ; then
	1219	echo "IGCM_sys_build_execution_scripts " $@
	1220	fi
[622]	1221
[623]	1222	EXECUTION=${HOST_MPIRUN_COMMAND}
[622]	1223
[623]	1224	if ( ${OK_PARA_MPMD} ) ; then
[622]	1225
[1390]	1226	# MPMD mode
	1227	# 1 MPI only
	1228	# 2 MPI/OpenMP mpirun method
	1229	# 3 MPI/OpenMP ccc_mprun method
	1230
[1381]	1231	if [ -f run_file ] ; then
	1232	IGCM_sys_Rm -f run_file
	1233	fi
	1234	touch run_file
	1235
[1390]	1236	# case 1 : Only MPI (MPMD)
[1078]	1237	if ( ! ${OK_PARA_OMP} ) ; then
[622]	1238
[1078]	1239	# Build run_file
[622]	1240
[1078]	1241	# First loop on the components for the coupler ie oasis (only if oasis3)
	1242	# the coupler ie oasis3 must be the first one
	1243	for comp in ${config_ListOfComponents[*]} ; do
[622]	1244
[1078]	1245	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1246	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1247
[1078]	1248	# for CPL component only
	1249	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1250
[1078]	1251	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1252	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1253	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1254	fi
	1255	done
[622]	1256
[1078]	1257	# Then second loop on the components
	1258	for comp in ${config_ListOfComponents[*]} ; do
[622]	1259
[1078]	1260	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1261	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1262
[1078]	1263	# Only if we really have an executable for the component and not the coupler ie oasis:
	1264	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
[622]	1265
[1078]	1266	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1267	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1268	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1269	fi
	1270	done
[622]	1271
[1390]	1272	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
	1273
	1274	IGCM_sys_Chmod u+x run_file
	1275	if ( $DEBUG_sys ) ; then
	1276	echo "run_file contains : "
	1277	cat run_file
	1278	fi
	1279
	1280	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
	1281
	1282	# 2 MPI/OpenMP mpirun method
	1283
	1284	# Use of mpirun instead of ccc_mprun
	1285	EXECUTION="time mpirun"
	1286
	1287	# Hosts treatment
	1288	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
	1289
	1290	i=0
	1291	rm -f hosts rankfile
	1292	IGCM_debug_Print 1 "sys Curie, Hosts available :"
	1293	for nodes in `cat hosts.tmp`
	1294	do
	1295	host[$i]=$nodes
	1296	echo "${host[$i]}" >> hosts
	1297	IGCM_debug_Print 1 ${host[$i]}
	1298	i=$((i+1))
	1299	done
	1300	rm -f hosts.tmp
	1301
	1302	listnodes=${host[*]}
	1303
	1304	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
	1305
	1306	# Initialisation
	1307	rank=0
	1308	current_core=0
	1309	init_exec=n
	1310
	1311	# Loop on the components
	1312	for comp in ${config_ListOfComponents[*]} ; do
	1313
	1314	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1315	eval ExeNameOut=\${config_Executable_${comp}[1]}
	1316
	1317	# Not possible if oasis has an executable (i.e old version of oasis3)
	1318	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
	1319	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
	1320	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
	1321	IGCM_debug_Verif_Exit
	1322	fi
	1323
	1324	# Only if we really have an executable for the component :
	1325	if [ "X${ExeNameOut}" != X\"\" ] ; then
	1326
	1327	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1328	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[1426]	1329	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
[1390]	1330
	1331	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1332	echo "" >> script_${ExeNameOut}.ksh
	1333	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
	1334
	1335	# Check if the number of threads is correct
	1336	case ${comp_proc_omp_loc} in
	1337	2\|4\|8\|16)
	1338	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
	1339	;;
	1340	*)
	1341	IGCM_debug_Exit "ERROR with OMP parameters !"
	1342	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
	1343	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
	1344	IGCM_debug_Verif_Exit
	1345	;;
	1346	esac
	1347	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1348	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1349	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1350	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1351	fi
	1352
	1353	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
	1354	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
	1355	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
	1356	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
	1357	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1358
	1359	if [ ${init_exec} = y ] ; then
	1360	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
	1361	else
	1362	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
	1363	init_exec=y
	1364	fi
	1365
	1366	# Build rankfile : method used to assign cores and nodes for the MPI process
	1367	# Ex :
	1368	#rank 0=curie5296 slot=0,1,2,3
	1369	#rank 1=curie5296 slot=4,5,6,7
	1370	# Example of final command :
	1371	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
	1372	# with script_lmdz.x.ksh :
	1373	# #!/bin/ksh
	1374	#export KMP_STACKSIZE=3g
	1375	#export KMP_LIBRARY=turnaround
	1376	#export MKL_SERIAL=YES
	1377	#OMP_NUM_THREADS=4
	1378	#./lmdz.x
[1426]	1379	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
[1430]	1380	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
[1426]	1381	else
	1382	(( offset_comp_proc_loc = comp_proc_omp_loc ))
	1383	fi
[1390]	1384	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
[1426]	1385	(( index_host = current_core / NB_CORE_PER_NODE ))
	1386	host_value=${host[${index_host}]}
	1387	(( slot = current_core % NB_CORE_PER_NODE ))
	1388	virg=","
	1389	string_final=""
	1390	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
	1391	string=$index$virg
	1392	string_final=$string_final$string
	1393	done
	1394	string_final=$( echo $string_final \| sed "s/.$//" )
	1395	echo "rank $rank=$host_value slot=$string_final" >> rankfile
	1396	(( rank = rank + 1 ))
	1397	(( current_core = current_core + offset_comp_proc_loc ))
[1390]	1398	done
	1399	fi
	1400	done
	1401
	1402	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
	1403
	1404	# 3 MPI/OpenMP ccc_mprun method
	1405
[1381]	1406	# MPI-OpenMP (MPMD)
	1407	# example of run_file
	1408	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
	1409	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
	1410	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
[622]	1411
[1078]	1412	# Loop on the components
	1413	for comp in ${config_ListOfComponents[*]} ; do
[1000]	1414
[1078]	1415	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1416	eval ExeNameOut=\${config_Executable_${comp}[1]}
[1000]	1417
[1078]	1418	# Not possible if oasis has an executable (i.e old version of oasis3)
	1419	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
	1420	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
	1421	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
	1422	IGCM_debug_Verif_Exit
	1423	fi
[1000]	1424
[1078]	1425	# Only if we really have an executable for the component :
	1426	if [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1427
[1078]	1428	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1429	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[1000]	1430
[1381]	1431	# Build script files
	1432
[1078]	1433	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1434	echo "" >> script_${ExeNameOut}.ksh
	1435	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
	1436
	1437	# Check if the number of threads is correct
	1438	case ${comp_proc_omp_loc} in
	1439	2\|4\|8\|16)
	1440	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
	1441	;;
	1442	*)
	1443	IGCM_debug_Exit "ERROR with OMP parameters !"
	1444	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
	1445	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
	1446	IGCM_debug_Verif_Exit
	1447	;;
	1448	esac
[1381]	1449	### only for LMDZ?
[1078]	1450	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1451	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1452	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1453	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
[1381]	1454
[1078]	1455	fi
[1318]	1456
[1381]	1457	# to have out/err per process on different files
	1458	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
[1317]	1459	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[1078]	1460
[1381]	1461	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
[1078]	1462
[1381]	1463	# Complete run_file
[1000]	1464
[1381]	1465	echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
	1466
[1078]	1467	fi
	1468	done
[1381]	1469
[1385]	1470	## variable added to stop after 60s instead of 600s by default.
	1471	## This is used when no error comes from executables and when something stopped an executable without notice.
	1472	export SLURM_WAIT=60
[1381]	1473
[1385]	1474	## module advised by TGCC (instead of 2 variables)
	1475	module load feature/bridge/heterogenous_mpmd
[1381]	1476
[1390]	1477	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
[1000]	1478
[1390]	1479	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
[1381]	1480
[1390]	1481	IGCM_sys_Chmod u+x run_file
	1482	if ( $DEBUG_sys ) ; then
	1483	echo "run_file contains : "
	1484	cat run_file
	1485	fi
[1381]	1486
[1390]	1487	else
[1381]	1488
[1390]	1489	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
	1490
	1491	fi # 1 2 or 3 if ${OK_PARA_MPMD}
	1492
	1493	else
[1078]	1494	# Only one executable (SPMD mode).
[1000]	1495
[622]	1496	for comp in ${config_ListOfComponents[*]} ; do
	1497
[685]	1498	# Only if we really have an executable for the component :
[623]	1499	eval ExeNameOut=\${config_Executable_${comp}[1]}
[685]	1500	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
[622]	1501
[1381]	1502	# Build script files
	1503
[685]	1504	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1505	echo "" >> script_${ExeNameOut}.ksh
	1506	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1507
	1508	if ( ${OK_PARA_OMP} ) ; then
	1509	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1510	echo "" >> script_${ExeNameOut}.ksh
	1511	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1512	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1513	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1514	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1515	fi
	1516
	1517	if ( ${OK_PARA_MPI} ) ; then
[623]	1518	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
[685]	1519	# Default : ccc_mprun used if nb_proc gt 1
	1520	# to have out/err per process on different files
[1317]	1521	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
	1522	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1523	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
[623]	1524	else
[685]	1525	# Default : ccc_mprun is NOT used if nb_proc eq 1
	1526	# to have out/err per process on different files
[1317]	1527	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
	1528	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1529	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
[622]	1530	fi
[685]	1531
	1532	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
	1533	cat script_${ExeNameOut}.ksh
	1534
[623]	1535	fi
[622]	1536
	1537	done
	1538
[1390]	1539	fi # ${OK_PARA_MPMD}
[622]	1540
[623]	1541	IGCM_debug_Print 1 "sys Curie : execution command is "
	1542	IGCM_debug_Print 1 "$EXECUTION"
[622]	1543
[623]	1544	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[622]	1545	}
	1546
[933]	1547	#D-#==================================================
	1548	#D-function IGCM_sys_check_path
	1549	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
	1550	#D-* do not point to an important use directory. Stop immediately in that case.
	1551	#D-* Examples:
	1552	#D-
	1553	function IGCM_sys_check_path {
	1554	IGCM_debug_PushStack "IGCM_sys_check_path"
	1555	if ( $DEBUG_sys ) ; then
	1556	echo "IGCM_sys_check_path"
	1557	fi
[811]	1558
[933]	1559	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
	1560	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
	1561	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
[936]	1562	IGCM_debug_Exit "This will stop the job"
[933]	1563	fi
	1564	IGCM_debug_PopStack "IGCM_sys_check_path"
	1565	}
	1566
[811]	1567	#D-#==================================================
	1568	#D-function IGCM_sys_check_quota
	1569	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
	1570	#D-* Examples:
	1571	#D-
[714]	1572	function IGCM_sys_check_quota {
[776]	1573	IGCM_debug_PushStack "IGCM_sys_check_quota"
	1574	if ( $DEBUG_sys ) ; then
	1575	echo "IGCM_sys_check_quota"
	1576	fi
	1577	# Limit of quota (in %)
	1578	limit_quota=90
	1579
	1580	# Check of the volume
[912]	1581	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
	1582	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
[776]	1583
	1584	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
	1585
	1586	unit_avail=${volume_avail: -1}
	1587	unit_quota=${volume_quota: -1}
	1588
	1589	if [ "${unit_quota}" = "*" ] ; then
[811]	1590	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1591	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1592	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1593	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1594	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1595	IGCM_debug_Verif_Exit
[714]	1596	fi
[776]	1597
	1598	temp_avail=${volume_avail%%${volume_avail: -1}*}
	1599	temp_quota=${volume_quota%%${volume_quota: -1}*}
	1600
	1601	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1602
	1603	# Convertion
	1604	if [ ${volume_avail: -1} = "T" ] ; then
	1605	(( temp_avail = temp_avail * 1000000000000 ))
	1606	elif [ ${volume_avail: -1} = "G" ] ; then
	1607	(( temp_avail = temp_avail * 1000000000 ))
	1608	elif [ ${volume_avail: -1} = "M" ] ; then
	1609	(( temp_avail = temp_avail * 1000000 ))
	1610	elif [ ${volume_avail: -1} = "k" ] ; then
	1611	(( temp_avail = temp_avail * 1000 ))
	1612	else
	1613	(( temp_avail = volume_avail ))
	1614	fi
	1615	if [ ${volume_quota: -1} = "T" ] ; then
	1616	(( temp_quota = temp_quota * 1000000000000 ))
	1617	elif [ ${volume_quota: -1} = "G" ] ; then
	1618	(( temp_quota = temp_quota * 1000000000 ))
	1619	elif [ ${volume_quota: -1} = "M" ] ; then
	1620	(( temp_quota = temp_quota * 1000000 ))
	1621	elif [ ${volume_quota: -1} = "k" ] ; then
	1622	(( temp_quota = temp_quota * 1000 ))
	1623	else
	1624	(( temp_quota = volume_quota ))
	1625	fi
	1626	fi
	1627
	1628	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1629	# echo "volume ratio is " $quota_volume
	1630
	1631	if [ ${quota_volume} -ge ${limit_quota} ] ; then
	1632	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1633	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
	1634	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1635	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1636	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1637	IGCM_debug_Verif_Exit
	1638	fi
	1639
	1640	fi
	1641
	1642	# Check of the number of inodes
	1643
[912]	1644	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
	1645	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
[776]	1646
	1647	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
	1648
	1649	unit_avail=${inode_avail: -1}
	1650	unit_quota=${inode_quota: -1}
	1651
	1652	if [ "${unit_quota}" = "*" ] ; then
[811]	1653	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1654	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1655	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1656	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1657	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1658	IGCM_debug_Verif_Exit
[776]	1659	fi
	1660
	1661	temp_avail=${inode_avail%%${inode_avail: -1}*}
	1662	temp_quota=${inode_quota%%${inode_quota: -1}*}
	1663
	1664	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1665
	1666	# Convertion
	1667	if [ ${inode_avail: -1} = "T" ] ; then
	1668	(( temp_avail = temp_avail * 1000000000000 ))
	1669	elif [ ${inode_avail: -1} = "G" ] ; then
	1670	(( temp_avail = temp_avail * 1000000000 ))
	1671	elif [ ${inode_avail: -1} = "M" ] ; then
	1672	(( temp_avail = temp_avail * 1000000 ))
	1673	elif [ ${inode_avail: -1} = "k" ] ; then
	1674	(( temp_avail = temp_avail * 1000 ))
	1675	else
	1676	(( temp_avail = inode_avail ))
	1677	fi
	1678
	1679	if [ ${inode_quota: -1} = "T" ] ; then
	1680	(( temp_quota = temp_quota * 1000000000000 ))
	1681	elif [ ${inode_quota: -1} = "G" ] ; then
	1682	(( temp_quota = temp_quota * 1000000000 ))
	1683	elif [ ${inode_quota: -1} = "M" ] ; then
	1684	(( temp_quota = temp_quota * 1000000 ))
	1685	elif [ ${inode_quota: -1} = "k" ] ; then
	1686	(( temp_quota = temp_quota * 1000 ))
	1687	else
	1688	(( temp_quota = inode_quota ))
	1689	fi
	1690	fi
	1691	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1692	# echo "inode ratio is " $quota_inode
	1693
	1694	if [ ${quota_inode} -ge ${limit_quota} ] ; then
	1695	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1696	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
	1697	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1698	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1699	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1700	IGCM_debug_Verif_Exit
	1701	fi
	1702	fi
	1703	IGCM_debug_PopStack "IGCM_sys_check_quota"
[714]	1704	}
	1705
[773]	1706	#D-#==================================================
[1286]	1707	#D-function IGCM_sys_projectAccounting
	1708	#D-* Purpose: store project accounting information in a file
	1709	#D-* Examples:
	1710	#D-
	1711	function IGCM_sys_projectAccounting {
	1712	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
	1713	if ( $DEBUG_sys ) ; then
	1714	echo "IGCM_sys_check_quota"
	1715	fi
	1716
[1288]	1717	ssh curie91 /usr/bin/ccc_myproject > $1
[1286]	1718
	1719	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
	1720	}
	1721
	1722	#D-#==================================================
[1244]	1723	#D-function IGCM_sys_getJobSchedulerID
	1724	#D-* Purpose: Get the job ID during execution
	1725	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
	1726	#D-
	1727	function IGCM_sys_getJobSchedulerID {
	1728	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
	1729	if ( $DEBUG_sys ) ; then
	1730	echo "IGCM_sys_getJobSchedulerID"
	1731	fi
	1732
	1733	eval ${1}=${BRIDGE_MSUB_JOBID}
	1734
	1735	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
	1736	}
	1737
	1738	#D-#==================================================
[958]	1739	#D-function IGCM_sys_GetJobID
[1244]	1740	#D-* Purpose: Get the job ID from the JobName
[958]	1741	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
	1742	#D-
	1743	function IGCM_sys_GetJobID {
	1744	IGCM_debug_PushStack "IGCM_sys_GetJobID"
	1745	if ( $DEBUG_sys ) ; then
	1746	echo "IGCM_sys_GetJobID"
	1747	fi
	1748
	1749	# With -f option, the full job name is given in the last column
	1750	ID=$( ccc_mstat -f -u $2 \| \
	1751	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
	1752
	1753	eval ${3}=${ID}
	1754	IGCM_debug_PopStack "IGCM_sys_GetJobID"
	1755	}
	1756
	1757	#D-#==================================================
[773]	1758	#D-function IGCM_sys_CountJobInQueue
[1244]	1759	#D-* Purpose: Count number of users job
[773]	1760	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
	1761	#D-
	1762	function IGCM_sys_CountJobInQueue {
	1763	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
	1764	if ( $DEBUG_sys ) ; then
	1765	echo "IGCM_sys_CountJobInQueue"
	1766	fi
	1767
	1768	# With -f option, the full job name is given in the last column
[834]	1769	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
[773]	1770
	1771	eval ${2}=${NbRun}
	1772
	1773	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
	1774	}
	1775
[955]	1776	#D-#==================================================
	1777	#D-function IGCM_sys_ListJobInQueue
[1244]	1778	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
[955]	1779	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
	1780	#D-
	1781	function IGCM_sys_ListJobInQueue {
	1782	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
	1783	if ( $DEBUG_sys ) ; then
	1784	echo "IGCM_sys_ListJobInQueue"
	1785	fi
	1786
	1787	# With -f option, the full job name is given in the last column
	1788	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
	1789	'( $2 == User && \
	1790	$NF != /TS/ && \
	1791	$NF !~ /PACK/ && \
	1792	$NF !~ /REBUILD/ && \
	1793	$NF !~ /pack/ ) \
	1794	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
	1795
	1796	eval set -A ${2} ${JobList[*]}
	1797
	1798	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
	1799	}
[1272]	1800
	1801	#D-#==================================================
	1802	#D-function IGCM_sys_atlas
	1803	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
	1804	#D-* Examples:
	1805	#D-
	1806	function IGCM_sys_atlas {
	1807	IGCM_debug_PushStack "IGCM_sys_atlas" $@
	1808	if ( $DEBUG_sys ) ; then
	1809	echo "IGCM_sys_atlas :" $@
	1810	fi
	1811
	1812	typeset status
	1813
	1814	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	1815	status=$?
	1816	if [ ${status} -gt 0 ] ; then
	1817	echo "IGCM_sys_atlas : error code ${status}"
	1818	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1819	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1820	IGCM_debug_PopStack "IGCM_sys_atlas"
	1821	return 1
	1822	else
	1823	IGCM_debug_PopStack "IGCM_sys_atlas"
	1824	return 0
	1825	fi
	1826
	1827	IGCM_debug_PopStack "IGCM_sys_atlas"
	1828	}
	1829
[1380]	1830	#D-#==================================================
	1831	#D-function IGCM_sys_rebuild_nemo
	1832	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
	1833	#D-* Examples:
	1834	#D-
	1835
	1836	function IGCM_sys_rebuild_nemo {
	1837	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
	1838	if ( $DEBUG_sys ) ; then
	1839	echo "IGCM_sys_rebuild_nemo :" $@
	1840	fi
	1841
	1842	$( ccc_home -u p86ipsl )/REBUILD_NEMO/rebuild_nemo ${1} ${2}
	1843
	1844	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
	1845	}
	1846

Note: See TracBrowser for help on using the repository browser.

Download in other formats: