Context Navigation

libIGCM_sys_irene-amd.ksh @ 1638

Last change on this file since 1638 was 1638, checked in by aclsce, 2 months ago
Added functionality to run concurrent execution with 2 or 3 model instances.
File size: 76.4 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: 1512 $ Revision of last commit
7	# $Author:: acosce $ Author of last commit
8	# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Irene-amd
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41	typeset -r DRYRUN=0
42	typeset -r DRYRUN_DEBUG=4
43	else
44	typeset -r DRYRUN=${DRYRUN:=0}
45	typeset -r DRYRUN_DEBUG=0
46	fi
47
48	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
49	# -------------------------------------------------------------------------------------
50	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
51	# \| \| Cp/Exe/param/files \| \| \| \|
52	# \| \| Chmod Qsub \| \| \| \|
53	# -------------------------------------------------------------------------------------
54	# \| 0 \| yes \| yes \| yes \| yes \|
55	# -------------------------------------------------------------------------------------
56	# \| 1 \| yes \| yes \| yes \| no \|
57	# -------------------------------------------------------------------------------------
58	# \| 2 \| yes \| yes \| no \| no \|
59	# -------------------------------------------------------------------------------------
60	# \| 3 \| yes \| no \| no \| no \|
61	# -------------------------------------------------------------------------------------
62	# \| 4 \| yes \| yes \| creation of RUN_DIR and Job_debug \|
63	# -------------------------------------------------------------------------------------
64
65
66	#=====================================================
67	# Global Variables :
68	#=====================================================
69	# Language : "fr" or "en"
70	typeset -r MYLANG="fr"
71
72	#=====================================================
73	# Host user names project maxCpuTime
74	# $hostname ou hostname
75	typeset HOST=${HOST:=$( hostname )}
76	# $username ou whoami
77	typeset LOGIN=${LOGIN:=$( whoami )}
78	# $hostname of the MASTER job
79	typeset MASTER=irene-amd
80	# add default project on irene-amd
81	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} \| cut -d@ -f1 )
82	# jobWarningDelay in seconds
83	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
84
85	#D-
86	#D-#==================================================
87	#D-Program used in libIGCM
88	#D-#==================================================
89
90	# Submit command
91	typeset SUBMIT=${SUBMIT:=ccc_msub}
92	# rsync with path
93	typeset -r RSYNC=/usr/bin/rsync
94	# RSYNC_opt args to rsync
95	typeset -r RSYNC_opt="-va"
96	# ie storage filesystem
97	typeset -r STOREHOST=${MASTER}
98	typeset -r REMOTE_RSYNC=/usr/bin/rsync
99
100	#====================================================
101	# Set environment tools (ferret, nco, cdo, rebuild, ...)
102	#====================================================
103	typeset CCCOS=${CCCOS:=$( ccc_os )}
104
105	if [ X${TaskType} = Xcomputing ] ; then
106	IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
107	else
108	module purge > /dev/null 2>&1
109	if [ X${CCCOS} = X"Rhel_8__x86_64" ] ; then
110	. $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
111	else
112	. $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_irene-amd > /dev/null 2>&1
113	. $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_atlas_irene-amd > /dev/null 2>&1
114	fi
115
116	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
117	export UVCDAT_ANONYMOUS_LOG=FALSE
118	fi
119
120	# FYI
121	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
122	[ ! X${TaskType} = Xchecking ] && module list
123
124	# For AddNoise and AddPertu3DOCE
125	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
126
127	# For rebuild
128	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl/bin
129
130	#====================================================
131	# Host specific DIRECTORIES
132	#====================================================
133
134	# ============ CESIUM START ============ #
135
136	#====================================================
137	#- Mirror libIGCM from titane to cesium if needed
138	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
139	#if [ ! ${ROOTSYS} = "home" ] ; then
140	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
141	#else
142	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
143	#fi
144
145	#====================================================
146	#- libIGCM_POST
147	#if ( ${MirrorlibIGCM} ) ; then
148	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
149	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
150	#else
151	# typeset -r libIGCM_POST=${libIGCM}
152	#fi
153
154	# ============ CESIUM END ============ #
155
156	#====================================================
157	#- MirrorlibIGCM for frontend
158	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
159
160	#====================================================
161	#- libIGCM_POST for frontend
162	typeset -r libIGCM_POST=${libIGCM}
163
164	#====================================================
165	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
166	typeset -r R_EXE="${MODIPSL}/bin"
167
168	#====================================================
169	#- SUBMIT_DIR : submission dir
170	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
171
172	#====================================================
173	#- IN
174	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
175
176	#====================================================
177	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
178	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
179
180	#====================================================
181	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
182	typeset -r OUTCOMMAND_PATH=/tmp
183
184	#====================================================
185	#- HOST_MPIRUN_COMMAND
186	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
187
188	#====================================================
189	#- Max number of arguments passed to nco operator or demigration command
190	UNIX_MAX_LIMIT=360
191
192	#====================================================
193	#- set PackDefault to true on irene-amd
194	PackDefault=true
195
196	#====================================================
197	#- Default number of MPI task for IPSL coupled model
198	#- required for backward compatibility
199	#-
200	DEFAULT_NUM_PROC_OCE=5
201	DEFAULT_NUM_PROC_CPL=1
202	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
203
204	#====================================================
205	#- Number of computing cores per node
206	#-
207	NB_CORE_PER_NODE=128
208
209	#D-#==================================================
210	#D-function IGCM_sys_defineArchives
211	#D-* Purpose:
212	#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
213	#D-* Define ARCHIVE : Dedicated to large files
214	#D-* Define STORAGE : Dedicated to small/medium files
215	#D-* Define R_OUT : Output tree located on ARCHIVE
216	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
217	#D-* Define R_BUF : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
218	#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
219	#D-* Examples:
220	#D-
221	function IGCM_sys_defineArchives {
222	IGCM_debug_PushStack "IGCM_sys_defineArchives"
223
224	# Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
225	if [ X${config_UserChoices_DataProject} = X ] \|\| [ X${config_UserChoices_DataProject} = XDEFAULT ]; then
226	# Default option: Change the dfldatadir according to the project used for submission
227	# The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
228	if [ X"$( echo ${PROJECT} \| grep cmip6 )" != "X" ] ; then
229	module switch dfldatadir dfldatadir/gencmip6
230	IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
231	else
232	module switch dfldatadir dfldatadir/${PROJECT}
233	IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
234	fi
235	else
236	# Use the project set by the variable DataProject in section UserChoices in config.card
237	module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
238	IGCM_debug_Print 1 "Load dfldatadir for DataProject read in config.card : ${config_UserChoices_DataProject}"
239	fi
240
241	#====================================================
242	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
243	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
244
245	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
246	#====================================================
247	#- ARCHIVE (dedicated to large files)
248	ARCHIVE=${config_UserChoices_ARCHIVE}
249	else
250	#====================================================
251	#- ARCHIVE (dedicated to large files)
252	ARCHIVE=${CCCSTOREDIR}
253	fi
254
255	if [ ! X${config_UserChoices_STORAGE} = X ]; then
256	#====================================================
257	#- STORAGE (dedicated to small/medium files)
258	STORAGE=${config_UserChoices_STORAGE}
259	else
260	#====================================================
261	#- STORAGE (dedicated to small/medium files)
262	STORAGE=${CCCWORKDIR}
263	fi
264
265	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
266	#====================================================
267	#- R_OUT
268	R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
269
270	#====================================================
271	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
272	R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
273
274	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
275
276	else
277	#====================================================
278	#- R_OUT
279	R_OUT=${ARCHIVE}/IGCM_OUT
280
281	#====================================================
282	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
283	R_FIG=${STORAGE}/IGCM_OUT
284	fi
285
286	#====================================================
287	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
288	CMIP6_BUF=${STORAGE}/IGCM_OUT
289
290	#====================================================
291	#- R_BUF (ONLY FOR double copy an scratch)
292	R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
293
294	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
295	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
296	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
297
298	IGCM_debug_PopStack "IGCM_sys_defineArchives"
299	}
300
301	#D-#==================================================
302	#D-function IGCM_sys_RshArchive
303	#D-* Purpose: Archive rsh command
304	#D-* Examples:
305	#D-
306	function IGCM_sys_RshArchive {
307	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
308	/bin/ksh <<-EOF
309	${@}
310	EOF
311	status=$?
312	if [ ${status} -gt 0 ] ; then
313	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
314	IGCM_debug_Exit "IGCM_sys_RshArchive"
315	fi
316	IGCM_debug_PopStack "IGCM_sys_RshArchive"
317	}
318
319	#D-#==================================================
320	#D-function IGCM_sys_RshArchive_NoError
321	#D-* Purpose: Archive rsh command, without error
322	#D-* used only in monitoring.job
323	#D-* Examples:
324	#D-
325	function IGCM_sys_RshArchive_NoError {
326	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
327	/bin/ksh <<-EOF
328	${@} 2> /dev/null
329	EOF
330	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
331	}
332
333	#D-#==================================================
334	#D-function IGCM_sys_MkdirArchive
335	#D-* Purpose: Mkdir on Archive
336	#D-* Examples:
337	#D-
338	function IGCM_sys_MkdirArchive {
339	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
340	if ( $DEBUG_sys ) ; then
341	echo "IGCM_sys_MkdirArchive :" $@
342	fi
343	#- creation de repertoire sur le serveur fichier
344	if [ ! -d ${1} ]; then
345	\mkdir -p $1
346	status=$?
347
348	if [ ${status} -gt 0 ] ; then
349	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
350	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
351	fi
352	fi
353	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
354	}
355
356	#D-#==================================================
357	#D-function IGCM_sys_TestDirArchive
358	#D-* Purpose: Test Directory that must exists on Archive
359	#D-* Examples:
360	#D-
361	function IGCM_sys_TestDirArchive {
362	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
363	if ( $DEBUG_sys ) ; then
364	echo "IGCM_sys_TestDirArchive :" $@
365	fi
366	typeset ExistFlag
367	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
368	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
369	return ${ExistFlag}
370	}
371
372	#D-#==================================================
373	#D-function IGCM_sys_IsFileArchived
374	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
375	#D-* Examples:
376	#D-
377	function IGCM_sys_IsFileArchived {
378	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
379	if ( $DEBUG_sys ) ; then
380	echo "IGCM_sys_IsFileArchived :" $@
381	fi
382	typeset IsArchivedFlag
383	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
384	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
385
386	return ${IsArchivedFlag}
387	}
388
389	#D-#==================================================
390	#D-function IGCM_sys_TestFileArchive
391	#D-* Purpose: Test file that must NOT EXISTS on Archive
392	#D-* Examples:
393	#D-
394	function IGCM_sys_TestFileArchive {
395	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
396	typeset ExistFlag
397	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
398	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
399
400	return ${ExistFlag}
401	}
402
403	#D-#==================================================
404	#D-function IGCM_sys_CountFileArchive
405	#D-* Purpose: Count files on Archive filesystem
406	#D-* Examples:
407	#D-
408	function IGCM_sys_CountFileArchive {
409	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
410	ls ${@} 2>/dev/null \| wc -l
411	if [ $? -gt 0 ] ; then
412	echo "IGCM_sys_CountFileArchive : erreur."
413	fi
414	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
415	}
416
417	#D-#==================================================
418	#D-function IGCM_sys_Tree
419	#D-* Purpose: Tree directories with files on ${ARCHIVE}
420	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
421	#D-
422	function IGCM_sys_Tree {
423	IGCM_debug_PushStack "IGCM_sys_Tree" $@
424	if ( $DEBUG_sys ) ; then
425	echo "IGCM_sys_Tree :" $@
426	fi
427
428	\ls -lR ${@}
429
430	IGCM_debug_PopStack "IGCM_sys_Tree"
431	}
432
433	#D-#==================================================
434	#D-function IGCM_sys_Qsub
435	#D-* Purpose: Qsub new job
436	#D-* Examples:
437	#D-
438	function IGCM_sys_Qsub {
439	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
440	if ( $DEBUG_sys ) ; then
441	echo "IGCM_sys_Qsub :" $@
442	fi
443	typeset options status
444	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
445
446	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
447	status=$?
448
449	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
450	if [ ${status} -gt 0 ] ; then
451	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
452	IGCM_debug_Exit "IGCM_sys_Qsub"
453	else
454	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
455	fi
456	IGCM_debug_PopStack "IGCM_sys_Qsub"
457	}
458
459	#D-#==================================================
460	#D-function IGCM_sys_QsubPost
461	#D-* Purpose: Qsub new job on scalaire
462	#D-* Examples:
463	#D-
464	function IGCM_sys_QsubPost {
465	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
466	if ( $DEBUG_sys ) ; then
467	echo "IGCM_sys_QsubPost :" $@
468	fi
469	typeset options status
470	options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
471
472	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
473	status=$?
474
475	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
476	if [ ${status} -gt 0 ] ; then
477	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
478	IGCM_debug_Exit "IGCM_sys_QsubPost"
479	else
480	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
481	fi
482	IGCM_debug_PopStack "IGCM_sys_QsubPost"
483	}
484
485	#D-*************************
486	#D- File transfer functions
487	#D-*************************
488	#D-
489
490	#D-#==================================================
491	#D-function IGCM_sys_RmRunDir
492	#D-* Purpose: rm tmpdir (dummy function most of the time batch
493	#D- scheduler will do the job)
494	#D-* Examples:
495	#D-
496	function IGCM_sys_RmRunDir {
497	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
498	if ( $DEBUG_sys ) ; then
499	echo "IGCM_sys_RmRunDir :" $@
500	fi
501
502	typeset status
503
504	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
505	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
506	status=$?
507
508	if [ ${status} -gt 0 ] ; then
509	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
510	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
511	IGCM_debug_Exit "IGCM_sys_RmRunDir"
512	else
513	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
514	fi
515	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
516	}
517
518	#D-#==================================================
519	#D-function IGCM_sys_Put_Dir
520	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
521	#D-* Examples:
522	#D-
523	function IGCM_sys_Put_Dir {
524	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
525	if ( $DEBUG_sys ) ; then
526	echo "IGCM_sys_Put_Dir :" $@
527	fi
528	if [ $DRYRUN = 0 ]; then
529	if [ ! -d ${1} ] ; then
530	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
531	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
532	return
533	fi
534
535	typeset status
536
537	# Only if we use rsync
538	#IGCM_sys_TestDirArchive $( dirname $2 )
539	#
540	#USUAL WAY
541	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
542	status=$?
543
544	if [ ${status} -gt 0 ] ; then
545	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
546	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
547	IGCM_debug_Exit "IGCM_sys_Put_Dir"
548	else
549	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
550	fi
551	fi
552	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
553	}
554
555	#D-#==================================================
556	#D-function IGCM_sys_Get_Dir
557	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
558	#D-* Examples:
559	#D-
560	function IGCM_sys_Get_Dir {
561	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
562	if ( $DEBUG_sys ) ; then
563	echo "IGCM_sys_Get_Dir :" $@
564	fi
565	if [ $DRYRUN = 0 ]; then
566	typeset NB_ESSAI DELAI status i
567	# number of tentative
568	NB_ESSAI=3
569	# time delay between tentative
570	DELAI=2
571
572	#
573	# USUAL WAY
574	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
575	ccc_hsm get -r $1
576
577	i=0
578	while [ $i -lt $NB_ESSAI ] ; do
579	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
580	status=$?
581	if [ ${status} -gt 0 ] ; then
582	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
583	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
584	sleep $DELAI
585	else
586	break
587	fi
588	(( i = i + 1 ))
589	done
590
591	if [ ${status} -gt 0 ] ; then
592	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
593	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
594	IGCM_debug_Exit "IGCM_sys_Get_Dir"
595	else
596	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
597	fi
598	fi
599	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
600	}
601
602	#D-#==================================================
603	#D-function IGCM_sys_Put_Rest
604	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
605	#D- File and target directory must exist.
606	#D-* Examples:
607	#D-
608	function IGCM_sys_Put_Rest {
609	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
610	if ( $DEBUG_sys ) ; then
611	echo "IGCM_sys_Put_Rest :" $@
612	fi
613	if [ $DRYRUN = 0 ]; then
614	if [ ! -f ${1} ] ; then
615	# The file is missing
616	if [ X${3} = X'OPTIONAL' ] ; then
617	echo "WARINING : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
618	IGCM_debug_Print 1 "IGCM_sys_Put_Rest: Optional restart file ${1} is missing, now continue for the next"
619	else
620	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
621	IGCM_debug_Exit "IGCM_sys_Put_Rest"
622	fi
623
624	else
625	# The file exist
626
627	typeset status
628	#
629	# USUAL WAY
630	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631	status=$?
632
633	# #RSYNC WITH NETWORK SSH CALL
634	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
635	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
636
637	# #RSYNC WITH NFS USE
638	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
639	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
640
641	# status=$?
642	# IGCM_sys_Rsync_out $status
643
644	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
645	# (( status=status+$? ))
646
647	if [ ${status} -gt 0 ] ; then
648	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
649	[ -f ${1} ] && ls -l ${1}
650	[ -f ${2} ] && ls -l ${2}
651	[ -f ${2}/${1} ] && ls -l ${2}/${1}
652	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
653	IGCM_debug_Exit "IGCM_sys_Put_Rest"
654	else
655
656	if [ X${JobType} = XRUN ] ; then
657	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
658	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
659	fi
660
661	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
662	fi
663	fi
664	fi
665	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
666	}
667
668	#D-#==================================================
669	#D-function IGCM_sys_Put_Out
670	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
671	#D-* Examples:
672	#D-
673	function IGCM_sys_Put_Out {
674	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
675	if ( $DEBUG_sys ) ; then
676	echo "IGCM_sys_Put_Out :" $@
677	fi
678
679	typeset NB_ESSAI DELAI status i exist skip
680	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
681
682	# number of tentative
683	NB_ESSAI=3
684	# time delay between tentative
685	DELAI=2
686
687	if [ $DRYRUN = 0 ]; then
688	if [ ! -f ${1} ] ; then
689	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
690	IGCM_debug_PopStack "IGCM_sys_Put_Out"
691	return 1
692	fi
693	#
694	IGCM_sys_MkdirArchive $( dirname $2 )
695	#
696	exist=false
697	skip=false
698	if [ -f $2 ] ; then
699	IGCM_debug_Print 1 "$2 already exist"
700	ccc_hsm get $2
701	exist=true
702	if [ "X$( diff $1 $2 )" = X ] ; then
703	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
704	skip=true
705	else
706	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
707	skip=false
708	fi
709	fi
710	#
711	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
712	IGCM_sys_Chmod u+w $2
713	fi
714
715	if [ X${skip} = Xfalse ] ; then
716	i=0
717	while [ $i -lt $NB_ESSAI ] ; do
718	# Identify file system
719	fileDeviceNumberInHex=$( stat -c %d $1 )
720	status=$?
721	if [ ${status} -gt 0 ] ; then
722	IGCM_debug_Exit "IGCM_sys_Put_Out"
723	fi
724	# Identify file system
725	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
726	status=$?
727	if [ ${status} -gt 0 ] ; then
728	IGCM_debug_Exit "IGCM_sys_Put_Out"
729	fi
730
731	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
732	# They are not on the same device. USUAL WAY
733	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
734	status=$?
735	else
736	# They are on the same device. NOT SO USUAL WAY
737	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
738	status=$?
739	fi
740	if [ ${status} -gt 0 ]; then
741	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
742	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
743	[ -f ${1} ] && ls -l ${1}
744	[ -f ${2} ] && ls -l ${2}
745	[ -f ${2}/${1} ] && ls -l ${2}/${1}
746	sleep $DELAI
747	else
748	break
749	fi
750	(( i = i + 1 ))
751	done
752	fi
753
754	# #RSYNC WITH NETWORK SSH CALL
755	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
756	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
757
758	# #RSYNC WITH NFS USE
759	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
760	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
761
762	# status=$?
763	# IGCM_sys_Rsync_out $status
764
765	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
766	# (( status=status+$? ))
767
768	if [ ${status} -gt 0 ] ; then
769	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
770	[ -f ${1} ] && ls -l ${1}
771	[ -f ${2} ] && ls -l ${2}
772	[ -f ${2}/${1} ] && ls -l ${2}/${1}
773	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
774	IGCM_debug_Exit "IGCM_sys_Put_Out"
775	else
776
777	if [ X${JobType} = XRUN ] ; then
778	if [ X${3} = X ] ; then
779	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
780	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
781	fi
782	fi
783
784	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
785	fi
786	fi
787	IGCM_debug_PopStack "IGCM_sys_Put_Out"
788	return 0
789	}
790
791	#D-#==================================================
792	#D-function IGCM_sys_Get
793	#D-* Purpose: Get a file from ${ARCHIVE}
794	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
795	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
796	function IGCM_sys_Get {
797	IGCM_debug_PushStack "IGCM_sys_Get" $@
798
799	typeset DEST dm_liste target file_work
800	typeset NB_ESSAI DELAI status i
801
802	if ( $DEBUG_sys ) ; then
803	echo "IGCM_sys_Get :" $@
804	fi
805
806	# number of tentative
807	NB_ESSAI=3
808	# time delay between tentative
809	DELAI=2
810
811	if [ $DRYRUN -le 2 ]; then
812	if [ X${1} = X'/l' ] ; then
813	eval set +A dm_liste \${${2}}
814	else
815	eval set +A dm_liste ${1}
816	fi
817	eval DEST=\${${#}}
818	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
819	status=$?
820	if [ ${status} -gt 0 ] ; then
821	echo "WARNING IGCM_sys_Get : error code ${status}"
822	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
823	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
824	fi
825
826	#if [ ${status} -gt 0 ] ; then
827	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
828	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
829	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
830	# sleep 30
831	# echo "We try another time"
832	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
833	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
834	# status=$?
835	# if [ ${status} -gt 0 ] ; then
836	# echo "ERROR IGCM_sys_Get : again demigration error :"
837	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
838	# IGCM_debug_Exit "IGCM_sys_Get"
839	# fi
840	# else
841	# echo "ERROR IGCM_sys_Get : demigration error :"
842	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
843	# IGCM_debug_Exit "IGCM_sys_Get"
844	# fi
845	#fi
846
847	# #RSYNC WITH NETWORK SSH CALL
848	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
849	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
850
851	# #RSYNC WITH NFS USE
852	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
853	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
854
855	# status=$?
856	# IGCM_sys_Rsync_out $status
857
858	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
859	# (( status=status+$? ))
860
861	#USUAL WAY
862	if [ X${1} = X'/l' ] ; then
863	for target in ${dm_liste[*]} ; do
864	local_file=$( basename ${target} )
865	# test if the target file is present before the loop
866	IGCM_sys_TestFileArchive ${target}
867	status=$?
868	if [ ${status} -gt 0 ] ; then
869	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
870	IGCM_debug_Exit "IGCM_sys_Get"
871	else
872	i=0
873	while [ $i -lt $NB_ESSAI ] ; do
874	#if [ X${DoLink} = Xtrue ] ; then
875	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
876	# status=$?
877	# else
878	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
879	# status=$?
880	#fi
881	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
882	status=$?
883	if [ ${status} -gt 0 ]; then
884	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
885	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
886	sleep $DELAI
887	else
888	break
889	fi
890	(( i = i + 1 ))
891	done
892	if [ ${status} -gt 0 ] ; then
893	echo "IGCM_sys_Get : error"
894	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
895	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
896	IGCM_debug_Exit "IGCM_sys_Get"
897	else
898	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
899	fi
900	fi
901	done
902	else
903	i=0
904	while [ $i -lt $NB_ESSAI ] ; do
905	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
906	status=$?
907	if [ ${status} -gt 0 ]; then
908	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
909	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
910	sleep $DELAI
911	else
912	break
913	fi
914	(( i = i + 1 ))
915	done
916	if [ ${status} -gt 0 ] ; then
917	echo "IGCM_sys_Get : error"
918	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
919	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
920	IGCM_debug_Exit "IGCM_sys_Get"
921	else
922	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
923	fi
924	fi
925	fi
926	IGCM_debug_PopStack "IGCM_sys_Get"
927	}
928
929	#D-#==================================================
930	#D-function IGCM_sys_GetDate_Monitoring
931	#D-* Purpose: get the last year for which the monitoring has been computed
932	#D-* Examples:
933	#D-
934	function IGCM_sys_GetDate_Monitoring {
935	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
936	if ( $DEBUG_sys ) ; then
937	echo "IGCM_sys_GetDate_Monitoring :" $@
938	fi
939
940	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
941
942	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
943	}
944
945	#D-#==================================================
946	#D-function IGCM_sys_Dods_Rm
947	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
948	#D-* Examples:
949	#D-
950	function IGCM_sys_Dods_Rm {
951	if ( $DEBUG_sys ) ; then
952	echo "IGCM_sys_Dods_Rm :" $@
953	fi
954	typeset status
955	if [ $DRYRUN = 0 ]; then
956
957	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
958	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
959	# echo "Nothing has been done."
960	# return
961	# fi
962
963	if [ "$#" -eq 1 ]; then
964	$(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
965	status=$?
966	else
967	$(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
968	status=$?
969	fi
970
971	# if [ ${status} -gt 0 ] ; then
972	# echo "IGCM_sys_Dods_Rm : error."
973	# cat out_dods_rm
974	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
975	# else
976	# rm out_dods_rm
977	# fi
978
979	fi
980	return $status
981	}
982
983	#D-#==================================================
984	#D-function IGCM_sys_Dods_Cp
985	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
986	#D-* Examples:
987	#D-
988	function IGCM_sys_Dods_Cp {
989	if ( $DEBUG_sys ) ; then
990	echo "IGCM_sys_Dods_Cp :" $@
991	fi
992	typeset status
993	if [ $DRYRUN = 0 ]; then
994
995	# if [ ! -d ${R_SAVE}/${1} ] ; then
996	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
997	# echo "Nothing has been done."
998	# return
999	# fi
1000
1001	$(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1002	status=$?
1003
1004	# if [ ${status} -gt 0 ] ; then
1005	# echo "IGCM_sys_Dods_Cp : error."
1006	# cat out_dods_cp
1007	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1008	# else
1009	# rm out_dods_cp
1010	# fi
1011
1012	fi
1013	return $status
1014	}
1015
1016	#D-#==================================================
1017	#D-function IGCM_sys_Put_Dods
1018	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1019	#D-* Examples:
1020	#D-
1021	function IGCM_sys_Put_Dods {
1022	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1023	if ( $DEBUG_sys ) ; then
1024	echo "IGCM_sys_Put_Dods :" $@
1025	fi
1026	typeset status
1027	if [ $DRYRUN = 0 ]; then
1028	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1029	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1030	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1031	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1032	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1033	return
1034	fi
1035
1036	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1037	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1038	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1039	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1040	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1041	return
1042	fi
1043	#
1044	if [ -d ${R_SAVE}/${1} ] ; then
1045	cd ${R_SAVE}
1046	elif [ -d ${R_FIGR}/${1} ] ; then
1047	cd ${R_FIGR}
1048	fi
1049
1050	IGCM_sys_Dods_Rm ${1}
1051	IGCM_sys_Dods_Cp ${1}
1052	status=0
1053
1054	if [ ${status} -gt 0 ] ; then
1055	echo "IGCM_sys_Put_Dods : error."
1056	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1057	fi
1058	fi
1059	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1060	}
1061
1062	##############################################################
1063	# REBUILD OPERATOR
1064
1065	#D-#==================================================
1066	#D-function IGCM_sys_sync
1067	#D-* Purpose: flush buffer on disk
1068	#D-* Examples:
1069	#D-
1070	function IGCM_sys_sync {
1071	IGCM_debug_PushStack "IGCM_sys_sync" $@
1072	if ( $DEBUG_sys ) ; then
1073	echo "IGCM_sys_sync :" $@
1074	fi
1075
1076	/bin/sync
1077
1078	IGCM_debug_PopStack "IGCM_sys_sync"
1079	}
1080
1081	############################################################
1082	# Activate Running Environnment Variables
1083
1084	#D-#==================================================
1085	#D-function IGCM_sys_activ_variables
1086	#D-* Purpose: set environement variables prior to execution
1087	#D-* Examples:
1088	#D-
1089	function IGCM_sys_activ_variables {
1090	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1091	if ( $DEBUG_sys ) ; then
1092	echo "IGCM_sys_activ_variables"
1093	fi
1094
1095	# --------------------------------------------------------------------
1096	#D- MODULE specifications
1097	# --------------------------------------------------------------------
1098	# Source the file EnvFile containing all module load needed to run the model.
1099	# EnvFile can be specified in config.card. If this is the case and if the file exists,
1100	# it will be used. Otherwise a default file will be used.
1101	if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1102	# EnvFile is set in config.card and the file exists
1103	IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1104	EnvFile=${config_UserChoices_EnvFile}
1105	else
1106	IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1107	module purge > /dev/null 2>&1
1108	if [ X${compilerVersion} = XV19 ] ; then
1109	IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V19"
1110	EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
1111	else
1112	IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1113	fi
1114	fi
1115	IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :"
1116	IGCM_debug_Print 1 ${EnvFile}
1117	. ${EnvFile}
1118
1119	IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene-amd. "
1120	module list
1121
1122	# --------------------------------------------------------------------
1123	#D- MPI specifications
1124	# --------------------------------------------------------------------
1125
1126	# --------------------------------------------------------------------
1127	#D- Other specifications
1128	# --------------------------------------------------------------------
1129
1130	ulimit -s unlimited
1131
1132	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1133	}
1134
1135	############################################################
1136	# Desactivate Running Environnment Variables
1137
1138	#D-#==================================================
1139	#D-function IGCM_sys_desactiv_variables
1140	#D-* Purpose: unset environement variables after execution
1141	#D-* Examples:
1142	#D-
1143	function IGCM_sys_desactiv_variables {
1144	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1145	if ( $DEBUG_sys ) ; then
1146	echo "IGCM_sys_desactiv_variables"
1147	fi
1148	# --------------------------------------------------------------------
1149	#D- MPI specifications
1150	# --------------------------------------------------------------------
1151
1152	# --------------------------------------------------------------------
1153	#D- Other specifications
1154	# --------------------------------------------------------------------
1155
1156	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1157	}
1158
1159	############################################################
1160	# Update job headers to be used by the scheduler
1161
1162	#D-#==================================================
1163	#D-function IGCM_sys_updateHeaders
1164	#D-* Purpose: Update job headers to be used by the scheduler
1165	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1166	#D-
1167	function IGCM_sys_updateHeaders {
1168	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1169	if ( $DEBUG_sys ) ; then
1170	echo "IGCM_sys_updateHeaders"
1171	fi
1172	typeset file
1173	file=$1
1174	# MPMD + MPI + OMP : mpirun/ccc_mprun/error
1175	(( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
1176	[ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
1177	sed -e "/::openMPthreads::/d" \
1178	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1179	-e "s/::NodeNumber::/${nodeNumber}/" \
1180	-e "/--cpu_bind=none/d" \
1181	${file} > ${file}.tmp
1182	IGCM_sys_Mv ${file}.tmp ${file}
1183
1184	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1185	}
1186
1187	############################################################
1188	# Build MPI/OMP scripts run file (dummy function)
1189
1190	#D-#==================================================
1191	#D-function IGCM_sys_build_run_file
1192	#D-* Purpose: build run file (deprecated)
1193	#D-* Examples:
1194	#D-
1195	function IGCM_sys_build_run_file {
1196	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1197	}
1198
1199	############################################################
1200	# Build MPI/OMP scripts
1201
1202	#D-#==================================================
1203	#D-function IGCM_sys_build_execution_scripts
1204	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1205	#D-* Examples:
1206	#D-
1207	function IGCM_sys_build_execution_scripts
1208	{
1209	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1210	if ( $DEBUG_sys ) ; then
1211	echo "IGCM_sys_build_execution_scripts " $@
1212	fi
1213	if ( [ "X${config_UserChoices_ExecutionMode}" = "Xslurm" ] ) ; then
1214
1215	EXECUTION="/usr/bin/time srun "
1216
1217	if ( ${OK_PARA_MPMD} ) ; then
1218
1219	# MPMD mode
1220	# 1 MPI only : executionType=1
1221	# 2 MPI/OpenMP : executionType=2
1222
1223	if [ -f run_file ] ; then
1224	IGCM_sys_Rm -f run_file
1225	fi
1226	if [ -f RUNDIR_2/run_file ] ; then
1227	IGCM_sys_Rm -f RUNDIR_2/run_file
1228	fi
1229	touch run_file
1230
1231	# case 1 : Only MPI (MPMD)
1232	# if ( ! ${OK_PARA_OMP} ) ; then
1233	# first_slurm_comp=0
1234	# # Build run_file
1235	# current_core=0
1236	# # First loop on the components for the coupler ie oasis (only if oasis3)
1237	# # the coupler ie oasis3 must be the first one
1238	# for comp in ${config_ListOfComponents[*]} ; do
1239	# number_rundir=$(echo ${comp} \| sed 's/[^0-9]*//g')
1240	# if [ X${number_rundir} != X ] ; then
1241	# [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
1242	# cd RUNDIR_${number_rundir}
1243	# if [ ${first_slurm_comp} = "0" ] ; then
1244	# current_core=0 ; first_slurm_comp=1 ;
1245	# fi
1246	# fi
1247	# eval ExeNameIn=\${config_Executable_${comp}[0]}
1248	# eval ExeNameOut=\${config_Executable_${comp}[1]}
1249
1250	# # Only if the component has an executable
1251	# if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1252
1253	# eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1254	# (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
1255	# echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
1256	# (( current_core = ${end_core} + 1 ))
1257	# fi
1258	# if [ X${number_rundir} != X ] ; then
1259	# cd $RUN_DIR ;
1260	# fi
1261	# done
1262
1263	# if [ X${number_rundir} != X ] ; then
1264	# echo "cd $RUN_DIR ; /usr/bin/time srun --ntasks=${config_UserChoices_NbMPItasks_run1} --multi-prog ./run_file > out_execution 2>&1 &" > EXECUTION.exe
1265	# echo "cd RUNDIR_2 ; /usr/bin/time srun --ntasks=${config_UserChoices_NbMPItasks_run2} --multi-prog ./run_file > out_execution 2>&1 &" >> EXECUTION.exe
1266	# echo "wait" >> EXECUTION.exe
1267	# chmod u+x EXECUTION.exe
1268	# else
1269	# EXECUTION="/usr/bin/time srun --multi-prog ./run_file"
1270	# fi
1271
1272	# IGCM_sys_Chmod u+x run_file
1273	# if ( $DEBUG_sys ) ; then
1274	# echo "run_file contains : "
1275	# cat run_file
1276	# fi
1277
1278	# else
1279
1280	# 2 MPI/OpenMP : executionType=2
1281
1282	# MPI-OpenMP (MPMD)
1283	# export SLURM_HOSTFILE=./hostlist
1284	# srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
1285	# example of hostlist file :
1286	# r3i3n33
1287	# r3i3n33
1288	# ...
1289	# example of run_file :
1290	# 0-70 ./prog_lmdz.x.sh %o %t
1291	# 71-430 ./prog_opa.xx.sh %o %t
1292	# 431-431 ./prog_xios.x.sh %o %t
1293	# examples of prog_file :
1294	# prog_lmdz.x.sh :
1295	# (( init = 0 + $1 ))
1296	# (( index = init * 10 ))
1297	# (( slot = index % 40 ))
1298	# taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
1299	# that will become
1300	# taskset -c 0-9 ./script_lmdz.x.ksh
1301	# ...
1302	# with script_lmdz.x.ksh
1303	# export OMP_STACKSIZE=3g
1304	# export OMP_PLACES=cores
1305	# export OMP_NUM_THREADS=10
1306	# ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
1307
1308	# Hosts treatment
1309	_bkIFS=$IFS;
1310	IFS=$'\n'; set -f
1311	listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1312	IFS=$_bkIFS; set +f
1313	rm -f hostlist
1314
1315	# Loop on the components to build run_file and script_exec files
1316	rank=0
1317	current_core=0
1318	current_core_mpi=0
1319	current_core_tmp=0
1320	current_core_mpi_tmp=0
1321	first_slurm_comp=0
1322	second_slurm_comp=0
1323
1324	for comp in ${config_ListOfComponents[*]} ; do
1325
1326	number_rundir=$(echo ${comp} \| sed 's/[^0-9]*//g')
1327	if [ X${number_rundir} != X ] ; then
1328	if [ X${number_rundir} = X2 ] ; then
1329	[ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
1330	cd RUNDIR_${number_rundir}
1331	if [ ${first_slurm_comp} = "0" ] ; then
1332	(( NbMPItasks_run1 = current_core_mpi_tmp ))
1333	### On change de noeud pour le prochain srun
1334	if [ $(( $current_core % $NB_CORE_PER_NODE )) -ne 0 ] ; then
1335	(( current_core = current_core + NB_CORE_PER_NODE - current_core % NB_CORE_PER_NODE ))
1336	fi
1337	current_core_tmp=0 ; current_core_mpi_tmp=0 ; first_slurm_comp=1 ;
1338	fi
1339	elif [ X${number_rundir} = X3 ] ; then
1340	[ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
1341	cd RUNDIR_${number_rundir}
1342	if [ ${second_slurm_comp} = "0" ] ; then
1343	(( NbMPItasks_run2 = current_core_mpi_tmp ))
1344	### On change de noeud pour le prochain srun
1345	if [ $(( $current_core % $NB_CORE_PER_NODE )) -ne 0 ] ; then
1346	(( current_core = current_core + NB_CORE_PER_NODE - current_core % NB_CORE_PER_NODE ))
1347	fi
1348	current_core_tmp=0 ; current_core_mpi_tmp=0 ; second_slurm_comp=1 ;
1349	fi
1350	fi
1351	fi
1352
1353	eval ExeNameIn=\${config_Executable_${comp}[0]}
1354	eval ExeNameOut=\${config_Executable_${comp}[1]}
1355
1356	# Not possible if oasis has an executable (i.e old version of oasis3)
1357	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1358	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1359	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1360	IGCM_debug_Verif_Exit
1361	fi
1362
1363	# Only if we really have an executable for the component :
1364	if [ "X${ExeNameOut}" != X\"\" ] ; then
1365
1366	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1367	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1368	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1369
1370
1371	# Build script files
1372
1373	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1374	echo "" >> script_${ExeNameOut}.ksh
1375	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1376	echo "export OMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1377	echo "export OMP_PLACES=cores" >> script_${ExeNameOut}.ksh
1378	echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1379	fi
1380
1381	# to have out/err per process on different files
1382	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1383
1384	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1385
1386	# Build run_file
1387	# Only if the component has an executable
1388	if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1389
1390	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1391	(( end_core = ${current_core_mpi_tmp} + ${comp_proc_mpi_loc} - 1 ))
1392	echo "${current_core_mpi_tmp}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
1393	(( current_core_mpi_tmp = ${end_core} + 1 ))
1394	fi
1395
1396	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1397	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1398	else
1399	(( offset_comp_proc_loc = comp_proc_omp_loc ))
1400	fi
1401
1402	# Build configuration file
1403
1404	echo "#!/bin/sh" > prog_${ExeNameOut}.sh
1405	echo "(( init = $current_core_tmp + \$1 ))" >> prog_${ExeNameOut}.sh
1406	echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
1407	echo "(( slot = index % $NB_CORE_PER_NODE ))" >> prog_${ExeNameOut}.sh
1408	echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
1409	echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
1410
1411	IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
1412
1413	# Build hostlist file
1414
1415	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1416	(( index_host = current_core / NB_CORE_PER_NODE ))
1417	host_value=${listnodes[${index_host}]}
1418	echo "$host_value" >> hostlist
1419	if [ ${DRYRUN_DEBUG} = 4 ] ; then
1420	echo "node_${index_host}_X" >> hostlist_template
1421	fi
1422	(( current_core = current_core + offset_comp_proc_loc ))
1423	(( current_core_tmp = current_core_tmp + offset_comp_proc_loc ))
1424	done
1425	fi
1426	if [ X${number_rundir} != X ] ; then
1427	cd $RUN_DIR ;
1428	fi
1429	done
1430
1431	## variable added to stop after 60s instead of 600s by default.
1432	## This is used when no error comes from executables and when something stopped an executable without notice.
1433	export SLURM_WAIT=60
1434
1435	if [ X${number_rundir} = X2 ] ; then
1436	echo "cd $RUN_DIR ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${NbMPItasks_run1} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution 2>&1 &" > EXECUTION.exe
1437	echo "cd RUNDIR_2 ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${current_core_mpi_tmp} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution 2>&1 &" >> EXECUTION.exe
1438	echo "wait" >> EXECUTION.exe
1439	chmod u+x EXECUTION.exe
1440	elif [ X${number_rundir} = X3 ] ; then
1441	echo "cd $RUN_DIR ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${NbMPItasks_run1} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution 2>&1 &" > EXECUTION.exe
1442	echo "cd RUNDIR_2 ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${NbMPItasks_run2} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution 2>&1 &" >> EXECUTION.exe
1443	echo "cd ../RUNDIR_3 ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${current_core_mpi_tmp} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution 2>&1 &" >> EXECUTION.exe
1444	echo "wait" >> EXECUTION.exe
1445	chmod u+x EXECUTION.exe
1446	else
1447	EXECUTION="/usr/bin/time srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1448	fi
1449	IGCM_sys_Chmod u+x run_file
1450	if ( $DEBUG_sys ) ; then
1451	echo "run_file contains : "
1452	cat run_file
1453	fi
1454
1455	# fi # if ${OK_PARA_MPMD}
1456
1457	else
1458	# Only one executable (SPMD mode): executionType=3, 4, 5 and 6
1459
1460	for comp in ${config_ListOfComponents[*]} ; do
1461
1462	# Only if we really have an executable for the component :
1463	eval ExeNameOut=\${config_Executable_${comp}[1]}
1464	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1465
1466	# Build script files
1467
1468	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1469	echo "" >> script_${ExeNameOut}.ksh
1470	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1471
1472	if ( ${OK_PARA_OMP} ) ; then
1473	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1474	# Check if the number of threads is correct
1475	case ${comp_proc_omp_loc} in
1476	2\|4\|5\|10\|20)
1477	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1478	;;
1479	*)
1480	IGCM_debug_Exit "ERROR with OMP parameters !"
1481	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1482	IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1483	IGCM_debug_Verif_Exit
1484	;;
1485	esac
1486	echo "" >> script_${ExeNameOut}.ksh
1487	echo "export OMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1488	echo "export OMP_PLACES=cores" >> script_${ExeNameOut}.ksh
1489	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1490	fi
1491
1492	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1493
1494	# To have out/err per process on different files
1495	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1496	EXECUTION="/usr/bin/time srun ./script_${ExeNameOut}.ksh"
1497
1498	IGCM_debug_Print 1 "sys Jean-Zay : script_${ExeNameOut}.ksh contains"
1499	cat script_${ExeNameOut}.ksh
1500
1501	fi
1502
1503	done
1504
1505	fi # ${OK_PARA_MPMD}
1506
1507	else
1508
1509	EXECUTION=${HOST_MPIRUN_COMMAND}
1510
1511	if ( ${OK_PARA_MPMD} ) ; then
1512
1513	# MPMD mode
1514	# 1 MPI only
1515	# 2 MPI/OpenMP mpirun method
1516	# 3 MPI/OpenMP ccc_mprun method
1517
1518	if [ -f run_file ] ; then
1519	IGCM_sys_Rm -f run_file
1520	fi
1521	touch run_file
1522
1523	# case 1 : Only MPI (MPMD)
1524	if ( ! ${OK_PARA_OMP} ) ; then
1525
1526	# Build run_file
1527
1528	# First loop on the components for the coupler ie oasis (only if oasis3)
1529	# the coupler ie oasis3 must be the first one
1530	for comp in ${config_ListOfComponents[*]} ; do
1531
1532	eval ExeNameIn=\${config_Executable_${comp}[0]}
1533	eval ExeNameOut=\${config_Executable_${comp}[1]}
1534
1535	# for CPL component only
1536	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1537
1538	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1539	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1540	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1541	fi
1542	done
1543
1544	# Then second loop on the components
1545	for comp in ${config_ListOfComponents[*]} ; do
1546
1547	eval ExeNameIn=\${config_Executable_${comp}[0]}
1548	eval ExeNameOut=\${config_Executable_${comp}[1]}
1549
1550	# Only if we really have an executable for the component and not the coupler ie oasis:
1551	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1552
1553	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1554	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1555	eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1556
1557	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1558	# Compute the number of core to add to force the use of a new node for xios servers
1559	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XIOS" ] ) ; then
1560	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1561	(( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_dep_loc ))
1562	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1563	(( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1564	if [ ${comp_proc_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1565	echo "1-${first_comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1566	if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1567	echo "${second_comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1568	fi
1569	else
1570	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1571	fi
1572	else
1573	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1574	fi
1575	else
1576	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1577	fi
1578
1579
1580	fi
1581	done
1582
1583	## module advised by TGCC (instead of 2 variables)
1584	module load feature/bridge/heterogenous_mpmd
1585
1586	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1587
1588	IGCM_sys_Chmod u+x run_file
1589	if ( $DEBUG_sys ) ; then
1590	echo "run_file contains : "
1591	cat run_file
1592	fi
1593
1594	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1595
1596	# 2 MPI/OpenMP mpirun method
1597
1598	# Use of mpirun instead of ccc_mprun
1599	EXECUTION="time mpirun"
1600
1601	# Hosts treatment
1602	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
1603
1604	i=0
1605	rm -f hosts rankfile
1606	IGCM_debug_Print 1 "sys Irene-amd, Hosts available :"
1607	for nodes in `cat hosts.tmp`
1608	do
1609	host[$i]=$nodes
1610	echo "${host[$i]}" >> hosts
1611	IGCM_debug_Print 1 ${host[$i]}
1612	i=$((i+1))
1613	done
1614	rm -f hosts.tmp
1615
1616	listnodes=${host[*]}
1617
1618	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1619
1620	# Initialisation
1621	rank=0
1622	current_core=0
1623	init_exec=n
1624
1625	# Loop on the components
1626	for comp in ${config_ListOfComponents[*]} ; do
1627
1628	eval ExeNameIn=\${config_Executable_${comp}[0]}
1629	eval ExeNameOut=\${config_Executable_${comp}[1]}
1630
1631	# Not possible if oasis has an executable (i.e old version of oasis3)
1632	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1633	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1634	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1635	IGCM_debug_Verif_Exit
1636	fi
1637
1638	# Only if we really have an executable for the component :
1639	if [ "X${ExeNameOut}" != X\"\" ] ; then
1640
1641	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1642	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1643	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1644
1645	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1646	echo "" >> script_${ExeNameOut}.ksh
1647	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1648
1649	# Check if the number of threads is correct
1650	case ${comp_proc_omp_loc} in
1651	2\|4\|8\|16)
1652	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1653	;;
1654	*)
1655	IGCM_debug_Exit "ERROR with OMP parameters !"
1656	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1657	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1658	IGCM_debug_Verif_Exit
1659	;;
1660	esac
1661	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1662	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1663	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1664	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1665	fi
1666
1667	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1668	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1669	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1670	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1671	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1672
1673	if [ ${init_exec} = y ] ; then
1674	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1675	else
1676	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1677	init_exec=y
1678	fi
1679
1680	# Build rankfile : method used to assign cores and nodes for the MPI process
1681	# Ex :
1682	#rank 0=curie5296 slot=0,1,2,3
1683	#rank 1=curie5296 slot=4,5,6,7
1684	# Example of final command :
1685	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1686	# with script_lmdz.x.ksh :
1687	# #!/bin/ksh
1688	#export KMP_STACKSIZE=3g
1689	#export KMP_LIBRARY=turnaround
1690	#export MKL_SERIAL=YES
1691	#OMP_NUM_THREADS=4
1692	#./lmdz.x
1693	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1694	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1695	else
1696	(( offset_comp_proc_loc = comp_proc_omp_loc ))
1697	fi
1698	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1699	(( index_host = current_core / NB_CORE_PER_NODE ))
1700	host_value=${host[${index_host}]}
1701	(( slot = current_core % NB_CORE_PER_NODE ))
1702	virg=","
1703	string_final=""
1704	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1705	string=$index$virg
1706	string_final=$string_final$string
1707	done
1708	string_final=$( echo $string_final \| sed "s/.$//" )
1709	echo "rank $rank=$host_value slot=$string_final" >> rankfile
1710	(( rank = rank + 1 ))
1711	(( current_core = current_core + offset_comp_proc_loc ))
1712	done
1713	fi
1714	done
1715
1716	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1717
1718	# 3 MPI/OpenMP ccc_mprun method
1719
1720	# MPI-OpenMP (MPMD)
1721	# example of run_file
1722	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1723	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1724	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1725
1726	# Loop on the components
1727	for comp in ${config_ListOfComponents[*]} ; do
1728
1729	eval ExeNameIn=\${config_Executable_${comp}[0]}
1730	eval ExeNameOut=\${config_Executable_${comp}[1]}
1731
1732	# Not possible if oasis has an executable (i.e old version of oasis3)
1733	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1734	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1735	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1736	IGCM_debug_Verif_Exit
1737	fi
1738
1739	# Only if we really have an executable for the component :
1740	if [ "X${ExeNameOut}" != X\"\" ] ; then
1741
1742	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1743	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1744	eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1745
1746	# Compute number of cores per process
1747	(( comp_proc_omp_dep_loc = comp_proc_omp_loc * comp_proc_dep_loc ))
1748
1749	# Build script files
1750
1751	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1752	echo "" >> script_${ExeNameOut}.ksh
1753	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1754
1755	# Check if the number of threads is correct
1756	case ${comp_proc_omp_loc} in
1757	2\|4\|8\|16)
1758	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1759	;;
1760	*)
1761	IGCM_debug_Exit "ERROR with OMP parameters !"
1762	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1763	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1764	IGCM_debug_Verif_Exit
1765	;;
1766	esac
1767	### only for LMDZ?
1768	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1769	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1770	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1771	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1772
1773	fi
1774
1775	# to have out/err per process on different files
1776	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1777	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1778
1779	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1780
1781	# Complete run_file
1782
1783	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1784	# Compute the number of core to add to force the use of a new node for xios servers
1785	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XIOS" ] ) ; then
1786	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1787	(( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_omp_dep_loc ))
1788	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1789	(( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1790	if [ ${comp_proc_omp_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1791	echo "1-${first_comp_proc_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1792	if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1793	echo "${second_comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1794	fi
1795	else
1796	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1797	fi
1798	else
1799	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1800	fi
1801	else
1802	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1803	fi
1804
1805	fi
1806	done
1807
1808	## variable added to stop after 60s instead of 600s by default.
1809	## This is used when no error comes from executables and when something stopped an executable without notice.
1810	export SLURM_WAIT=60
1811
1812	## module advised by TGCC (instead of 2 variables)
1813	module load feature/bridge/heterogenous_mpmd
1814
1815	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1816
1817	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1818
1819	IGCM_sys_Chmod u+x run_file
1820	if ( $DEBUG_sys ) ; then
1821	echo "run_file contains : "
1822	cat run_file
1823	fi
1824
1825	else
1826
1827	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1828
1829	fi # 1 2 or 3 if ${OK_PARA_MPMD}
1830
1831	else
1832	# Only one executable (SPMD mode).
1833
1834	for comp in ${config_ListOfComponents[*]} ; do
1835
1836	# Only if we really have an executable for the component :
1837	eval ExeNameOut=\${config_Executable_${comp}[1]}
1838	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1839
1840	# Build script files
1841
1842	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1843	echo "" >> script_${ExeNameOut}.ksh
1844	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1845
1846	if ( ${OK_PARA_OMP} ) ; then
1847	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1848	echo "" >> script_${ExeNameOut}.ksh
1849	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1850	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1851	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1852	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1853	fi
1854
1855	if ( ${OK_PARA_MPI} ) ; then
1856	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1857	# Default : ccc_mprun used if nb_proc gt 1
1858	# to have out/err per process on different files
1859	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1860	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1861	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1862	else
1863	# Default : ccc_mprun is NOT used if nb_proc eq 1
1864	# to have out/err per process on different files
1865	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1866	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1867	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1868	fi
1869
1870	IGCM_debug_Print 1 "sys Irene-amd : script_${ExeNameOut}.ksh contains"
1871	cat script_${ExeNameOut}.ksh
1872
1873	fi
1874
1875	done
1876
1877	fi # ${OK_PARA_MPMD}
1878
1879	fi
1880	IGCM_debug_Print 1 "sys Irene-amd : execution command is "
1881	IGCM_debug_Print 1 "$EXECUTION"
1882
1883	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1884	}
1885
1886	#D-#==================================================
1887	#D-function IGCM_sys_check_path
1888	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1889	#D-* do not point to an important use directory. Stop immediately in that case.
1890	#D-* Examples:
1891	#D-
1892	function IGCM_sys_check_path {
1893	IGCM_debug_PushStack "IGCM_sys_check_path"
1894	if ( $DEBUG_sys ) ; then
1895	echo "IGCM_sys_check_path"
1896	fi
1897
1898	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1899	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1900	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1901	IGCM_debug_Exit "This will stop the job"
1902	fi
1903	IGCM_debug_PopStack "IGCM_sys_check_path"
1904	}
1905
1906	#D-#==================================================
1907	#D-function IGCM_sys_check_quota
1908	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1909	#D-* Examples:
1910	#D-
1911	function IGCM_sys_check_quota {
1912	IGCM_debug_PushStack "IGCM_sys_check_quota"
1913	if ( $DEBUG_sys ) ; then
1914	echo "IGCM_sys_check_quota"
1915	fi
1916	# Limit of quota (in %)
1917	limit_quota=90
1918
1919	# Check of the volume
1920	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
1921	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
1922
1923	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1924
1925	unit_avail=${volume_avail: -1}
1926	unit_quota=${volume_quota: -1}
1927
1928	if [ "${unit_quota}" = "*" ] ; then
1929	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1930	IGCM_debug_Print 1 "More than 100% of your quota is used"
1931	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1932	IGCM_debug_Print 1 "You must have more than 10% available to run"
1933	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1934	IGCM_debug_Verif_Exit
1935	fi
1936
1937	temp_avail=${volume_avail%%${volume_avail: -1}*}
1938	temp_quota=${volume_quota%%${volume_quota: -1}*}
1939
1940	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1941
1942	# Convertion
1943	if [ ${volume_avail: -1} = "T" ] ; then
1944	(( temp_avail = temp_avail * 1000000000000 ))
1945	elif [ ${volume_avail: -1} = "G" ] ; then
1946	(( temp_avail = temp_avail * 1000000000 ))
1947	elif [ ${volume_avail: -1} = "M" ] ; then
1948	(( temp_avail = temp_avail * 1000000 ))
1949	elif [ ${volume_avail: -1} = "k" ] ; then
1950	(( temp_avail = temp_avail * 1000 ))
1951	else
1952	(( temp_avail = volume_avail ))
1953	fi
1954	if [ ${volume_quota: -1} = "T" ] ; then
1955	(( temp_quota = temp_quota * 1000000000000 ))
1956	elif [ ${volume_quota: -1} = "G" ] ; then
1957	(( temp_quota = temp_quota * 1000000000 ))
1958	elif [ ${volume_quota: -1} = "M" ] ; then
1959	(( temp_quota = temp_quota * 1000000 ))
1960	elif [ ${volume_quota: -1} = "k" ] ; then
1961	(( temp_quota = temp_quota * 1000 ))
1962	else
1963	(( temp_quota = volume_quota ))
1964	fi
1965	fi
1966
1967	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1968	# echo "volume ratio is " $quota_volume
1969
1970	if [ ${quota_volume} -ge ${limit_quota} ] ; then
1971	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1972	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1973	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1974	IGCM_debug_Print 1 "You must have more than 10% available to run"
1975	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1976	IGCM_debug_Verif_Exit
1977	fi
1978
1979	fi
1980
1981	# Check of the number of inodes
1982
1983	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
1984	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
1985
1986	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1987
1988	unit_avail=${inode_avail: -1}
1989	unit_quota=${inode_quota: -1}
1990
1991	if [ "${unit_quota}" = "*" ] ; then
1992	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1993	IGCM_debug_Print 1 "More than 100% of your quota is used"
1994	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1995	IGCM_debug_Print 1 "You must have more than 10% available to run"
1996	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1997	IGCM_debug_Verif_Exit
1998	fi
1999
2000	temp_avail=${inode_avail%%${inode_avail: -1}*}
2001	temp_quota=${inode_quota%%${inode_quota: -1}*}
2002
2003	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2004
2005	# Convertion
2006	if [ ${inode_avail: -1} = "T" ] ; then
2007	(( temp_avail = temp_avail * 1000000000000 ))
2008	elif [ ${inode_avail: -1} = "G" ] ; then
2009	(( temp_avail = temp_avail * 1000000000 ))
2010	elif [ ${inode_avail: -1} = "M" ] ; then
2011	(( temp_avail = temp_avail * 1000000 ))
2012	elif [ ${inode_avail: -1} = "k" ] ; then
2013	(( temp_avail = temp_avail * 1000 ))
2014	else
2015	(( temp_avail = inode_avail ))
2016	fi
2017
2018	if [ ${inode_quota: -1} = "T" ] ; then
2019	(( temp_quota = temp_quota * 1000000000000 ))
2020	elif [ ${inode_quota: -1} = "G" ] ; then
2021	(( temp_quota = temp_quota * 1000000000 ))
2022	elif [ ${inode_quota: -1} = "M" ] ; then
2023	(( temp_quota = temp_quota * 1000000 ))
2024	elif [ ${inode_quota: -1} = "k" ] ; then
2025	(( temp_quota = temp_quota * 1000 ))
2026	else
2027	(( temp_quota = inode_quota ))
2028	fi
2029	fi
2030	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2031	# echo "inode ratio is " $quota_inode
2032
2033	if [ ${quota_inode} -ge ${limit_quota} ] ; then
2034	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2035	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
2036	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2037	IGCM_debug_Print 1 "You must have more than 10% available to run"
2038	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2039	IGCM_debug_Verif_Exit
2040	fi
2041	fi
2042	IGCM_debug_PopStack "IGCM_sys_check_quota"
2043	}
2044
2045	#D-#==================================================
2046	#D-function IGCM_sys_projectAccounting
2047	#D-* Purpose: store project accounting information in a file
2048	#D-* Examples:
2049	#D-
2050	function IGCM_sys_projectAccounting {
2051	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
2052	if ( $DEBUG_sys ) ; then
2053	echo "IGCM_sys_check_quota"
2054	fi
2055
2056	ssh irene194 /usr/bin/ccc_myproject > $1
2057
2058	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
2059	}
2060
2061	#D-#==================================================
2062	#D-function IGCM_sys_getJobSchedulerID
2063	#D-* Purpose: Get the job ID during execution
2064	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
2065	#D-
2066	function IGCM_sys_getJobSchedulerID {
2067	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
2068	if ( $DEBUG_sys ) ; then
2069	echo "IGCM_sys_getJobSchedulerID"
2070	fi
2071	eval ${1}=${BRIDGE_MSUB_JOBID}
2072
2073	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
2074	}
2075
2076	#D-#==================================================
2077	#D-function IGCM_sys_GetJobID
2078	#D-* Purpose: Get the job ID from the JobName
2079	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
2080	#D-
2081	function IGCM_sys_GetJobID {
2082	IGCM_debug_PushStack "IGCM_sys_GetJobID"
2083	if ( $DEBUG_sys ) ; then
2084	echo "IGCM_sys_GetJobID"
2085	fi
2086
2087	# With -f option, the full job name is given in the last column
2088	ID=$( ccc_mstat -f -u $2 \| \
2089	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
2090
2091	eval ${3}=${ID}
2092	IGCM_debug_PopStack "IGCM_sys_GetJobID"
2093	}
2094
2095	#D-#==================================================
2096	#D-function IGCM_sys_CountJobInQueue
2097	#D-* Purpose: Count number of users job
2098	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
2099	#D-
2100	function IGCM_sys_CountJobInQueue {
2101	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
2102	if ( $DEBUG_sys ) ; then
2103	echo "IGCM_sys_CountJobInQueue"
2104	fi
2105
2106	# With -f option, the full job name is given in the last column
2107	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
2108
2109	eval ${2}=${NbRun}
2110
2111	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
2112	}
2113
2114	#D-#==================================================
2115	#D-function IGCM_sys_ListJobInQueue
2116	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
2117	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
2118	#D-
2119	function IGCM_sys_ListJobInQueue {
2120	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
2121	if ( $DEBUG_sys ) ; then
2122	echo "IGCM_sys_ListJobInQueue"
2123	fi
2124
2125	# With -f option, the full job name is given in the last column
2126	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
2127	'( $2 == User && \
2128	$NF != /TS/ && \
2129	$NF !~ /PACK/ && \
2130	$NF !~ /REBUILD/ && \
2131	$NF !~ /pack/ ) \
2132	{ print $NF }' \| sed -e "s/$.$\.[0-9]/\1/" )
2133
2134	eval set -A ${2} ${JobList[*]}
2135
2136	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
2137	}
2138
2139	#D-#==================================================
2140	#D-function IGCM_sys_atlas
2141	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
2142	#D-* Examples:
2143	#D-
2144	function IGCM_sys_atlas {
2145	IGCM_debug_PushStack "IGCM_sys_atlas" $@
2146	if ( $DEBUG_sys ) ; then
2147	echo "IGCM_sys_atlas :" $@
2148	fi
2149
2150	typeset status
2151
2152	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
2153	status=$?
2154	if [ ${status} -gt 0 ] ; then
2155	echo "IGCM_sys_atlas : error code ${status}"
2156	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
2157	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
2158	IGCM_debug_PopStack "IGCM_sys_atlas"
2159	return 1
2160	else
2161	IGCM_debug_PopStack "IGCM_sys_atlas"
2162	return 0
2163	fi
2164
2165	IGCM_debug_PopStack "IGCM_sys_atlas"
2166	}
2167
2168	#D-#==================================================
2169	#D-function IGCM_sys_rebuild_nemo
2170	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
2171	#D-* Examples:
2172	#D-
2173
2174	function IGCM_sys_rebuild_nemo {
2175	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
2176	if ( $DEBUG_sys ) ; then
2177	echo "IGCM_sys_rebuild_nemo :" $@
2178	fi
2179
2180	$( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
2181
2182	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
2183	}
2184

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: branches/libIGCM_concurrent/libIGCM_sys/libIGCM_sys_irene-amd.ksh @ 1638

Download in other formats: