Context Navigation

source: branches/libIGCM_CESMEP/libIGCM_sys/libIGCM_sys_irene-amd.ksh @ 1564

Last change on this file since 1564 was 1563, checked in by ssenesi, 19 months ago
Better gues default_project on irene(-amd) in libIGCM_sys
File size: 65.8 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: 1512 $ Revision of last commit
7	# $Author:: acosce $ Author of last commit
8	# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Irene-amd
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host user names project maxCpuTime
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=irene-amd
71	# add default project on irene-amd
72	if [ ! -z $BRIDGE_MSUB_PROJECT ]; then
73	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT} \| cut -d@ -f1 )
74	else
75	typeset PROJECT=$(echo ${CCCHOME} \| cut -d/ -f5 )
76	fi
77	# jobWarningDelay in seconds
78	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
79
80	#D-
81	#D-#==================================================
82	#D-Program used in libIGCM
83	#D-#==================================================
84
85	# Submit command
86	typeset SUBMIT=${SUBMIT:=ccc_msub}
87	# rsync with path
88	typeset -r RSYNC=/usr/bin/rsync
89	# RSYNC_opt args to rsync
90	typeset -r RSYNC_opt="-va"
91	# ie storage filesystem
92	typeset -r STOREHOST=${MASTER}
93	typeset -r REMOTE_RSYNC=/usr/bin/rsync
94
95	#====================================================
96	# Set environment tools (ferret, nco, cdo, rebuild, ...)
97	#====================================================
98	if [ X${TaskType} = Xcomputing ] ; then
99	IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
100	else
101	module purge > /dev/null 2>&1
102	. $( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
103	. $( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_atlas_irene-amd > /dev/null 2>&1
104
105	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
106	export UVCDAT_ANONYMOUS_LOG=FALSE
107	fi
108
109	# FYI
110	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
111	[ ! X${TaskType} = Xchecking ] && module list
112
113	# For AddNoise and AddPertu3DOCE
114	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene-amd/bin
115
116	# For rebuild
117	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene-amd/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
118
119	#====================================================
120	# Host specific DIRECTORIES
121	#====================================================
122
123	# ============ CESIUM START ============ #
124
125	#====================================================
126	#- Mirror libIGCM from titane to cesium if needed
127	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
128	#if [ ! ${ROOTSYS} = "home" ] ; then
129	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
130	#else
131	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
132	#fi
133
134	#====================================================
135	#- libIGCM_POST
136	#if ( ${MirrorlibIGCM} ) ; then
137	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
138	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
139	#else
140	# typeset -r libIGCM_POST=${libIGCM}
141	#fi
142
143	# ============ CESIUM END ============ #
144
145	#====================================================
146	#- MirrorlibIGCM for frontend
147	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
148
149	#====================================================
150	#- libIGCM_POST for frontend
151	typeset -r libIGCM_POST=${libIGCM}
152
153	#====================================================
154	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
155	typeset -r R_EXE="${MODIPSL}/bin"
156
157	#====================================================
158	#- SUBMIT_DIR : submission dir
159	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
160
161	#====================================================
162	#- IN
163	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
164
165	#====================================================
166	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
167	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
168
169	#====================================================
170	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
171	typeset -r OUTCOMMAND_PATH=/tmp
172
173	#====================================================
174	#- HOST_MPIRUN_COMMAND
175	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
176
177	#====================================================
178	#- Max number of arguments passed to nco operator or demigration command
179	UNIX_MAX_LIMIT=360
180
181	#====================================================
182	#- set PackDefault to true on irene-amd
183	PackDefault=true
184
185	#====================================================
186	#- Default number of MPI task for IPSL coupled model
187	#- required for backward compatibility
188	#-
189	DEFAULT_NUM_PROC_OCE=5
190	DEFAULT_NUM_PROC_CPL=1
191	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
192
193	#====================================================
194	#- Number of computing cores per node
195	#-
196	NB_CORE_PER_NODE=128
197
198	#D-#==================================================
199	#D-function IGCM_sys_defineArchives
200	#D-* Purpose:
201	#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
202	#D-* Define ARCHIVE : Dedicated to large files
203	#D-* Define STORAGE : Dedicated to small/medium files
204	#D-* Define R_OUT : Output tree located on ARCHIVE
205	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
206	#D-* Define R_BUF : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
207	#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
208	#D-* Examples:
209	#D-
210	function IGCM_sys_defineArchives {
211	IGCM_debug_PushStack "IGCM_sys_defineArchives"
212
213	# Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
214	if [ X${config_UserChoices_DataProject} = X ] \|\| [ X${config_UserChoices_DataProject} = DEFAULT ]; then
215	# Default option: Change the dfldatadir according to the project used for submission
216	# The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
217	if [ X"$( echo ${PROJECT} \| grep cmip6 )" != "X" ] ; then
218	module switch dfldatadir dfldatadir/gencmip6
219	IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
220	else
221	module switch dfldatadir dfldatadir/${PROJECT}
222	IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
223	fi
224	else
225	# Use the project set by the variable DataProject in section UserChoices in config.card
226	module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
227	IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
228	fi
229
230	#====================================================
231	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
232	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
233
234	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
235	#====================================================
236	#- ARCHIVE (dedicated to large files)
237	ARCHIVE=${config_UserChoices_ARCHIVE}
238	else
239	#====================================================
240	#- ARCHIVE (dedicated to large files)
241	ARCHIVE=${CCCSTOREDIR}
242	fi
243
244	if [ ! X${config_UserChoices_STORAGE} = X ]; then
245	#====================================================
246	#- STORAGE (dedicated to small/medium files)
247	STORAGE=${config_UserChoices_STORAGE}
248	else
249	#====================================================
250	#- STORAGE (dedicated to small/medium files)
251	STORAGE=${CCCWORKDIR}
252	fi
253
254	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
255	#====================================================
256	#- R_OUT
257	R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
258
259	#====================================================
260	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
261	R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
262
263	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
264
265	else
266	#====================================================
267	#- R_OUT
268	R_OUT=${ARCHIVE}/IGCM_OUT
269
270	#====================================================
271	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
272	R_FIG=${STORAGE}/IGCM_OUT
273	fi
274
275	#====================================================
276	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
277	CMIP6_BUF=${STORAGE}/IGCM_OUT
278
279	#====================================================
280	#- R_BUF (ONLY FOR double copy an scratch)
281	R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
282
283	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
284	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
285	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
286
287	IGCM_debug_PopStack "IGCM_sys_defineArchives"
288	}
289
290	#D-#==================================================
291	#D-function IGCM_sys_RshArchive
292	#D-* Purpose: Archive rsh command
293	#D-* Examples:
294	#D-
295	function IGCM_sys_RshArchive {
296	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
297	/bin/ksh <<-EOF
298	${@}
299	EOF
300	status=$?
301	if [ ${status} -gt 0 ] ; then
302	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
303	IGCM_debug_Exit "IGCM_sys_RshArchive"
304	fi
305	IGCM_debug_PopStack "IGCM_sys_RshArchive"
306	}
307
308	#D-#==================================================
309	#D-function IGCM_sys_RshArchive_NoError
310	#D-* Purpose: Archive rsh command, without error
311	#D-* used only in monitoring.job
312	#D-* Examples:
313	#D-
314	function IGCM_sys_RshArchive_NoError {
315	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
316	/bin/ksh <<-EOF
317	${@} 2> /dev/null
318	EOF
319	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
320	}
321
322	#D-#==================================================
323	#D-function IGCM_sys_MkdirArchive
324	#D-* Purpose: Mkdir on Archive
325	#D-* Examples:
326	#D-
327	function IGCM_sys_MkdirArchive {
328	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
329	if ( $DEBUG_sys ) ; then
330	echo "IGCM_sys_MkdirArchive :" $@
331	fi
332	#- creation de repertoire sur le serveur fichier
333	if [ ! -d ${1} ]; then
334	\mkdir -p $1
335	status=$?
336
337	if [ ${status} -gt 0 ] ; then
338	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
339	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
340	fi
341	fi
342	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
343	}
344
345	#D-#==================================================
346	#D-function IGCM_sys_TestDirArchive
347	#D-* Purpose: Test Directory that must exists on Archive
348	#D-* Examples:
349	#D-
350	function IGCM_sys_TestDirArchive {
351	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
352	if ( $DEBUG_sys ) ; then
353	echo "IGCM_sys_TestDirArchive :" $@
354	fi
355	typeset ExistFlag
356	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
357	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
358	return ${ExistFlag}
359	}
360
361	#D-#==================================================
362	#D-function IGCM_sys_IsFileArchived
363	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
364	#D-* Examples:
365	#D-
366	function IGCM_sys_IsFileArchived {
367	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
368	if ( $DEBUG_sys ) ; then
369	echo "IGCM_sys_IsFileArchived :" $@
370	fi
371	typeset IsArchivedFlag
372	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
373	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
374
375	return ${IsArchivedFlag}
376	}
377
378	#D-#==================================================
379	#D-function IGCM_sys_TestFileArchive
380	#D-* Purpose: Test file that must NOT EXISTS on Archive
381	#D-* Examples:
382	#D-
383	function IGCM_sys_TestFileArchive {
384	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
385	typeset ExistFlag
386	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
387	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
388
389	return ${ExistFlag}
390	}
391
392	#D-#==================================================
393	#D-function IGCM_sys_CountFileArchive
394	#D-* Purpose: Count files on Archive filesystem
395	#D-* Examples:
396	#D-
397	function IGCM_sys_CountFileArchive {
398	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
399	ls ${@} 2>/dev/null \| wc -l
400	if [ $? -gt 0 ] ; then
401	echo "IGCM_sys_CountFileArchive : erreur."
402	fi
403	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
404	}
405
406	#D-#==================================================
407	#D-function IGCM_sys_Tree
408	#D-* Purpose: Tree directories with files on ${ARCHIVE}
409	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
410	#D-
411	function IGCM_sys_Tree {
412	IGCM_debug_PushStack "IGCM_sys_Tree" $@
413	if ( $DEBUG_sys ) ; then
414	echo "IGCM_sys_Tree :" $@
415	fi
416
417	\ls -lR ${@}
418
419	IGCM_debug_PopStack "IGCM_sys_Tree"
420	}
421
422	#D-#==================================================
423	#D-function IGCM_sys_Qsub
424	#D-* Purpose: Qsub new job
425	#D-* Examples:
426	#D-
427	function IGCM_sys_Qsub {
428	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
429	if ( $DEBUG_sys ) ; then
430	echo "IGCM_sys_Qsub :" $@
431	fi
432	typeset options status
433	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
434
435	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
436	status=$?
437
438	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
439	if [ ${status} -gt 0 ] ; then
440	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
441	IGCM_debug_Exit "IGCM_sys_Qsub"
442	else
443	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
444	fi
445	IGCM_debug_PopStack "IGCM_sys_Qsub"
446	}
447
448	#D-#==================================================
449	#D-function IGCM_sys_QsubPost
450	#D-* Purpose: Qsub new job on scalaire
451	#D-* Examples:
452	#D-
453	function IGCM_sys_QsubPost {
454	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
455	if ( $DEBUG_sys ) ; then
456	echo "IGCM_sys_QsubPost :" $@
457	fi
458	typeset options status
459	options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
460
461	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
462	status=$?
463
464	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
465	if [ ${status} -gt 0 ] ; then
466	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
467	IGCM_debug_Exit "IGCM_sys_QsubPost"
468	else
469	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
470	fi
471	IGCM_debug_PopStack "IGCM_sys_QsubPost"
472	}
473
474	#D-*************************
475	#D- File transfer functions
476	#D-*************************
477	#D-
478
479	#D-#==================================================
480	#D-function IGCM_sys_RmRunDir
481	#D-* Purpose: rm tmpdir (dummy function most of the time batch
482	#D- scheduler will do the job)
483	#D-* Examples:
484	#D-
485	function IGCM_sys_RmRunDir {
486	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
487	if ( $DEBUG_sys ) ; then
488	echo "IGCM_sys_RmRunDir :" $@
489	fi
490
491	typeset status
492
493	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
494	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
495	status=$?
496
497	if [ ${status} -gt 0 ] ; then
498	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
499	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
500	IGCM_debug_Exit "IGCM_sys_RmRunDir"
501	else
502	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
503	fi
504	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
505	}
506
507	#D-#==================================================
508	#D-function IGCM_sys_Put_Dir
509	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
510	#D-* Examples:
511	#D-
512	function IGCM_sys_Put_Dir {
513	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
514	if ( $DEBUG_sys ) ; then
515	echo "IGCM_sys_Put_Dir :" $@
516	fi
517	if [ $DRYRUN = 0 ]; then
518	if [ ! -d ${1} ] ; then
519	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
520	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
521	return
522	fi
523
524	typeset status
525
526	# Only if we use rsync
527	#IGCM_sys_TestDirArchive $( dirname $2 )
528	#
529	#USUAL WAY
530	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
531	status=$?
532
533	if [ ${status} -gt 0 ] ; then
534	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
535	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
536	IGCM_debug_Exit "IGCM_sys_Put_Dir"
537	else
538	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
539	fi
540	fi
541	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
542	}
543
544	#D-#==================================================
545	#D-function IGCM_sys_Get_Dir
546	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
547	#D-* Examples:
548	#D-
549	function IGCM_sys_Get_Dir {
550	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
551	if ( $DEBUG_sys ) ; then
552	echo "IGCM_sys_Get_Dir :" $@
553	fi
554	if [ $DRYRUN = 0 ]; then
555	typeset NB_ESSAI DELAI status i
556	# number of tentative
557	NB_ESSAI=3
558	# time delay between tentative
559	DELAI=2
560
561	#
562	# USUAL WAY
563	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
564	ccc_hsm get -r $1
565
566	i=0
567	while [ $i -lt $NB_ESSAI ] ; do
568	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
569	status=$?
570	if [ ${status} -gt 0 ] ; then
571	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
572	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
573	sleep $DELAI
574	else
575	break
576	fi
577	(( i = i + 1 ))
578	done
579
580	if [ ${status} -gt 0 ] ; then
581	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
582	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
583	IGCM_debug_Exit "IGCM_sys_Get_Dir"
584	else
585	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
586	fi
587	fi
588	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
589	}
590
591	#D-#==================================================
592	#D-function IGCM_sys_Put_Rest
593	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
594	#D- File and target directory must exist.
595	#D-* Examples:
596	#D-
597	function IGCM_sys_Put_Rest {
598	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
599	if ( $DEBUG_sys ) ; then
600	echo "IGCM_sys_Put_Rest :" $@
601	fi
602	if [ $DRYRUN = 0 ]; then
603	if [ ! -f ${1} ] ; then
604	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
605	IGCM_debug_Exit "IGCM_sys_Put_Rest"
606	fi
607
608	typeset status
609	#
610	# USUAL WAY
611	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
612	status=$?
613
614	# #RSYNC WITH NETWORK SSH CALL
615	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
616	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
617
618	# #RSYNC WITH NFS USE
619	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
620	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
621
622	# status=$?
623	# IGCM_sys_Rsync_out $status
624
625	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
626	# (( status=status+$? ))
627
628	if [ ${status} -gt 0 ] ; then
629	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
630	[ -f ${1} ] && ls -l ${1}
631	[ -f ${2} ] && ls -l ${2}
632	[ -f ${2}/${1} ] && ls -l ${2}/${1}
633	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
634	IGCM_debug_Exit "IGCM_sys_Put_Rest"
635	else
636
637	if [ X${JobType} = XRUN ] ; then
638	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
639	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
640	fi
641
642	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
643	fi
644	fi
645	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
646	}
647
648	#D-#==================================================
649	#D-function IGCM_sys_Put_Out
650	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
651	#D-* Examples:
652	#D-
653	function IGCM_sys_Put_Out {
654	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
655	if ( $DEBUG_sys ) ; then
656	echo "IGCM_sys_Put_Out :" $@
657	fi
658
659	typeset NB_ESSAI DELAI status i exist skip
660	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
661
662	# number of tentative
663	NB_ESSAI=3
664	# time delay between tentative
665	DELAI=2
666
667	if [ $DRYRUN = 0 ]; then
668	if [ ! -f ${1} ] ; then
669	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
670	IGCM_debug_PopStack "IGCM_sys_Put_Out"
671	return 1
672	fi
673	#
674	IGCM_sys_MkdirArchive $( dirname $2 )
675	#
676	exist=false
677	skip=false
678	if [ -f $2 ] ; then
679	IGCM_debug_Print 1 "$2 already exist"
680	ccc_hsm get $2
681	exist=true
682	if [ "X$( diff $1 $2 )" = X ] ; then
683	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
684	skip=true
685	else
686	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
687	skip=false
688	fi
689	fi
690	#
691	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
692	IGCM_sys_Chmod u+w $2
693	fi
694
695	if [ X${skip} = Xfalse ] ; then
696	i=0
697	while [ $i -lt $NB_ESSAI ] ; do
698	# Identify file system
699	fileDeviceNumberInHex=$( stat -c %d $1 )
700	status=$?
701	if [ ${status} -gt 0 ] ; then
702	IGCM_debug_Exit "IGCM_sys_Put_Out"
703	fi
704	# Identify file system
705	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
706	status=$?
707	if [ ${status} -gt 0 ] ; then
708	IGCM_debug_Exit "IGCM_sys_Put_Out"
709	fi
710
711	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
712	# They are not on the same device. USUAL WAY
713	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
714	status=$?
715	else
716	# They are on the same device. NOT SO USUAL WAY
717	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
718	status=$?
719	fi
720	if [ ${status} -gt 0 ]; then
721	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
722	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
723	[ -f ${1} ] && ls -l ${1}
724	[ -f ${2} ] && ls -l ${2}
725	[ -f ${2}/${1} ] && ls -l ${2}/${1}
726	sleep $DELAI
727	else
728	break
729	fi
730	(( i = i + 1 ))
731	done
732	fi
733
734	# #RSYNC WITH NETWORK SSH CALL
735	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
736	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
737
738	# #RSYNC WITH NFS USE
739	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
740	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
741
742	# status=$?
743	# IGCM_sys_Rsync_out $status
744
745	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
746	# (( status=status+$? ))
747
748	if [ ${status} -gt 0 ] ; then
749	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
750	[ -f ${1} ] && ls -l ${1}
751	[ -f ${2} ] && ls -l ${2}
752	[ -f ${2}/${1} ] && ls -l ${2}/${1}
753	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
754	IGCM_debug_Exit "IGCM_sys_Put_Out"
755	else
756
757	if [ X${JobType} = XRUN ] ; then
758	if [ X${3} = X ] ; then
759	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
760	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
761	fi
762	fi
763
764	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
765	fi
766	fi
767	IGCM_debug_PopStack "IGCM_sys_Put_Out"
768	return 0
769	}
770
771	#D-#==================================================
772	#D-function IGCM_sys_Get
773	#D-* Purpose: Get a file from ${ARCHIVE}
774	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
775	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
776	function IGCM_sys_Get {
777	IGCM_debug_PushStack "IGCM_sys_Get" $@
778
779	typeset DEST dm_liste target file_work
780	typeset NB_ESSAI DELAI status i
781
782	if ( $DEBUG_sys ) ; then
783	echo "IGCM_sys_Get :" $@
784	fi
785
786	# number of tentative
787	NB_ESSAI=3
788	# time delay between tentative
789	DELAI=2
790
791	if [ $DRYRUN -le 2 ]; then
792	if [ X${1} = X'/l' ] ; then
793	eval set +A dm_liste \${${2}}
794	else
795	eval set +A dm_liste ${1}
796	fi
797	eval DEST=\${${#}}
798	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
799	status=$?
800	if [ ${status} -gt 0 ] ; then
801	echo "WARNING IGCM_sys_Get : error code ${status}"
802	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
803	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
804	fi
805
806	#if [ ${status} -gt 0 ] ; then
807	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
808	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
809	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
810	# sleep 30
811	# echo "We try another time"
812	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
813	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
814	# status=$?
815	# if [ ${status} -gt 0 ] ; then
816	# echo "ERROR IGCM_sys_Get : again demigration error :"
817	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
818	# IGCM_debug_Exit "IGCM_sys_Get"
819	# fi
820	# else
821	# echo "ERROR IGCM_sys_Get : demigration error :"
822	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
823	# IGCM_debug_Exit "IGCM_sys_Get"
824	# fi
825	#fi
826
827	# #RSYNC WITH NETWORK SSH CALL
828	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
829	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
830
831	# #RSYNC WITH NFS USE
832	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
833	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
834
835	# status=$?
836	# IGCM_sys_Rsync_out $status
837
838	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
839	# (( status=status+$? ))
840
841	#USUAL WAY
842	if [ X${1} = X'/l' ] ; then
843	for target in ${dm_liste[*]} ; do
844	local_file=$( basename ${target} )
845	# test if the target file is present before the loop
846	IGCM_sys_TestFileArchive ${target}
847	status=$?
848	if [ ${status} -gt 0 ] ; then
849	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
850	IGCM_debug_Exit "IGCM_sys_Get"
851	else
852	i=0
853	while [ $i -lt $NB_ESSAI ] ; do
854	#if [ X${DoLink} = Xtrue ] ; then
855	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
856	# status=$?
857	# else
858	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
859	# status=$?
860	#fi
861	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
862	status=$?
863	if [ ${status} -gt 0 ]; then
864	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
865	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
866	sleep $DELAI
867	else
868	break
869	fi
870	(( i = i + 1 ))
871	done
872	if [ ${status} -gt 0 ] ; then
873	echo "IGCM_sys_Get : error"
874	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
875	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
876	IGCM_debug_Exit "IGCM_sys_Get"
877	else
878	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
879	fi
880	fi
881	done
882	else
883	i=0
884	while [ $i -lt $NB_ESSAI ] ; do
885	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
886	status=$?
887	if [ ${status} -gt 0 ]; then
888	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
889	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
890	sleep $DELAI
891	else
892	break
893	fi
894	(( i = i + 1 ))
895	done
896	if [ ${status} -gt 0 ] ; then
897	echo "IGCM_sys_Get : error"
898	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
899	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
900	IGCM_debug_Exit "IGCM_sys_Get"
901	else
902	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
903	fi
904	fi
905	fi
906	IGCM_debug_PopStack "IGCM_sys_Get"
907	}
908
909	#D-#==================================================
910	#D-function IGCM_sys_GetDate_Monitoring
911	#D-* Purpose: get the last year for which the monitoring has been computed
912	#D-* Examples:
913	#D-
914	function IGCM_sys_GetDate_Monitoring {
915	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
916	if ( $DEBUG_sys ) ; then
917	echo "IGCM_sys_GetDate_Monitoring :" $@
918	fi
919
920	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
921
922	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
923	}
924
925	#D-#==================================================
926	#D-function IGCM_sys_Dods_Rm
927	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
928	#D-* Examples:
929	#D-
930	function IGCM_sys_Dods_Rm {
931	if ( $DEBUG_sys ) ; then
932	echo "IGCM_sys_Dods_Rm :" $@
933	fi
934	typeset status
935	if [ $DRYRUN = 0 ]; then
936
937	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
938	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
939	# echo "Nothing has been done."
940	# return
941	# fi
942
943	if [ "$#" -eq 1 ]; then
944	$(ccc_home -u igcmg)/Tools/irene-amd/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
945	status=$?
946	else
947	$(ccc_home -u igcmg)/Tools/irene-amd/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
948	status=$?
949	fi
950
951	# if [ ${status} -gt 0 ] ; then
952	# echo "IGCM_sys_Dods_Rm : error."
953	# cat out_dods_rm
954	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
955	# else
956	# rm out_dods_rm
957	# fi
958
959	fi
960	return $status
961	}
962
963	#D-#==================================================
964	#D-function IGCM_sys_Dods_Cp
965	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
966	#D-* Examples:
967	#D-
968	function IGCM_sys_Dods_Cp {
969	if ( $DEBUG_sys ) ; then
970	echo "IGCM_sys_Dods_Cp :" $@
971	fi
972	typeset status
973	if [ $DRYRUN = 0 ]; then
974
975	# if [ ! -d ${R_SAVE}/${1} ] ; then
976	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
977	# echo "Nothing has been done."
978	# return
979	# fi
980
981	$(ccc_home -u igcmg)/Tools/irene-amd/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
982	status=$?
983
984	# if [ ${status} -gt 0 ] ; then
985	# echo "IGCM_sys_Dods_Cp : error."
986	# cat out_dods_cp
987	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
988	# else
989	# rm out_dods_cp
990	# fi
991
992	fi
993	return $status
994	}
995
996	#D-#==================================================
997	#D-function IGCM_sys_Put_Dods
998	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
999	#D-* Examples:
1000	#D-
1001	function IGCM_sys_Put_Dods {
1002	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1003	if ( $DEBUG_sys ) ; then
1004	echo "IGCM_sys_Put_Dods :" $@
1005	fi
1006	typeset status
1007	if [ $DRYRUN = 0 ]; then
1008	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1009	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1010	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1011	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1012	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1013	return
1014	fi
1015
1016	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1017	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1018	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1019	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1020	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1021	return
1022	fi
1023	#
1024	if [ -d ${R_SAVE}/${1} ] ; then
1025	cd ${R_SAVE}
1026	elif [ -d ${R_FIGR}/${1} ] ; then
1027	cd ${R_FIGR}
1028	fi
1029
1030	IGCM_sys_Dods_Rm ${1}
1031	IGCM_sys_Dods_Cp ${1}
1032	status=0
1033
1034	if [ ${status} -gt 0 ] ; then
1035	echo "IGCM_sys_Put_Dods : error."
1036	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1037	fi
1038	fi
1039	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1040	}
1041
1042	##############################################################
1043	# REBUILD OPERATOR
1044
1045	#D-#==================================================
1046	#D-function IGCM_sys_sync
1047	#D-* Purpose: flush buffer on disk
1048	#D-* Examples:
1049	#D-
1050	function IGCM_sys_sync {
1051	IGCM_debug_PushStack "IGCM_sys_sync" $@
1052	if ( $DEBUG_sys ) ; then
1053	echo "IGCM_sys_sync :" $@
1054	fi
1055
1056	/bin/sync
1057
1058	IGCM_debug_PopStack "IGCM_sys_sync"
1059	}
1060
1061	############################################################
1062	# Activate Running Environnment Variables
1063
1064	#D-#==================================================
1065	#D-function IGCM_sys_activ_variables
1066	#D-* Purpose: set environement variables prior to execution
1067	#D-* Examples:
1068	#D-
1069	function IGCM_sys_activ_variables {
1070	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1071	if ( $DEBUG_sys ) ; then
1072	echo "IGCM_sys_activ_variables"
1073	fi
1074
1075	# --------------------------------------------------------------------
1076	#D- MODULE specifications
1077	# --------------------------------------------------------------------
1078	# Source the file EnvFile containing all module load needed to run the model.
1079	# EnvFile can be specified in config.card. If this is the case and if the file exists,
1080	# it will be used. Otherwise a default file will be used.
1081	if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1082	# EnvFile is set in config.card and the file exists
1083	IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1084	EnvFile=${config_UserChoices_EnvFile}
1085	else
1086	IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1087	module purge > /dev/null 2>&1
1088	if [ X${compilerVersion} = XV19 ] ; then
1089	IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V19"
1090	EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
1091	else
1092	IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1093	fi
1094	fi
1095	IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :"
1096	IGCM_debug_Print 1 ${EnvFile}
1097	. ${EnvFile}
1098
1099	IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene-amd. "
1100	module list
1101
1102	# --------------------------------------------------------------------
1103	#D- MPI specifications
1104	# --------------------------------------------------------------------
1105
1106	# --------------------------------------------------------------------
1107	#D- Other specifications
1108	# --------------------------------------------------------------------
1109
1110	ulimit -s unlimited
1111
1112	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1113	}
1114
1115	############################################################
1116	# Desactivate Running Environnment Variables
1117
1118	#D-#==================================================
1119	#D-function IGCM_sys_desactiv_variables
1120	#D-* Purpose: unset environement variables after execution
1121	#D-* Examples:
1122	#D-
1123	function IGCM_sys_desactiv_variables {
1124	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1125	if ( $DEBUG_sys ) ; then
1126	echo "IGCM_sys_desactiv_variables"
1127	fi
1128	# --------------------------------------------------------------------
1129	#D- MPI specifications
1130	# --------------------------------------------------------------------
1131
1132	# --------------------------------------------------------------------
1133	#D- Other specifications
1134	# --------------------------------------------------------------------
1135
1136	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1137	}
1138
1139	############################################################
1140	# Update job headers to be used by the scheduler
1141
1142	#D-#==================================================
1143	#D-function IGCM_sys_updateHeaders
1144	#D-* Purpose: Update job headers to be used by the scheduler
1145	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1146	#D-
1147	function IGCM_sys_updateHeaders {
1148	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1149	if ( $DEBUG_sys ) ; then
1150	echo "IGCM_sys_updateHeaders"
1151	fi
1152	typeset file
1153	file=$1
1154
1155	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1156	# Compute the number of core to add to force the use of a new node for xios servers
1157
1158	if [ "X${IOS_OK_DEDICATED}" == "X" ] ; then
1159	(( IOS_OK_DEDICATED = 0 ))
1160	fi
1161
1162
1163	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1164	(( current_core_noxios = coreNumber - IOS_PROC_MPI * IOS_PROC_OMP * IOS_PROC_DEP ))
1165	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1166	(( second_comp_proc_mpi_loc = IOS_PROC_MPI - 1 ))
1167	(( coreNumber = current_core_noxios - 1 + first_comp_proc_dep_loc * 2 + second_comp_proc_mpi_loc * IOS_PROC_OMP * IOS_PROC_DEP ))
1168	fi
1169
1170	if [ ${executionType} -eq 1 ] ; then
1171	# MPMD + MPI
1172	sed -e "/::openMPthreads::/d" \
1173	-e "s/::JobNumProcTot::/${coreNumber}/" \
1174	${file} > ${file}.tmp
1175
1176	elif [ ${executionType} -eq 2 ] ; then
1177	# MPMD + MPI + OMP :ccc_mprun
1178	sed -e "/::openMPthreads::/d" \
1179	-e "s/::JobNumProcTot::/${coreNumber}/" \
1180	${file} > ${file}.tmp
1181
1182	elif [ ${executionType} -eq 3 ] ; then
1183	# SPMD + MPI/OMP
1184	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1185	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1186	-e "/#MSUB -x/d" \
1187	${file} > ${file}.tmp
1188
1189	elif [ ${executionType} -eq 4 ] ; then
1190	# SPMD + MPI only
1191	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
1192	-e "/::openMPthreads::/d" \
1193	-e "/#MSUB -x/d" \
1194	${file} > ${file}.tmp
1195
1196	elif [ ${executionType} -eq 5 ] ; then
1197	# SPMD + OMP only
1198	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1199	-e "/::JobNumProcTot::/d" \
1200	-e "/#MSUB -x/d" \
1201	${file} > ${file}.tmp
1202
1203	elif [ ${executionType} -eq 6 ] ; then
1204	# SEQUENTIAL THEN
1205	sed -e "s/::JobNumProcTot::/1/" \
1206	-e "/::openMPthreads::/d" \
1207	-e "/#MSUB -x/d" \
1208	${file} > ${file}.tmp
1209
1210	fi
1211
1212	IGCM_sys_Mv ${file}.tmp ${file}
1213
1214	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1215	}
1216
1217	############################################################
1218	# Build MPI/OMP scripts run file (dummy function)
1219
1220	#D-#==================================================
1221	#D-function IGCM_sys_build_run_file
1222	#D-* Purpose: build run file (deprecated)
1223	#D-* Examples:
1224	#D-
1225	function IGCM_sys_build_run_file {
1226	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1227	}
1228
1229	############################################################
1230	# Build MPI/OMP scripts
1231
1232	#D-#==================================================
1233	#D-function IGCM_sys_build_execution_scripts
1234	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1235	#D-* Examples:
1236	#D-
1237	function IGCM_sys_build_execution_scripts
1238	{
1239	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1240	if ( $DEBUG_sys ) ; then
1241	echo "IGCM_sys_build_execution_scripts " $@
1242	fi
1243
1244	EXECUTION=${HOST_MPIRUN_COMMAND}
1245
1246	if ( ${OK_PARA_MPMD} ) ; then
1247
1248	# MPMD mode
1249	# 1 MPI only
1250	# 2 MPI/OpenMP mpirun method
1251	# 3 MPI/OpenMP ccc_mprun method
1252
1253	if [ -f run_file ] ; then
1254	IGCM_sys_Rm -f run_file
1255	fi
1256	touch run_file
1257
1258	# case 1 : Only MPI (MPMD)
1259	if ( ! ${OK_PARA_OMP} ) ; then
1260
1261	# Build run_file
1262
1263	# First loop on the components for the coupler ie oasis (only if oasis3)
1264	# the coupler ie oasis3 must be the first one
1265	for comp in ${config_ListOfComponents[*]} ; do
1266
1267	eval ExeNameIn=\${config_Executable_${comp}[0]}
1268	eval ExeNameOut=\${config_Executable_${comp}[1]}
1269
1270	# for CPL component only
1271	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1272
1273	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1274	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1275	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1276	fi
1277	done
1278
1279	# Then second loop on the components
1280	for comp in ${config_ListOfComponents[*]} ; do
1281
1282	eval ExeNameIn=\${config_Executable_${comp}[0]}
1283	eval ExeNameOut=\${config_Executable_${comp}[1]}
1284
1285	# Only if we really have an executable for the component and not the coupler ie oasis:
1286	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1287
1288	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1289	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1290	eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1291
1292	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1293	# Compute the number of core to add to force the use of a new node for xios servers
1294	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XIOS" ] ) ; then
1295	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1296	(( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_dep_loc ))
1297	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1298	(( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1299	if [ ${comp_proc_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1300	echo "1-${first_comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1301	if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1302	echo "${second_comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1303	fi
1304	else
1305	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1306	fi
1307	else
1308	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1309	fi
1310	else
1311	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1312	fi
1313
1314
1315	fi
1316	done
1317
1318	## module advised by TGCC (instead of 2 variables)
1319	module load feature/bridge/heterogenous_mpmd
1320
1321	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1322
1323	IGCM_sys_Chmod u+x run_file
1324	if ( $DEBUG_sys ) ; then
1325	echo "run_file contains : "
1326	cat run_file
1327	fi
1328
1329	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1330
1331	# 2 MPI/OpenMP mpirun method
1332
1333	# Use of mpirun instead of ccc_mprun
1334	EXECUTION="time mpirun"
1335
1336	# Hosts treatment
1337	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
1338
1339	i=0
1340	rm -f hosts rankfile
1341	IGCM_debug_Print 1 "sys Irene-amd, Hosts available :"
1342	for nodes in `cat hosts.tmp`
1343	do
1344	host[$i]=$nodes
1345	echo "${host[$i]}" >> hosts
1346	IGCM_debug_Print 1 ${host[$i]}
1347	i=$((i+1))
1348	done
1349	rm -f hosts.tmp
1350
1351	listnodes=${host[*]}
1352
1353	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1354
1355	# Initialisation
1356	rank=0
1357	current_core=0
1358	init_exec=n
1359
1360	# Loop on the components
1361	for comp in ${config_ListOfComponents[*]} ; do
1362
1363	eval ExeNameIn=\${config_Executable_${comp}[0]}
1364	eval ExeNameOut=\${config_Executable_${comp}[1]}
1365
1366	# Not possible if oasis has an executable (i.e old version of oasis3)
1367	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1368	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1369	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1370	IGCM_debug_Verif_Exit
1371	fi
1372
1373	# Only if we really have an executable for the component :
1374	if [ "X${ExeNameOut}" != X\"\" ] ; then
1375
1376	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1377	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1378	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1379
1380	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1381	echo "" >> script_${ExeNameOut}.ksh
1382	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1383
1384	# Check if the number of threads is correct
1385	case ${comp_proc_omp_loc} in
1386	2\|4\|8\|16)
1387	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1388	;;
1389	*)
1390	IGCM_debug_Exit "ERROR with OMP parameters !"
1391	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1392	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1393	IGCM_debug_Verif_Exit
1394	;;
1395	esac
1396	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1397	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1398	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1399	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1400	fi
1401
1402	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1403	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1404	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1405	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1406	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1407
1408	if [ ${init_exec} = y ] ; then
1409	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1410	else
1411	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1412	init_exec=y
1413	fi
1414
1415	# Build rankfile : method used to assign cores and nodes for the MPI process
1416	# Ex :
1417	#rank 0=curie5296 slot=0,1,2,3
1418	#rank 1=curie5296 slot=4,5,6,7
1419	# Example of final command :
1420	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1421	# with script_lmdz.x.ksh :
1422	# #!/bin/ksh
1423	#export KMP_STACKSIZE=3g
1424	#export KMP_LIBRARY=turnaround
1425	#export MKL_SERIAL=YES
1426	#OMP_NUM_THREADS=4
1427	#./lmdz.x
1428	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1429	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1430	else
1431	(( offset_comp_proc_loc = comp_proc_omp_loc ))
1432	fi
1433	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1434	(( index_host = current_core / NB_CORE_PER_NODE ))
1435	host_value=${host[${index_host}]}
1436	(( slot = current_core % NB_CORE_PER_NODE ))
1437	virg=","
1438	string_final=""
1439	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1440	string=$index$virg
1441	string_final=$string_final$string
1442	done
1443	string_final=$( echo $string_final \| sed "s/.$//" )
1444	echo "rank $rank=$host_value slot=$string_final" >> rankfile
1445	(( rank = rank + 1 ))
1446	(( current_core = current_core + offset_comp_proc_loc ))
1447	done
1448	fi
1449	done
1450
1451	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1452
1453	# 3 MPI/OpenMP ccc_mprun method
1454
1455	# MPI-OpenMP (MPMD)
1456	# example of run_file
1457	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1458	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1459	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1460
1461	# Loop on the components
1462	for comp in ${config_ListOfComponents[*]} ; do
1463
1464	eval ExeNameIn=\${config_Executable_${comp}[0]}
1465	eval ExeNameOut=\${config_Executable_${comp}[1]}
1466
1467	# Not possible if oasis has an executable (i.e old version of oasis3)
1468	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1469	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1470	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1471	IGCM_debug_Verif_Exit
1472	fi
1473
1474	# Only if we really have an executable for the component :
1475	if [ "X${ExeNameOut}" != X\"\" ] ; then
1476
1477	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1478	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1479	eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1480
1481	# Compute number of cores per process
1482	(( comp_proc_omp_dep_loc = comp_proc_omp_loc * comp_proc_dep_loc ))
1483
1484	# Build script files
1485
1486	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1487	echo "" >> script_${ExeNameOut}.ksh
1488	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1489
1490	# Check if the number of threads is correct
1491	case ${comp_proc_omp_loc} in
1492	2\|4\|8\|16)
1493	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1494	;;
1495	*)
1496	IGCM_debug_Exit "ERROR with OMP parameters !"
1497	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1498	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1499	IGCM_debug_Verif_Exit
1500	;;
1501	esac
1502	### only for LMDZ?
1503	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1504	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1505	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1506	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1507
1508	fi
1509
1510	# to have out/err per process on different files
1511	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1512	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1513
1514	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1515
1516	# Complete run_file
1517
1518	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1519	# Compute the number of core to add to force the use of a new node for xios servers
1520	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XIOS" ] ) ; then
1521	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1522	(( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_omp_dep_loc ))
1523	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1524	(( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1525	if [ ${comp_proc_omp_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1526	echo "1-${first_comp_proc_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1527	if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1528	echo "${second_comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1529	fi
1530	else
1531	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1532	fi
1533	else
1534	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1535	fi
1536	else
1537	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1538	fi
1539
1540	fi
1541	done
1542
1543	## variable added to stop after 60s instead of 600s by default.
1544	## This is used when no error comes from executables and when something stopped an executable without notice.
1545	export SLURM_WAIT=60
1546
1547	## module advised by TGCC (instead of 2 variables)
1548	module load feature/bridge/heterogenous_mpmd
1549
1550	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1551
1552	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1553
1554	IGCM_sys_Chmod u+x run_file
1555	if ( $DEBUG_sys ) ; then
1556	echo "run_file contains : "
1557	cat run_file
1558	fi
1559
1560	else
1561
1562	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1563
1564	fi # 1 2 or 3 if ${OK_PARA_MPMD}
1565
1566	else
1567	# Only one executable (SPMD mode).
1568
1569	for comp in ${config_ListOfComponents[*]} ; do
1570
1571	# Only if we really have an executable for the component :
1572	eval ExeNameOut=\${config_Executable_${comp}[1]}
1573	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1574
1575	# Build script files
1576
1577	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1578	echo "" >> script_${ExeNameOut}.ksh
1579	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1580
1581	if ( ${OK_PARA_OMP} ) ; then
1582	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1583	echo "" >> script_${ExeNameOut}.ksh
1584	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1585	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1586	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1587	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1588	fi
1589
1590	if ( ${OK_PARA_MPI} ) ; then
1591	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1592	# Default : ccc_mprun used if nb_proc gt 1
1593	# to have out/err per process on different files
1594	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1595	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1596	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1597	else
1598	# Default : ccc_mprun is NOT used if nb_proc eq 1
1599	# to have out/err per process on different files
1600	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1601	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1602	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1603	fi
1604
1605	IGCM_debug_Print 1 "sys Irene-amd : script_${ExeNameOut}.ksh contains"
1606	cat script_${ExeNameOut}.ksh
1607
1608	fi
1609
1610	done
1611
1612	fi # ${OK_PARA_MPMD}
1613
1614	IGCM_debug_Print 1 "sys Irene-amd : execution command is "
1615	IGCM_debug_Print 1 "$EXECUTION"
1616
1617	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1618	}
1619
1620	#D-#==================================================
1621	#D-function IGCM_sys_check_path
1622	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1623	#D-* do not point to an important use directory. Stop immediately in that case.
1624	#D-* Examples:
1625	#D-
1626	function IGCM_sys_check_path {
1627	IGCM_debug_PushStack "IGCM_sys_check_path"
1628	if ( $DEBUG_sys ) ; then
1629	echo "IGCM_sys_check_path"
1630	fi
1631
1632	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1633	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1634	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1635	IGCM_debug_Exit "This will stop the job"
1636	fi
1637	IGCM_debug_PopStack "IGCM_sys_check_path"
1638	}
1639
1640	#D-#==================================================
1641	#D-function IGCM_sys_check_quota
1642	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1643	#D-* Examples:
1644	#D-
1645	function IGCM_sys_check_quota {
1646	IGCM_debug_PushStack "IGCM_sys_check_quota"
1647	if ( $DEBUG_sys ) ; then
1648	echo "IGCM_sys_check_quota"
1649	fi
1650	# Limit of quota (in %)
1651	limit_quota=90
1652
1653	# Check of the volume
1654	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
1655	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
1656
1657	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1658
1659	unit_avail=${volume_avail: -1}
1660	unit_quota=${volume_quota: -1}
1661
1662	if [ "${unit_quota}" = "*" ] ; then
1663	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1664	IGCM_debug_Print 1 "More than 100% of your quota is used"
1665	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1666	IGCM_debug_Print 1 "You must have more than 10% available to run"
1667	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1668	IGCM_debug_Verif_Exit
1669	fi
1670
1671	temp_avail=${volume_avail%%${volume_avail: -1}*}
1672	temp_quota=${volume_quota%%${volume_quota: -1}*}
1673
1674	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1675
1676	# Convertion
1677	if [ ${volume_avail: -1} = "T" ] ; then
1678	(( temp_avail = temp_avail * 1000000000000 ))
1679	elif [ ${volume_avail: -1} = "G" ] ; then
1680	(( temp_avail = temp_avail * 1000000000 ))
1681	elif [ ${volume_avail: -1} = "M" ] ; then
1682	(( temp_avail = temp_avail * 1000000 ))
1683	elif [ ${volume_avail: -1} = "k" ] ; then
1684	(( temp_avail = temp_avail * 1000 ))
1685	else
1686	(( temp_avail = volume_avail ))
1687	fi
1688	if [ ${volume_quota: -1} = "T" ] ; then
1689	(( temp_quota = temp_quota * 1000000000000 ))
1690	elif [ ${volume_quota: -1} = "G" ] ; then
1691	(( temp_quota = temp_quota * 1000000000 ))
1692	elif [ ${volume_quota: -1} = "M" ] ; then
1693	(( temp_quota = temp_quota * 1000000 ))
1694	elif [ ${volume_quota: -1} = "k" ] ; then
1695	(( temp_quota = temp_quota * 1000 ))
1696	else
1697	(( temp_quota = volume_quota ))
1698	fi
1699	fi
1700
1701	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1702	# echo "volume ratio is " $quota_volume
1703
1704	if [ ${quota_volume} -ge ${limit_quota} ] ; then
1705	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1706	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1707	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1708	IGCM_debug_Print 1 "You must have more than 10% available to run"
1709	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1710	IGCM_debug_Verif_Exit
1711	fi
1712
1713	fi
1714
1715	# Check of the number of inodes
1716
1717	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
1718	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
1719
1720	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1721
1722	unit_avail=${inode_avail: -1}
1723	unit_quota=${inode_quota: -1}
1724
1725	if [ "${unit_quota}" = "*" ] ; then
1726	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1727	IGCM_debug_Print 1 "More than 100% of your quota is used"
1728	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1729	IGCM_debug_Print 1 "You must have more than 10% available to run"
1730	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1731	IGCM_debug_Verif_Exit
1732	fi
1733
1734	temp_avail=${inode_avail%%${inode_avail: -1}*}
1735	temp_quota=${inode_quota%%${inode_quota: -1}*}
1736
1737	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1738
1739	# Convertion
1740	if [ ${inode_avail: -1} = "T" ] ; then
1741	(( temp_avail = temp_avail * 1000000000000 ))
1742	elif [ ${inode_avail: -1} = "G" ] ; then
1743	(( temp_avail = temp_avail * 1000000000 ))
1744	elif [ ${inode_avail: -1} = "M" ] ; then
1745	(( temp_avail = temp_avail * 1000000 ))
1746	elif [ ${inode_avail: -1} = "k" ] ; then
1747	(( temp_avail = temp_avail * 1000 ))
1748	else
1749	(( temp_avail = inode_avail ))
1750	fi
1751
1752	if [ ${inode_quota: -1} = "T" ] ; then
1753	(( temp_quota = temp_quota * 1000000000000 ))
1754	elif [ ${inode_quota: -1} = "G" ] ; then
1755	(( temp_quota = temp_quota * 1000000000 ))
1756	elif [ ${inode_quota: -1} = "M" ] ; then
1757	(( temp_quota = temp_quota * 1000000 ))
1758	elif [ ${inode_quota: -1} = "k" ] ; then
1759	(( temp_quota = temp_quota * 1000 ))
1760	else
1761	(( temp_quota = inode_quota ))
1762	fi
1763	fi
1764	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1765	# echo "inode ratio is " $quota_inode
1766
1767	if [ ${quota_inode} -ge ${limit_quota} ] ; then
1768	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1769	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1770	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1771	IGCM_debug_Print 1 "You must have more than 10% available to run"
1772	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1773	IGCM_debug_Verif_Exit
1774	fi
1775	fi
1776	IGCM_debug_PopStack "IGCM_sys_check_quota"
1777	}
1778
1779	#D-#==================================================
1780	#D-function IGCM_sys_projectAccounting
1781	#D-* Purpose: store project accounting information in a file
1782	#D-* Examples:
1783	#D-
1784	function IGCM_sys_projectAccounting {
1785	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1786	if ( $DEBUG_sys ) ; then
1787	echo "IGCM_sys_check_quota"
1788	fi
1789
1790	ssh irene194 /usr/bin/ccc_myproject > $1
1791
1792	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1793	}
1794
1795	#D-#==================================================
1796	#D-function IGCM_sys_getJobSchedulerID
1797	#D-* Purpose: Get the job ID during execution
1798	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1799	#D-
1800	function IGCM_sys_getJobSchedulerID {
1801	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1802	if ( $DEBUG_sys ) ; then
1803	echo "IGCM_sys_getJobSchedulerID"
1804	fi
1805	eval ${1}=${BRIDGE_MSUB_JOBID}
1806
1807	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1808	}
1809
1810	#D-#==================================================
1811	#D-function IGCM_sys_GetJobID
1812	#D-* Purpose: Get the job ID from the JobName
1813	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1814	#D-
1815	function IGCM_sys_GetJobID {
1816	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1817	if ( $DEBUG_sys ) ; then
1818	echo "IGCM_sys_GetJobID"
1819	fi
1820
1821	# With -f option, the full job name is given in the last column
1822	ID=$( ccc_mstat -f -u $2 \| \
1823	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1824
1825	eval ${3}=${ID}
1826	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1827	}
1828
1829	#D-#==================================================
1830	#D-function IGCM_sys_CountJobInQueue
1831	#D-* Purpose: Count number of users job
1832	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1833	#D-
1834	function IGCM_sys_CountJobInQueue {
1835	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1836	if ( $DEBUG_sys ) ; then
1837	echo "IGCM_sys_CountJobInQueue"
1838	fi
1839
1840	# With -f option, the full job name is given in the last column
1841	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1842
1843	eval ${2}=${NbRun}
1844
1845	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1846	}
1847
1848	#D-#==================================================
1849	#D-function IGCM_sys_ListJobInQueue
1850	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1851	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1852	#D-
1853	function IGCM_sys_ListJobInQueue {
1854	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1855	if ( $DEBUG_sys ) ; then
1856	echo "IGCM_sys_ListJobInQueue"
1857	fi
1858
1859	# With -f option, the full job name is given in the last column
1860	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
1861	'( $2 == User && \
1862	$NF != /TS/ && \
1863	$NF !~ /PACK/ && \
1864	$NF !~ /REBUILD/ && \
1865	$NF !~ /pack/ ) \
1866	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1867
1868	eval set -A ${2} ${JobList[*]}
1869
1870	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1871	}
1872
1873	#D-#==================================================
1874	#D-function IGCM_sys_atlas
1875	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1876	#D-* Examples:
1877	#D-
1878	function IGCM_sys_atlas {
1879	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1880	if ( $DEBUG_sys ) ; then
1881	echo "IGCM_sys_atlas :" $@
1882	fi
1883
1884	typeset status
1885
1886	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1887	status=$?
1888	if [ ${status} -gt 0 ] ; then
1889	echo "IGCM_sys_atlas : error code ${status}"
1890	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1891	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1892	IGCM_debug_PopStack "IGCM_sys_atlas"
1893	return 1
1894	else
1895	IGCM_debug_PopStack "IGCM_sys_atlas"
1896	return 0
1897	fi
1898
1899	IGCM_debug_PopStack "IGCM_sys_atlas"
1900	}
1901
1902	#D-#==================================================
1903	#D-function IGCM_sys_rebuild_nemo
1904	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1905	#D-* Examples:
1906	#D-
1907
1908	function IGCM_sys_rebuild_nemo {
1909	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1910	if ( $DEBUG_sys ) ; then
1911	echo "IGCM_sys_rebuild_nemo :" $@
1912	fi
1913
1914	$( ccc_home -u igcmg)/Tools/irene-amd/bin/rebuild_nemo ${1} ${2}
1915
1916	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1917	}
1918

Note: See TracBrowser for help on using the repository browser.

Download in other formats: