Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene.ksh @ 1440

Last change on this file since 1440 was 1440, checked in by jgipsl, 6 years ago
Added more possibilities at irene
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 62.1 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Curie
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host user names project maxCpuTime
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=curie
71	# add default project on curie
72	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} \| cut -d@ -f1 )
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=ccc_msub}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=${MASTER}
89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
93	#====================================================
94	if [ X${TaskType} = Xcomputing ] ; then
95	IGCM_debug_Print 1 "Module will be loaded once we know the compiler version"
96	else
97	module purge > /dev/null 2>&1
98	. /ccc/cont003/home/dsm/p86ipsl/.env_irene > /dev/null 2>&1
99	. /ccc/cont003/home/dsm/p86ipsl/.env_atlas_irene > /dev/null 2>&1
100
101	# . /ccc/cont003/home/dsm/p86ipsl/.env_intel17.0.2_curie > /dev/null 2>&1
102	# . /ccc/cont003/home/dsm/p86ipsl/.env_netcdf4.3.3.1_curie > /dev/null 2>&1
103	# . /ccc/cont003/home/dsm/p86ipsl/.env_atlas_curie > /dev/null 2>&1
104	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
105	export UVCDAT_ANONYMOUS_LOG=FALSE
106	fi
107
108	# Use CMIP6 storage space when using project id gch0316
109	[[ "X${PROJECT}" = "Xgch0316" && ! $(module list --terse 2>&1 \| grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
110
111	# FYI
112	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
113	[ ! X${TaskType} = Xchecking ] && module list
114
115	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
116	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
117	export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
118
119	#====================================================
120	# Specific for ocean additionnal diagnostic
121	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
122	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
123
124	#====================================================
125	# Host specific DIRECTORIES
126	#====================================================
127
128	# ============ CESIUM START ============ #
129
130	#====================================================
131	#- Mirror libIGCM from titane to cesium if needed
132	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
133	#if [ ! ${ROOTSYS} = "home" ] ; then
134	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
135	#else
136	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
137	#fi
138
139	#====================================================
140	#- libIGCM_POST
141	#if ( ${MirrorlibIGCM} ) ; then
142	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
143	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
144	#else
145	# typeset -r libIGCM_POST=${libIGCM}
146	#fi
147
148	# ============ CESIUM END ============ #
149
150	#====================================================
151	#- MirrorlibIGCM for frontend
152	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
153
154	#====================================================
155	#- libIGCM_POST for frontend
156	typeset -r libIGCM_POST=${libIGCM}
157
158	#====================================================
159	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
160	typeset -r R_EXE="${MODIPSL}/bin"
161
162	#====================================================
163	#- SUBMIT_DIR : submission dir
164	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
165
166	#====================================================
167	#- IN
168	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
169	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
170
171	#====================================================
172	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
173	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
174
175	#====================================================
176	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
177	typeset -r OUTCOMMAND_PATH=/tmp
178
179	#====================================================
180	#- HOST_MPIRUN_COMMAND
181	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
182
183	#====================================================
184	#- Max number of arguments passed to nco operator or demigration command
185	UNIX_MAX_LIMIT=360
186
187	#====================================================
188	#- set PackDefault to true on curie
189	PackDefault=true
190
191	#====================================================
192	#- Default number of MPI task for IPSL coupled model
193	#- required for backward compatibility
194	#-
195	DEFAULT_NUM_PROC_OCE=5
196	DEFAULT_NUM_PROC_CPL=1
197	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
198
199	#====================================================
200	#- Number of computing cores per node
201	#-
202	NB_CORE_PER_NODE=16
203
204	#D-#==================================================
205	#D-function IGCM_sys_defineArchives
206	#D-* Purpose:
207	#D-* Define ARCHIVE : Dedicated to large files
208	#D-* Define STORAGE : Dedicated to small/medium files
209	#D-* Define R_OUT : Output tree located on ARCHIVE
210	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
211	#D-* Define R_BUF : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
212	#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
213	#D-* Examples:
214	#D-
215	function IGCM_sys_defineArchives {
216	IGCM_debug_PushStack "IGCM_sys_defineArchives"
217
218	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
219	#====================================================
220	#- ARCHIVE (dedicated to large files)
221	ARCHIVE=${config_UserChoices_ARCHIVE}
222	else
223	#====================================================
224	#- ARCHIVE (dedicated to large files)
225	ARCHIVE=${CCCSTOREDIR}
226	fi
227
228	if [ ! X${config_UserChoices_STORAGE} = X ]; then
229	#====================================================
230	#- STORAGE (dedicated to small/medium files)
231	STORAGE=${config_UserChoices_STORAGE}
232	else
233	#====================================================
234	#- STORAGE (dedicated to small/medium files)
235	STORAGE=${CCCWORKDIR}
236	fi
237
238	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
239	#====================================================
240	#- R_OUT
241	R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
242
243	#====================================================
244	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
245	R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
246
247	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
248
249	else
250	#====================================================
251	#- R_OUT
252	R_OUT=${ARCHIVE}/IGCM_OUT
253
254	#====================================================
255	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
256	R_FIG=${STORAGE}/IGCM_OUT
257	fi
258
259	#====================================================
260	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
261	CMIP6_BUF=${STORAGE}/IGCM_OUT
262
263	#====================================================
264	#- R_BUF (ONLY FOR double copy an scratch)
265	R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
266
267	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
268	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
269	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
270
271	IGCM_debug_PopStack "IGCM_sys_defineArchives"
272	}
273
274	#D-#==================================================
275	#D-function IGCM_sys_RshArchive
276	#D-* Purpose: Archive rsh command
277	#D-* Examples:
278	#D-
279	function IGCM_sys_RshArchive {
280	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
281	/bin/ksh <<-EOF
282	${@}
283	EOF
284	status=$?
285	if [ ${status} -gt 0 ] ; then
286	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
287	IGCM_debug_Exit "IGCM_sys_RshArchive"
288	fi
289	IGCM_debug_PopStack "IGCM_sys_RshArchive"
290	}
291
292	#D-#==================================================
293	#D-function IGCM_sys_RshArchive_NoError
294	#D-* Purpose: Archive rsh command, without error
295	#D-* used only in monitoring.job
296	#D-* Examples:
297	#D-
298	function IGCM_sys_RshArchive_NoError {
299	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
300	/bin/ksh <<-EOF
301	${@} 2> /dev/null
302	EOF
303	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
304	}
305
306	#D-#==================================================
307	#D-function IGCM_sys_MkdirArchive
308	#D-* Purpose: Mkdir on Archive
309	#D-* Examples:
310	#D-
311	function IGCM_sys_MkdirArchive {
312	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
313	if ( $DEBUG_sys ) ; then
314	echo "IGCM_sys_MkdirArchive :" $@
315	fi
316	#- creation de repertoire sur le serveur fichier
317	if [ ! -d ${1} ]; then
318	\mkdir -p $1
319	status=$?
320
321	if [ ${status} -gt 0 ] ; then
322	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
323	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
324	fi
325	fi
326	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
327	}
328
329	#D-#==================================================
330	#D-function IGCM_sys_TestDirArchive
331	#D-* Purpose: Test Directory that must exists on Archive
332	#D-* Examples:
333	#D-
334	function IGCM_sys_TestDirArchive {
335	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
336	if ( $DEBUG_sys ) ; then
337	echo "IGCM_sys_TestDirArchive :" $@
338	fi
339	typeset ExistFlag
340	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
341	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
342	return ${ExistFlag}
343	}
344
345	#D-#==================================================
346	#D-function IGCM_sys_IsFileArchived
347	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
348	#D-* Examples:
349	#D-
350	function IGCM_sys_IsFileArchived {
351	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
352	if ( $DEBUG_sys ) ; then
353	echo "IGCM_sys_IsFileArchived :" $@
354	fi
355	typeset IsArchivedFlag
356	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
357	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
358
359	return ${IsArchivedFlag}
360	}
361
362	#D-#==================================================
363	#D-function IGCM_sys_TestFileArchive
364	#D-* Purpose: Test file that must NOT EXISTS on Archive
365	#D-* Examples:
366	#D-
367	function IGCM_sys_TestFileArchive {
368	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
369	typeset ExistFlag
370	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
371	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
372
373	return ${ExistFlag}
374	}
375
376	#D-#==================================================
377	#D-function IGCM_sys_CountFileArchive
378	#D-* Purpose: Count files on Archive filesystem
379	#D-* Examples:
380	#D-
381	function IGCM_sys_CountFileArchive {
382	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
383	ls ${@} 2>/dev/null \| wc -l
384	if [ $? -gt 0 ] ; then
385	echo "IGCM_sys_CountFileArchive : erreur."
386	fi
387	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
388	}
389
390	#D-#==================================================
391	#D-function IGCM_sys_Tree
392	#D-* Purpose: Tree directories with files on ${ARCHIVE}
393	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
394	#D-
395	function IGCM_sys_Tree {
396	IGCM_debug_PushStack "IGCM_sys_Tree" $@
397	if ( $DEBUG_sys ) ; then
398	echo "IGCM_sys_Tree :" $@
399	fi
400
401	\ls -lR ${@}
402
403	IGCM_debug_PopStack "IGCM_sys_Tree"
404	}
405
406	#D-#==================================================
407	#D-function IGCM_sys_Qsub
408	#D-* Purpose: Qsub new job
409	#D-* Examples:
410	#D-
411	function IGCM_sys_Qsub {
412	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
413	if ( $DEBUG_sys ) ; then
414	echo "IGCM_sys_Qsub :" $@
415	fi
416	typeset options status
417	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
418
419	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
420	status=$?
421
422	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
423	if [ ${status} -gt 0 ] ; then
424	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
425	IGCM_debug_Exit "IGCM_sys_Qsub"
426	else
427	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
428	fi
429	IGCM_debug_PopStack "IGCM_sys_Qsub"
430	}
431
432	#D-#==================================================
433	#D-function IGCM_sys_QsubPost
434	#D-* Purpose: Qsub new job on scalaire
435	#D-* Examples:
436	#D-
437	function IGCM_sys_QsubPost {
438	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
439	if ( $DEBUG_sys ) ; then
440	echo "IGCM_sys_QsubPost :" $@
441	fi
442	typeset options status
443
444	# EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
445	case $( hostname -s ) in
446	curie*)
447	options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
448	airain*)
449	options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
450	esac
451
452	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
453	status=$?
454
455	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
456	if [ ${status} -gt 0 ] ; then
457	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
458	IGCM_debug_Exit "IGCM_sys_QsubPost"
459	else
460	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
461	fi
462	IGCM_debug_PopStack "IGCM_sys_QsubPost"
463	}
464
465	#D-*************************
466	#D- File transfer functions
467	#D-*************************
468	#D-
469
470	#D-#==================================================
471	#D-function IGCM_sys_RmRunDir
472	#D-* Purpose: rm tmpdir (dummy function most of the time batch
473	#D- scheduler will do the job)
474	#D-* Examples:
475	#D-
476	function IGCM_sys_RmRunDir {
477	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
478	if ( $DEBUG_sys ) ; then
479	echo "IGCM_sys_RmRunDir :" $@
480	fi
481
482	typeset status
483
484	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
485	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
486	status=$?
487
488	if [ ${status} -gt 0 ] ; then
489	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
490	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
491	IGCM_debug_Exit "IGCM_sys_RmRunDir"
492	else
493	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
494	fi
495	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
496	}
497
498	#D-#==================================================
499	#D-function IGCM_sys_Put_Dir
500	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
501	#D-* Examples:
502	#D-
503	function IGCM_sys_Put_Dir {
504	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
505	if ( $DEBUG_sys ) ; then
506	echo "IGCM_sys_Put_Dir :" $@
507	fi
508	if [ $DRYRUN = 0 ]; then
509	if [ ! -d ${1} ] ; then
510	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
511	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
512	return
513	fi
514
515	typeset status
516
517	# Only if we use rsync
518	#IGCM_sys_TestDirArchive $( dirname $2 )
519	#
520	#USUAL WAY
521	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
522	status=$?
523
524	if [ ${status} -gt 0 ] ; then
525	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
526	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
527	IGCM_debug_Exit "IGCM_sys_Put_Dir"
528	else
529	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
530	fi
531	fi
532	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
533	}
534
535	#D-#==================================================
536	#D-function IGCM_sys_Get_Dir
537	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
538	#D-* Examples:
539	#D-
540	function IGCM_sys_Get_Dir {
541	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
542	if ( $DEBUG_sys ) ; then
543	echo "IGCM_sys_Get_Dir :" $@
544	fi
545	if [ $DRYRUN = 0 ]; then
546	typeset NB_ESSAI DELAI status i
547	# number of tentative
548	NB_ESSAI=3
549	# time delay between tentative
550	DELAI=2
551
552	#
553	# USUAL WAY
554	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
555	ccc_hsm get -r $1
556
557	i=0
558	while [ $i -lt $NB_ESSAI ] ; do
559	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
560	status=$?
561	if [ ${status} -gt 0 ] ; then
562	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
563	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
564	sleep $DELAI
565	else
566	break
567	fi
568	(( i = i + 1 ))
569	done
570
571	if [ ${status} -gt 0 ] ; then
572	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
573	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
574	IGCM_debug_Exit "IGCM_sys_Get_Dir"
575	else
576	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
577	fi
578	fi
579	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
580	}
581
582	#D-#==================================================
583	#D-function IGCM_sys_Put_Rest
584	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
585	#D- File and target directory must exist.
586	#D-* Examples:
587	#D-
588	function IGCM_sys_Put_Rest {
589	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
590	if ( $DEBUG_sys ) ; then
591	echo "IGCM_sys_Put_Rest :" $@
592	fi
593	if [ $DRYRUN = 0 ]; then
594	if [ ! -f ${1} ] ; then
595	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
596	IGCM_debug_Exit "IGCM_sys_Put_Rest"
597	fi
598
599	typeset status
600	#
601	# USUAL WAY
602	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
603	status=$?
604
605	# #RSYNC WITH NETWORK SSH CALL
606	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
607	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
608
609	# #RSYNC WITH NFS USE
610	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
611	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
612
613	# status=$?
614	# IGCM_sys_Rsync_out $status
615
616	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
617	# (( status=status+$? ))
618
619	if [ ${status} -gt 0 ] ; then
620	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
621	[ -f ${1} ] && ls -l ${1}
622	[ -f ${2} ] && ls -l ${2}
623	[ -f ${2}/${1} ] && ls -l ${2}/${1}
624	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
625	IGCM_debug_Exit "IGCM_sys_Put_Rest"
626	else
627
628	if [ X${JobType} = XRUN ] ; then
629	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
630	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
631	fi
632
633	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
634	fi
635	fi
636	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
637	}
638
639	#D-#==================================================
640	#D-function IGCM_sys_Put_Out
641	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
642	#D-* Examples:
643	#D-
644	function IGCM_sys_Put_Out {
645	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
646	if ( $DEBUG_sys ) ; then
647	echo "IGCM_sys_Put_Out :" $@
648	fi
649
650	typeset NB_ESSAI DELAI status i exist skip
651	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
652
653	# number of tentative
654	NB_ESSAI=3
655	# time delay between tentative
656	DELAI=2
657
658	if [ $DRYRUN = 0 ]; then
659	if [ ! -f ${1} ] ; then
660	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
661	IGCM_debug_PopStack "IGCM_sys_Put_Out"
662	return 1
663	fi
664	#
665	IGCM_sys_MkdirArchive $( dirname $2 )
666	#
667	exist=false
668	skip=false
669	if [ -f $2 ] ; then
670	IGCM_debug_Print 1 "$2 already exist"
671	ccc_hsm get $2
672	exist=true
673	if [ "X$( diff $1 $2 )" = X ] ; then
674	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
675	skip=true
676	else
677	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
678	skip=false
679	fi
680	fi
681	#
682	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
683	IGCM_sys_Chmod u+w $2
684	fi
685
686	if [ X${skip} = Xfalse ] ; then
687	i=0
688	while [ $i -lt $NB_ESSAI ] ; do
689	# Identify file system
690	fileDeviceNumberInHex=$( stat -c %d $1 )
691	status=$?
692	if [ ${status} -gt 0 ] ; then
693	IGCM_debug_Exit "IGCM_sys_Put_Out"
694	fi
695	# Identify file system
696	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
697	status=$?
698	if [ ${status} -gt 0 ] ; then
699	IGCM_debug_Exit "IGCM_sys_Put_Out"
700	fi
701
702	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
703	# They are not on the same device. USUAL WAY
704	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
705	status=$?
706	else
707	# They are on the same device. NOT SO USUAL WAY
708	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
709	status=$?
710	fi
711	if [ ${status} -gt 0 ]; then
712	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
713	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
714	[ -f ${1} ] && ls -l ${1}
715	[ -f ${2} ] && ls -l ${2}
716	[ -f ${2}/${1} ] && ls -l ${2}/${1}
717	sleep $DELAI
718	else
719	break
720	fi
721	(( i = i + 1 ))
722	done
723	fi
724
725	# #RSYNC WITH NETWORK SSH CALL
726	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
727	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
728
729	# #RSYNC WITH NFS USE
730	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
731	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
732
733	# status=$?
734	# IGCM_sys_Rsync_out $status
735
736	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
737	# (( status=status+$? ))
738
739	if [ ${status} -gt 0 ] ; then
740	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
741	[ -f ${1} ] && ls -l ${1}
742	[ -f ${2} ] && ls -l ${2}
743	[ -f ${2}/${1} ] && ls -l ${2}/${1}
744	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
745	IGCM_debug_Exit "IGCM_sys_Put_Out"
746	else
747
748	if [ X${JobType} = XRUN ] ; then
749	if [ X${3} = X ] ; then
750	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
751	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
752	fi
753	fi
754
755	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
756	fi
757	fi
758	IGCM_debug_PopStack "IGCM_sys_Put_Out"
759	return 0
760	}
761
762	#D-#==================================================
763	#D-function IGCM_sys_Get
764	#D-* Purpose: Get a file from ${ARCHIVE}
765	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
766	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
767	function IGCM_sys_Get {
768	IGCM_debug_PushStack "IGCM_sys_Get" $@
769
770	typeset DEST dm_liste target file_work
771	typeset NB_ESSAI DELAI status i
772
773	if ( $DEBUG_sys ) ; then
774	echo "IGCM_sys_Get :" $@
775	fi
776
777	# number of tentative
778	NB_ESSAI=3
779	# time delay between tentative
780	DELAI=2
781
782	if [ $DRYRUN -le 2 ]; then
783	if [ X${1} = X'/l' ] ; then
784	eval set +A dm_liste \${${2}}
785	else
786	eval set +A dm_liste ${1}
787	fi
788	eval DEST=\${${#}}
789	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
790	status=$?
791	if [ ${status} -gt 0 ] ; then
792	echo "WARNING IGCM_sys_Get : error code ${status}"
793	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
794	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
795	fi
796
797	#if [ ${status} -gt 0 ] ; then
798	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
799	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
800	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
801	# sleep 30
802	# echo "We try another time"
803	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
804	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
805	# status=$?
806	# if [ ${status} -gt 0 ] ; then
807	# echo "ERROR IGCM_sys_Get : again demigration error :"
808	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
809	# IGCM_debug_Exit "IGCM_sys_Get"
810	# fi
811	# else
812	# echo "ERROR IGCM_sys_Get : demigration error :"
813	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
814	# IGCM_debug_Exit "IGCM_sys_Get"
815	# fi
816	#fi
817
818	# #RSYNC WITH NETWORK SSH CALL
819	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
820	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
821
822	# #RSYNC WITH NFS USE
823	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
824	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
825
826	# status=$?
827	# IGCM_sys_Rsync_out $status
828
829	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
830	# (( status=status+$? ))
831
832	#USUAL WAY
833	if [ X${1} = X'/l' ] ; then
834	for target in ${dm_liste[*]} ; do
835	local_file=$( basename ${target} )
836	# test if the target file is present before the loop
837	IGCM_sys_TestFileArchive ${target}
838	status=$?
839	if [ ${status} -gt 0 ] ; then
840	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
841	IGCM_debug_Exit "IGCM_sys_Get"
842	else
843	i=0
844	while [ $i -lt $NB_ESSAI ] ; do
845	#if [ X${DoLink} = Xtrue ] ; then
846	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
847	# status=$?
848	# else
849	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
850	# status=$?
851	#fi
852	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
853	status=$?
854	if [ ${status} -gt 0 ]; then
855	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
856	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
857	sleep $DELAI
858	else
859	break
860	fi
861	(( i = i + 1 ))
862	done
863	if [ ${status} -gt 0 ] ; then
864	echo "IGCM_sys_Get : error"
865	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
866	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
867	IGCM_debug_Exit "IGCM_sys_Get"
868	else
869	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
870	fi
871	fi
872	done
873	else
874	i=0
875	while [ $i -lt $NB_ESSAI ] ; do
876	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
877	status=$?
878	if [ ${status} -gt 0 ]; then
879	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
880	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
881	sleep $DELAI
882	else
883	break
884	fi
885	(( i = i + 1 ))
886	done
887	if [ ${status} -gt 0 ] ; then
888	echo "IGCM_sys_Get : error"
889	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
890	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
891	IGCM_debug_Exit "IGCM_sys_Get"
892	else
893	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
894	fi
895	fi
896	fi
897	IGCM_debug_PopStack "IGCM_sys_Get"
898	}
899
900	#D-#==================================================
901	#D-function IGCM_sys_GetDate_Monitoring
902	#D-* Purpose: get the last year for which the monitoring has been computed
903	#D-* Examples:
904	#D-
905	function IGCM_sys_GetDate_Monitoring {
906	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
907	if ( $DEBUG_sys ) ; then
908	echo "IGCM_sys_GetDate_Monitoring :" $@
909	fi
910
911	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
912
913	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
914	}
915
916	#D-#==================================================
917	#D-function IGCM_sys_Dods_Rm
918	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
919	#D-* Examples:
920	#D-
921	function IGCM_sys_Dods_Rm {
922	if ( $DEBUG_sys ) ; then
923	echo "IGCM_sys_Dods_Rm :" $@
924	fi
925	typeset status
926	if [ $DRYRUN = 0 ]; then
927
928	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
929	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
930	# echo "Nothing has been done."
931	# return
932	# fi
933
934	if [ "$#" -eq 1 ]; then
935	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
936	status=$?
937	else
938	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
939	status=$?
940	fi
941
942	# if [ ${status} -gt 0 ] ; then
943	# echo "IGCM_sys_Dods_Rm : error."
944	# cat out_dods_rm
945	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
946	# else
947	# rm out_dods_rm
948	# fi
949
950	fi
951	return $status
952	}
953
954	#D-#==================================================
955	#D-function IGCM_sys_Dods_Cp
956	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
957	#D-* Examples:
958	#D-
959	function IGCM_sys_Dods_Cp {
960	if ( $DEBUG_sys ) ; then
961	echo "IGCM_sys_Dods_Cp :" $@
962	fi
963	typeset status
964	if [ $DRYRUN = 0 ]; then
965
966	# if [ ! -d ${R_SAVE}/${1} ] ; then
967	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
968	# echo "Nothing has been done."
969	# return
970	# fi
971
972	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
973	status=$?
974
975	# if [ ${status} -gt 0 ] ; then
976	# echo "IGCM_sys_Dods_Cp : error."
977	# cat out_dods_cp
978	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
979	# else
980	# rm out_dods_cp
981	# fi
982
983	fi
984	return $status
985	}
986
987	#D-#==================================================
988	#D-function IGCM_sys_Put_Dods
989	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
990	#D-* Examples:
991	#D-
992	function IGCM_sys_Put_Dods {
993	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
994	if ( $DEBUG_sys ) ; then
995	echo "IGCM_sys_Put_Dods :" $@
996	fi
997	typeset status
998	if [ $DRYRUN = 0 ]; then
999	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1000	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1001	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1002	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1003	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1004	return
1005	fi
1006
1007	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1008	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1009	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1010	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1011	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1012	return
1013	fi
1014	#
1015	if [ -d ${R_SAVE}/${1} ] ; then
1016	cd ${R_SAVE}
1017	elif [ -d ${R_FIGR}/${1} ] ; then
1018	cd ${R_FIGR}
1019	fi
1020
1021	IGCM_sys_Dods_Rm ${1}
1022	IGCM_sys_Dods_Cp ${1}
1023	status=0
1024
1025	if [ ${status} -gt 0 ] ; then
1026	echo "IGCM_sys_Put_Dods : error."
1027	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1028	fi
1029	fi
1030	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1031	}
1032
1033	##############################################################
1034	# REBUILD OPERATOR
1035
1036	#D-#==================================================
1037	#D-function IGCM_sys_sync
1038	#D-* Purpose: flush buffer on disk
1039	#D-* Examples:
1040	#D-
1041	function IGCM_sys_sync {
1042	IGCM_debug_PushStack "IGCM_sys_sync" $@
1043	if ( $DEBUG_sys ) ; then
1044	echo "IGCM_sys_sync :" $@
1045	fi
1046
1047	/bin/sync
1048
1049	IGCM_debug_PopStack "IGCM_sys_sync"
1050	}
1051
1052	############################################################
1053	# Activate Running Environnment Variables
1054
1055	#D-#==================================================
1056	#D-function IGCM_sys_activ_variables
1057	#D-* Purpose: set environement variables prior to execution
1058	#D-* Examples:
1059	#D-
1060	function IGCM_sys_activ_variables {
1061	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1062	if ( $DEBUG_sys ) ; then
1063	echo "IGCM_sys_activ_variables"
1064	fi
1065
1066	# --------------------------------------------------------------------
1067	#D- MODULE specifications
1068	# --------------------------------------------------------------------
1069	module purge > /dev/null 2>&1
1070	if [ X${compilerVersion} = XV17 ] ; then
1071	IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17"
1072	else
1073	IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1074	fi
1075	. /ccc/cont003/home/dsm/p86ipsl/.env_irene > /dev/null 2>&1
1076	IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded module for Irene. "
1077	module list
1078
1079	# Keep track of this
1080	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
1081	[ ! X${TaskType} = Xchecking ] && module list
1082
1083	# --------------------------------------------------------------------
1084	#D- MPI specifications
1085	# --------------------------------------------------------------------
1086
1087	# --------------------------------------------------------------------
1088	#D- Other specifications
1089	# --------------------------------------------------------------------
1090
1091	ulimit -s unlimited
1092
1093	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1094	}
1095
1096	############################################################
1097	# Desactivate Running Environnment Variables
1098
1099	#D-#==================================================
1100	#D-function IGCM_sys_desactiv_variables
1101	#D-* Purpose: unset environement variables after execution
1102	#D-* Examples:
1103	#D-
1104	function IGCM_sys_desactiv_variables {
1105	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1106	if ( $DEBUG_sys ) ; then
1107	echo "IGCM_sys_desactiv_variables"
1108	fi
1109	# --------------------------------------------------------------------
1110	#D- MPI specifications
1111	# --------------------------------------------------------------------
1112
1113	# --------------------------------------------------------------------
1114	#D- Other specifications
1115	# --------------------------------------------------------------------
1116
1117	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1118	}
1119
1120	############################################################
1121	# Update job headers to be used by the scheduler
1122
1123	#D-#==================================================
1124	#D-function IGCM_sys_updateHeaders
1125	#D-* Purpose: Update job headers to be used by the scheduler
1126	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1127	#D-
1128	function IGCM_sys_updateHeaders {
1129	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1130	if ( $DEBUG_sys ) ; then
1131	echo "IGCM_sys_updateHeaders"
1132	fi
1133	typeset file
1134	file=$1
1135
1136	if [ ${executionType} -eq 1 ] ; then
1137	# MPMD + MPI
1138	sed -e "/::openMPthreads::/d" \
1139	-e "s/::JobNumProcTot::/${coreNumber}/" \
1140	-e "/#MSUB -x/d" \
1141	-e "/--cpu_bind=none/d" \
1142	${file} > ${file}.tmp
1143
1144	elif [ ${executionType} -eq 2 ] ; then
1145	# MPMD + MPI + OMP : mpirun/ccc_mprun/error
1146	if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1147	sed -e "/::openMPthreads::/d" \
1148	-e "s/::JobNumProcTot::/${coreNumber}/" \
1149	${file} > ${file}.tmp
1150	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1151	sed -e "/::openMPthreads::/d" \
1152	-e "s/::JobNumProcTot::/${coreNumber}/" \
1153	-e "/--cpu_bind=none/d" \
1154	${file} > ${file}.tmp
1155	else
1156	IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
1157	IGCM_debug_Exit "IGCM_sys_updateHeaders"
1158	fi
1159	elif [ ${executionType} -eq 3 ] ; then
1160	# SPMD + MPI/OMP
1161	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1162	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1163	-e "/#MSUB -x/d" \
1164	-e "/--cpu_bind=none/d" \
1165	${file} > ${file}.tmp
1166
1167	elif [ ${executionType} -eq 4 ] ; then
1168	# SPMD + MPI only
1169	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
1170	-e "/::openMPthreads::/d" \
1171	-e "/#MSUB -x/d" \
1172	-e "/--cpu_bind=none/d" \
1173	${file} > ${file}.tmp
1174
1175	elif [ ${executionType} -eq 5 ] ; then
1176	# SPMD + OMP only
1177	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1178	-e "/::JobNumProcTot::/d" \
1179	-e "/#MSUB -x/d" \
1180	-e "/--cpu_bind=none/d" \
1181	${file} > ${file}.tmp
1182
1183	elif [ ${executionType} -eq 6 ] ; then
1184	# SEQUENTIAL THEN
1185	sed -e "s/::JobNumProcTot::/1/" \
1186	-e "/::openMPthreads::/d" \
1187	-e "/#MSUB -x/d" \
1188	-e "/--cpu_bind=none/d" \
1189	${file} > ${file}.tmp
1190
1191	fi
1192
1193	IGCM_sys_Mv ${file}.tmp ${file}
1194
1195	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1196	}
1197
1198	############################################################
1199	# Build MPI/OMP scripts run file (dummy function)
1200
1201	#D-#==================================================
1202	#D-function IGCM_sys_build_run_file
1203	#D-* Purpose: build run file (deprecated)
1204	#D-* Examples:
1205	#D-
1206	function IGCM_sys_build_run_file {
1207	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1208	}
1209
1210	############################################################
1211	# Build MPI/OMP scripts
1212
1213	#D-#==================================================
1214	#D-function IGCM_sys_build_execution_scripts
1215	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1216	#D-* Examples:
1217	#D-
1218	function IGCM_sys_build_execution_scripts
1219	{
1220	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1221	if ( $DEBUG_sys ) ; then
1222	echo "IGCM_sys_build_execution_scripts " $@
1223	fi
1224
1225	EXECUTION=${HOST_MPIRUN_COMMAND}
1226
1227	if ( ${OK_PARA_MPMD} ) ; then
1228
1229	# MPMD mode
1230	# 1 MPI only
1231	# 2 MPI/OpenMP mpirun method
1232	# 3 MPI/OpenMP ccc_mprun method
1233
1234	if [ -f run_file ] ; then
1235	IGCM_sys_Rm -f run_file
1236	fi
1237	touch run_file
1238
1239	# case 1 : Only MPI (MPMD)
1240	if ( ! ${OK_PARA_OMP} ) ; then
1241
1242	# Build run_file
1243
1244	# First loop on the components for the coupler ie oasis (only if oasis3)
1245	# the coupler ie oasis3 must be the first one
1246	for comp in ${config_ListOfComponents[*]} ; do
1247
1248	eval ExeNameIn=\${config_Executable_${comp}[0]}
1249	eval ExeNameOut=\${config_Executable_${comp}[1]}
1250
1251	# for CPL component only
1252	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1253
1254	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1255	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1256	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1257	fi
1258	done
1259
1260	# Then second loop on the components
1261	for comp in ${config_ListOfComponents[*]} ; do
1262
1263	eval ExeNameIn=\${config_Executable_${comp}[0]}
1264	eval ExeNameOut=\${config_Executable_${comp}[1]}
1265
1266	# Only if we really have an executable for the component and not the coupler ie oasis:
1267	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1268
1269	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1270	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1271	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1272	fi
1273	done
1274
1275	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1276
1277	IGCM_sys_Chmod u+x run_file
1278	if ( $DEBUG_sys ) ; then
1279	echo "run_file contains : "
1280	cat run_file
1281	fi
1282
1283	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1284
1285	# 2 MPI/OpenMP mpirun method
1286
1287	# Use of mpirun instead of ccc_mprun
1288	EXECUTION="time mpirun"
1289
1290	# Hosts treatment
1291	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
1292
1293	i=0
1294	rm -f hosts rankfile
1295	IGCM_debug_Print 1 "sys Curie, Hosts available :"
1296	for nodes in `cat hosts.tmp`
1297	do
1298	host[$i]=$nodes
1299	echo "${host[$i]}" >> hosts
1300	IGCM_debug_Print 1 ${host[$i]}
1301	i=$((i+1))
1302	done
1303	rm -f hosts.tmp
1304
1305	listnodes=${host[*]}
1306
1307	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1308
1309	# Initialisation
1310	rank=0
1311	current_core=0
1312	init_exec=n
1313
1314	# Loop on the components
1315	for comp in ${config_ListOfComponents[*]} ; do
1316
1317	eval ExeNameIn=\${config_Executable_${comp}[0]}
1318	eval ExeNameOut=\${config_Executable_${comp}[1]}
1319
1320	# Not possible if oasis has an executable (i.e old version of oasis3)
1321	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1322	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1323	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1324	IGCM_debug_Verif_Exit
1325	fi
1326
1327	# Only if we really have an executable for the component :
1328	if [ "X${ExeNameOut}" != X\"\" ] ; then
1329
1330	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1331	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1332	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1333
1334	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1335	echo "" >> script_${ExeNameOut}.ksh
1336	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1337
1338	# Check if the number of threads is correct
1339	case ${comp_proc_omp_loc} in
1340	2\|4\|8\|16)
1341	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1342	;;
1343	*)
1344	IGCM_debug_Exit "ERROR with OMP parameters !"
1345	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1346	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1347	IGCM_debug_Verif_Exit
1348	;;
1349	esac
1350	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1351	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1352	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1353	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1354	fi
1355
1356	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1357	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1358	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1359	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1360	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1361
1362	if [ ${init_exec} = y ] ; then
1363	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1364	else
1365	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1366	init_exec=y
1367	fi
1368
1369	# Build rankfile : method used to assign cores and nodes for the MPI process
1370	# Ex :
1371	#rank 0=curie5296 slot=0,1,2,3
1372	#rank 1=curie5296 slot=4,5,6,7
1373	# Example of final command :
1374	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1375	# with script_lmdz.x.ksh :
1376	# #!/bin/ksh
1377	#export KMP_STACKSIZE=3g
1378	#export KMP_LIBRARY=turnaround
1379	#export MKL_SERIAL=YES
1380	#OMP_NUM_THREADS=4
1381	#./lmdz.x
1382	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1383	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1384	else
1385	(( offset_comp_proc_loc = comp_proc_omp_loc ))
1386	fi
1387	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1388	(( index_host = current_core / NB_CORE_PER_NODE ))
1389	host_value=${host[${index_host}]}
1390	(( slot = current_core % NB_CORE_PER_NODE ))
1391	virg=","
1392	string_final=""
1393	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1394	string=$index$virg
1395	string_final=$string_final$string
1396	done
1397	string_final=$( echo $string_final \| sed "s/.$//" )
1398	echo "rank $rank=$host_value slot=$string_final" >> rankfile
1399	(( rank = rank + 1 ))
1400	(( current_core = current_core + offset_comp_proc_loc ))
1401	done
1402	fi
1403	done
1404
1405	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1406
1407	# 3 MPI/OpenMP ccc_mprun method
1408
1409	# MPI-OpenMP (MPMD)
1410	# example of run_file
1411	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1412	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1413	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1414
1415	# Loop on the components
1416	for comp in ${config_ListOfComponents[*]} ; do
1417
1418	eval ExeNameIn=\${config_Executable_${comp}[0]}
1419	eval ExeNameOut=\${config_Executable_${comp}[1]}
1420
1421	# Not possible if oasis has an executable (i.e old version of oasis3)
1422	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1423	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1424	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1425	IGCM_debug_Verif_Exit
1426	fi
1427
1428	# Only if we really have an executable for the component :
1429	if [ "X${ExeNameOut}" != X\"\" ] ; then
1430
1431	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1432	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1433
1434	# Build script files
1435
1436	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1437	echo "" >> script_${ExeNameOut}.ksh
1438	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1439
1440	# Check if the number of threads is correct
1441	case ${comp_proc_omp_loc} in
1442	2\|4\|8\|12\|16\|24)
1443	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1444	;;
1445	*)
1446	IGCM_debug_Exit "ERROR with OMP parameters !"
1447	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1448	IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible "
1449	IGCM_debug_Verif_Exit
1450	;;
1451	esac
1452	### only for LMDZ?
1453	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1454	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1455	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1456	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1457
1458	fi
1459
1460	# to have out/err per process on different files
1461	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1462	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1463
1464	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1465
1466	# Complete run_file
1467
1468	echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1469
1470	fi
1471	done
1472
1473	## variable added to stop after 60s instead of 600s by default.
1474	## This is used when no error comes from executables and when something stopped an executable without notice.
1475	export SLURM_WAIT=60
1476
1477	## module advised by TGCC (instead of 2 variables)
1478	module load feature/bridge/heterogenous_mpmd
1479
1480	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1481
1482	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1483
1484	IGCM_sys_Chmod u+x run_file
1485	if ( $DEBUG_sys ) ; then
1486	echo "run_file contains : "
1487	cat run_file
1488	fi
1489
1490	else
1491
1492	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1493
1494	fi # 1 2 or 3 if ${OK_PARA_MPMD}
1495
1496	else
1497	# Only one executable (SPMD mode).
1498
1499	for comp in ${config_ListOfComponents[*]} ; do
1500
1501	# Only if we really have an executable for the component :
1502	eval ExeNameOut=\${config_Executable_${comp}[1]}
1503	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1504
1505	# Build script files
1506
1507	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1508	echo "" >> script_${ExeNameOut}.ksh
1509	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1510
1511	if ( ${OK_PARA_OMP} ) ; then
1512	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1513	echo "" >> script_${ExeNameOut}.ksh
1514	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1515	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1516	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1517	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1518	fi
1519
1520	if ( ${OK_PARA_MPI} ) ; then
1521	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1522	# Default : ccc_mprun used if nb_proc gt 1
1523	# to have out/err per process on different files
1524	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1525	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1526	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1527	else
1528	# Default : ccc_mprun is NOT used if nb_proc eq 1
1529	# to have out/err per process on different files
1530	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1531	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1532	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1533	fi
1534
1535	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
1536	cat script_${ExeNameOut}.ksh
1537
1538	fi
1539
1540	done
1541
1542	fi # ${OK_PARA_MPMD}
1543
1544	IGCM_debug_Print 1 "sys Curie : execution command is "
1545	IGCM_debug_Print 1 "$EXECUTION"
1546
1547	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1548	}
1549
1550	#D-#==================================================
1551	#D-function IGCM_sys_check_path
1552	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1553	#D-* do not point to an important use directory. Stop immediately in that case.
1554	#D-* Examples:
1555	#D-
1556	function IGCM_sys_check_path {
1557	IGCM_debug_PushStack "IGCM_sys_check_path"
1558	if ( $DEBUG_sys ) ; then
1559	echo "IGCM_sys_check_path"
1560	fi
1561
1562	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1563	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1564	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1565	IGCM_debug_Exit "This will stop the job"
1566	fi
1567	IGCM_debug_PopStack "IGCM_sys_check_path"
1568	}
1569
1570	#D-#==================================================
1571	#D-function IGCM_sys_check_quota
1572	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1573	#D-* Examples:
1574	#D-
1575	function IGCM_sys_check_quota {
1576	IGCM_debug_PushStack "IGCM_sys_check_quota"
1577	if ( $DEBUG_sys ) ; then
1578	echo "IGCM_sys_check_quota"
1579	fi
1580	# Limit of quota (in %)
1581	limit_quota=90
1582
1583	# Check of the volume
1584	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
1585	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
1586
1587	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1588
1589	unit_avail=${volume_avail: -1}
1590	unit_quota=${volume_quota: -1}
1591
1592	if [ "${unit_quota}" = "*" ] ; then
1593	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1594	IGCM_debug_Print 1 "More than 100% of your quota is used"
1595	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1596	IGCM_debug_Print 1 "You must have more than 10% available to run"
1597	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1598	IGCM_debug_Verif_Exit
1599	fi
1600
1601	temp_avail=${volume_avail%%${volume_avail: -1}*}
1602	temp_quota=${volume_quota%%${volume_quota: -1}*}
1603
1604	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1605
1606	# Convertion
1607	if [ ${volume_avail: -1} = "T" ] ; then
1608	(( temp_avail = temp_avail * 1000000000000 ))
1609	elif [ ${volume_avail: -1} = "G" ] ; then
1610	(( temp_avail = temp_avail * 1000000000 ))
1611	elif [ ${volume_avail: -1} = "M" ] ; then
1612	(( temp_avail = temp_avail * 1000000 ))
1613	elif [ ${volume_avail: -1} = "k" ] ; then
1614	(( temp_avail = temp_avail * 1000 ))
1615	else
1616	(( temp_avail = volume_avail ))
1617	fi
1618	if [ ${volume_quota: -1} = "T" ] ; then
1619	(( temp_quota = temp_quota * 1000000000000 ))
1620	elif [ ${volume_quota: -1} = "G" ] ; then
1621	(( temp_quota = temp_quota * 1000000000 ))
1622	elif [ ${volume_quota: -1} = "M" ] ; then
1623	(( temp_quota = temp_quota * 1000000 ))
1624	elif [ ${volume_quota: -1} = "k" ] ; then
1625	(( temp_quota = temp_quota * 1000 ))
1626	else
1627	(( temp_quota = volume_quota ))
1628	fi
1629	fi
1630
1631	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1632	# echo "volume ratio is " $quota_volume
1633
1634	if [ ${quota_volume} -ge ${limit_quota} ] ; then
1635	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1636	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1637	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1638	IGCM_debug_Print 1 "You must have more than 10% available to run"
1639	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1640	IGCM_debug_Verif_Exit
1641	fi
1642
1643	fi
1644
1645	# Check of the number of inodes
1646
1647	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
1648	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
1649
1650	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1651
1652	unit_avail=${inode_avail: -1}
1653	unit_quota=${inode_quota: -1}
1654
1655	if [ "${unit_quota}" = "*" ] ; then
1656	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1657	IGCM_debug_Print 1 "More than 100% of your quota is used"
1658	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1659	IGCM_debug_Print 1 "You must have more than 10% available to run"
1660	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1661	IGCM_debug_Verif_Exit
1662	fi
1663
1664	temp_avail=${inode_avail%%${inode_avail: -1}*}
1665	temp_quota=${inode_quota%%${inode_quota: -1}*}
1666
1667	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1668
1669	# Convertion
1670	if [ ${inode_avail: -1} = "T" ] ; then
1671	(( temp_avail = temp_avail * 1000000000000 ))
1672	elif [ ${inode_avail: -1} = "G" ] ; then
1673	(( temp_avail = temp_avail * 1000000000 ))
1674	elif [ ${inode_avail: -1} = "M" ] ; then
1675	(( temp_avail = temp_avail * 1000000 ))
1676	elif [ ${inode_avail: -1} = "k" ] ; then
1677	(( temp_avail = temp_avail * 1000 ))
1678	else
1679	(( temp_avail = inode_avail ))
1680	fi
1681
1682	if [ ${inode_quota: -1} = "T" ] ; then
1683	(( temp_quota = temp_quota * 1000000000000 ))
1684	elif [ ${inode_quota: -1} = "G" ] ; then
1685	(( temp_quota = temp_quota * 1000000000 ))
1686	elif [ ${inode_quota: -1} = "M" ] ; then
1687	(( temp_quota = temp_quota * 1000000 ))
1688	elif [ ${inode_quota: -1} = "k" ] ; then
1689	(( temp_quota = temp_quota * 1000 ))
1690	else
1691	(( temp_quota = inode_quota ))
1692	fi
1693	fi
1694	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1695	# echo "inode ratio is " $quota_inode
1696
1697	if [ ${quota_inode} -ge ${limit_quota} ] ; then
1698	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1699	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1700	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1701	IGCM_debug_Print 1 "You must have more than 10% available to run"
1702	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1703	IGCM_debug_Verif_Exit
1704	fi
1705	fi
1706	IGCM_debug_PopStack "IGCM_sys_check_quota"
1707	}
1708
1709	#D-#==================================================
1710	#D-function IGCM_sys_projectAccounting
1711	#D-* Purpose: store project accounting information in a file
1712	#D-* Examples:
1713	#D-
1714	function IGCM_sys_projectAccounting {
1715	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1716	if ( $DEBUG_sys ) ; then
1717	echo "IGCM_sys_check_quota"
1718	fi
1719
1720	ssh curie91 /usr/bin/ccc_myproject > $1
1721
1722	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1723	}
1724
1725	#D-#==================================================
1726	#D-function IGCM_sys_getJobSchedulerID
1727	#D-* Purpose: Get the job ID during execution
1728	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1729	#D-
1730	function IGCM_sys_getJobSchedulerID {
1731	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1732	if ( $DEBUG_sys ) ; then
1733	echo "IGCM_sys_getJobSchedulerID"
1734	fi
1735
1736	eval ${1}=${BRIDGE_MSUB_JOBID}
1737
1738	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1739	}
1740
1741	#D-#==================================================
1742	#D-function IGCM_sys_GetJobID
1743	#D-* Purpose: Get the job ID from the JobName
1744	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1745	#D-
1746	function IGCM_sys_GetJobID {
1747	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1748	if ( $DEBUG_sys ) ; then
1749	echo "IGCM_sys_GetJobID"
1750	fi
1751
1752	# With -f option, the full job name is given in the last column
1753	ID=$( ccc_mstat -f -u $2 \| \
1754	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1755
1756	eval ${3}=${ID}
1757	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1758	}
1759
1760	#D-#==================================================
1761	#D-function IGCM_sys_CountJobInQueue
1762	#D-* Purpose: Count number of users job
1763	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1764	#D-
1765	function IGCM_sys_CountJobInQueue {
1766	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1767	if ( $DEBUG_sys ) ; then
1768	echo "IGCM_sys_CountJobInQueue"
1769	fi
1770
1771	# With -f option, the full job name is given in the last column
1772	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1773
1774	eval ${2}=${NbRun}
1775
1776	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1777	}
1778
1779	#D-#==================================================
1780	#D-function IGCM_sys_ListJobInQueue
1781	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1782	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1783	#D-
1784	function IGCM_sys_ListJobInQueue {
1785	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1786	if ( $DEBUG_sys ) ; then
1787	echo "IGCM_sys_ListJobInQueue"
1788	fi
1789
1790	# With -f option, the full job name is given in the last column
1791	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
1792	'( $2 == User && \
1793	$NF != /TS/ && \
1794	$NF !~ /PACK/ && \
1795	$NF !~ /REBUILD/ && \
1796	$NF !~ /pack/ ) \
1797	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1798
1799	eval set -A ${2} ${JobList[*]}
1800
1801	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1802	}
1803
1804	#D-#==================================================
1805	#D-function IGCM_sys_atlas
1806	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1807	#D-* Examples:
1808	#D-
1809	function IGCM_sys_atlas {
1810	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1811	if ( $DEBUG_sys ) ; then
1812	echo "IGCM_sys_atlas :" $@
1813	fi
1814
1815	typeset status
1816
1817	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1818	status=$?
1819	if [ ${status} -gt 0 ] ; then
1820	echo "IGCM_sys_atlas : error code ${status}"
1821	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1822	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1823	IGCM_debug_PopStack "IGCM_sys_atlas"
1824	return 1
1825	else
1826	IGCM_debug_PopStack "IGCM_sys_atlas"
1827	return 0
1828	fi
1829
1830	IGCM_debug_PopStack "IGCM_sys_atlas"
1831	}
1832
1833	#D-#==================================================
1834	#D-function IGCM_sys_rebuild_nemo
1835	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1836	#D-* Examples:
1837	#D-
1838
1839	function IGCM_sys_rebuild_nemo {
1840	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1841	if ( $DEBUG_sys ) ; then
1842	echo "IGCM_sys_rebuild_nemo :" $@
1843	fi
1844
1845	$( ccc_home -u p86ipsl )/REBUILD_NEMO/rebuild_nemo ${1} ${2}
1846
1847	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1848	}
1849

Note: See TracBrowser for help on using the repository browser.

Download in other formats: