Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1439

Last change on this file since 1439 was 1431, checked in by sdipsl, 6 years ago
CMIP6 files go in CCCWORKDIR and alike.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 62.0 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Curie
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host user names project maxCpuTime
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=curie
71	# add default project on curie
72	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} \| cut -d@ -f1 )
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=ccc_msub}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=${MASTER}
89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
93	#====================================================
94	if [ X${TaskType} = Xcomputing ] ; then
95	IGCM_debug_Print 1 "Module will be loaded once we know the compiler version"
96	else
97	module purge > /dev/null 2>&1
98	. /ccc/cont003/home/dsm/p86ipsl/.env_intel17.0.2_curie > /dev/null 2>&1
99	. /ccc/cont003/home/dsm/p86ipsl/.env_netcdf4.3.3.1_curie > /dev/null 2>&1
100	. /ccc/cont003/home/dsm/p86ipsl/.env_atlas_curie > /dev/null 2>&1
101	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
102	export UVCDAT_ANONYMOUS_LOG=FALSE
103	fi
104
105	# Use CMIP6 storage space when using CMIP6 cpu ressources
106	[[ "X${PROJECT}" = "Xdevcmip6" && ! $(module list --terse 2>&1 \| grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
107
108	# FYI
109	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
110	[ ! X${TaskType} = Xchecking ] && module list
111
112	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
113	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
114	export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
115
116	#====================================================
117	# Specific for ocean additionnal diagnostic
118	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
119	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
120
121	#====================================================
122	# Host specific DIRECTORIES
123	#====================================================
124
125	# ============ CESIUM START ============ #
126
127	#====================================================
128	#- Mirror libIGCM from titane to cesium if needed
129	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
130	#if [ ! ${ROOTSYS} = "home" ] ; then
131	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
132	#else
133	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
134	#fi
135
136	#====================================================
137	#- libIGCM_POST
138	#if ( ${MirrorlibIGCM} ) ; then
139	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
140	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
141	#else
142	# typeset -r libIGCM_POST=${libIGCM}
143	#fi
144
145	# ============ CESIUM END ============ #
146
147	#====================================================
148	#- MirrorlibIGCM for frontend
149	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
150
151	#====================================================
152	#- libIGCM_POST for frontend
153	typeset -r libIGCM_POST=${libIGCM}
154
155	#====================================================
156	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
157	typeset -r R_EXE="${MODIPSL}/bin"
158
159	#====================================================
160	#- SUBMIT_DIR : submission dir
161	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
162
163	#====================================================
164	#- IN
165	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
166	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
167
168	#====================================================
169	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
170	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
171
172	#====================================================
173	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
174	typeset -r OUTCOMMAND_PATH=/tmp
175
176	#====================================================
177	#- HOST_MPIRUN_COMMAND
178	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
179
180	#====================================================
181	#- Max number of arguments passed to nco operator or demigration command
182	UNIX_MAX_LIMIT=360
183
184	#====================================================
185	#- set PackDefault to true on curie
186	PackDefault=true
187
188	#====================================================
189	#- Default number of MPI task for IPSL coupled model
190	#- required for backward compatibility
191	#-
192	DEFAULT_NUM_PROC_OCE=5
193	DEFAULT_NUM_PROC_CPL=1
194	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
195
196	#====================================================
197	#- Number of computing cores per node
198	#-
199	NB_CORE_PER_NODE=16
200
201	#D-#==================================================
202	#D-function IGCM_sys_defineArchives
203	#D-* Purpose:
204	#D-* Define ARCHIVE : Dedicated to large files
205	#D-* Define STORAGE : Dedicated to small/medium files
206	#D-* Define R_OUT : Output tree located on ARCHIVE
207	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
208	#D-* Define R_BUF : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
209	#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
210	#D-* Examples:
211	#D-
212	function IGCM_sys_defineArchives {
213	IGCM_debug_PushStack "IGCM_sys_defineArchives"
214
215	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
216	#====================================================
217	#- ARCHIVE (dedicated to large files)
218	ARCHIVE=${config_UserChoices_ARCHIVE}
219	else
220	#====================================================
221	#- ARCHIVE (dedicated to large files)
222	ARCHIVE=${CCCSTOREDIR}
223	fi
224
225	if [ ! X${config_UserChoices_STORAGE} = X ]; then
226	#====================================================
227	#- STORAGE (dedicated to small/medium files)
228	STORAGE=${config_UserChoices_STORAGE}
229	else
230	#====================================================
231	#- STORAGE (dedicated to small/medium files)
232	STORAGE=${CCCWORKDIR}
233	fi
234
235	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
236	#====================================================
237	#- R_OUT
238	R_OUT=${SCRATCHDIR}/IGCM_OUT
239
240	#====================================================
241	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
242	R_FIG=${SCRATCHDIR}/IGCM_OUT
243
244	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
245
246	else
247	#====================================================
248	#- R_OUT
249	R_OUT=${ARCHIVE}/IGCM_OUT
250
251	#====================================================
252	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
253	R_FIG=${STORAGE}/IGCM_OUT
254	fi
255
256	#====================================================
257	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
258	CMIP6_BUF=${STORAGE}/IGCM_OUT
259
260	#====================================================
261	#- R_BUF (ONLY FOR double copy an scratch)
262	R_BUF=${SCRATCHDIR}/IGCM_OUT
263
264	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
265	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
266	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
267
268	IGCM_debug_PopStack "IGCM_sys_defineArchives"
269	}
270
271	#D-#==================================================
272	#D-function IGCM_sys_RshArchive
273	#D-* Purpose: Archive rsh command
274	#D-* Examples:
275	#D-
276	function IGCM_sys_RshArchive {
277	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
278	/bin/ksh <<-EOF
279	${@}
280	EOF
281	status=$?
282	if [ ${status} -gt 0 ] ; then
283	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
284	IGCM_debug_Exit "IGCM_sys_RshArchive"
285	fi
286	IGCM_debug_PopStack "IGCM_sys_RshArchive"
287	}
288
289	#D-#==================================================
290	#D-function IGCM_sys_RshArchive_NoError
291	#D-* Purpose: Archive rsh command, without error
292	#D-* used only in monitoring.job
293	#D-* Examples:
294	#D-
295	function IGCM_sys_RshArchive_NoError {
296	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
297	/bin/ksh <<-EOF
298	${@} 2> /dev/null
299	EOF
300	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
301	}
302
303	#D-#==================================================
304	#D-function IGCM_sys_MkdirArchive
305	#D-* Purpose: Mkdir on Archive
306	#D-* Examples:
307	#D-
308	function IGCM_sys_MkdirArchive {
309	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
310	if ( $DEBUG_sys ) ; then
311	echo "IGCM_sys_MkdirArchive :" $@
312	fi
313	#- creation de repertoire sur le serveur fichier
314	if [ ! -d ${1} ]; then
315	\mkdir -p $1
316	status=$?
317
318	if [ ${status} -gt 0 ] ; then
319	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
320	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
321	fi
322	fi
323	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
324	}
325
326	#D-#==================================================
327	#D-function IGCM_sys_TestDirArchive
328	#D-* Purpose: Test Directory that must exists on Archive
329	#D-* Examples:
330	#D-
331	function IGCM_sys_TestDirArchive {
332	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
333	if ( $DEBUG_sys ) ; then
334	echo "IGCM_sys_TestDirArchive :" $@
335	fi
336	typeset ExistFlag
337	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
338	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
339	return ${ExistFlag}
340	}
341
342	#D-#==================================================
343	#D-function IGCM_sys_IsFileArchived
344	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
345	#D-* Examples:
346	#D-
347	function IGCM_sys_IsFileArchived {
348	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
349	if ( $DEBUG_sys ) ; then
350	echo "IGCM_sys_IsFileArchived :" $@
351	fi
352	typeset IsArchivedFlag
353	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
354	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
355
356	return ${IsArchivedFlag}
357	}
358
359	#D-#==================================================
360	#D-function IGCM_sys_TestFileArchive
361	#D-* Purpose: Test file that must NOT EXISTS on Archive
362	#D-* Examples:
363	#D-
364	function IGCM_sys_TestFileArchive {
365	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
366	typeset ExistFlag
367	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
368	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
369
370	return ${ExistFlag}
371	}
372
373	#D-#==================================================
374	#D-function IGCM_sys_CountFileArchive
375	#D-* Purpose: Count files on Archive filesystem
376	#D-* Examples:
377	#D-
378	function IGCM_sys_CountFileArchive {
379	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
380	ls ${@} 2>/dev/null \| wc -l
381	if [ $? -gt 0 ] ; then
382	echo "IGCM_sys_CountFileArchive : erreur."
383	fi
384	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
385	}
386
387	#D-#==================================================
388	#D-function IGCM_sys_Tree
389	#D-* Purpose: Tree directories with files on ${ARCHIVE}
390	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
391	#D-
392	function IGCM_sys_Tree {
393	IGCM_debug_PushStack "IGCM_sys_Tree" $@
394	if ( $DEBUG_sys ) ; then
395	echo "IGCM_sys_Tree :" $@
396	fi
397
398	\ls -lR ${@}
399
400	IGCM_debug_PopStack "IGCM_sys_Tree"
401	}
402
403	#D-#==================================================
404	#D-function IGCM_sys_Qsub
405	#D-* Purpose: Qsub new job
406	#D-* Examples:
407	#D-
408	function IGCM_sys_Qsub {
409	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
410	if ( $DEBUG_sys ) ; then
411	echo "IGCM_sys_Qsub :" $@
412	fi
413	typeset options status
414	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
415
416	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
417	status=$?
418
419	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
420	if [ ${status} -gt 0 ] ; then
421	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
422	IGCM_debug_Exit "IGCM_sys_Qsub"
423	else
424	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
425	fi
426	IGCM_debug_PopStack "IGCM_sys_Qsub"
427	}
428
429	#D-#==================================================
430	#D-function IGCM_sys_QsubPost
431	#D-* Purpose: Qsub new job on scalaire
432	#D-* Examples:
433	#D-
434	function IGCM_sys_QsubPost {
435	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
436	if ( $DEBUG_sys ) ; then
437	echo "IGCM_sys_QsubPost :" $@
438	fi
439	typeset options status
440
441	# EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
442	case $( hostname -s ) in
443	curie*)
444	options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
445	airain*)
446	options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
447	esac
448
449	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
450	status=$?
451
452	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
453	if [ ${status} -gt 0 ] ; then
454	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
455	IGCM_debug_Exit "IGCM_sys_QsubPost"
456	else
457	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
458	fi
459	IGCM_debug_PopStack "IGCM_sys_QsubPost"
460	}
461
462	#D-*************************
463	#D- File transfer functions
464	#D-*************************
465	#D-
466
467	#D-#==================================================
468	#D-function IGCM_sys_RmRunDir
469	#D-* Purpose: rm tmpdir (dummy function most of the time batch
470	#D- scheduler will do the job)
471	#D-* Examples:
472	#D-
473	function IGCM_sys_RmRunDir {
474	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
475	if ( $DEBUG_sys ) ; then
476	echo "IGCM_sys_RmRunDir :" $@
477	fi
478
479	typeset status
480
481	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
482	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
483	status=$?
484
485	if [ ${status} -gt 0 ] ; then
486	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
487	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
488	IGCM_debug_Exit "IGCM_sys_RmRunDir"
489	else
490	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
491	fi
492	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
493	}
494
495	#D-#==================================================
496	#D-function IGCM_sys_Put_Dir
497	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
498	#D-* Examples:
499	#D-
500	function IGCM_sys_Put_Dir {
501	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
502	if ( $DEBUG_sys ) ; then
503	echo "IGCM_sys_Put_Dir :" $@
504	fi
505	if [ $DRYRUN = 0 ]; then
506	if [ ! -d ${1} ] ; then
507	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
508	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
509	return
510	fi
511
512	typeset status
513
514	# Only if we use rsync
515	#IGCM_sys_TestDirArchive $( dirname $2 )
516	#
517	#USUAL WAY
518	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
519	status=$?
520
521	if [ ${status} -gt 0 ] ; then
522	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
523	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
524	IGCM_debug_Exit "IGCM_sys_Put_Dir"
525	else
526	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
527	fi
528	fi
529	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
530	}
531
532	#D-#==================================================
533	#D-function IGCM_sys_Get_Dir
534	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
535	#D-* Examples:
536	#D-
537	function IGCM_sys_Get_Dir {
538	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
539	if ( $DEBUG_sys ) ; then
540	echo "IGCM_sys_Get_Dir :" $@
541	fi
542	if [ $DRYRUN = 0 ]; then
543	typeset NB_ESSAI DELAI status i
544	# number of tentative
545	NB_ESSAI=3
546	# time delay between tentative
547	DELAI=2
548
549	#
550	# USUAL WAY
551	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
552	ccc_hsm get -r $1
553
554	i=0
555	while [ $i -lt $NB_ESSAI ] ; do
556	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
557	status=$?
558	if [ ${status} -gt 0 ] ; then
559	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
560	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
561	sleep $DELAI
562	else
563	break
564	fi
565	(( i = i + 1 ))
566	done
567
568	if [ ${status} -gt 0 ] ; then
569	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
570	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
571	IGCM_debug_Exit "IGCM_sys_Get_Dir"
572	else
573	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
574	fi
575	fi
576	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
577	}
578
579	#D-#==================================================
580	#D-function IGCM_sys_Put_Rest
581	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
582	#D- File and target directory must exist.
583	#D-* Examples:
584	#D-
585	function IGCM_sys_Put_Rest {
586	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
587	if ( $DEBUG_sys ) ; then
588	echo "IGCM_sys_Put_Rest :" $@
589	fi
590	if [ $DRYRUN = 0 ]; then
591	if [ ! -f ${1} ] ; then
592	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
593	IGCM_debug_Exit "IGCM_sys_Put_Rest"
594	fi
595
596	typeset status
597	#
598	# USUAL WAY
599	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
600	status=$?
601
602	# #RSYNC WITH NETWORK SSH CALL
603	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
604	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
605
606	# #RSYNC WITH NFS USE
607	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
608	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
609
610	# status=$?
611	# IGCM_sys_Rsync_out $status
612
613	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
614	# (( status=status+$? ))
615
616	if [ ${status} -gt 0 ] ; then
617	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
618	[ -f ${1} ] && ls -l ${1}
619	[ -f ${2} ] && ls -l ${2}
620	[ -f ${2}/${1} ] && ls -l ${2}/${1}
621	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
622	IGCM_debug_Exit "IGCM_sys_Put_Rest"
623	else
624
625	if [ X${JobType} = XRUN ] ; then
626	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
627	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
628	fi
629
630	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
631	fi
632	fi
633	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
634	}
635
636	#D-#==================================================
637	#D-function IGCM_sys_Put_Out
638	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
639	#D-* Examples:
640	#D-
641	function IGCM_sys_Put_Out {
642	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
643	if ( $DEBUG_sys ) ; then
644	echo "IGCM_sys_Put_Out :" $@
645	fi
646
647	typeset NB_ESSAI DELAI status i exist skip
648	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
649
650	# number of tentative
651	NB_ESSAI=3
652	# time delay between tentative
653	DELAI=2
654
655	if [ $DRYRUN = 0 ]; then
656	if [ ! -f ${1} ] ; then
657	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
658	IGCM_debug_PopStack "IGCM_sys_Put_Out"
659	return 1
660	fi
661	#
662	IGCM_sys_MkdirArchive $( dirname $2 )
663	#
664	exist=false
665	skip=false
666	if [ -f $2 ] ; then
667	IGCM_debug_Print 1 "$2 already exist"
668	ccc_hsm get $2
669	exist=true
670	if [ "X$( diff $1 $2 )" = X ] ; then
671	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
672	skip=true
673	else
674	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
675	skip=false
676	fi
677	fi
678	#
679	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
680	IGCM_sys_Chmod u+w $2
681	fi
682
683	if [ X${skip} = Xfalse ] ; then
684	i=0
685	while [ $i -lt $NB_ESSAI ] ; do
686	# Identify file system
687	fileDeviceNumberInHex=$( stat -c %d $1 )
688	status=$?
689	if [ ${status} -gt 0 ] ; then
690	IGCM_debug_Exit "IGCM_sys_Put_Out"
691	fi
692	# Identify file system
693	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
694	status=$?
695	if [ ${status} -gt 0 ] ; then
696	IGCM_debug_Exit "IGCM_sys_Put_Out"
697	fi
698
699	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
700	# They are not on the same device. USUAL WAY
701	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
702	status=$?
703	else
704	# They are on the same device. NOT SO USUAL WAY
705	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
706	status=$?
707	fi
708	if [ ${status} -gt 0 ]; then
709	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
710	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
711	[ -f ${1} ] && ls -l ${1}
712	[ -f ${2} ] && ls -l ${2}
713	[ -f ${2}/${1} ] && ls -l ${2}/${1}
714	sleep $DELAI
715	else
716	break
717	fi
718	(( i = i + 1 ))
719	done
720	fi
721
722	# #RSYNC WITH NETWORK SSH CALL
723	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
724	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
725
726	# #RSYNC WITH NFS USE
727	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
728	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
729
730	# status=$?
731	# IGCM_sys_Rsync_out $status
732
733	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
734	# (( status=status+$? ))
735
736	if [ ${status} -gt 0 ] ; then
737	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
738	[ -f ${1} ] && ls -l ${1}
739	[ -f ${2} ] && ls -l ${2}
740	[ -f ${2}/${1} ] && ls -l ${2}/${1}
741	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
742	IGCM_debug_Exit "IGCM_sys_Put_Out"
743	else
744
745	if [ X${JobType} = XRUN ] ; then
746	if [ X${3} = X ] ; then
747	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
748	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
749	fi
750	fi
751
752	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
753	fi
754	fi
755	IGCM_debug_PopStack "IGCM_sys_Put_Out"
756	return 0
757	}
758
759	#D-#==================================================
760	#D-function IGCM_sys_Get
761	#D-* Purpose: Get a file from ${ARCHIVE}
762	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
763	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
764	function IGCM_sys_Get {
765	IGCM_debug_PushStack "IGCM_sys_Get" $@
766
767	typeset DEST dm_liste target file_work
768	typeset NB_ESSAI DELAI status i
769
770	if ( $DEBUG_sys ) ; then
771	echo "IGCM_sys_Get :" $@
772	fi
773
774	# number of tentative
775	NB_ESSAI=3
776	# time delay between tentative
777	DELAI=2
778
779	if [ $DRYRUN -le 2 ]; then
780	if [ X${1} = X'/l' ] ; then
781	eval set +A dm_liste \${${2}}
782	else
783	eval set +A dm_liste ${1}
784	fi
785	eval DEST=\${${#}}
786	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
787	status=$?
788	if [ ${status} -gt 0 ] ; then
789	echo "WARNING IGCM_sys_Get : error code ${status}"
790	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
791	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
792	fi
793
794	#if [ ${status} -gt 0 ] ; then
795	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
796	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
797	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
798	# sleep 30
799	# echo "We try another time"
800	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
801	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
802	# status=$?
803	# if [ ${status} -gt 0 ] ; then
804	# echo "ERROR IGCM_sys_Get : again demigration error :"
805	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
806	# IGCM_debug_Exit "IGCM_sys_Get"
807	# fi
808	# else
809	# echo "ERROR IGCM_sys_Get : demigration error :"
810	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
811	# IGCM_debug_Exit "IGCM_sys_Get"
812	# fi
813	#fi
814
815	# #RSYNC WITH NETWORK SSH CALL
816	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
817	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
818
819	# #RSYNC WITH NFS USE
820	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
821	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
822
823	# status=$?
824	# IGCM_sys_Rsync_out $status
825
826	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
827	# (( status=status+$? ))
828
829	#USUAL WAY
830	if [ X${1} = X'/l' ] ; then
831	for target in ${dm_liste[*]} ; do
832	local_file=$( basename ${target} )
833	# test if the target file is present before the loop
834	IGCM_sys_TestFileArchive ${target}
835	status=$?
836	if [ ${status} -gt 0 ] ; then
837	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
838	IGCM_debug_Exit "IGCM_sys_Get"
839	else
840	i=0
841	while [ $i -lt $NB_ESSAI ] ; do
842	#if [ X${DoLink} = Xtrue ] ; then
843	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
844	# status=$?
845	# else
846	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
847	# status=$?
848	#fi
849	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
850	status=$?
851	if [ ${status} -gt 0 ]; then
852	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
853	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
854	sleep $DELAI
855	else
856	break
857	fi
858	(( i = i + 1 ))
859	done
860	if [ ${status} -gt 0 ] ; then
861	echo "IGCM_sys_Get : error"
862	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
863	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
864	IGCM_debug_Exit "IGCM_sys_Get"
865	else
866	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
867	fi
868	fi
869	done
870	else
871	i=0
872	while [ $i -lt $NB_ESSAI ] ; do
873	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
874	status=$?
875	if [ ${status} -gt 0 ]; then
876	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
877	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
878	sleep $DELAI
879	else
880	break
881	fi
882	(( i = i + 1 ))
883	done
884	if [ ${status} -gt 0 ] ; then
885	echo "IGCM_sys_Get : error"
886	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
887	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
888	IGCM_debug_Exit "IGCM_sys_Get"
889	else
890	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
891	fi
892	fi
893	fi
894	IGCM_debug_PopStack "IGCM_sys_Get"
895	}
896
897	#D-#==================================================
898	#D-function IGCM_sys_GetDate_Monitoring
899	#D-* Purpose: get the last year for which the monitoring has been computed
900	#D-* Examples:
901	#D-
902	function IGCM_sys_GetDate_Monitoring {
903	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
904	if ( $DEBUG_sys ) ; then
905	echo "IGCM_sys_GetDate_Monitoring :" $@
906	fi
907
908	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
909
910	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
911	}
912
913	#D-#==================================================
914	#D-function IGCM_sys_Dods_Rm
915	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
916	#D-* Examples:
917	#D-
918	function IGCM_sys_Dods_Rm {
919	if ( $DEBUG_sys ) ; then
920	echo "IGCM_sys_Dods_Rm :" $@
921	fi
922	typeset status
923	if [ $DRYRUN = 0 ]; then
924
925	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
926	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
927	# echo "Nothing has been done."
928	# return
929	# fi
930
931	if [ "$#" -eq 1 ]; then
932	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
933	status=$?
934	else
935	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
936	status=$?
937	fi
938
939	# if [ ${status} -gt 0 ] ; then
940	# echo "IGCM_sys_Dods_Rm : error."
941	# cat out_dods_rm
942	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
943	# else
944	# rm out_dods_rm
945	# fi
946
947	fi
948	return $status
949	}
950
951	#D-#==================================================
952	#D-function IGCM_sys_Dods_Cp
953	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
954	#D-* Examples:
955	#D-
956	function IGCM_sys_Dods_Cp {
957	if ( $DEBUG_sys ) ; then
958	echo "IGCM_sys_Dods_Cp :" $@
959	fi
960	typeset status
961	if [ $DRYRUN = 0 ]; then
962
963	# if [ ! -d ${R_SAVE}/${1} ] ; then
964	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
965	# echo "Nothing has been done."
966	# return
967	# fi
968
969	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
970	status=$?
971
972	# if [ ${status} -gt 0 ] ; then
973	# echo "IGCM_sys_Dods_Cp : error."
974	# cat out_dods_cp
975	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
976	# else
977	# rm out_dods_cp
978	# fi
979
980	fi
981	return $status
982	}
983
984	#D-#==================================================
985	#D-function IGCM_sys_Put_Dods
986	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
987	#D-* Examples:
988	#D-
989	function IGCM_sys_Put_Dods {
990	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
991	if ( $DEBUG_sys ) ; then
992	echo "IGCM_sys_Put_Dods :" $@
993	fi
994	typeset status
995	if [ $DRYRUN = 0 ]; then
996	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
997	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
998	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
999	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1000	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1001	return
1002	fi
1003
1004	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1005	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1006	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1007	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1008	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1009	return
1010	fi
1011	#
1012	if [ -d ${R_SAVE}/${1} ] ; then
1013	cd ${R_SAVE}
1014	elif [ -d ${R_FIGR}/${1} ] ; then
1015	cd ${R_FIGR}
1016	fi
1017
1018	IGCM_sys_Dods_Rm ${1}
1019	IGCM_sys_Dods_Cp ${1}
1020	status=0
1021
1022	if [ ${status} -gt 0 ] ; then
1023	echo "IGCM_sys_Put_Dods : error."
1024	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1025	fi
1026	fi
1027	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1028	}
1029
1030	##############################################################
1031	# REBUILD OPERATOR
1032
1033	#D-#==================================================
1034	#D-function IGCM_sys_sync
1035	#D-* Purpose: flush buffer on disk
1036	#D-* Examples:
1037	#D-
1038	function IGCM_sys_sync {
1039	IGCM_debug_PushStack "IGCM_sys_sync" $@
1040	if ( $DEBUG_sys ) ; then
1041	echo "IGCM_sys_sync :" $@
1042	fi
1043
1044	/bin/sync
1045
1046	IGCM_debug_PopStack "IGCM_sys_sync"
1047	}
1048
1049	############################################################
1050	# Activate Running Environnment Variables
1051
1052	#D-#==================================================
1053	#D-function IGCM_sys_activ_variables
1054	#D-* Purpose: set environement variables prior to execution
1055	#D-* Examples:
1056	#D-
1057	function IGCM_sys_activ_variables {
1058	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1059	if ( $DEBUG_sys ) ; then
1060	echo "IGCM_sys_activ_variables"
1061	fi
1062
1063	# --------------------------------------------------------------------
1064	#D- MODULE specifications
1065	# --------------------------------------------------------------------
1066	module purge > /dev/null 2>&1
1067	if [ X${compilerVersion} = XV17 ] ; then
1068	. /ccc/cont003/home/dsm/p86ipsl/.env_intel17.0.2_curie > /dev/null 2>&1
1069	elif [ X${compilerVersion} = XV14 ] ; then
1070	. /ccc/cont003/home/dsm/p86ipsl/.env_intel14.0.3_curie > /dev/null 2>&1
1071	else
1072	IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1073	fi
1074	. /ccc/cont003/home/dsm/p86ipsl/.env_netcdf4.3.3.1_curie > /dev/null 2>&1
1075
1076	# Keep track of this
1077	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
1078	[ ! X${TaskType} = Xchecking ] && module list
1079
1080	# --------------------------------------------------------------------
1081	#D- MPI specifications
1082	# --------------------------------------------------------------------
1083
1084	# --------------------------------------------------------------------
1085	#D- Other specifications
1086	# --------------------------------------------------------------------
1087
1088	ulimit -s unlimited
1089
1090	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1091	}
1092
1093	############################################################
1094	# Desactivate Running Environnment Variables
1095
1096	#D-#==================================================
1097	#D-function IGCM_sys_desactiv_variables
1098	#D-* Purpose: unset environement variables after execution
1099	#D-* Examples:
1100	#D-
1101	function IGCM_sys_desactiv_variables {
1102	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1103	if ( $DEBUG_sys ) ; then
1104	echo "IGCM_sys_desactiv_variables"
1105	fi
1106	# --------------------------------------------------------------------
1107	#D- MPI specifications
1108	# --------------------------------------------------------------------
1109
1110	# --------------------------------------------------------------------
1111	#D- Other specifications
1112	# --------------------------------------------------------------------
1113
1114	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1115	}
1116
1117	############################################################
1118	# Update job headers to be used by the scheduler
1119
1120	#D-#==================================================
1121	#D-function IGCM_sys_updateHeaders
1122	#D-* Purpose: Update job headers to be used by the scheduler
1123	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1124	#D-
1125	function IGCM_sys_updateHeaders {
1126	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1127	if ( $DEBUG_sys ) ; then
1128	echo "IGCM_sys_updateHeaders"
1129	fi
1130	typeset file
1131	file=$1
1132
1133	if [ ${executionType} -eq 1 ] ; then
1134	# MPMD + MPI
1135	sed -e "/::openMPthreads::/d" \
1136	-e "s/::JobNumProcTot::/${coreNumber}/" \
1137	-e "/#MSUB -x/d" \
1138	-e "/--cpu_bind=none/d" \
1139	${file} > ${file}.tmp
1140
1141	elif [ ${executionType} -eq 2 ] ; then
1142	# MPMD + MPI + OMP : mpirun/ccc_mprun/error
1143	if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1144	sed -e "/::openMPthreads::/d" \
1145	-e "s/::JobNumProcTot::/${coreNumber}/" \
1146	${file} > ${file}.tmp
1147	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1148	sed -e "/::openMPthreads::/d" \
1149	-e "s/::JobNumProcTot::/${coreNumber}/" \
1150	-e "/--cpu_bind=none/d" \
1151	${file} > ${file}.tmp
1152	else
1153	IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
1154	IGCM_debug_Exit "IGCM_sys_updateHeaders"
1155	fi
1156	elif [ ${executionType} -eq 3 ] ; then
1157	# SPMD + MPI/OMP
1158	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1159	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1160	-e "/#MSUB -x/d" \
1161	-e "/--cpu_bind=none/d" \
1162	${file} > ${file}.tmp
1163
1164	elif [ ${executionType} -eq 4 ] ; then
1165	# SPMD + MPI only
1166	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
1167	-e "/::openMPthreads::/d" \
1168	-e "/#MSUB -x/d" \
1169	-e "/--cpu_bind=none/d" \
1170	${file} > ${file}.tmp
1171
1172	elif [ ${executionType} -eq 5 ] ; then
1173	# SPMD + OMP only
1174	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1175	-e "/::JobNumProcTot::/d" \
1176	-e "/#MSUB -x/d" \
1177	-e "/--cpu_bind=none/d" \
1178	${file} > ${file}.tmp
1179
1180	elif [ ${executionType} -eq 6 ] ; then
1181	# SEQUENTIAL THEN
1182	sed -e "s/::JobNumProcTot::/1/" \
1183	-e "/::openMPthreads::/d" \
1184	-e "/#MSUB -x/d" \
1185	-e "/--cpu_bind=none/d" \
1186	${file} > ${file}.tmp
1187
1188	fi
1189
1190	IGCM_sys_Mv ${file}.tmp ${file}
1191
1192	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1193	}
1194
1195	############################################################
1196	# Build MPI/OMP scripts run file (dummy function)
1197
1198	#D-#==================================================
1199	#D-function IGCM_sys_build_run_file
1200	#D-* Purpose: build run file (deprecated)
1201	#D-* Examples:
1202	#D-
1203	function IGCM_sys_build_run_file {
1204	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1205	}
1206
1207	############################################################
1208	# Build MPI/OMP scripts
1209
1210	#D-#==================================================
1211	#D-function IGCM_sys_build_execution_scripts
1212	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1213	#D-* Examples:
1214	#D-
1215	function IGCM_sys_build_execution_scripts
1216	{
1217	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1218	if ( $DEBUG_sys ) ; then
1219	echo "IGCM_sys_build_execution_scripts " $@
1220	fi
1221
1222	EXECUTION=${HOST_MPIRUN_COMMAND}
1223
1224	if ( ${OK_PARA_MPMD} ) ; then
1225
1226	# MPMD mode
1227	# 1 MPI only
1228	# 2 MPI/OpenMP mpirun method
1229	# 3 MPI/OpenMP ccc_mprun method
1230
1231	if [ -f run_file ] ; then
1232	IGCM_sys_Rm -f run_file
1233	fi
1234	touch run_file
1235
1236	# case 1 : Only MPI (MPMD)
1237	if ( ! ${OK_PARA_OMP} ) ; then
1238
1239	# Build run_file
1240
1241	# First loop on the components for the coupler ie oasis (only if oasis3)
1242	# the coupler ie oasis3 must be the first one
1243	for comp in ${config_ListOfComponents[*]} ; do
1244
1245	eval ExeNameIn=\${config_Executable_${comp}[0]}
1246	eval ExeNameOut=\${config_Executable_${comp}[1]}
1247
1248	# for CPL component only
1249	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1250
1251	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1252	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1253	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1254	fi
1255	done
1256
1257	# Then second loop on the components
1258	for comp in ${config_ListOfComponents[*]} ; do
1259
1260	eval ExeNameIn=\${config_Executable_${comp}[0]}
1261	eval ExeNameOut=\${config_Executable_${comp}[1]}
1262
1263	# Only if we really have an executable for the component and not the coupler ie oasis:
1264	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1265
1266	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1267	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1268	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1269	fi
1270	done
1271
1272	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1273
1274	IGCM_sys_Chmod u+x run_file
1275	if ( $DEBUG_sys ) ; then
1276	echo "run_file contains : "
1277	cat run_file
1278	fi
1279
1280	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1281
1282	# 2 MPI/OpenMP mpirun method
1283
1284	# Use of mpirun instead of ccc_mprun
1285	EXECUTION="time mpirun"
1286
1287	# Hosts treatment
1288	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
1289
1290	i=0
1291	rm -f hosts rankfile
1292	IGCM_debug_Print 1 "sys Curie, Hosts available :"
1293	for nodes in `cat hosts.tmp`
1294	do
1295	host[$i]=$nodes
1296	echo "${host[$i]}" >> hosts
1297	IGCM_debug_Print 1 ${host[$i]}
1298	i=$((i+1))
1299	done
1300	rm -f hosts.tmp
1301
1302	listnodes=${host[*]}
1303
1304	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1305
1306	# Initialisation
1307	rank=0
1308	current_core=0
1309	init_exec=n
1310
1311	# Loop on the components
1312	for comp in ${config_ListOfComponents[*]} ; do
1313
1314	eval ExeNameIn=\${config_Executable_${comp}[0]}
1315	eval ExeNameOut=\${config_Executable_${comp}[1]}
1316
1317	# Not possible if oasis has an executable (i.e old version of oasis3)
1318	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1319	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1320	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1321	IGCM_debug_Verif_Exit
1322	fi
1323
1324	# Only if we really have an executable for the component :
1325	if [ "X${ExeNameOut}" != X\"\" ] ; then
1326
1327	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1328	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1329	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1330
1331	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1332	echo "" >> script_${ExeNameOut}.ksh
1333	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1334
1335	# Check if the number of threads is correct
1336	case ${comp_proc_omp_loc} in
1337	2\|4\|8\|16)
1338	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1339	;;
1340	*)
1341	IGCM_debug_Exit "ERROR with OMP parameters !"
1342	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1343	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1344	IGCM_debug_Verif_Exit
1345	;;
1346	esac
1347	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1348	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1349	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1350	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1351	fi
1352
1353	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1354	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1355	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1356	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1357	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1358
1359	if [ ${init_exec} = y ] ; then
1360	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1361	else
1362	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1363	init_exec=y
1364	fi
1365
1366	# Build rankfile : method used to assign cores and nodes for the MPI process
1367	# Ex :
1368	#rank 0=curie5296 slot=0,1,2,3
1369	#rank 1=curie5296 slot=4,5,6,7
1370	# Example of final command :
1371	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1372	# with script_lmdz.x.ksh :
1373	# #!/bin/ksh
1374	#export KMP_STACKSIZE=3g
1375	#export KMP_LIBRARY=turnaround
1376	#export MKL_SERIAL=YES
1377	#OMP_NUM_THREADS=4
1378	#./lmdz.x
1379	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1380	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1381	else
1382	(( offset_comp_proc_loc = comp_proc_omp_loc ))
1383	fi
1384	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1385	(( index_host = current_core / NB_CORE_PER_NODE ))
1386	host_value=${host[${index_host}]}
1387	(( slot = current_core % NB_CORE_PER_NODE ))
1388	virg=","
1389	string_final=""
1390	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1391	string=$index$virg
1392	string_final=$string_final$string
1393	done
1394	string_final=$( echo $string_final \| sed "s/.$//" )
1395	echo "rank $rank=$host_value slot=$string_final" >> rankfile
1396	(( rank = rank + 1 ))
1397	(( current_core = current_core + offset_comp_proc_loc ))
1398	done
1399	fi
1400	done
1401
1402	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1403
1404	# 3 MPI/OpenMP ccc_mprun method
1405
1406	# MPI-OpenMP (MPMD)
1407	# example of run_file
1408	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1409	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1410	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1411
1412	# Loop on the components
1413	for comp in ${config_ListOfComponents[*]} ; do
1414
1415	eval ExeNameIn=\${config_Executable_${comp}[0]}
1416	eval ExeNameOut=\${config_Executable_${comp}[1]}
1417
1418	# Not possible if oasis has an executable (i.e old version of oasis3)
1419	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1420	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1421	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1422	IGCM_debug_Verif_Exit
1423	fi
1424
1425	# Only if we really have an executable for the component :
1426	if [ "X${ExeNameOut}" != X\"\" ] ; then
1427
1428	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1429	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1430
1431	# Build script files
1432
1433	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1434	echo "" >> script_${ExeNameOut}.ksh
1435	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1436
1437	# Check if the number of threads is correct
1438	case ${comp_proc_omp_loc} in
1439	2\|4\|8\|16)
1440	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1441	;;
1442	*)
1443	IGCM_debug_Exit "ERROR with OMP parameters !"
1444	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1445	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1446	IGCM_debug_Verif_Exit
1447	;;
1448	esac
1449	### only for LMDZ?
1450	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1451	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1452	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1453	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1454
1455	fi
1456
1457	# to have out/err per process on different files
1458	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1459	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1460
1461	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1462
1463	# Complete run_file
1464
1465	echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1466
1467	fi
1468	done
1469
1470	## variable added to stop after 60s instead of 600s by default.
1471	## This is used when no error comes from executables and when something stopped an executable without notice.
1472	export SLURM_WAIT=60
1473
1474	## module advised by TGCC (instead of 2 variables)
1475	module load feature/bridge/heterogenous_mpmd
1476
1477	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1478
1479	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1480
1481	IGCM_sys_Chmod u+x run_file
1482	if ( $DEBUG_sys ) ; then
1483	echo "run_file contains : "
1484	cat run_file
1485	fi
1486
1487	else
1488
1489	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1490
1491	fi # 1 2 or 3 if ${OK_PARA_MPMD}
1492
1493	else
1494	# Only one executable (SPMD mode).
1495
1496	for comp in ${config_ListOfComponents[*]} ; do
1497
1498	# Only if we really have an executable for the component :
1499	eval ExeNameOut=\${config_Executable_${comp}[1]}
1500	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1501
1502	# Build script files
1503
1504	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1505	echo "" >> script_${ExeNameOut}.ksh
1506	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1507
1508	if ( ${OK_PARA_OMP} ) ; then
1509	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1510	echo "" >> script_${ExeNameOut}.ksh
1511	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1512	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1513	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1514	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1515	fi
1516
1517	if ( ${OK_PARA_MPI} ) ; then
1518	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1519	# Default : ccc_mprun used if nb_proc gt 1
1520	# to have out/err per process on different files
1521	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1522	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1523	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1524	else
1525	# Default : ccc_mprun is NOT used if nb_proc eq 1
1526	# to have out/err per process on different files
1527	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1528	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1529	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1530	fi
1531
1532	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
1533	cat script_${ExeNameOut}.ksh
1534
1535	fi
1536
1537	done
1538
1539	fi # ${OK_PARA_MPMD}
1540
1541	IGCM_debug_Print 1 "sys Curie : execution command is "
1542	IGCM_debug_Print 1 "$EXECUTION"
1543
1544	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1545	}
1546
1547	#D-#==================================================
1548	#D-function IGCM_sys_check_path
1549	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1550	#D-* do not point to an important use directory. Stop immediately in that case.
1551	#D-* Examples:
1552	#D-
1553	function IGCM_sys_check_path {
1554	IGCM_debug_PushStack "IGCM_sys_check_path"
1555	if ( $DEBUG_sys ) ; then
1556	echo "IGCM_sys_check_path"
1557	fi
1558
1559	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1560	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1561	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1562	IGCM_debug_Exit "This will stop the job"
1563	fi
1564	IGCM_debug_PopStack "IGCM_sys_check_path"
1565	}
1566
1567	#D-#==================================================
1568	#D-function IGCM_sys_check_quota
1569	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1570	#D-* Examples:
1571	#D-
1572	function IGCM_sys_check_quota {
1573	IGCM_debug_PushStack "IGCM_sys_check_quota"
1574	if ( $DEBUG_sys ) ; then
1575	echo "IGCM_sys_check_quota"
1576	fi
1577	# Limit of quota (in %)
1578	limit_quota=90
1579
1580	# Check of the volume
1581	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
1582	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
1583
1584	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1585
1586	unit_avail=${volume_avail: -1}
1587	unit_quota=${volume_quota: -1}
1588
1589	if [ "${unit_quota}" = "*" ] ; then
1590	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1591	IGCM_debug_Print 1 "More than 100% of your quota is used"
1592	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1593	IGCM_debug_Print 1 "You must have more than 10% available to run"
1594	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1595	IGCM_debug_Verif_Exit
1596	fi
1597
1598	temp_avail=${volume_avail%%${volume_avail: -1}*}
1599	temp_quota=${volume_quota%%${volume_quota: -1}*}
1600
1601	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1602
1603	# Convertion
1604	if [ ${volume_avail: -1} = "T" ] ; then
1605	(( temp_avail = temp_avail * 1000000000000 ))
1606	elif [ ${volume_avail: -1} = "G" ] ; then
1607	(( temp_avail = temp_avail * 1000000000 ))
1608	elif [ ${volume_avail: -1} = "M" ] ; then
1609	(( temp_avail = temp_avail * 1000000 ))
1610	elif [ ${volume_avail: -1} = "k" ] ; then
1611	(( temp_avail = temp_avail * 1000 ))
1612	else
1613	(( temp_avail = volume_avail ))
1614	fi
1615	if [ ${volume_quota: -1} = "T" ] ; then
1616	(( temp_quota = temp_quota * 1000000000000 ))
1617	elif [ ${volume_quota: -1} = "G" ] ; then
1618	(( temp_quota = temp_quota * 1000000000 ))
1619	elif [ ${volume_quota: -1} = "M" ] ; then
1620	(( temp_quota = temp_quota * 1000000 ))
1621	elif [ ${volume_quota: -1} = "k" ] ; then
1622	(( temp_quota = temp_quota * 1000 ))
1623	else
1624	(( temp_quota = volume_quota ))
1625	fi
1626	fi
1627
1628	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1629	# echo "volume ratio is " $quota_volume
1630
1631	if [ ${quota_volume} -ge ${limit_quota} ] ; then
1632	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1633	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1634	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1635	IGCM_debug_Print 1 "You must have more than 10% available to run"
1636	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1637	IGCM_debug_Verif_Exit
1638	fi
1639
1640	fi
1641
1642	# Check of the number of inodes
1643
1644	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
1645	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
1646
1647	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1648
1649	unit_avail=${inode_avail: -1}
1650	unit_quota=${inode_quota: -1}
1651
1652	if [ "${unit_quota}" = "*" ] ; then
1653	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1654	IGCM_debug_Print 1 "More than 100% of your quota is used"
1655	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1656	IGCM_debug_Print 1 "You must have more than 10% available to run"
1657	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1658	IGCM_debug_Verif_Exit
1659	fi
1660
1661	temp_avail=${inode_avail%%${inode_avail: -1}*}
1662	temp_quota=${inode_quota%%${inode_quota: -1}*}
1663
1664	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1665
1666	# Convertion
1667	if [ ${inode_avail: -1} = "T" ] ; then
1668	(( temp_avail = temp_avail * 1000000000000 ))
1669	elif [ ${inode_avail: -1} = "G" ] ; then
1670	(( temp_avail = temp_avail * 1000000000 ))
1671	elif [ ${inode_avail: -1} = "M" ] ; then
1672	(( temp_avail = temp_avail * 1000000 ))
1673	elif [ ${inode_avail: -1} = "k" ] ; then
1674	(( temp_avail = temp_avail * 1000 ))
1675	else
1676	(( temp_avail = inode_avail ))
1677	fi
1678
1679	if [ ${inode_quota: -1} = "T" ] ; then
1680	(( temp_quota = temp_quota * 1000000000000 ))
1681	elif [ ${inode_quota: -1} = "G" ] ; then
1682	(( temp_quota = temp_quota * 1000000000 ))
1683	elif [ ${inode_quota: -1} = "M" ] ; then
1684	(( temp_quota = temp_quota * 1000000 ))
1685	elif [ ${inode_quota: -1} = "k" ] ; then
1686	(( temp_quota = temp_quota * 1000 ))
1687	else
1688	(( temp_quota = inode_quota ))
1689	fi
1690	fi
1691	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1692	# echo "inode ratio is " $quota_inode
1693
1694	if [ ${quota_inode} -ge ${limit_quota} ] ; then
1695	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1696	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1697	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1698	IGCM_debug_Print 1 "You must have more than 10% available to run"
1699	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1700	IGCM_debug_Verif_Exit
1701	fi
1702	fi
1703	IGCM_debug_PopStack "IGCM_sys_check_quota"
1704	}
1705
1706	#D-#==================================================
1707	#D-function IGCM_sys_projectAccounting
1708	#D-* Purpose: store project accounting information in a file
1709	#D-* Examples:
1710	#D-
1711	function IGCM_sys_projectAccounting {
1712	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1713	if ( $DEBUG_sys ) ; then
1714	echo "IGCM_sys_check_quota"
1715	fi
1716
1717	ssh curie91 /usr/bin/ccc_myproject > $1
1718
1719	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1720	}
1721
1722	#D-#==================================================
1723	#D-function IGCM_sys_getJobSchedulerID
1724	#D-* Purpose: Get the job ID during execution
1725	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1726	#D-
1727	function IGCM_sys_getJobSchedulerID {
1728	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1729	if ( $DEBUG_sys ) ; then
1730	echo "IGCM_sys_getJobSchedulerID"
1731	fi
1732
1733	eval ${1}=${BRIDGE_MSUB_JOBID}
1734
1735	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1736	}
1737
1738	#D-#==================================================
1739	#D-function IGCM_sys_GetJobID
1740	#D-* Purpose: Get the job ID from the JobName
1741	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1742	#D-
1743	function IGCM_sys_GetJobID {
1744	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1745	if ( $DEBUG_sys ) ; then
1746	echo "IGCM_sys_GetJobID"
1747	fi
1748
1749	# With -f option, the full job name is given in the last column
1750	ID=$( ccc_mstat -f -u $2 \| \
1751	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1752
1753	eval ${3}=${ID}
1754	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1755	}
1756
1757	#D-#==================================================
1758	#D-function IGCM_sys_CountJobInQueue
1759	#D-* Purpose: Count number of users job
1760	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1761	#D-
1762	function IGCM_sys_CountJobInQueue {
1763	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1764	if ( $DEBUG_sys ) ; then
1765	echo "IGCM_sys_CountJobInQueue"
1766	fi
1767
1768	# With -f option, the full job name is given in the last column
1769	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1770
1771	eval ${2}=${NbRun}
1772
1773	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1774	}
1775
1776	#D-#==================================================
1777	#D-function IGCM_sys_ListJobInQueue
1778	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1779	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1780	#D-
1781	function IGCM_sys_ListJobInQueue {
1782	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1783	if ( $DEBUG_sys ) ; then
1784	echo "IGCM_sys_ListJobInQueue"
1785	fi
1786
1787	# With -f option, the full job name is given in the last column
1788	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
1789	'( $2 == User && \
1790	$NF != /TS/ && \
1791	$NF !~ /PACK/ && \
1792	$NF !~ /REBUILD/ && \
1793	$NF !~ /pack/ ) \
1794	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1795
1796	eval set -A ${2} ${JobList[*]}
1797
1798	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1799	}
1800
1801	#D-#==================================================
1802	#D-function IGCM_sys_atlas
1803	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1804	#D-* Examples:
1805	#D-
1806	function IGCM_sys_atlas {
1807	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1808	if ( $DEBUG_sys ) ; then
1809	echo "IGCM_sys_atlas :" $@
1810	fi
1811
1812	typeset status
1813
1814	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1815	status=$?
1816	if [ ${status} -gt 0 ] ; then
1817	echo "IGCM_sys_atlas : error code ${status}"
1818	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1819	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1820	IGCM_debug_PopStack "IGCM_sys_atlas"
1821	return 1
1822	else
1823	IGCM_debug_PopStack "IGCM_sys_atlas"
1824	return 0
1825	fi
1826
1827	IGCM_debug_PopStack "IGCM_sys_atlas"
1828	}
1829
1830	#D-#==================================================
1831	#D-function IGCM_sys_rebuild_nemo
1832	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1833	#D-* Examples:
1834	#D-
1835
1836	function IGCM_sys_rebuild_nemo {
1837	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1838	if ( $DEBUG_sys ) ; then
1839	echo "IGCM_sys_rebuild_nemo :" $@
1840	fi
1841
1842	$( ccc_home -u p86ipsl )/REBUILD_NEMO/rebuild_nemo ${1} ${2}
1843
1844	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1845	}
1846

Note: See TracBrowser for help on using the repository browser.

Download in other formats: