Context Navigation

source: tags/libIGCM_v2.0/libIGCM_sys/libIGCM_sys_titane.ksh @ 1170

Last change on this file since 1170 was 853, checked in by sdipsl, 11 years ago
Clean up job submission
Property svn:keywords set to `Revision Author Date`
File size: 83.9 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Titane
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe param files \| \| Chmod \| \|
46	# \| \| Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=titane
71
72	#D-
73	#D-#==================================================
74	#D-Program used in libIGCM
75	#D-#==================================================
76
77	# Submit command
78	typeset SUBMIT=${SUBMIT:=ccc_msub}
79	# rsync with path
80	typeset -r RSYNC=/usr/bin/rsync
81	# RSYNC_opt args to rsync
82	typeset -r RSYNC_opt="-va"
83	# ie storage filesystem
84	typeset -r STOREHOST=${MASTER}
85
86	#====================================================
87	# Source default environment
88	#====================================================
89	. /etc/profile
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo)
93	#====================================================
94	. /home/cont003/p86ipsl/.atlas_env_titane_ksh
95	if ( [ X${LSB_QUEUE} = Xmono ] \|\| [ X${LSB_QUEUE} = Xmonoext ] ) ; then
96	module switch nco/3.9.4 nco/3.9.4_netcdf4
97	fi
98
99	#====================================================
100	# Specific for ocean additionnal diagnostic
101	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
102	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
103
104	#====================================================
105	# Host specific DIRECTORIES
106	#====================================================
107
108	# ============ CESIUM START ============ #
109
110	#====================================================
111	#- Mirror libIGCM from titane to cesium if needed
112	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
113	#if [ ! ${ROOTSYS} = "home" ] ; then
114	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
115	#else
116	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
117	#fi
118
119	#====================================================
120	#- libIGCM_POST
121	#if ( ${MirrorlibIGCM} ) ; then
122	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
123	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
124	#else
125	# typeset -r libIGCM_POST=${libIGCM}
126	#fi
127
128	# ============ CESIUM END ============ #
129
130	#====================================================
131	#- MirrorlibIGCM for frontend
132	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
133
134	#====================================================
135	#- libIGCM_POST for frontend
136	typeset -r libIGCM_POST=${libIGCM}
137
138	#====================================================
139	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
140	typeset -r R_EXE="${MODIPSL}/bin"
141
142	#====================================================
143	#- SUBMIT_DIR : submission dir
144	typeset SUBMIT_DIR=${SUBMIT_DIR:=${LS_SUBCWD}}
145
146	#====================================================
147	#- IN
148	typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
149	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
150
151	#====================================================
152	#- ARCHIVE (dedicated to large files)
153	typeset ARCHIVE=${CCCSTOREDIR}
154
155	#====================================================
156	#- STORAGE (dedicated to small/medium files)
157	typeset STORAGE=${CCCWORKDIR}
158
159	#====================================================
160	#- R_OUT
161	typeset R_OUT=${ARCHIVE}/IGCM_OUT
162
163	#====================================================
164	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
165	typeset R_FIG=${STORAGE}/IGCM_OUT
166
167	#====================================================
168	#- R_BUF (ONLY FOR double copy an scratch)
169	typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
170
171	#====================================================
172	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
173	if ( [ ! X${LSB_QUEUE} = Xmono ] && [ ! X${LSB_QUEUE} = Xmonoext ] ) ; then
174	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${LSB_JOBID}}
175	else
176	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${LSB_JOBID}}
177	fi
178
179	#====================================================
180	#- HOST_MPIRUN_COMMAND
181	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
182
183	#====================================================
184	#- Max number of arguments passed to nco operator or demigration command
185	UNIX_MAX_LIMIT=360
186
187	#====================================================
188	#- set PackDefault true on titane
189	PackDefault=true
190
191	#====================================================
192	#- Number of core per node (max number of OpenMP task)
193	NUM_COREPERNODE=8
194
195	#====================================================
196	#- Default number of MPI task for IPSL coupled model
197	#- required for backward compatibility
198	#-
199	DEFAULT_NUM_PROC_OCE=5
200	DEFAULT_NUM_PROC_CPL=1
201	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
202	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
203
204	#D-#==================================================
205	#D-function IGCM_sys_ChangeArchive
206	#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
207	#D-* Examples:
208	#D-
209	function IGCM_sys_ChangeArchive {
210	IGCM_debug_PushStack "IGCM_sys_ChangeArchive"
211
212	R_OUT=${SCRATCHDIR}/IGCM_OUT
213	R_FIG=${SCRATCHDIR}/IGCM_OUT
214
215	IGCM_debug_Print 1 "R_OUT has been redefined = ${R_OUT}"
216	IGCM_debug_Print 1 "R_FIG has been redefined = ${R_FIG}"
217
218	IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
219	}
220
221	#D-#==================================================
222	#D-function IGCM_sys_RshMaster
223	#D-* Purpose: Connection to frontend machine.
224	#D-* Examples:
225	#D-
226	function IGCM_sys_RshMaster {
227	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
228	/bin/ksh <<-EOF
229	export libIGCM=${libIGCM}
230	export DEBUG_debug=${DEBUG_debug}
231	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
232	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
233	${@}
234	EOF
235	if [ $? -gt 0 ] ; then
236	echo "IGCM_sys_RshMaster : erreur."
237	IGCM_debug_Exit "IGCM_sys_RshMaster"
238	fi
239	IGCM_debug_PopStack "IGCM_sys_RshMaster"
240	}
241
242	#D-#==================================================
243	#D-function IGCM_sys_RshArchive
244	#D-* Purpose: Archive rsh command
245	#D-* Examples:
246	#D-
247	function IGCM_sys_RshArchive {
248	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
249	/bin/ksh <<-EOF
250	${@}
251	EOF
252	if [ $? -gt 0 ] ; then
253	echo "IGCM_sys_RshArchive : erreur."
254	IGCM_debug_Exit "IGCM_sys_RshArchive"
255	fi
256	IGCM_debug_PopStack "IGCM_sys_RshArchive"
257	}
258
259	#D-#==================================================
260	#D-function IGCM_sys_RshPost
261	#D-* Purpose: Post-process rsh command
262	#D-* Examples:
263	#D-
264	function IGCM_sys_RshPost {
265	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
266	if ( $DEBUG_sys ) ; then
267	echo "IGCM_sys_RshPost :" $@
268	fi
269
270	#echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
271	#cat tmp_IGCM_sys_RshPost_$$
272	# keep standard input (stdin) for the loop onto temporary file
273	cat >tmp_IGCM_sys_RshPost_$$
274
275	# ============ FRONTEND START ============ #
276
277	/bin/ksh <tmp_IGCM_sys_RshPost_$$
278	if [ $? -gt 0 ] ; then
279	echo "IGCM_sys_RshPost : erreur."
280	IGCM_debug_Exit "IGCM_sys_RshPost"
281	fi
282	# delete temporary file
283	\rm tmp_IGCM_sys_RshPost_$$
284
285	# ============ FRONTEND END ============ #
286
287	# ============ CESIUM START ============ #
288	# typeset NB_ESSAI DELAI status i
289	# if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
290	# #little hack so that rebuild submission is done on titane not an cesium
291	#
292	# libIGCM_POST_sed=$( echo $libIGCM_POST \| sed 's/\//\\\//g' )
293	# POST_DIR_sed=$( echo ${POST_DIR} \| sed 's/\//\\\//g' )
294	# sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
295	# sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
296	# sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
297	# sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
298	# \mv tmp.txt tmp_IGCM_sys_RshPost_$$
299	#
300	# echo cat tmp_IGCM_sys_RshPost_$$ AFTER
301	# cat tmp_IGCM_sys_RshPost_$$
302	#
303	# /bin/ksh <tmp_IGCM_sys_RshPost_$$
304	# if [ $? -gt 0 ] ; then
305	# echo "IGCM_sys_RshPost : erreur."
306	# IGCM_debug_Exit "IGCM_sys_RshPost"
307	# fi
308	# # delete temporary file
309	# \rm tmp_IGCM_sys_RshPost_$$
310	#
311	# else
312	# # number of tentative
313	# NB_ESSAI=10
314	# # time delay between tentative
315	# DELAI=10
316	# i=0
317	# while [ $i -ne $NB_ESSAI ] ; do
318	# ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
319	# status=$?
320	# if [ ${status} -ne 0 ];
321	# then
322	# sleep $DELAI
323	# else
324	# break
325	# fi
326	# let i=$i+1
327	# done
328	# # delete temporary file
329	# \rm tmp_IGCM_sys_RshPost_$$
330	#
331	# if [ ${status} -gt 0 ] ; then
332	# echo "IGCM_sys_RshPost : erreur."
333	# IGCM_debug_Exit "IGCM_sys_RshPost"
334	# fi
335	# fi
336
337	# ============ CESIUM END ============ #
338
339	IGCM_debug_PopStack "IGCM_sys_RshPost"
340	}
341
342	#D-#==================================================
343	#D-function IGCM_sys_SendMail
344	#D-* Purpose: Send mail when simulation is over
345	#D-* Examples:
346	#D-
347	function IGCM_sys_SendMail {
348	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
349	if ( $DEBUG_sys ) ; then
350	echo "IGCM_sys_SendMail :" $@
351	fi
352
353	if ( ${ExitFlag} ) ; then
354	status=failed
355	else
356	status=completed
357	fi
358
359	cat << END_MAIL > job_end.mail
360	Dear ${LOGIN},
361
362	Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
363	Job started : ${DateBegin}
364	Job ended : ${DateEnd}
365	Output files are available in ${R_SAVE}
366	Files to be rebuild are temporarily available in ${REBUILD_DIR}
367	Pre-packed files are temporarily available in ${R_BUFR}
368	Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
369	END_MAIL
370
371	if [ ! -z ${config_UserChoices_MailName} ] ; then
372	mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
373	elif [ -f ~/.forward ] ; then
374	mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
375	fi
376	sleep 10
377	rm -f job_end.mail
378
379	if [ $? -gt 0 ] ; then
380	echo "IGCM_sys_SendMail : erreur."
381	IGCM_debug_Exit "IGCM_sys_SendMail"
382	fi
383	IGCM_debug_PopStack "IGCM_sys_SendMail"
384	}
385
386	#D-#==================================================
387	#D-function IGCM_sys_Mkdir
388	#D-* Purpose: Master locale mkdir command
389	#D-* Examples:
390	#D-
391	function IGCM_sys_Mkdir {
392	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
393	if ( $DEBUG_sys ) ; then
394	echo "IGCM_sys_Mkdir :" $@
395	fi
396	if [ ! -d ${1} ]; then
397	\mkdir -p $1
398	if [ $? -gt 0 ] ; then
399	echo "IGCM_sys_Mkdir : erreur."
400	IGCM_debug_Exit "IGCM_sys_Mkdir"
401	fi
402	fi
403	# vérification :
404	if [ ! -d ${1} ] ; then
405	echo "IGCM_sys_Mkdir : erreur."
406	IGCM_debug_Exit "IGCM_sys_Mkdir"
407	fi
408	IGCM_debug_PopStack "IGCM_sys_Mkdir"
409	}
410
411	#D-#==================================================
412	#D-function IGCM_sys_MkdirArchive
413	#D-* Purpose: Mkdir on Archive
414	#D-* Examples:
415	#D-
416	function IGCM_sys_MkdirArchive {
417	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
418	if ( $DEBUG_sys ) ; then
419	echo "IGCM_sys_MkdirArchive :" $@
420	fi
421	#- creation de repertoire sur le serveur fichier
422	if [ ! -d ${1} ]; then
423	\mkdir -p $1
424	if [ $? -gt 0 ] ; then
425	echo "IGCM_sys_MkdirArchive : erreur."
426	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
427	fi
428	fi
429	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
430	}
431
432	#D-#==================================================
433	#D-function IGCM_sys_MkdirWork
434	#D-* Purpose: Mkdir on Work
435	#D-* Examples:
436	#D-
437	function IGCM_sys_MkdirWork {
438	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
439	if ( $DEBUG_sys ) ; then
440	echo "IGCM_sys_MkdirWork :" $@
441	fi
442	#- creation de repertoire sur le serveur fichier
443	if [ ! -d ${1} ]; then
444	\mkdir -p $1
445	if [ $? -gt 0 ] ; then
446	echo "IGCM_sys_MkdirWork : erreur."
447	IGCM_debug_Exit "IGCM_sys_MkdirWork"
448	fi
449	fi
450	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
451	}
452
453	#D-#==================================================
454	#D-function IGCM_sys_Cd
455	#D-* Purpose: master cd command
456	#D-* Examples:
457	#D-
458	function IGCM_sys_Cd {
459	IGCM_debug_PushStack "IGCM_sys_Cd" $@
460	if ( $DEBUG_sys ) ; then
461	echo "IGCM_sys_Cd :" $@
462	fi
463	\cd $1
464	if [ $? -gt 0 ] ; then
465	echo "IGCM_sys_Cd : erreur."
466	IGCM_debug_Exit "IGCM_sys_Cd"
467	fi
468	IGCM_debug_PopStack "IGCM_sys_Cd"
469	}
470
471	#D-#==================================================
472	#D-function IGCM_sys_Chmod
473	#D-* Purpose: Chmod
474	#D-* Examples:
475	#D-
476	function IGCM_sys_Chmod {
477	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
478	if ( $DEBUG_sys ) ; then
479	echo "IGCM_sys_Chmod :" $@
480	fi
481	if [ $DRYRUN -le 1 ]; then
482	\chmod $@
483	if [ $? -gt 0 ] ; then
484	echo "IGCM_sys_Chmod : erreur."
485	IGCM_debug_Exit "IGCM_sys_Chmod"
486	fi
487	fi
488	IGCM_debug_PopStack "IGCM_sys_Chmod"
489	}
490
491	#D-#==================================================
492	#D-function IGCM_sys_FileSize
493	#D-* Purpose: Filesize
494	#D-* Examples:
495	#D-
496	function IGCM_sys_FileSize {
497	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
498
499	typeset sizeF
500	set +A sizeF -- $( ls -la ${1} )
501	if [ $? -gt 0 ] ; then
502	IGCM_debug_Exit "IGCM_sys_FileSize"
503	fi
504	eval ${2}=${sizeF[4]}
505
506	IGCM_debug_PopStack "IGCM_sys_FileSize"
507	}
508
509	#D-#==================================================
510	#D-function IGCM_sys_TestDir
511	#D-* Purpose: Test Directory that must exists
512	#D-* Examples:
513	#D-
514	function IGCM_sys_TestDir {
515	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
516	if ( $DEBUG_sys ) ; then
517	echo "IGCM_sys_TestDir :" $@
518	fi
519	typeset ExistFlag
520	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
521	IGCM_debug_PopStack "IGCM_sys_TestDir"
522
523	return ${ExistFlag}
524	}
525
526	#D-#==================================================
527	#D-function IGCM_sys_TestDirArchive
528	#D-* Purpose: Test Directory that must exists on Archive
529	#D-* Examples:
530	#D-
531	function IGCM_sys_TestDirArchive {
532	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
533	if ( $DEBUG_sys ) ; then
534	echo "IGCM_sys_TestDirArchive :" $@
535	fi
536	typeset ExistFlag
537	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
538	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
539
540	return ${ExistFlag}
541	}
542
543	#D-#==================================================
544	#D-function IGCM_sys_IsFileArchived
545	#D-* Purpose: Test file that must NOT EXISTS on Archive
546	#D-* Examples:
547	#D-
548	function IGCM_sys_IsFileArchived {
549	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
550	if ( $DEBUG_sys ) ; then
551	echo "IGCM_sys_IsFileArchived :" $@
552	fi
553	typeset IsArchivedFlag
554	IsArchivedFlag=$( [ X$( echo $1 \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
555	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
556
557	return ${IsArchivedFlag}
558	}
559
560	#D-#==================================================
561	#D-function IGCM_sys_TestFileArchive
562	#D-* Purpose: Test file that must NOT EXISTS on Archive
563	#D-* Examples:
564	#D-
565	function IGCM_sys_TestFileArchive {
566	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
567	typeset ExistFlag
568	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
569	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
570
571	return ${ExistFlag}
572	}
573
574	#D-#==================================================
575	#D-function IGCM_sys_TestFileBuffer
576	#D-* Purpose: Test file that must NOT EXISTS on Buffer
577	#D-* Examples:
578	#D-
579	function IGCM_sys_TestFileBuffer {
580	IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
581	typeset ExistFlag
582	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
583	IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
584
585	return ${ExistFlag}
586	}
587
588	#D-#==================================================
589	#D-function IGCM_sys_CountFileArchive
590	#D-* Purpose: Count files on Archive filesystem
591	#D-* Examples:
592	#D-
593	function IGCM_sys_CountFileArchive {
594	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
595	ls ${@} 2>/dev/null \| wc -l
596	if [ $? -gt 0 ] ; then
597	echo "IGCM_sys_CountFileArchive : erreur."
598	fi
599	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
600	}
601
602	#D-#==================================================
603	#D-function IGCM_sys_CountFileBuffer
604	#D-* Purpose: Count files on Scratch filesystem
605	#D-* Examples:
606	#D-
607	function IGCM_sys_CountFileBuffer {
608	IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
609	ls ${@} 2>/dev/null \| wc -l
610	if [ $? -gt 0 ] ; then
611	echo "IGCM_sys_CountFileBuffer : erreur."
612	fi
613	IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
614	}
615
616	#D-#==================================================
617	#D-function IGCM_sys_Tree
618	#D-* Purpose: Tree directories with files on ${ARCHIVE}
619	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
620	#D-
621	function IGCM_sys_Tree {
622	IGCM_debug_PushStack "IGCM_sys_Tree" $@
623	if ( $DEBUG_sys ) ; then
624	echo "IGCM_sys_Tree :" $@
625	fi
626
627	\ls -lR ${@}
628
629	IGCM_debug_PopStack "IGCM_sys_Tree"
630	}
631
632	#D-#==================================================
633	#D-function IGCM_sys_Tar
634	#D-* Purpose: master tar command
635	#D-* Examples:
636	#D-
637	function IGCM_sys_Tar {
638	IGCM_debug_PushStack "IGCM_sys_Tar" $@
639	if ( $DEBUG_sys ) ; then
640	echo "IGCM_sys_Tar :" $@
641	fi
642	\tar cf $@
643	if [ $? -gt 0 ] ; then
644	echo "IGCM_sys_Tar : erreur."
645	IGCM_debug_Exit "IGCM_sys_Tar"
646	fi
647	IGCM_debug_PopStack "IGCM_sys_Tar"
648	}
649
650	#D-#==================================================
651	#D-function IGCM_sys_UnTar
652	#D-* Purpose: master un-tar command
653	#D-* Examples:
654	#D-
655	function IGCM_sys_UnTar {
656	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
657	if ( $DEBUG_sys ) ; then
658	echo "IGCM_sys_UnTar :" $@
659	fi
660	\tar xvf $1
661	if [ $? -gt 0 ] ; then
662	echo "IGCM_sys_UnTar : erreur."
663	IGCM_debug_Exit "IGCM_sys_UnTar"
664	fi
665	IGCM_debug_PopStack "IGCM_sys_UnTar"
666	}
667
668	#D-#==================================================
669	#D-function IGCM_sys_Qsub
670	#D-* Purpose: Qsub new job
671	#D-* Examples:
672	#D-
673	function IGCM_sys_Qsub {
674	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
675	if ( $DEBUG_sys ) ; then
676	echo "IGCM_sys_Qsub :" $@
677	fi
678	typeset options
679	if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
680	options=" -o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
681	else
682	options=" -o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod}"
683	fi
684
685	/usr/local/bin/ccc_msub ${options} < $1
686	if [ $? -gt 0 ] ; then
687	echo "IGCM_sys_Qsub : erreur ${options} $1"
688	IGCM_debug_Exit "IGCM_sys_Qsub"
689	fi
690	IGCM_debug_PopStack "IGCM_sys_Qsub"
691	}
692
693	#D-#==================================================
694	#D-function IGCM_sys_QsubPost
695	#D-* Purpose: Qsub new job on scalaire
696	#D-* Examples:
697	#D-
698	function IGCM_sys_QsubPost {
699	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
700	if ( $DEBUG_sys ) ; then
701	echo "IGCM_sys_QsubPost :" $@
702	fi
703
704	# Need to export listVarEnv for cesium
705	# NO Need to export listVarEnv for titane
706	# ============ FRONTEND START ============ #
707
708	/usr/local/bin/ccc_msub -p ${BRIDGE_MSUB_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job
709
710	# ============ FRONTEND END ============ #
711
712	# ============ CESIUM START ============ #
713	# typeset NB_ESSAI DELAI status i
714	# # number of tentative
715	# NB_ESSAI=10
716	# # time delay between tentative
717	# DELAI=10
718	# (( i = 0 ))
719	# while [ $i -lt $NB_ESSAI ] ; do
720	# /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
721	# status=$?
722	# if [ ${status} -gt 0 ]; then
723	# sleep $DELAI
724	# else
725	# break
726	# fi
727	# (( i = i + 1 ))
728	# done
729	# ============ CESIUM END ============ #
730
731	if [ $? -gt 0 ] ; then
732	echo "IGCM_sys_QsubPost : erreur " $@
733	IGCM_debug_Exit "IGCM_sys_QsubPost"
734	fi
735	IGCM_debug_PopStack "IGCM_sys_QsubPost"
736	}
737
738	#D-*************************
739	#D- File transfer functions
740	#D-*************************
741	#D-
742
743	#D-#==================================================
744	#D-function IGCM_sys_Rsync_out
745	#D-* Purpose: treat return val of rsync
746	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
747	#D- Error values and explanations can depend on your system version.
748	function IGCM_sys_Rsync_out {
749	status=$1
750	if [ ! $status ] ; then
751	echo "rsync error !"
752	fi
753
754	if [ $MYLANG = "fr" ]; then
755	case $status in
756	0) return ;;
757	1) echo "Erreur de rsync ; RERR_SYNTAX : "
758	echo "Erreur de syntaxe ou d'utilisation."
759	return;;
760	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
761	echo "Incompatibilité de protocole."
762	return;;
763	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
764	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
765	echo "répertoires"
766	return;;
767	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
768	echo "Action demandée non supportée : une tentative de manipulation de"
769	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
770	echo "été faite ; ou une option qui est supportée par le client mais"
771	echo "pas par le serveur a été spécifiée."
772	return;;
773	10) echo "Erreur de rsync ; RERR_SOCKETIO"
774	echo "Erreur dans le socket d'entrée sortie"
775	return;;
776	11) echo "Erreur de rsync ; RERR_FILEIO"
777	echo "Erreur d'entrée sortie fichier"
778	return;;
779	12) echo "Erreur de rsync ; RERR_STREAMIO"
780	echo "Erreur dans flux de donnée du protocole rsync"
781	return;;
782	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
783	echo "Erreur avec les diagnostics du programme"
784	return;;
785	14) echo "Erreur de rsync ; RERR_IPC"
786	echo "Erreur dans le code IPC"
787	return;;
788	20) echo "Erreur de rsync ; RERR_SIGNAL"
789	echo "SIGUSR1 ou SIGINT reçu"
790	return;;
791	21) echo "Erreur de rsync ; RERR_WAITCHILD"
792	echo "Une erreur retournée par waitpid()"
793	return;;
794	22) echo "Erreur de rsync ; RERR_MALLOC"
795	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
796	return;;
797	23) echo ""
798	echo "Erreur fichier inexistant"
799	return;;
800	30) echo "Erreur de rsync ; RERR_TIMEOUT"
801	echo "Temps d'attente écoulé dans l'envoi/réception de données"
802	return;;
803	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $status
804	return;;
805	esac
806	elif [ $MYLANG = "en" ] ; then
807	case $status in
808	0) return;;
809	1) echo "rsync error : Syntax or usage error "
810	return;;
811	2) echo "rsync error : Protocol incompatibility "
812	return;;
813	3) echo "rsync error : Errors selecting input/output files, dirs"
814	return;;
815	4) echo "rsync error : Requested action not supported: an attempt"
816	echo "was made to manipulate 64-bit files on a platform that cannot support"
817	echo "them; or an option was specified that is supported by the client and"
818	echo "not by the server."
819	return;;
820	5) echo "rsync error : Error starting client-server protocol"
821	return;;
822	10) echo "rsync error : Error in socket I/O "
823	return;;
824	11) echo "rsync error : Error in file I/O "
825	return;;
826	12) echo "rsync error : Error in rsync protocol data stream "
827	return;;
828	13) echo "rsync error : Errors with program diagnostics "
829	return;;
830	14) echo "rsync error : Error in IPC code "
831	return;;
832	20) echo "rsync error : Received SIGUSR1 or SIGINT "
833	return;;
834	21) echo "rsync error : Some error returned by waitpid() "
835	return;;
836	22) echo "rsync error : Error allocating core memory buffers "
837	return;;
838	23) echo "rsync error : Partial transfer due to error"
839	return;;
840	24) echo "rsync error : Partial transfer due to vanished source files"
841	return;;
842	30) echo "rsync error : Timeout in data send/receive "
843	return;;
844	*) echo "rsync error : return code of rsync unknown :" $status
845	return;;
846	esac
847	else
848	echo "unknown language $MYLANG."
849	return
850	fi
851	}
852
853	#D-#==================================================
854	#D-function IGCM_sys_Miror_libIGCM
855	#D-* Purpose: Mirror libIGCM PATH and lib to cesium
856	#D-* Examples:
857	#D-
858	function IGCM_sys_Mirror_libIGCM {
859	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
860	if ( $DEBUG_sys ) ; then
861	echo "IGCM_sys_Mirror_libIGCM"
862	fi
863
864	typeset status DEST
865
866	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
867
868	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
869	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
870	status=$?
871
872	if [ ${status} -gt 0 ] ; then
873	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
874	cat out_rsync
875	fi
876	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
877	}
878
879	#D-#==================================================
880	#D-function IGCM_sys_Cp
881	#D-* Purpose: generic cp
882	#D-* Examples:
883	#D-
884	function IGCM_sys_Cp {
885	IGCM_debug_PushStack "IGCM_sys_Cp" $@
886	if ( $DEBUG_sys ) ; then
887	echo "IGCM_sys_Cp :" $@
888	fi
889
890	typeset status
891
892	echo cp $@ > out_rsync 2>&1
893	\cp $@ >> out_rsync 2>&1
894	status=$?
895
896	if [ ${status} -gt 0 ] ; then
897	echo "IGCM_sys_Cp : error code ${status}"
898	cat out_rsync
899	IGCM_debug_Exit "IGCM_sys_Cp"
900	else
901	\rm out_rsync
902	fi
903	IGCM_debug_PopStack "IGCM_sys_Cp"
904	}
905
906	#D-#==================================================
907	#D-function IGCM_sys_Rm
908	#D-* Purpose: generic rm
909	#D-* Examples:
910	#D-
911	function IGCM_sys_Rm {
912	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
913	if ( $DEBUG_sys ) ; then
914	echo "IGCM_sys_Rm :" $@
915	fi
916
917	typeset status
918
919	echo rm $@ > out_rsync 2>&1
920	\rm $@ >> out_rsync 2>&1
921	status=$?
922
923	if [ ${status} -gt 0 ] ; then
924	echo "IGCM_sys_Rm : error code ${status}"
925	cat out_rsync
926	IGCM_debug_Exit "IGCM_sys_Rm"
927	else
928	\rm out_rsync
929	fi
930	IGCM_debug_PopStack "IGCM_sys_Rm"
931	}
932
933	#D-#==================================================
934	#D-function IGCM_sys_RmRunDir
935	#D-* Purpose: rm tmpdir (dummy function most of the time batch
936	#D- scheduler will do the job)
937	#D-* Examples:
938	#D-
939	function IGCM_sys_RmRunDir {
940	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
941	if ( $DEBUG_sys ) ; then
942	echo "IGCM_sys_RmRunDir :" $@
943	fi
944
945	typeset status
946
947	echo rm $@ > out_rsync 2>&1
948	\rm $@ >> out_rsync 2>&1
949	status=$?
950
951	if [ ${status} -gt 0 ] ; then
952	echo "IGCM_sys_RmRunDir : error code ${status}"
953	cat out_rsync
954	IGCM_debug_Exit "IGCM_sys_RmRunDir"
955	else
956	\rm out_rsync
957	fi
958	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
959	}
960
961	#D-#==================================================
962	#D-function IGCM_sys_Mv
963	#D-* Purpose: generic move
964	#D-* Examples:
965	#D-
966	function IGCM_sys_Mv {
967	IGCM_debug_PushStack "IGCM_sys_Mv" $@
968	if ( $DEBUG_sys ) ; then
969	echo "IGCM_sys_Mv :" $@
970	fi
971
972	if [ $DRYRUN = 0 ]; then
973
974	typeset status
975
976	echo mv $@ > out_rsync 2>&1
977	\mv $@ >> out_rsync 2>&1
978	status=$?
979
980	if [ ${status} -gt 0 ] ; then
981	echo "IGCM_sys_Mv : error code ${status}"
982	cat out_rsync
983	IGCM_debug_Exit "IGCM_sys_Mv"
984	else
985	\rm out_rsync
986	fi
987	fi
988	IGCM_debug_PopStack "IGCM_sys_Mv"
989	}
990
991	#D-#==================================================
992	#D-function IGCM_sys_Put_Dir
993	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
994	#D-* Examples:
995	#D-
996	function IGCM_sys_Put_Dir {
997	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
998	if ( $DEBUG_sys ) ; then
999	echo "IGCM_sys_Put_Dir :" $@
1000	fi
1001	if [ $DRYRUN = 0 ]; then
1002	if [ ! -d ${1} ] ; then
1003	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
1004	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1005	return
1006	fi
1007
1008	typeset status
1009
1010	# Only if we use rsync
1011	#IGCM_sys_TestDirArchive $( dirname $2 )
1012	#
1013	#USUAL WAY
1014	\cp -r $1 $2 > out_rsync 2>&1
1015	status=$?
1016
1017	if [ ${status} -gt 0 ] ; then
1018	echo "IGCM_sys_Put_Dir : error code ${status}"
1019	cat out_rsync
1020	IGCM_debug_Exit "IGCM_sys_Put_Dir"
1021	else
1022	\rm out_rsync
1023	fi
1024	fi
1025	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1026	}
1027
1028	#D-#==================================================
1029	#D-function IGCM_sys_Get_Dir
1030	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
1031	#D-* Examples:
1032	#D-
1033	function IGCM_sys_Get_Dir {
1034	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
1035	if ( $DEBUG_sys ) ; then
1036	echo "IGCM_sys_Get_Dir :" $@
1037	fi
1038	if [ $DRYRUN = 0 ]; then
1039	# if [ ! -d ${1} ] ; then
1040	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
1041	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1042	# return
1043	# fi
1044
1045	typeset NB_ESSAI DELAI status i
1046	# number of tentative
1047	NB_ESSAI=3
1048	# time delay between tentative
1049	DELAI=2
1050
1051	# Only if we use rsync
1052	#IGCM_sys_TestDirArchive $( dirname $2 )
1053	#
1054	# USUAL WAY
1055	# add dmfind/dmget (to demigrate all offline files) :
1056	#dmfind $1 -state MIG -o -state OFL -o -state PAR \| dmget
1057	ccc_hsm get -r $1
1058
1059	i=0
1060	while [ $i -lt $NB_ESSAI ] ; do
1061	\cp -ur $1 $2 >> out_rsync 2>&1
1062	status=$?
1063	if [ ${status} -gt 0 ]; then
1064	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
1065	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
1066	sleep $DELAI
1067	else
1068	break
1069	fi
1070	(( i = i + 1 ))
1071	done
1072
1073	if [ ${status} -gt 0 ] ; then
1074	echo "IGCM_sys_Get_Dir : error."
1075	cat out_rsync
1076	IGCM_debug_Exit "IGCM_sys_Get_Dir"
1077	else
1078	\rm out_rsync
1079	fi
1080	fi
1081	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1082	}
1083
1084	#D-#==================================================
1085	#D-function IGCM_sys_Get_Master
1086	#D-* Purpose: Copy a complete directory from MASTER filesystem
1087	#D-* Examples:
1088	#D-
1089	function IGCM_sys_Get_Master {
1090	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
1091	if ( $DEBUG_sys ) ; then
1092	echo "IGCM_sys_Get_Master :" $@
1093	fi
1094	if [ $DRYRUN = 0 ]; then
1095	if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
1096	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
1097	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1098	return
1099	fi
1100
1101	typeset NB_ESSAI DELAI status i
1102	# number of tentative
1103	NB_ESSAI=3
1104	# time delay between tentative
1105	DELAI=2
1106
1107	i=0
1108	while [ $i -lt $NB_ESSAI ] ; do
1109	\cp -urL $1 $2 > out_rsync 2>&1
1110	status=$?
1111	if [ ${status} -gt 0 ]; then
1112	IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
1113	IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
1114	sleep $DELAI
1115	else
1116	break
1117	fi
1118	(( i = i + 1 ))
1119	done
1120
1121	if [ ${status} -gt 0 ] ; then
1122	echo "IGCM_sys_Get_Master : error."
1123	cat out_rsync
1124	IGCM_debug_Exit "IGCM_sys_Get_Master"
1125	else
1126	\rm out_rsync
1127	fi
1128	fi
1129	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1130	}
1131
1132	#====================================================
1133	#- Call IGCM_sys_Mirror_libIGCM now !
1134	if ( $MirrorlibIGCM ) ; then
1135	IGCM_sys_Mirror_libIGCM
1136	fi
1137
1138	#D-#==================================================
1139	#D-function IGCM_sys_Put_Rest
1140	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1141	#D- File and target directory must exist.
1142	#D-* Examples:
1143	#D-
1144	function IGCM_sys_Put_Rest {
1145	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1146	if ( $DEBUG_sys ) ; then
1147	echo "IGCM_sys_Put_Rest :" $@
1148	fi
1149	if [ $DRYRUN = 0 ]; then
1150	if [ ! -f ${1} ] ; then
1151	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1152	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1153	fi
1154
1155	typeset status
1156	#
1157	if [ X${JobType} = XRUN ] ; then
1158	IGCM_sys_Chmod 444 ${1}
1159	fi
1160
1161	#
1162	# USUAL WAY
1163	\cp $1 $2 > out_rsync 2>&1
1164	status=$?
1165
1166	# #RSYNC WITH NETWORK SSH CALL
1167	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1168	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1169
1170	# #RSYNC WITH NFS USE
1171	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1172	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1173
1174	# status=$?
1175	# IGCM_sys_Rsync_out $status
1176
1177	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1178	# (( status=status+$? ))
1179
1180	if [ ${status} -gt 0 ] ; then
1181	echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1182	cat out_rsync
1183	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1184	else
1185	\rm out_rsync
1186	fi
1187	fi
1188	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1189	}
1190
1191	#D-#==================================================
1192	#D-function IGCM_sys_PutBuffer_Rest
1193	#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1194	#D- File and target directory must exist.
1195	#D-* Examples:
1196	#D-
1197	function IGCM_sys_PutBuffer_Rest {
1198	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1199	if ( $DEBUG_sys ) ; then
1200	echo "IGCM_sys_PutBuffer_Rest :" $@
1201	fi
1202	if [ $DRYRUN = 0 ]; then
1203	if [ ! -f ${1} ] ; then
1204	echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1205	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1206	fi
1207
1208	typeset status
1209	#
1210	if [ X${JobType} = XRUN ] ; then
1211	IGCM_sys_Chmod 444 ${1}
1212	fi
1213
1214	#
1215	# USUAL WAY
1216	\cp $1 $2 > out_rsync 2>&1
1217	status=$?
1218
1219	if [ ${status} -gt 0 ] ; then
1220	echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1221	cat out_rsync
1222	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1223	else
1224	\rm out_rsync
1225	fi
1226	fi
1227	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1228	}
1229
1230	#D-#==================================================
1231	#D-function IGCM_sys_PrepareTaredRestart
1232	#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
1233	#D-* Examples:
1234	#D-
1235	function IGCM_sys_PrepareTaredRestart {
1236	IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
1237	echo $1
1238	IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1239	}
1240
1241	#D-#==================================================
1242	#D-function IGCM_sys_Put_Out
1243	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1244	#D-* Examples:
1245	#D-
1246	function IGCM_sys_Put_Out {
1247	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1248	if ( $DEBUG_sys ) ; then
1249	echo "IGCM_sys_Put_Out :" $@
1250	fi
1251
1252	typeset NB_ESSAI DELAI status i exist skip
1253
1254	# number of tentative
1255	NB_ESSAI=3
1256	# time delay between tentative
1257	DELAI=2
1258
1259	if [ $DRYRUN = 0 ]; then
1260	if [ ! -f ${1} ] ; then
1261	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1262	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1263	return 1
1264	fi
1265	#
1266	IGCM_sys_MkdirArchive $( dirname $2 )
1267	#
1268
1269	#=====================================================
1270	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1271	#=====================================================
1272
1273	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
1274	#if [ $? -eq 0 ] ; then
1275	# typeset WORKPATH FILEPATH
1276	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_BUF}\|" )
1277	# IGCM_sys_MkdirWork ${WORKPATH}
1278	# FILEPATH=${WORKPATH}/$( basename $2 )
1279	# #
1280	# IGCM_sys_Cp ${1} ${FILEPATH}
1281	#fi
1282
1283	if [ X${JobType} = XRUN ] ; then
1284	if [ X${3} = X ] ; then
1285	IGCM_sys_Chmod 444 ${1}
1286	fi
1287	fi
1288	#
1289
1290	exist=false
1291	skip=false
1292	if [ -f $2 ] ; then
1293	IGCM_debug_Print 1 "$2 already exist"
1294	#dmget $2
1295	ccc_hsm get $2
1296	exist=true
1297	if [ "X$( diff $1 $2 )" = X ] ; then
1298	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1299	skip=true
1300	else
1301	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1302	skip=false
1303	fi
1304	fi
1305	#
1306	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1307	IGCM_sys_Chmod u+w $2
1308	fi
1309
1310	if [ X${skip} = Xfalse ] ; then
1311	i=0
1312	while [ $i -lt $NB_ESSAI ] ; do
1313	# USUAL WAY
1314	\cp $1 $2 > out_rsync 2>&1
1315	status=$?
1316	if [ ${status} -gt 0 ]; then
1317	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1318	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
1319	if [ -f $2 ] ; then
1320	if [ "X$( diff $1 $2 )" = X ] ; then
1321	IGCM_debug_Print 2 "$1 and $2 are the same file"
1322	IGCM_debug_Print 2 "Cp gave an error but the file has been copied properly!"
1323	status=0
1324	\rm out_rsync
1325	break
1326	else
1327	IGCM_debug_Print 2 "$1 and $2 are not the same file, we continue to try"
1328	\rm $2
1329	fi
1330	fi
1331	sleep $DELAI
1332	else
1333	break
1334	fi
1335	(( i = i + 1 ))
1336	done
1337	fi
1338
1339	# #RSYNC WITH NETWORK SSH CALL
1340	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1341	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1342
1343	# #RSYNC WITH NFS USE
1344	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1345	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1346
1347	# status=$?
1348	# IGCM_sys_Rsync_out $status
1349
1350	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1351	# (( status=status+$? ))
1352
1353	if [ ${status} -gt 0 ] ; then
1354	echo "IGCM_sys_Put_Out : error."
1355	cat out_rsync
1356	IGCM_debug_Exit "IGCM_sys_Put_Out"
1357	else
1358	\rm out_rsync
1359	fi
1360	fi
1361	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1362	return 0
1363	}
1364
1365	#D-#==================================================
1366	#D-function IGCM_sys_PutBuffer_Out
1367	#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1368	#D-* Examples:
1369	#D-
1370	function IGCM_sys_PutBuffer_Out {
1371	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1372	if ( $DEBUG_sys ) ; then
1373	echo "IGCM_sys_PutBuffer_Out :" $@
1374	fi
1375
1376	typeset NB_ESSAI DELAI status i exist skip
1377
1378	# number of tentative
1379	NB_ESSAI=3
1380	# time delay between tentative
1381	DELAI=2
1382
1383	if [ $DRYRUN = 0 ]; then
1384	if [ ! -f ${1} ] ; then
1385	echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1386	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1387	return 1
1388	fi
1389	#
1390	IGCM_sys_Mkdir $( dirname $2 )
1391	#
1392
1393	if [ X${JobType} = XRUN ] ; then
1394	if [ X${3} = X ] ; then
1395	IGCM_sys_Chmod 444 ${1}
1396	fi
1397	fi
1398
1399	exist=false
1400	skip=false
1401	if [ -f $2 ] ; then
1402	IGCM_debug_Print 1 "$2 already exist"
1403	exist=true
1404	if [ "X$( diff $1 $2 )" = X ] ; then
1405	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1406	skip=true
1407	else
1408	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1409	skip=false
1410	fi
1411	fi
1412	#
1413	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1414	IGCM_sys_Chmod u+w $2
1415	fi
1416
1417	if [ X${skip} = Xfalse ] ; then
1418	i=0
1419	while [ $i -lt $NB_ESSAI ] ; do
1420	# USUAL WAY
1421	\cp $1 $2 > out_rsync 2>&1
1422	status=$?
1423	if [ ${status} -gt 0 ]; then
1424	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1425	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1426	sleep $DELAI
1427	else
1428	break
1429	fi
1430	(( i = i + 1 ))
1431	done
1432	fi
1433
1434	if [ ${status} -gt 0 ] ; then
1435	echo "IGCM_sys_PutBuffer_Out : error."
1436	cat out_rsync
1437	IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1438	else
1439	\rm out_rsync
1440	fi
1441	fi
1442	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1443	return 0
1444	}
1445
1446	#D-#==================================================
1447	#D-function IGCM_sys_Get
1448	#D-* Purpose: Get a file from ${ARCHIVE}
1449	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1450	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
1451	function IGCM_sys_Get {
1452	IGCM_debug_PushStack "IGCM_sys_Get" $@
1453
1454	typeset DEST dm_liste ifile target file_work
1455	typeset NB_ESSAI DELAI status i
1456
1457	if ( $DEBUG_sys ) ; then
1458	echo "IGCM_sys_Get :" $@
1459	fi
1460
1461	# number of tentative
1462	NB_ESSAI=3
1463	# time delay between tentative
1464	DELAI=2
1465
1466	if [ $DRYRUN -le 2 ]; then
1467	if [ X${1} = X'/l' ] ; then
1468	# test if the first file is present in the old computation :
1469	eval set +A dm_liste \${${2}}
1470	else
1471	eval set +A dm_liste ${1}
1472	fi
1473	eval DEST=\${${#}}
1474
1475	#=====================================================
1476	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1477	#=====================================================
1478
1479	# Is it an R_OUT file (not R_IN) ?
1480	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
1481	#if [ $? -eq 0 ] ; then
1482	# # Yes ? then we try to get it in SCRATCHDIR
1483	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_BUF}\|g" )
1484	# if [ -f ${file_work[0]} ] ; then
1485	# IGCM_sys_Cp ${file_work[*]} ${DEST}
1486	# IGCM_debug_PopStack "IGCM_sys_Get"
1487	# return
1488	# fi
1489	#fi
1490
1491	ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1492	status=$?
1493	if [ ${status} -gt 0 ] ; then
1494	echo "WARNING IGCM_sys_Get : error code ${status}"
1495	cat out_rsync
1496	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1497	fi
1498
1499	#if [ ${status} -gt 0 ] ; then
1500	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1501	# cat out_rsync
1502	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1503	# sleep 30
1504	# echo "We try another time"
1505	## dmget ${dm_liste[*]} > out_rsync 2>&1
1506	# ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1507	# status=$?
1508	# if [ ${status} -gt 0 ] ; then
1509	# echo "ERROR IGCM_sys_Get : again demigration error :"
1510	# cat out_rsync
1511	# IGCM_debug_Exit "IGCM_sys_Get"
1512	# fi
1513	# else
1514	# echo "ERROR IGCM_sys_Get : demigration error :"
1515	# cat out_rsync
1516	# IGCM_debug_Exit "IGCM_sys_Get"
1517	# fi
1518	#fi
1519
1520	# #RSYNC WITH NETWORK SSH CALL
1521	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1522	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1523
1524	# #RSYNC WITH NFS USE
1525	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1526	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1527
1528	# status=$?
1529	# IGCM_sys_Rsync_out $status
1530
1531	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1532	# (( status=status+$? ))
1533
1534	#USUAL WAY
1535	if [ X${1} = X'/l' ] ; then
1536	for target in ${dm_liste[*]} ; do
1537	local_file=$( basename ${target} )
1538	# test if the target file is present before the loop
1539	IGCM_sys_TestFileArchive ${target}
1540	status=$?
1541	if [ ${status} -gt 0 ] ; then
1542	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
1543	IGCM_debug_Exit "IGCM_sys_Get"
1544	else
1545	i=0
1546	while [ $i -lt $NB_ESSAI ] ; do
1547	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1548	status=$?
1549	if [ ${status} -gt 0 ]; then
1550	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1551	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1552	sleep $DELAI
1553	else
1554	break
1555	fi
1556	(( i = i + 1 ))
1557	done
1558	if [ ${status} -gt 0 ] ; then
1559	echo "IGCM_sys_Get : error"
1560	cat out_rsync
1561	\rm out_rsync
1562	IGCM_debug_Exit "IGCM_sys_Get"
1563	else
1564	\rm out_rsync
1565	fi
1566	fi
1567	done
1568	else
1569	i=0
1570	while [ $i -lt $NB_ESSAI ] ; do
1571	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1572	status=$?
1573	if [ ${status} -gt 0 ]; then
1574	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1575	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1576	sleep $DELAI
1577	else
1578	break
1579	fi
1580	(( i = i + 1 ))
1581	done
1582	if [ ${status} -gt 0 ] ; then
1583	echo "IGCM_sys_Get : error"
1584	cat out_rsync
1585	\rm out_rsync
1586	IGCM_debug_Exit "IGCM_sys_Get"
1587	else
1588	\rm out_rsync
1589	fi
1590	fi
1591	fi
1592	IGCM_debug_PopStack "IGCM_sys_Get"
1593	}
1594
1595	#D-#==================================================
1596	#D-function IGCM_sys_GetBuffer
1597	#D-* Purpose: Get a file from ${SCRATCHDIR}
1598	#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1599	#D- IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1600	function IGCM_sys_GetBuffer {
1601	IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1602
1603	typeset DEST buf_liste ifile target file_work
1604	typeset NB_ESSAI DELAI status i
1605
1606	if ( $DEBUG_sys ) ; then
1607	echo "IGCM_sys_GetBuffer :" $@
1608	fi
1609
1610	# number of tentative
1611	NB_ESSAI=3
1612	# time delay between tentative
1613	DELAI=2
1614
1615	if [ $DRYRUN -le 2 ]; then
1616	if [ X${1} = X'/l' ] ; then
1617	# test if the first file is present in the old computation :
1618	eval set +A buf_liste \${${2}}
1619	else
1620	eval set +A buf_liste ${1}
1621	fi
1622	eval DEST=\${${#}}
1623
1624	#USUAL WAY
1625	if [ X${1} = X'/l' ] ; then
1626	for target in ${buf_liste[*]} ; do
1627	local_file=$( basename ${target} )
1628	i=0
1629	while [ $i -lt $NB_ESSAI ] ; do
1630	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1631	status=$?
1632	if [ ${status} -gt 0 ]; then
1633	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1634	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1635	sleep $DELAI
1636	else
1637	break
1638	fi
1639	(( i = i + 1 ))
1640	done
1641	if [ ${status} -gt 0 ] ; then
1642	echo "IGCM_sys_Get : error"
1643	cat out_rsync
1644	\rm out_rsync
1645	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1646	else
1647	\rm out_rsync
1648	fi
1649	done
1650	else
1651	i=0
1652	while [ $i -lt $NB_ESSAI ] ; do
1653	\cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1654	status=$?
1655	if [ ${status} -gt 0 ]; then
1656	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1657	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1658	sleep $DELAI
1659	else
1660	break
1661	fi
1662	(( i = i + 1 ))
1663	done
1664	if [ ${status} -gt 0 ] ; then
1665	echo "IGCM_sys_Get : error"
1666	cat out_rsync
1667	\rm out_rsync
1668	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1669	else
1670	\rm out_rsync
1671	fi
1672	fi
1673	fi
1674	IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1675	}
1676
1677	#D-#==================================================
1678	#D-function IGCM_sys_GetDate_FichWork
1679	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1680	#D-* Examples:
1681	#D-
1682	function IGCM_sys_GetDate_FichWork {
1683	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1684	if ( $DEBUG_sys ) ; then
1685	echo "IGCM_sys_GetDate_FichWork :" $@
1686	fi
1687	typeset dateF
1688	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1689	eval ${2}=${dateF[5]}
1690
1691	# donne la date filesys d'un fichier sur la machine work
1692	IGCM_debug_PopStack "IGCM_sys_FichWork"
1693	}
1694
1695	#D-#==================================================
1696	#D-function IGCM_sys_GetDate_FichArchive
1697	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1698	#D-* Examples:
1699	#D-
1700	function IGCM_sys_GetDate_FichArchive {
1701	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1702	if ( $DEBUG_sys ) ; then
1703	echo "IGCM_sys_GetDate_FichArchive :" $@
1704	fi
1705	typeset dateF
1706	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1707	eval ${2}=${dateF[5]}
1708
1709	IGCM_debug_PopStack "IGCM_sys_FichArchive"
1710	}
1711
1712	#D-#==================================================
1713	#D-function IGCM_sys_Dods_Rm
1714	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1715	#D-* Examples:
1716	#D-
1717	function IGCM_sys_Dods_Rm {
1718	if ( $DEBUG_sys ) ; then
1719	echo "IGCM_sys_Dods_Rm :" $@
1720	fi
1721	typeset status
1722	if [ $DRYRUN = 0 ]; then
1723
1724	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1725	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1726	# echo "Nothing has been done."
1727	# return
1728	# fi
1729
1730	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
1731	status=$?
1732
1733	# if [ ${status} -gt 0 ] ; then
1734	# echo "IGCM_sys_Dods_Rm : error."
1735	# cat out_dods_rm
1736	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1737	# else
1738	# rm out_dods_rm
1739	# fi
1740
1741	fi
1742	return $status
1743	}
1744
1745	#D-#==================================================
1746	#D-function IGCM_sys_Dods_Cp
1747	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1748	#D-* Examples:
1749	#D-
1750	function IGCM_sys_Dods_Cp {
1751	if ( $DEBUG_sys ) ; then
1752	echo "IGCM_sys_Dods_Cp :" $@
1753	fi
1754	typeset status
1755	if [ $DRYRUN = 0 ]; then
1756
1757	# if [ ! -d ${R_SAVE}/${1} ] ; then
1758	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1759	# echo "Nothing has been done."
1760	# return
1761	# fi
1762
1763	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1764	status=$?
1765
1766	# if [ ${status} -gt 0 ] ; then
1767	# echo "IGCM_sys_Dods_Cp : error."
1768	# cat out_dods_cp
1769	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1770	# else
1771	# rm out_dods_cp
1772	# fi
1773
1774	fi
1775	return $status
1776	}
1777
1778	#D-#==================================================
1779	#D-function IGCM_sys_Put_Dods
1780	#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1781	#D-* Examples:
1782	#D-
1783	function IGCM_sys_Put_Dods {
1784	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1785	if ( $DEBUG_sys ) ; then
1786	echo "IGCM_sys_Put_Dods :" $@
1787	fi
1788	#set -vx
1789	typeset status
1790	if [ $DRYRUN = 0 ]; then
1791	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1792	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1793	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1794	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1795	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1796	return
1797	fi
1798
1799	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1800	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1801	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1802	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1803	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1804	return
1805	fi
1806	#
1807	if [ -d ${R_SAVE}/${1} ] ; then
1808	cd ${R_SAVE}
1809	elif [ -d ${R_FIGR}/${1} ] ; then
1810	cd ${R_FIGR}
1811	fi
1812
1813	IGCM_sys_Dods_Rm ${1}
1814	IGCM_sys_Dods_Cp ${1}
1815	status=0
1816
1817	if [ ${status} -gt 0 ] ; then
1818	echo "IGCM_sys_Put_Dods : error."
1819	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1820	fi
1821	fi
1822	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1823	}
1824
1825	##############################################################
1826	# REBUILD OPERATOR
1827
1828	function IGCM_sys_rebuild {
1829	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1830	if ( $DEBUG_sys ) ; then
1831	echo "IGCM_sys_rebuild :" $@
1832	fi
1833
1834	typeset NB_ESSAI DELAI status i firstArg
1835	# number of tentative
1836	NB_ESSAI=3
1837	# time delay between tentative
1838	DELAI=2
1839
1840	i=0
1841	while [ $i -lt $NB_ESSAI ] ; do
1842	/home/cont003/p86ipsl/X64/bin/rebuild -f -o $@ > out_rsync 2>&1
1843	status=$?
1844	if [ ${status} -gt 0 ] ; then
1845	IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1846	cat out_rsync
1847	\rm out_rsync
1848	IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1849	firstArg=${1}
1850	\rm ${firstArg}
1851	sleep $DELAI
1852	else
1853	\rm out_rsync
1854	break
1855	fi
1856	(( i = i + 1 ))
1857	done
1858
1859	if [ ${status} -gt 0 ] ; then
1860	echo "IGCM_sys_rebuild : rebuild error"
1861	IGCM_debug_Exit "rebuild"
1862	fi
1863
1864	IGCM_debug_PopStack "IGCM_sys_rebuild"
1865	}
1866
1867	function IGCM_sys_rebuild_station {
1868	IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1869	typeset i list_opt file_in file_out prefix_invert list_invert
1870	if ( $DEBUG_sys ) ; then
1871	echo "IGCM_sys_rebuild_station :" $@
1872	fi
1873	list_opt=$@
1874
1875	# Invert Axis : t,x -> x,t
1876	# t,pres,x -> x,t,pres
1877	# So that we can concatenate along x
1878	i=0
1879	for file_in in ${list_opt} ; do
1880	(( i = i + 1))
1881	[ ${i} = 1 ] && file_out=${file_in} && continue
1882	prefix_invert=$( basename ${file_in} .nc )
1883	IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1884	list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1885	done
1886
1887	# Concatenate
1888	IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1889
1890	# Re-ivert file
1891	IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1892
1893	# Station re-ordering is too expansive to be run within libICGM
1894	# This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1895	# This re-ordering must be done "in memory" by the cmorization process
1896	# Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1897	# BEGIN reordering
1898
1899	# Only LMDZ text output contains the exact ordering of the station.
1900	# We isolate this in the code below:
1901	# 0 38 -157.5000000000000 70.98591549295774
1902	# 0 54 27.49999999999999 67.18309859154928
1903	# 0 56 -62.50000000000001 82.39436619718309
1904	# 0 79 12.49999999999999 78.59154929577466
1905	# 0 116 -165.0000000000000 76.05633802816901
1906	# 0 117 130.0000000000000 70.98591549295774
1907	# 0 118 110.0000000000000 87.46478873239437
1908	# 1 40 4.999999999999995 51.97183098591550
1909	# typeset iStation iProc list_opt file_in file_out prefix_invert
1910	# typeset -Z4 j4
1911	# typeset -Z3 j3
1912
1913	# unset list_opt
1914	# set +A list_opt $@
1915
1916	# Filename after rebuild
1917	# file_out=${list_opt[0]}
1918	# Prefix of output files
1919	# prefix_invert=$( basename ${file_out} .nc )
1920	# Number of procs
1921	# num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} \| wc -l )
1922
1923	# iProc=0
1924	# while [ ${iProc} -lt ${num_proc} ] ; do
1925	# Array containing Station as a number
1926	# unset proc_stn
1927	# set +A proc_stn $( grep "iophy_mpi rank ip lon lat $iProc" ${PREFIX}_${Exe_Output} \| sed -e "s/iophy_mpi rank ip lon lat //g" \| awk ' {print $2}' )
1928	# Number of stations produced by processor proc
1929	# stationLast=${#proc_stn[*]}
1930	# Proc number on 4 digits
1931	# j4=${iProc}
1932	# Init
1933	# iStation=0
1934	# while [ ${iStation} -lt ${stationLast} ] ; do
1935	# Station number on 3 digits
1936	# j3=${proc_stn[${iStation}]}
1937	# Extract station
1938	# Invert Axis : t,x -> x,t
1939	# t,pres,x -> x,t,pres
1940	# So that we can concatenate along x
1941	# IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1942	# (( iStation = iStation + 1 ))
1943	# done
1944	# (( iProc = iProc + 1 ))
1945	# done
1946
1947	# Concatenate all station along x
1948	# IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1949
1950	# Re-invert file
1951	# IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1952
1953	# END reordering
1954
1955	IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1956	}
1957
1958	############################################################
1959	# Activate Running Environnment Variables
1960
1961	function IGCM_sys_activ_variables {
1962	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1963	if ( $DEBUG_sys ) ; then
1964	echo "IGCM_sys_activ_variables"
1965	fi
1966
1967	ulimit -s unlimited
1968
1969	export KMP_STACKSIZE=3g
1970	export KMP_LIBRARY=turnaround
1971	export MKL_SERIAL=YES
1972	unset MPI_PLACING_MODE
1973
1974	# add module load nb of threads for core placement
1975	(( max_omp = 0 ))
1976
1977	for comp in ${config_ListOfComponents[*]} ; do
1978	eval ExeNameIn=\${config_Executable_${comp}[0]}
1979	eval ExeNameOut=\${config_Executable_${comp}[1]}
1980
1981	# Only if we really have an executable for the component :
1982	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1983	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1984	if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
1985	(( max_omp = comp_proc_omp_loc ))
1986	fi
1987	fi
1988	done
1989
1990	( [ X${max_omp} = X4 ] \|\| [ X${max_omp} = X8 ] ) && { module load openmp/${max_omp}thds ; IGCM_debug_Print 1 "module load openmp/${max_omp}thds" ; }
1991
1992	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1993	}
1994
1995	############################################################
1996	# Desactivate Running Environnment Variables
1997
1998	function IGCM_sys_desactiv_variables {
1999	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
2000	if ( $DEBUG_sys ) ; then
2001	echo "IGCM_sys_desactiv_variables"
2002	fi
2003	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
2004	}
2005
2006	############################################################
2007	# Build MPI/OMP scripts run file (dummy function)
2008
2009	function IGCM_sys_build_run_file {
2010
2011	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
2012
2013	}
2014
2015	############################################################
2016	# Build MPI/OMP scripts
2017	function IGCM_sys_build_execution_scripts
2018	{
2019	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
2020	if ( $DEBUG_sys ) ; then
2021	echo "IGCM_sys_build_execution_scripts " $@
2022	fi
2023	typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
2024	typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
2025	typeset num_corempi nombre_restant_node nombre_restant_comp
2026	typeset IGCM_OMP
2027
2028	if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ] ; then
2029	IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
2030	fi
2031
2032	if ( ${OK_PARA_MPMD} ) ; then
2033
2034	if [ -f run_file ] ; then
2035	IGCM_sys_Rm -f run_file
2036	fi
2037	touch run_file
2038
2039	if ( ${OK_PARA_OMP} ) ; then
2040
2041	# Hosts treatment
2042
2043	${HOST_MPIRUN_COMMAND} hostname \| sort \| uniq > hosts.tmp
2044
2045	i=0
2046	rm -f hosts
2047	IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
2048	for nodes in `cat hosts.tmp` ; do
2049	host[$i]=$nodes
2050	echo "${host[$i]} slots=8 max_slots=8" >> hosts
2051	IGCM_debug_Print 1 ${host[$i]}
2052	i=$((i+1))
2053	done
2054	rm -f hosts.tmp
2055
2056	listnodes=${host[*]}
2057
2058	EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
2059
2060	# Initialisation
2061
2062	init_node=y
2063	node_num_current=0
2064	start_num=0
2065	init_exec=n
2066
2067	# Test : if oasis is there, we put it at the first position
2068
2069	for comp in ${config_ListOfComponents[*]} ; do
2070
2071	if [ "X${comp}" = "XCPL" ] ; then
2072
2073	eval ExeNameIn=\${config_Executable_${comp}[0]}
2074	eval ExeNameOut=\${config_Executable_${comp}[1]}
2075
2076	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2077	echo "" >> script_${ExeNameOut}.ksh
2078	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
2079	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
2080	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
2081	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2082	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2083
2084	init_node=n
2085
2086	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
2087	node_num_current=0
2088	node_current=${host[${node_num_current}]}
2089
2090	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
2091
2092	init_exec=y
2093	start_num=1
2094
2095	fi
2096
2097	done
2098
2099	# Then loop on the components (except for oasis)
2100
2101	for comp in ${config_ListOfComponents[*]} ; do
2102
2103	eval ExeNameIn=\${config_Executable_${comp}[0]}
2104	eval ExeNameOut=\${config_Executable_${comp}[1]}
2105
2106	# Only if we really have an executable for the component :
2107	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
2108
2109	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2110	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2111
2112	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2113	# echo "set -vx" >> script_${ExeNameOut}.ksh
2114	echo "" >> script_${ExeNameOut}.ksh
2115	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
2116	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
2117	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
2118	echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
2119	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
2120	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
2121	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2122
2123	node_num=0
2124
2125	# We define the number of MPI process to be assigned for the component
2126
2127	nombre_restant_comp=${comp_proc_mpi_loc}
2128
2129	# Loop on the allocated nodes
2130
2131	for node in ${listnodes} ; do
2132
2133	# We go to the current node
2134	if [ ${node_num} = ${node_num_current} ] ; then
2135
2136	node_current=${host[${node_num_current}]}
2137
2138	# If first time on the node : initialisation
2139
2140	if [ ${init_node} = y ] ; then
2141	nombre_restant_node=${NUM_COREPERNODE}
2142	fi
2143
2144	# Test on the number of OMP threads
2145
2146	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
2147	(( node_num = node_num + 1 ))
2148	node_num_current=${node_num}
2149	init_node=y
2150	continue
2151	fi
2152
2153	# Number of MPI process to assign
2154
2155	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
2156
2157	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
2158	num_corempi=${nombre_restant_comp}
2159	fi
2160
2161	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
2162	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
2163
2164	if [ ${init_exec} = y ] ; then
2165	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
2166	else
2167	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
2168	init_exec=y
2169	fi
2170
2171	(( start_num = num_corempi + start_num ))
2172
2173	else
2174
2175	(( node_num = node_num + 1 ))
2176	continue
2177	fi
2178
2179	# Test on the number of core/process remaining on the node/component
2180
2181	if [ ${nombre_restant_node} = 0 ] ; then
2182	(( node_num = node_num + 1 ))
2183	node_num_current=${node_num}
2184	init_node=y
2185
2186	if [ ${nombre_restant_comp} = 0 ] ; then
2187	break 1
2188	fi
2189	else
2190
2191	node_num_current=${node_num}
2192	init_node=n
2193
2194	if [ ${nombre_restant_comp} = 0 ] ; then
2195	break 1
2196	fi
2197	fi
2198	done
2199	fi
2200	done
2201
2202	else
2203
2204	# Then first loop on the components for the coupler ie oasis
2205
2206	## the coupler ie oasis must be the first one
2207	for comp in ${config_ListOfComponents[*]} ; do
2208
2209	eval ExeNameOut=\${config_Executable_${comp}[1]}
2210
2211	# for CPL component only
2212	if [ "X${comp}" = "XCPL" ] ; then
2213	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2214	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
2215	fi
2216	done
2217
2218	# Then second loop on the components
2219
2220	for comp in ${config_ListOfComponents[*]} ; do
2221
2222	eval ExeNameOut=\${config_Executable_${comp}[1]}
2223
2224	# Only if we really have an executable for the component and not the coupler ie oasis:
2225	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
2226	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2227	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
2228	fi
2229	done
2230	IGCM_sys_Chmod u+x run_file
2231
2232	EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
2233
2234	fi
2235
2236	else # Only one executable.
2237	# Check has to be added with :
2238	# Job :
2239	# #MSUB -n 8 # reservation des processeurs pour le job
2240	# #MSUB -N 4 # reservation des nodes pour le job
2241	# BATCH_NUM_PROC_TOT=32
2242	#
2243	# config.card :
2244	# ATM= (gcm.e, lmdz.x, 8MPI, 4OMP)
2245	#
2246	# tested with :
2247	# Job :
2248	# #MSUB -n 32
2249	# BATCH_NUM_PROC_TOT=32
2250	#
2251	# config.card :
2252	# ATM= (gcm.e, lmdz.x)
2253
2254
2255	for comp in ${config_ListOfComponents[*]} ; do
2256
2257	# Only if we really have an executable for the component :
2258	eval ExeNameOut=\${config_Executable_${comp}[1]}
2259	### if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
2260	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
2261
2262	IGCM_OMP=1
2263
2264	if ( ${OK_PARA_OMP} ) ; then
2265	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2266	export OMP_NUM_THREADS=${comp_proc_omp_loc}
2267	IGCMG_OMP=${comp_proc_omp_loc}
2268	fi
2269	if ( ${OK_PARA_MPI} ) ; then
2270	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2271	# Default : mpirun used if nb_proc gt 1
2272	# pour sortie out/err par process ? more test required
2273	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}" >> script_${ExeNameOut}.ksh
2274	EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
2275	else
2276	# Check if BATCH_NUM_PROC_TOT = IGCMG_OMP
2277	if ( [ ${IGCM_OMP} -ne ${BATCH_NUM_PROC_TOT} ] ) ; then
2278	IGCM_debug_Print 1 "sys Titane : BATCH_NUM_PROC_TOT <> ${comp}_PROC_OMP : ${BATCH_NUM_PROC_TOT} <> ${BRIDGE_MSUB_NPROC} "
2279	IGCM_debug_Print 1 "sys Titane : Check #MSUB -n xx into Job"
2280	IGCM_debug_Exit "IGCM_sys_build_execution_scripts"
2281	fi
2282	# Default : mpirun is NOT used if nb_proc eq 1
2283	# pour sortie out/err par process ? more test required
2284	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2285	EXECUTION="/usr/bin/time ./${ExeNameOut}"
2286	fi
2287	fi
2288	done
2289
2290	fi
2291
2292	IGCM_debug_Print 1 "sys Titane : La commande d execution est "
2293	IGCM_debug_Print 1 "$EXECUTION"
2294
2295	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
2296	}
2297
2298	############################################################
2299	# Check of space available on temporary filesytems
2300	function IGCM_sys_check_quota {
2301	IGCM_debug_PushStack "IGCM_sys_check_quota"
2302	if ( $DEBUG_sys ) ; then
2303	echo "IGCM_sys_check_quota "
2304	fi
2305	# Limit of quota (in %)
2306	limit_quota=90
2307
2308	# Check of the volume
2309	volume_quota=$(ccc_quota \| grep ' scratch' \| awk '{print $2}')
2310	volume_avail=$(ccc_quota \| grep ' scratch' \| awk '{print $3}')
2311
2312	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
2313
2314	unit_avail=${volume_avail: -1}
2315	unit_quota=${volume_quota: -1}
2316
2317	if [ "${unit_quota}" = "*" ] ; then
2318	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2319	IGCM_debug_Print 1 "More than 100% of your quota is used"
2320	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2321	IGCM_debug_Print 1 "You must have more than 10% available to run"
2322	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2323	IGCM_debug_Verif_Exit
2324	fi
2325
2326	temp_avail=${volume_avail%%${volume_avail: -1}*}
2327	temp_quota=${volume_quota%%${volume_quota: -1}*}
2328
2329	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2330
2331	# Convertion
2332	if [ ${volume_avail: -1} = "T" ] ; then
2333	(( temp_avail = temp_avail * 1000000000000 ))
2334	elif [ ${volume_avail: -1} = "G" ] ; then
2335	(( temp_avail = temp_avail * 1000000000 ))
2336	elif [ ${volume_avail: -1} = "M" ] ; then
2337	(( temp_avail = temp_avail * 1000000 ))
2338	elif [ ${volume_avail: -1} = "k" ] ; then
2339	(( temp_avail = temp_avail * 1000 ))
2340	else
2341	(( temp_avail = volume_avail ))
2342	fi
2343	if [ ${volume_quota: -1} = "T" ] ; then
2344	(( temp_quota = temp_quota * 1000000000000 ))
2345	elif [ ${volume_quota: -1} = "G" ] ; then
2346	(( temp_quota = temp_quota * 1000000000 ))
2347	elif [ ${volume_quota: -1} = "M" ] ; then
2348	(( temp_quota = temp_quota * 1000000 ))
2349	elif [ ${volume_quota: -1} = "k" ] ; then
2350	(( temp_quota = temp_quota * 1000 ))
2351	else
2352	(( temp_quota = volume_quota ))
2353	fi
2354	fi
2355
2356	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2357	# echo "volume ratio is " $quota_volume
2358
2359	if [ ${quota_volume} -ge ${limit_quota} ] ; then
2360	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2361	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
2362	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2363	IGCM_debug_Print 1 "You must have more than 10% available to run"
2364	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2365	IGCM_debug_Verif_Exit
2366	fi
2367
2368	fi
2369
2370	# Check of the number of inodes
2371
2372	inode_quota=$(ccc_quota \| grep ' scratch' \| awk '{print $6}')
2373	inode_avail=$(ccc_quota \| grep ' scratch' \| awk '{print $7}')
2374
2375	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
2376
2377	unit_avail=${inode_avail: -1}
2378	unit_quota=${inode_quota: -1}
2379
2380	if [ "${unit_quota}" = "*" ] ; then
2381	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2382	IGCM_debug_Print 1 "More than 100% of your quota is used"
2383	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2384	IGCM_debug_Print 1 "You must have more than 10% available to run"
2385	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2386	IGCM_debug_Verif_Exit
2387	fi
2388
2389	temp_avail=${inode_avail%%${inode_avail: -1}*}
2390	temp_quota=${inode_quota%%${inode_quota: -1}*}
2391
2392	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2393
2394	# Convertion
2395	if [ ${inode_avail: -1} = "T" ] ; then
2396	(( temp_avail = temp_avail * 1000000000000 ))
2397	elif [ ${inode_avail: -1} = "G" ] ; then
2398	(( temp_avail = temp_avail * 1000000000 ))
2399	elif [ ${inode_avail: -1} = "M" ] ; then
2400	(( temp_avail = temp_avail * 1000000 ))
2401	elif [ ${inode_avail: -1} = "k" ] ; then
2402	(( temp_avail = temp_avail * 1000 ))
2403	else
2404	(( temp_avail = inode_avail ))
2405	fi
2406
2407	if [ ${inode_quota: -1} = "T" ] ; then
2408	(( temp_quota = temp_quota * 1000000000000 ))
2409	elif [ ${inode_quota: -1} = "G" ] ; then
2410	(( temp_quota = temp_quota * 1000000000 ))
2411	elif [ ${inode_quota: -1} = "M" ] ; then
2412	(( temp_quota = temp_quota * 1000000 ))
2413	elif [ ${inode_quota: -1} = "k" ] ; then
2414	(( temp_quota = temp_quota * 1000 ))
2415	else
2416	(( temp_quota = inode_quota ))
2417	fi
2418	fi
2419	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2420	# echo "inode ratio is " $quota_inode
2421
2422	if [ ${quota_inode} -ge ${limit_quota} ] ; then
2423	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2424	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
2425	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2426	IGCM_debug_Print 1 "You must have more than 10% available to run"
2427	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2428	IGCM_debug_Verif_Exit
2429	fi
2430
2431	fi
2432	IGCM_debug_PopStack "IGCM_sys_check_quota"
2433	}
2434
2435	##############################################################
2436	# NCO OPERATOR
2437
2438	function IGCM_sys_ncap2 {
2439	IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
2440	if ( $DEBUG_sys ) ; then
2441	echo "IGCM_sys_ncap2 :" $@
2442	fi
2443
2444	typeset NB_ESSAI DELAI status i
2445	# number of tentative
2446	NB_ESSAI=3
2447	# time delay between tentative
2448	DELAI=2
2449
2450	i=0
2451	while [ $i -lt $NB_ESSAI ] ; do
2452	ncap2 "$@" > out_rsync 2>&1
2453	status=$?
2454	if [ ${status} -gt 0 ] ; then
2455	IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
2456	cat out_rsync
2457	\rm out_rsync
2458	IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2459	sleep $DELAI
2460	else
2461	\rm out_rsync
2462	break
2463	fi
2464	(( i = i + 1 ))
2465	done
2466
2467	if [ ${status} -gt 0 ] ; then
2468	echo "IGCM_sys_ncap2 : ncap2 error"
2469	IGCM_debug_Exit "ncap2"
2470	fi
2471
2472	IGCM_debug_PopStack "IGCM_sys_ncap2"
2473	}
2474
2475	function IGCM_sys_ncatted {
2476	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
2477	if ( $DEBUG_sys ) ; then
2478	echo "IGCM_sys_ncatted :" $@
2479	fi
2480
2481	typeset NB_ESSAI DELAI status i
2482	# number of tentative
2483	NB_ESSAI=3
2484	# time delay between tentative
2485	DELAI=2
2486
2487	i=0
2488	while [ $i -lt $NB_ESSAI ] ; do
2489	ncatted "$@" > out_rsync 2>&1
2490	status=$?
2491	if [ ${status} -gt 0 ] ; then
2492	IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
2493	cat out_rsync
2494	\rm out_rsync
2495	IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2496	sleep $DELAI
2497	else
2498	\rm out_rsync
2499	break
2500	fi
2501	(( i = i + 1 ))
2502	done
2503
2504	if [ ${status} -gt 0 ] ; then
2505	echo "IGCM_sys_ncatted : ncatted error"
2506	IGCM_debug_Exit "ncatted"
2507	fi
2508
2509	IGCM_debug_PopStack "IGCM_sys_ncatted"
2510	}
2511
2512	function IGCM_sys_ncbo {
2513	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2514	if ( $DEBUG_sys ) ; then
2515	echo "IGCM_sys_ncbo :" $@
2516	fi
2517
2518	typeset NB_ESSAI DELAI status i
2519	# number of tentative
2520	NB_ESSAI=3
2521	# time delay between tentative
2522	DELAI=2
2523
2524	i=0
2525	while [ $i -lt $NB_ESSAI ] ; do
2526	ncbo $@ > out_rsync 2>&1
2527	status=$?
2528	if [ ${status} -gt 0 ] ; then
2529	IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
2530	cat out_rsync
2531	\rm out_rsync
2532	IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2533	sleep $DELAI
2534	else
2535	\rm out_rsync
2536	break
2537	fi
2538	(( i = i + 1 ))
2539	done
2540
2541	if [ ${status} -gt 0 ] ; then
2542	echo "IGCM_sys_ncbo : ncbo error"
2543	IGCM_debug_Exit "ncbo"
2544	fi
2545
2546	IGCM_debug_PopStack "IGCM_sys_ncbo"
2547	}
2548
2549	function IGCM_sys_ncdiff {
2550	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2551	if ( $DEBUG_sys ) ; then
2552	echo "IGCM_sys_ncdiff :" $@
2553	fi
2554
2555	typeset NB_ESSAI DELAI status i
2556	# number of tentative
2557	NB_ESSAI=3
2558	# time delay between tentative
2559	DELAI=2
2560
2561	i=0
2562	while [ $i -lt $NB_ESSAI ] ; do
2563	ncdiff $@ > out_rsync 2>&1
2564	status=$?
2565	if [ ${status} -gt 0 ] ; then
2566	IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2567	cat out_rsync
2568	\rm out_rsync
2569	IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2570	sleep $DELAI
2571	else
2572	\rm out_rsync
2573	break
2574	fi
2575	(( i = i + 1 ))
2576	done
2577
2578	if [ ${status} -gt 0 ] ; then
2579	echo "IGCM_sys_ncdiff : ncdiff error"
2580	IGCM_debug_Exit "ncdiff"
2581	fi
2582
2583	IGCM_debug_PopStack "IGCM_sys_ncdiff"
2584	}
2585
2586	function IGCM_sys_ncea {
2587	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2588	if ( $DEBUG_sys ) ; then
2589	echo "IGCM_sys_ncea :" $@
2590	fi
2591
2592	typeset NB_ESSAI DELAI status i
2593	# number of tentative
2594	NB_ESSAI=3
2595	# time delay between tentative
2596	DELAI=2
2597
2598	i=0
2599	while [ $i -lt $NB_ESSAI ] ; do
2600	ncea $@ > out_rsync 2>&1
2601	status=$?
2602	if [ ${status} -gt 0 ] ; then
2603	IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2604	cat out_rsync
2605	\rm out_rsync
2606	IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2607	sleep $DELAI
2608	else
2609	\rm out_rsync
2610	break
2611	fi
2612	(( i = i + 1 ))
2613	done
2614
2615	if [ ${status} -gt 0 ] ; then
2616	echo "IGCM_sys_ncea : ncea error"
2617	IGCM_debug_Exit "ncea"
2618	fi
2619
2620	IGCM_debug_PopStack "IGCM_sys_ncea"
2621	}
2622
2623	function IGCM_sys_ncecat {
2624	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2625	if ( $DEBUG_sys ) ; then
2626	echo "IGCM_sys_ncecat :" $@
2627	fi
2628
2629	typeset NB_ESSAI DELAI status i
2630	# number of tentative
2631	NB_ESSAI=3
2632	# time delay between tentative
2633	DELAI=2
2634
2635	i=0
2636	while [ $i -lt $NB_ESSAI ] ; do
2637	ncecat $@ > out_rsync 2>&1
2638	status=$?
2639	if [ ${status} -gt 0 ] ; then
2640	IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2641	cat out_rsync
2642	\rm out_rsync
2643	IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2644	sleep $DELAI
2645	else
2646	\rm out_rsync
2647	break
2648	fi
2649	(( i = i + 1 ))
2650	done
2651
2652	if [ ${status} -gt 0 ] ; then
2653	echo "IGCM_sys_ncecat : ncecat error"
2654	IGCM_debug_Exit "ncecat"
2655	fi
2656
2657	IGCM_debug_PopStack "IGCM_sys_ncecat"
2658	}
2659
2660	function IGCM_sys_ncflint {
2661	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2662	if ( $DEBUG_sys ) ; then
2663	echo "IGCM_sys_ncflint :" $@
2664	fi
2665
2666	typeset NB_ESSAI DELAI status i
2667	# number of tentative
2668	NB_ESSAI=3
2669	# time delay between tentative
2670	DELAI=2
2671
2672	i=0
2673	while [ $i -lt $NB_ESSAI ] ; do
2674	ncflint $@ > out_rsync 2>&1
2675	status=$?
2676	if [ ${status} -gt 0 ] ; then
2677	IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2678	cat out_rsync
2679	\rm out_rsync
2680	IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2681	sleep $DELAI
2682	else
2683	\rm out_rsync
2684	break
2685	fi
2686	(( i = i + 1 ))
2687	done
2688
2689	if [ ${status} -gt 0 ] ; then
2690	echo "IGCM_sys_ncflint : ncflint error"
2691	IGCM_debug_Exit "ncflint"
2692	fi
2693
2694	IGCM_debug_PopStack "IGCM_sys_ncflint"
2695	}
2696
2697	function IGCM_sys_ncks {
2698	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2699	if ( $DEBUG_sys ) ; then
2700	echo "IGCM_sys_ncks :" $@
2701	fi
2702
2703	typeset NB_ESSAI DELAI status i
2704	# number of tentative
2705	NB_ESSAI=3
2706	# time delay between tentative
2707	DELAI=2
2708
2709	i=0
2710	while [ $i -lt $NB_ESSAI ] ; do
2711	ncks $@ > out_rsync 2>&1
2712	status=$?
2713	if [ ${status} -gt 0 ] ; then
2714	IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2715	cat out_rsync
2716	\rm out_rsync
2717	IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2718	sleep $DELAI
2719	else
2720	\rm out_rsync
2721	break
2722	fi
2723	(( i = i + 1 ))
2724	done
2725
2726	if [ ${status} -gt 0 ] ; then
2727	echo "IGCM_sys_ncks : ncks error"
2728	IGCM_debug_Exit "ncks"
2729	fi
2730
2731	IGCM_debug_PopStack "IGCM_sys_ncks"
2732	}
2733
2734	function IGCM_sys_ncpdq {
2735	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2736	if ( $DEBUG_sys ) ; then
2737	echo "IGCM_sys_ncpdq :" $@
2738	fi
2739
2740	typeset NB_ESSAI DELAI status i
2741	# number of tentative
2742	NB_ESSAI=3
2743	# time delay between tentative
2744	DELAI=2
2745
2746	i=0
2747	while [ $i -lt $NB_ESSAI ] ; do
2748	ncpdq $@ > out_rsync 2>&1
2749	status=$?
2750	if [ ${status} -gt 0 ] ; then
2751	IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2752	cat out_rsync
2753	\rm out_rsync
2754	IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2755	sleep $DELAI
2756	else
2757	\rm out_rsync
2758	break
2759	fi
2760	(( i = i + 1 ))
2761	done
2762
2763	if [ ${status} -gt 0 ] ; then
2764	echo "IGCM_sys_ncpdq : ncpdq error"
2765	IGCM_debug_Exit "ncpdq"
2766	fi
2767
2768	IGCM_debug_PopStack "IGCM_sys_ncpdq"
2769	}
2770
2771	function IGCM_sys_ncra {
2772	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2773	if ( $DEBUG_sys ) ; then
2774	echo "IGCM_sys_ncra :" $@
2775	fi
2776
2777	typeset NB_ESSAI DELAI status i
2778	# number of tentative
2779	NB_ESSAI=3
2780	# time delay between tentative
2781	DELAI=2
2782
2783	i=0
2784	while [ $i -lt $NB_ESSAI ] ; do
2785	ncra $@ > out_rsync 2>&1
2786	status=$?
2787	if [ ${status} -gt 0 ] ; then
2788	IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2789	cat out_rsync
2790	\rm out_rsync
2791	IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2792	sleep $DELAI
2793	else
2794	\rm out_rsync
2795	break
2796	fi
2797	(( i = i + 1 ))
2798	done
2799
2800	if [ ${status} -gt 0 ] ; then
2801	echo "IGCM_sys_ncra : ncra error"
2802	IGCM_debug_Exit "ncra"
2803	fi
2804
2805	IGCM_debug_PopStack "IGCM_sys_ncra"
2806	}
2807
2808	function IGCM_sys_ncrcat {
2809	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2810	if ( $DEBUG_sys ) ; then
2811	echo "IGCM_sys_ncrcat :" $@
2812	fi
2813
2814	typeset NB_ESSAI DELAI status i
2815	# number of tentative
2816	NB_ESSAI=3
2817	# time delay between tentative
2818	DELAI=2
2819
2820	i=0
2821	while [ $i -lt $NB_ESSAI ] ; do
2822	ncrcat $@ > out_rsync 2>&1
2823	status=$?
2824	if [ ${status} -gt 0 ] ; then
2825	IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2826	cat out_rsync
2827	\rm out_rsync
2828	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2829	sleep $DELAI
2830	else
2831	\rm out_rsync
2832	break
2833	fi
2834	(( i = i + 1 ))
2835	done
2836
2837	if [ ${status} -gt 0 ] ; then
2838	echo "IGCM_sys_ncrcat : ncrcat error"
2839	#IGCM_debug_Exit "ncrcat"
2840	fi
2841
2842	IGCM_debug_PopStack "IGCM_sys_ncrcat"
2843	}
2844
2845	function IGCM_sys_ncrename {
2846	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2847	if ( $DEBUG_sys ) ; then
2848	echo "IGCM_sys_ncrename :" $@
2849	fi
2850
2851	typeset NB_ESSAI DELAI status i
2852	# number of tentative
2853	NB_ESSAI=3
2854	# time delay between tentative
2855	DELAI=2
2856
2857	i=0
2858	while [ $i -lt $NB_ESSAI ] ; do
2859	ncrename $@ > out_rsync 2>&1
2860	status=$?
2861	if [ ${status} -gt 0 ] ; then
2862	IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2863	cat out_rsync
2864	\rm out_rsync
2865	IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2866	sleep $DELAI
2867	else
2868	\rm out_rsync
2869	break
2870	fi
2871	(( i = i + 1 ))
2872	done
2873
2874	if [ ${status} -gt 0 ] ; then
2875	echo "IGCM_sys_ncrename : ncrename error"
2876	IGCM_debug_Exit "ncrename"
2877	fi
2878
2879	IGCM_debug_PopStack "IGCM_sys_ncrename"
2880	}
2881
2882	function IGCM_sys_ncwa {
2883	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2884	if ( $DEBUG_sys ) ; then
2885	echo "IGCM_sys_ncwa :" $@
2886	fi
2887
2888	typeset NB_ESSAI DELAI status i
2889	# number of tentative
2890	NB_ESSAI=3
2891	# time delay between tentative
2892	DELAI=2
2893
2894	i=0
2895	while [ $i -lt $NB_ESSAI ] ; do
2896	ncwa $@ > out_rsync 2>&1
2897	status=$?
2898	if [ ${status} -gt 0 ] ; then
2899	IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2900	cat out_rsync
2901	\rm out_rsync
2902	IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2903	sleep $DELAI
2904	else
2905	\rm out_rsync
2906	break
2907	fi
2908	(( i = i + 1 ))
2909	done
2910
2911	if [ ${status} -gt 0 ] ; then
2912	echo "IGCM_sys_ncwa : ncwa error"
2913	IGCM_debug_Exit "ncwa"
2914	fi
2915
2916	IGCM_debug_PopStack "IGCM_sys_ncwa"
2917	}
2918
2919	##############################################################
2920	# CDO OPERATOR
2921
2922	function IGCM_sys_cdo {
2923	IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2924	if ( $DEBUG_sys ) ; then
2925	echo "IGCM_sys_cdo :" $@
2926	fi
2927
2928	typeset status
2929
2930	\cdo $@ > out_rsync 2>&1
2931	status=$?
2932	if [ ${status} -gt 0 ] ; then
2933	echo "IGCM_sys_cdo : error code ${status}"
2934	cat out_rsync
2935	\rm out_rsync
2936	IGCM_debug_PopStack "IGCM_sys_cdo"
2937	return 1
2938	else
2939	IGCM_debug_PopStack "IGCM_sys_cdo"
2940	return 0
2941	fi
2942
2943	IGCM_debug_PopStack "IGCM_sys_cdo"
2944	}
2945
2946	#D-#==================================================
2947	#D-function IGCM_sys_CountJobInQueue
2948	#D-* Purpose: Check if job_name is currently
2949	#D- running or in queue
2950	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
2951	#D-
2952
2953	function IGCM_sys_CountJobInQueue {
2954	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
2955	if ( $DEBUG_sys ) ; then
2956	echo "IGCM_sys_CountJobInQueue"
2957	fi
2958
2959	#NbRun=$( ccc_mstat -f \| grep -c ${JobName} )
2960
2961	# With -f option, the full job name is given in the last column
2962	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ($NF~String) { x=x+1 } END { print x }' )
2963
2964	eval ${2}=${NbRun}
2965
2966	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
2967	}
2968

Note: See TracBrowser for help on using the repository browser.

Download in other formats: