Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 937

Last change on this file since 937 was 936, checked in by sdipsl, 11 years ago
bugfix again with instrumentation. Should be fine now... syntax error in libIGCM_sys
Property svn:keywords set to `Revision Author Date`
File size: 82.9 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Curie
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=curie
71	# add default project on curie
72	typeset DEFAULT_PROJECT=gen0826
73
74	#D-
75	#D-#==================================================
76	#D-Program used in libIGCM
77	#D-#==================================================
78
79	# Submit command
80	typeset SUBMIT=${SUBMIT:=ccc_msub}
81	# rsync with path
82	typeset -r RSYNC=/usr/bin/rsync
83	# RSYNC_opt args to rsync
84	typeset -r RSYNC_opt="-va"
85	# ie storage filesystem
86	typeset -r STOREHOST=${MASTER}
87
88	#====================================================
89	# Source default environment
90	#====================================================
91	##. /etc/profile
92
93	#====================================================
94	# Set environment tools (ferret, nco, cdo)
95	#====================================================
96	if [ X${TaskType} = Xcomputing ] ; then
97	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_calcul_curie_ksh
98	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64/bin
99	else
100	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_post_curie_ksh
101	fi
102
103	#====================================================
104	# Specific for ocean additionnal diagnostic
105	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
106	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
107
108	#====================================================
109	# Host specific DIRECTORIES
110	#====================================================
111
112	# ============ CESIUM START ============ #
113
114	#====================================================
115	#- Mirror libIGCM from titane to cesium if needed
116	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
117	#if [ ! ${ROOTSYS} = "home" ] ; then
118	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
119	#else
120	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
121	#fi
122
123	#====================================================
124	#- libIGCM_POST
125	#if ( ${MirrorlibIGCM} ) ; then
126	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
127	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
128	#else
129	# typeset -r libIGCM_POST=${libIGCM}
130	#fi
131
132	# ============ CESIUM END ============ #
133
134	#====================================================
135	#- MirrorlibIGCM for frontend
136	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
137
138	#====================================================
139	#- libIGCM_POST for frontend
140	typeset -r libIGCM_POST=${libIGCM}
141
142	#====================================================
143	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
144	typeset -r R_EXE="${MODIPSL}/bin"
145
146	#====================================================
147	#- SUBMIT_DIR : submission dir
148	typeset SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
149
150	#====================================================
151	#- IN
152	typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
153	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
154
155	#====================================================
156	#- ARCHIVE (dedicated to large files)
157	typeset ARCHIVE=${CCCSTOREDIR}
158
159	#====================================================
160	#- STORAGE (dedicated to small/medium files)
161	typeset STORAGE=${CCCWORKDIR}
162
163	#====================================================
164	#- R_OUT
165	typeset R_OUT=${ARCHIVE}/IGCM_OUT
166
167	#====================================================
168	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
169	typeset R_FIG=${STORAGE}/IGCM_OUT
170
171	#====================================================
172	#- R_BUF (ONLY FOR double copy an scratch)
173	typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
174
175	#====================================================
176	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
177	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
178
179	#====================================================
180	#- HOST_MPIRUN_COMMAND
181	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}
182
183	#====================================================
184	#- Max number of arguments passed to nco operator or demigration command
185	UNIX_MAX_LIMIT=360
186
187	#====================================================
188	#- set PackDefault true on curie
189	PackDefault=true
190
191	#====================================================
192	#- Number of core per node (max number of OpenMP task)
193	NUM_COREPERNODE=8
194
195	#====================================================
196	#- Default number of MPI task for IPSL coupled model
197	#- required for backward compatibility
198	#-
199	DEFAULT_NUM_PROC_OCE=5
200	DEFAULT_NUM_PROC_CPL=1
201	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
202	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
203
204	#D-#==================================================
205	#D-function IGCM_sys_ChangeArchive
206	#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
207	#D-* Examples:
208	#D-
209	function IGCM_sys_ChangeArchive {
210	IGCM_debug_PushStack "IGCM_sys_ChangeArchive"
211
212	R_OUT=${SCRATCHDIR}/IGCM_OUT
213	R_FIG=${SCRATCHDIR}/IGCM_OUT
214
215	IGCM_debug_Print 1 "R_OUT has been redefined = ${R_OUT}"
216	IGCM_debug_Print 1 "R_FIG has been redefined = ${R_FIG}"
217
218	IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
219	}
220
221	#D-#==================================================
222	#D-function IGCM_sys_RshMaster
223	#D-* Purpose: Connection to frontend machine.
224	#D-* Examples:
225	#D-
226	function IGCM_sys_RshMaster {
227	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
228	/bin/ksh <<-EOF
229	export libIGCM=${libIGCM}
230	export DEBUG_debug=${DEBUG_debug}
231	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
232	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
233	${@}
234	EOF
235	if [ $? -gt 0 ] ; then
236	echo "IGCM_sys_RshMaster : erreur."
237	IGCM_debug_Exit "IGCM_sys_RshMaster"
238	fi
239	IGCM_debug_PopStack "IGCM_sys_RshMaster"
240	}
241
242	#D-#==================================================
243	#D-function IGCM_sys_RshArchive
244	#D-* Purpose: Archive rsh command
245	#D-* Examples:
246	#D-
247	function IGCM_sys_RshArchive {
248	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
249	/bin/ksh <<-EOF
250	${@}
251	EOF
252	status=$?
253	if [ ${status} -gt 0 ] ; then
254	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
255	IGCM_debug_Exit "IGCM_sys_RshArchive"
256	fi
257	IGCM_debug_PopStack "IGCM_sys_RshArchive"
258	}
259
260	#D-#==================================================
261	#D-function IGCM_sys_RshArchive_NoError
262	#D-* Purpose: Archive rsh command, without error
263	#D-* used only in monitoring.job
264	#D-* Examples:
265	#D-
266	function IGCM_sys_RshArchive_NoError {
267	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
268	/bin/ksh <<-EOF
269	${@} 2> \dev\null
270	EOF
271	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
272	}
273
274	#D-#==================================================
275	#D-function IGCM_sys_RshPost
276	#D-* Purpose: Post-process rsh command
277	#D-* Examples:
278	#D-
279	function IGCM_sys_RshPost {
280	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
281	if ( $DEBUG_sys ) ; then
282	echo "IGCM_sys_RshPost :" $@
283	fi
284	# keep standard input (stdin) for the loop onto temporary file
285	cat >/tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
286
287	/bin/ksh </tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
288	if [ $? -gt 0 ] ; then
289	echo "IGCM_sys_RshPost : erreur."
290	IGCM_debug_Exit "IGCM_sys_RshPost"
291	fi
292	# delete temporary file
293	\rm /tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
294
295	# ============ FRONTEND END ============ #
296
297	# ============ CESIUM START ============ #
298	# typeset NB_ESSAI DELAI status i
299	# if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
300	# #little hack so that rebuild submission is done on titane not an cesium
301	#
302	# libIGCM_POST_sed=$( echo $libIGCM_POST \| sed 's/\//\\\//g' )
303	# POST_DIR_sed=$( echo ${POST_DIR} \| sed 's/\//\\\//g' )
304	# sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
305	# sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
306	# sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
307	# sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
308	# \mv tmp.txt tmp_IGCM_sys_RshPost_$$
309	#
310	# echo cat tmp_IGCM_sys_RshPost_$$ AFTER
311	# cat tmp_IGCM_sys_RshPost_$$
312	#
313	# /bin/ksh <tmp_IGCM_sys_RshPost_$$
314	# if [ $? -gt 0 ] ; then
315	# echo "IGCM_sys_RshPost : erreur."
316	# IGCM_debug_Exit "IGCM_sys_RshPost"
317	# fi
318	# # delete temporary file
319	# \rm tmp_IGCM_sys_RshPost_$$
320	#
321	# else
322	# # number of tentative
323	# NB_ESSAI=10
324	# # time delay between tentative
325	# DELAI=10
326	# i=0
327	# while [ $i -ne $NB_ESSAI ] ; do
328	# ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
329	# status=$?
330	# if [ ${status} -ne 0 ];
331	# then
332	# sleep $DELAI
333	# else
334	# break
335	# fi
336	# let i=$i+1
337	# done
338	# # delete temporary file
339	# \rm tmp_IGCM_sys_RshPost_$$
340	#
341	# if [ ${status} -gt 0 ] ; then
342	# echo "IGCM_sys_RshPost : erreur."
343	# IGCM_debug_Exit "IGCM_sys_RshPost"
344	# fi
345	# fi
346
347	# ============ CESIUM END ============ #
348
349	IGCM_debug_PopStack "IGCM_sys_RshPost"
350	}
351
352	#D-#==================================================
353	#D-function IGCM_sys_SendMail
354	#D-* Purpose: Send mail when simulation is over
355	#D-* Examples:
356	#D-
357	function IGCM_sys_SendMail {
358	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
359
360	if ( $DEBUG_sys ) ; then
361	echo "IGCM_sys_SendMail :" $@
362	fi
363
364	if [ X${1} = XAccounting ] ; then
365	status=Accounting
366	mailText=jobAccounting.mail
367	elif ( ${ExitFlag} ) ; then
368	status=failed
369	mailText=jobEnd.mail
370	else
371	status=completed
372	mailText=jobEnd.mail
373	fi
374
375	# Update selected mail template
376	while read -r line; do
377	eval echo $line >> mail.txt ;
378	done < ${libIGCM}/libIGCM_sys/${mailText}
379
380	if [ ! -z ${config_UserChoices_MailName} ] ; then
381	mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < mail.txt
382	elif [ -f ~/.forward ] ; then
383	mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < mail.txt
384	fi
385
386	sleep 10
387	rm -f mail.txt
388
389	IGCM_debug_PopStack "IGCM_sys_SendMail"
390	}
391
392	#D-#==================================================
393	#D-function IGCM_sys_Mkdir
394	#D-* Purpose: Master locale mkdir command
395	#D-* Examples:
396	#D-
397	function IGCM_sys_Mkdir {
398	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
399	if ( $DEBUG_sys ) ; then
400	echo "IGCM_sys_Mkdir :" $@
401	fi
402	if [ ! -d ${1} ]; then
403	\mkdir -p $1
404	if [ $? -gt 0 ] ; then
405	echo "IGCM_sys_Mkdir : erreur."
406	IGCM_debug_Exit "IGCM_sys_Mkdir"
407	fi
408	fi
409	# vérification :
410	if [ ! -d ${1} ] ; then
411	echo "IGCM_sys_Mkdir : erreur."
412	IGCM_debug_Exit "IGCM_sys_Mkdir"
413	fi
414	IGCM_debug_PopStack "IGCM_sys_Mkdir"
415	}
416
417	#D-#==================================================
418	#D-function IGCM_sys_MkdirArchive
419	#D-* Purpose: Mkdir on Archive
420	#D-* Examples:
421	#D-
422	function IGCM_sys_MkdirArchive {
423	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
424	if ( $DEBUG_sys ) ; then
425	echo "IGCM_sys_MkdirArchive :" $@
426	fi
427	#- creation de repertoire sur le serveur fichier
428	if [ ! -d ${1} ]; then
429	\mkdir -p $1
430	status=$?
431
432	if [ ${status} -gt 0 ] ; then
433	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
434	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
435	fi
436	fi
437	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
438	}
439
440	#D-#==================================================
441	#D-function IGCM_sys_MkdirWork
442	#D-* Purpose: Mkdir on Work
443	#D-* Examples:
444	#D-
445	function IGCM_sys_MkdirWork {
446	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
447	if ( $DEBUG_sys ) ; then
448	echo "IGCM_sys_MkdirWork :" $@
449	fi
450	#- creation de repertoire sur le serveur fichier
451	if [ ! -d ${1} ]; then
452	\mkdir -p $1
453	if [ $? -gt 0 ] ; then
454	echo "IGCM_sys_MkdirWork : erreur."
455	IGCM_debug_Exit "IGCM_sys_MkdirWork"
456	fi
457	fi
458	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
459	}
460
461	#D-#==================================================
462	#D-function IGCM_sys_Cd
463	#D-* Purpose: master cd command
464	#D-* Examples:
465	#D-
466	function IGCM_sys_Cd {
467	IGCM_debug_PushStack "IGCM_sys_Cd" $@
468	if ( $DEBUG_sys ) ; then
469	echo "IGCM_sys_Cd :" $@
470	fi
471	\cd $1
472	if [ $? -gt 0 ] ; then
473	echo "IGCM_sys_Cd : erreur."
474	IGCM_debug_Exit "IGCM_sys_Cd"
475	fi
476	IGCM_debug_PopStack "IGCM_sys_Cd"
477	}
478
479	#D-#==================================================
480	#D-function IGCM_sys_Chmod
481	#D-* Purpose: Chmod
482	#D-* Examples:
483	#D-
484	function IGCM_sys_Chmod {
485	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
486	if ( $DEBUG_sys ) ; then
487	echo "IGCM_sys_Chmod :" $@
488	fi
489	\chmod $@
490	if [ $? -gt 0 ] ; then
491	echo "IGCM_sys_Chmod : erreur."
492	IGCM_debug_Exit "IGCM_sys_Chmod"
493	fi
494	IGCM_debug_PopStack "IGCM_sys_Chmod"
495	}
496
497	#D-#==================================================
498	#D-function IGCM_sys_FileSize
499	#D-* Purpose: Filesize
500	#D-* Examples:
501	#D-
502	function IGCM_sys_FileSize {
503	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
504
505	typeset sizeF
506	set +A sizeF -- $( ls -la ${1} )
507	if [ $? -gt 0 ] ; then
508	IGCM_debug_Exit "IGCM_sys_FileSize"
509	fi
510	eval ${2}=${sizeF[4]}
511
512	IGCM_debug_PopStack "IGCM_sys_FileSize"
513	}
514
515	#D-#==================================================
516	#D-function IGCM_sys_TestDir
517	#D-* Purpose: Test Directory that must exists
518	#D-* Examples:
519	#D-
520	function IGCM_sys_TestDir {
521	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
522	if ( $DEBUG_sys ) ; then
523	echo "IGCM_sys_TestDir :" $@
524	fi
525	typeset ExistFlag
526	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
527	IGCM_debug_PopStack "IGCM_sys_TestDir"
528
529	return ${ExistFlag}
530	}
531
532	#D-#==================================================
533	#D-function IGCM_sys_TestDirArchive
534	#D-* Purpose: Test Directory that must exists on Archive
535	#D-* Examples:
536	#D-
537	function IGCM_sys_TestDirArchive {
538	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
539	if ( $DEBUG_sys ) ; then
540	echo "IGCM_sys_TestDirArchive :" $@
541	fi
542	typeset ExistFlag
543	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
544	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
545	return ${ExistFlag}
546	}
547
548	#D-#==================================================
549	#D-function IGCM_sys_IsFileArchived
550	#D-* Purpose: Test file that must NOT EXISTS on Archive
551	#D-* Examples:
552	#D-
553	function IGCM_sys_IsFileArchived {
554	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
555	if ( $DEBUG_sys ) ; then
556	echo "IGCM_sys_IsFileArchived :" $@
557	fi
558	typeset IsArchivedFlag
559	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
560	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
561
562	return ${IsArchivedFlag}
563	}
564
565	#D-#==================================================
566	#D-function IGCM_sys_TestFileArchive
567	#D-* Purpose: Test file that must NOT EXISTS on Archive
568	#D-* Examples:
569	#D-
570	function IGCM_sys_TestFileArchive {
571	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
572	typeset ExistFlag
573	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
574	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
575
576	return ${ExistFlag}
577	}
578
579	#D-#==================================================
580	#D-function IGCM_sys_TestFileBuffer
581	#D-* Purpose: Test file that must NOT EXISTS on Buffer
582	#D-* Examples:
583	#D-
584	function IGCM_sys_TestFileBuffer {
585	IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
586	typeset ExistFlag
587	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
588	IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
589
590	return ${ExistFlag}
591	}
592
593	#D-#==================================================
594	#D-function IGCM_sys_CountFileArchive
595	#D-* Purpose: Count files on Archive filesystem
596	#D-* Examples:
597	#D-
598	function IGCM_sys_CountFileArchive {
599	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
600	ls ${@} 2>/dev/null \| wc -l
601	if [ $? -gt 0 ] ; then
602	echo "IGCM_sys_CountFileArchive : erreur."
603	fi
604	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
605	}
606
607	#D-#==================================================
608	#D-function IGCM_sys_CountFileBuffer
609	#D-* Purpose: Count files on Scratch filesystem
610	#D-* Examples:
611	#D-
612	function IGCM_sys_CountFileBuffer {
613	IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
614	ls ${@} 2>/dev/null \| wc -l
615	if [ $? -gt 0 ] ; then
616	echo "IGCM_sys_CountFileBuffer : erreur."
617	fi
618	IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
619	}
620
621	#D-#==================================================
622	#D-function IGCM_sys_Tree
623	#D-* Purpose: Tree directories with files on ${ARCHIVE}
624	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
625	#D-
626	function IGCM_sys_Tree {
627	IGCM_debug_PushStack "IGCM_sys_Tree" $@
628	if ( $DEBUG_sys ) ; then
629	echo "IGCM_sys_Tree :" $@
630	fi
631
632	\ls -lR ${@}
633
634	IGCM_debug_PopStack "IGCM_sys_Tree"
635	}
636
637	#D-#==================================================
638	#D-function IGCM_sys_Tar
639	#D-* Purpose: master tar command
640	#D-* Examples:
641	#D-
642	function IGCM_sys_Tar {
643	IGCM_debug_PushStack "IGCM_sys_Tar" $@
644	if ( $DEBUG_sys ) ; then
645	echo "IGCM_sys_Tar :" $@
646	fi
647	\tar cf $@
648	if [ $? -gt 0 ] ; then
649	echo "IGCM_sys_Tar : erreur."
650	IGCM_debug_Exit "IGCM_sys_Tar"
651	fi
652	IGCM_debug_PopStack "IGCM_sys_Tar"
653	}
654
655	#D-#==================================================
656	#D-function IGCM_sys_UnTar
657	#D-* Purpose: master un-tar command
658	#D-* Examples:
659	#D-
660	function IGCM_sys_UnTar {
661	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
662	if ( $DEBUG_sys ) ; then
663	echo "IGCM_sys_UnTar :" $@
664	fi
665	\tar xvf $1
666	if [ $? -gt 0 ] ; then
667	echo "IGCM_sys_UnTar : erreur."
668	IGCM_debug_Exit "IGCM_sys_UnTar"
669	fi
670	IGCM_debug_PopStack "IGCM_sys_UnTar"
671	}
672
673	#D-#==================================================
674	#D-function IGCM_sys_Qsub
675	#D-* Purpose: Qsub new job
676	#D-* Examples:
677	#D-
678	function IGCM_sys_Qsub {
679	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
680	if ( $DEBUG_sys ) ; then
681	echo "IGCM_sys_Qsub :" $@
682	fi
683	typeset options status
684	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
685
686	/usr/bin/ccc_msub ${options} < $1
687	status=$?
688	if [ ${status} -gt 0 ] ; then
689	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
690	IGCM_debug_Exit "IGCM_sys_Qsub"
691	fi
692	IGCM_debug_PopStack "IGCM_sys_Qsub"
693	}
694
695	#D-#==================================================
696	#D-function IGCM_sys_QsubPost
697	#D-* Purpose: Qsub new job on scalaire
698	#D-* Examples:
699	#D-
700	function IGCM_sys_QsubPost {
701	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
702	if ( $DEBUG_sys ) ; then
703	echo "IGCM_sys_QsubPost :" $@
704	fi
705	typeset options status
706	MY_PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=${DEFAULT_PROJECT}} \| cut -d@ -f1 )
707	options="-Q normal -A ${MY_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
708
709	/usr/bin/ccc_msub ${options} < ${libIGCM_POST}/$1.job
710	status=$?
711	if [ ${status} -gt 0 ] ; then
712	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
713	IGCM_debug_Exit "IGCM_sys_QsubPost"
714	fi
715	IGCM_debug_PopStack "IGCM_sys_QsubPost"
716	}
717
718	#D-*************************
719	#D- File transfer functions
720	#D-*************************
721	#D-
722
723	#D-#==================================================
724	#D-function IGCM_sys_Rsync_out
725	#D-* Purpose: treat return val of rsync
726	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
727	#D- Error values and explanations can depend on your system version.
728	function IGCM_sys_Rsync_out {
729	status=$1
730	if [ ! $status ] ; then
731	echo "rsync error !"
732	fi
733
734	if [ $MYLANG = "fr" ]; then
735	case $status in
736	0) return ;;
737	1) echo "Erreur de rsync ; RERR_SYNTAX : "
738	echo "Erreur de syntaxe ou d'utilisation."
739	return;;
740	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
741	echo "Incompatibilité de protocole."
742	return;;
743	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
744	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
745	echo "répertoires"
746	return;;
747	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
748	echo "Action demandée non supportée : une tentative de manipulation de"
749	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
750	echo "été faite ; ou une option qui est supportée par le client mais"
751	echo "pas par le serveur a été spécifiée."
752	return;;
753	10) echo "Erreur de rsync ; RERR_SOCKETIO"
754	echo "Erreur dans le socket d'entrée sortie"
755	return;;
756	11) echo "Erreur de rsync ; RERR_FILEIO"
757	echo "Erreur d'entrée sortie fichier"
758	return;;
759	12) echo "Erreur de rsync ; RERR_STREAMIO"
760	echo "Erreur dans flux de donnée du protocole rsync"
761	return;;
762	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
763	echo "Erreur avec les diagnostics du programme"
764	return;;
765	14) echo "Erreur de rsync ; RERR_IPC"
766	echo "Erreur dans le code IPC"
767	return;;
768	20) echo "Erreur de rsync ; RERR_SIGNAL"
769	echo "SIGUSR1 ou SIGINT reçu"
770	return;;
771	21) echo "Erreur de rsync ; RERR_WAITCHILD"
772	echo "Une erreur retournée par waitpid()"
773	return;;
774	22) echo "Erreur de rsync ; RERR_MALLOC"
775	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
776	return;;
777	23) echo ""
778	echo "Erreur fichier inexistant"
779	return;;
780	30) echo "Erreur de rsync ; RERR_TIMEOUT"
781	echo "Temps d'attente écoulé dans l'envoi/réception de données"
782	return;;
783	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $status
784	return;;
785	esac
786	elif [ $MYLANG = "en" ] ; then
787	case $status in
788	0) return;;
789	1) echo "rsync error : Syntax or usage error "
790	return;;
791	2) echo "rsync error : Protocol incompatibility "
792	return;;
793	3) echo "rsync error : Errors selecting input/output files, dirs"
794	return;;
795	4) echo "rsync error : Requested action not supported: an attempt"
796	echo "was made to manipulate 64-bit files on a platform that cannot support"
797	echo "them; or an option was specified that is supported by the client and"
798	echo "not by the server."
799	return;;
800	5) echo "rsync error : Error starting client-server protocol"
801	return;;
802	10) echo "rsync error : Error in socket I/O "
803	return;;
804	11) echo "rsync error : Error in file I/O "
805	return;;
806	12) echo "rsync error : Error in rsync protocol data stream "
807	return;;
808	13) echo "rsync error : Errors with program diagnostics "
809	return;;
810	14) echo "rsync error : Error in IPC code "
811	return;;
812	20) echo "rsync error : Received SIGUSR1 or SIGINT "
813	return;;
814	21) echo "rsync error : Some error returned by waitpid() "
815	return;;
816	22) echo "rsync error : Error allocating core memory buffers "
817	return;;
818	23) echo "rsync error : Partial transfer due to error"
819	return;;
820	24) echo "rsync error : Partial transfer due to vanished source files"
821	return;;
822	30) echo "rsync error : Timeout in data send/receive "
823	return;;
824	*) echo "rsync error : return code of rsync unknown :" $status
825	return;;
826	esac
827	else
828	echo "unknown language $MYLANG."
829	return
830	fi
831	}
832
833	#D-#==================================================
834	#D-function IGCM_sys_Miror_libIGCM
835	#D-* Purpose: Mirror libIGCM PATH and lib to frontend
836	#D-* Examples:
837	#D-
838	function IGCM_sys_Mirror_libIGCM {
839	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
840	if ( $DEBUG_sys ) ; then
841	echo "IGCM_sys_Mirror_libIGCM"
842	fi
843
844	typeset status
845
846	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
847
848	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
849	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
850	status=$?
851
852	if [ ${status} -gt 0 ] ; then
853	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
854	cat out_rsync
855	fi
856	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
857	}
858
859	#D-#==================================================
860	#D-function IGCM_sys_Cp
861	#D-* Purpose: generic cp
862	#D-* Examples:
863	#D-
864	function IGCM_sys_Cp {
865	IGCM_debug_PushStack "IGCM_sys_Cp" $@
866	if ( $DEBUG_sys ) ; then
867	echo "IGCM_sys_Cp :" $@
868	fi
869
870	typeset status
871
872	echo cp $@ > out_rsync 2>&1
873	\cp $@ >> out_rsync 2>&1
874	status=$?
875
876	if [ ${status} -gt 0 ] ; then
877	echo "IGCM_sys_Cp : error code ${status}"
878	cat out_rsync
879	IGCM_debug_Exit "IGCM_sys_Cp"
880	else
881	\rm out_rsync
882	fi
883	IGCM_debug_PopStack "IGCM_sys_Cp"
884	}
885
886	#D-#==================================================
887	#D-function IGCM_sys_Rm
888	#D-* Purpose: generic rm
889	#D-* Examples:
890	#D-
891	function IGCM_sys_Rm {
892	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
893	if ( $DEBUG_sys ) ; then
894	echo "IGCM_sys_Rm :" $@
895	fi
896
897	typeset status
898
899	echo rm $@ > out_rsync 2>&1
900	\rm $@ >> out_rsync 2>&1
901	status=$?
902
903	if [ ${status} -gt 0 ] ; then
904	echo "IGCM_sys_Rm : error code ${status}"
905	cat out_rsync
906	IGCM_debug_Exit "IGCM_sys_Rm"
907	else
908	\rm out_rsync
909	fi
910	IGCM_debug_PopStack "IGCM_sys_Rm"
911	}
912
913	#D-#==================================================
914	#D-function IGCM_sys_RmRunDir
915	#D-* Purpose: rm tmpdir (dummy function most of the time batch
916	#D- scheduler will do the job)
917	#D-* Examples:
918	#D-
919	function IGCM_sys_RmRunDir {
920	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
921	if ( $DEBUG_sys ) ; then
922	echo "IGCM_sys_RmRunDir :" $@
923	fi
924
925	typeset status
926
927	echo rm $@ > out_rsync 2>&1
928	\rm $@ >> out_rsync 2>&1
929	status=$?
930
931	if [ ${status} -gt 0 ] ; then
932	echo "IGCM_sys_RmRunDir : rm error code is ${status}."
933	cat out_rsync
934	IGCM_debug_Exit "IGCM_sys_RmRunDir"
935	else
936	\rm out_rsync
937	fi
938	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
939	}
940
941	#D-#==================================================
942	#D-function IGCM_sys_Mv
943	#D-* Purpose: generic move
944	#D-* Examples:
945	#D-
946	function IGCM_sys_Mv {
947	IGCM_debug_PushStack "IGCM_sys_Mv" $@
948	if ( $DEBUG_sys ) ; then
949	echo "IGCM_sys_Mv :" $@
950	fi
951
952	if [ $DRYRUN = 0 ]; then
953
954	typeset status
955
956	echo mv $@ > out_rsync 2>&1
957	\mv $@ >> out_rsync 2>&1
958	status=$?
959
960	if [ ${status} -gt 0 ] ; then
961	echo "IGCM_sys_Mv : error code ${status}"
962	cat out_rsync
963	IGCM_debug_Exit "IGCM_sys_Mv"
964	else
965	\rm out_rsync
966	fi
967	fi
968
969	IGCM_debug_PopStack "IGCM_sys_Mv"
970	}
971
972	#D-#==================================================
973	#D-function IGCM_sys_Put_Dir
974	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
975	#D-* Examples:
976	#D-
977	function IGCM_sys_Put_Dir {
978	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
979	if ( $DEBUG_sys ) ; then
980	echo "IGCM_sys_Put_Dir :" $@
981	fi
982	if [ $DRYRUN = 0 ]; then
983	if [ ! -d ${1} ] ; then
984	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
985	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
986	return
987	fi
988
989	typeset status
990
991	# Only if we use rsync
992	#IGCM_sys_TestDirArchive $( dirname $2 )
993	#
994	#USUAL WAY
995	\cp -r $1 $2 > out_rsync 2>&1
996	status=$?
997
998	if [ ${status} -gt 0 ] ; then
999	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
1000	cat out_rsync
1001	IGCM_debug_Exit "IGCM_sys_Put_Dir"
1002	else
1003	\rm out_rsync
1004	fi
1005	fi
1006	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1007	}
1008
1009	#D-#==================================================
1010	#D-function IGCM_sys_Get_Dir
1011	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
1012	#D-* Examples:
1013	#D-
1014	function IGCM_sys_Get_Dir {
1015	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
1016	if ( $DEBUG_sys ) ; then
1017	echo "IGCM_sys_Get_Dir :" $@
1018	fi
1019	if [ $DRYRUN = 0 ]; then
1020	# if [ ! -d ${1} ] ; then
1021	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
1022	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1023	# return
1024	# fi
1025
1026	typeset NB_ESSAI DELAI status i
1027	# number of tentative
1028	NB_ESSAI=3
1029	# time delay between tentative
1030	DELAI=2
1031
1032	# Only if we use rsync
1033	#IGCM_sys_TestDirArchive $( dirname $2 )
1034	#
1035	# USUAL WAY
1036	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
1037	ccc_hsm get -r $1
1038
1039	i=0
1040	while [ $i -lt $NB_ESSAI ] ; do
1041	\cp -ur $1 $2 >> out_rsync 2>&1
1042	status=$?
1043	if [ ${status} -gt 0 ] ; then
1044	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
1045	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
1046	sleep $DELAI
1047	else
1048	break
1049	fi
1050	(( i = i + 1 ))
1051	done
1052
1053	if [ ${status} -gt 0 ] ; then
1054	echo "IGCM_sys_Get_Dir : error."
1055	cat out_rsync
1056	IGCM_debug_Exit "IGCM_sys_Get_Dir"
1057	else
1058	\rm out_rsync
1059	fi
1060	fi
1061	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1062	}
1063
1064	#D-#==================================================
1065	#D-function IGCM_sys_Get_Master
1066	#D-* Purpose: Copy a complete directory from MASTER filesystem
1067	#D-* Examples:
1068	#D-
1069	function IGCM_sys_Get_Master {
1070	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
1071	if ( $DEBUG_sys ) ; then
1072	echo "IGCM_sys_Get_Master :" $@
1073	fi
1074	if [ $DRYRUN = 0 ]; then
1075	if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
1076	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
1077	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1078	return
1079	fi
1080
1081	typeset NB_ESSAI DELAI status i
1082	# number of tentative
1083	NB_ESSAI=3
1084	# time delay between tentative
1085	DELAI=2
1086
1087	i=0
1088	while [ $i -lt $NB_ESSAI ] ; do
1089	\cp -urL $1 $2 > out_rsync 2>&1
1090	status=$?
1091	if [ ${status} -gt 0 ]; then
1092	IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
1093	IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
1094	sleep $DELAI
1095	else
1096	break
1097	fi
1098	(( i = i + 1 ))
1099	done
1100
1101	if [ ${status} -gt 0 ] ; then
1102	echo "IGCM_sys_Get_Master : error."
1103	cat out_rsync
1104	IGCM_debug_Exit "IGCM_sys_Get_Master"
1105	else
1106	\rm out_rsync
1107	fi
1108	fi
1109	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1110	}
1111
1112	#====================================================
1113	#- Call IGCM_sys_Mirror_libIGCM now !
1114	if ( $MirrorlibIGCM ) ; then
1115	IGCM_sys_Mirror_libIGCM
1116	fi
1117
1118	#D-#==================================================
1119	#D-function IGCM_sys_Put_Rest
1120	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1121	#D- File and target directory must exist.
1122	#D-* Examples:
1123	#D-
1124	function IGCM_sys_Put_Rest {
1125	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1126	if ( $DEBUG_sys ) ; then
1127	echo "IGCM_sys_Put_Rest :" $@
1128	fi
1129	if [ $DRYRUN = 0 ]; then
1130	if [ ! -f ${1} ] ; then
1131	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1132	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1133	fi
1134
1135	typeset status
1136	#
1137	# USUAL WAY
1138	\cp $1 $2 > out_rsync 2>&1
1139	status=$?
1140
1141	# #RSYNC WITH NETWORK SSH CALL
1142	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1143	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1144
1145	# #RSYNC WITH NFS USE
1146	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1147	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1148
1149	# status=$?
1150	# IGCM_sys_Rsync_out $status
1151
1152	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1153	# (( status=status+$? ))
1154
1155	if [ ${status} -gt 0 ] ; then
1156	echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1157	[ -f ${1} ] && ls -l ${1}
1158	[ -f ${2} ] && ls -l ${2}
1159	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1160	cat out_rsync
1161	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1162	else
1163
1164	if [ X${JobType} = XRUN ] ; then
1165	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1166	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1167	fi
1168
1169	\rm out_rsync
1170	fi
1171	fi
1172	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1173	}
1174
1175	#D-#==================================================
1176	#D-function IGCM_sys_PutBuffer_Rest
1177	#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1178	#D- File and target directory must exist.
1179	#D-* Examples:
1180	#D-
1181	function IGCM_sys_PutBuffer_Rest {
1182	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1183	if ( $DEBUG_sys ) ; then
1184	echo "IGCM_sys_PutBuffer_Rest :" $@
1185	fi
1186	if [ $DRYRUN = 0 ]; then
1187	if [ ! -f ${1} ] ; then
1188	echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1189	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1190	fi
1191
1192	typeset status
1193	#
1194	# USUAL WAY
1195	\cp $1 $2 > out_rsync 2>&1
1196	status=$?
1197
1198	if [ ${status} -gt 0 ] ; then
1199	echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1200	[ -f ${2} ] && ls -l ${2}
1201	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1202	cat out_rsync
1203	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1204	else
1205
1206	if [ X${JobType} = XRUN ] ; then
1207	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1208	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1209	fi
1210
1211	\rm out_rsync
1212	fi
1213	fi
1214	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1215	}
1216
1217	#D-#==================================================
1218	#D-function IGCM_sys_PrepareTaredRestart
1219	#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
1220	#D-* Examples:
1221	#D-
1222	function IGCM_sys_PrepareTaredRestart {
1223	IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
1224	if [ $DRYRUN = 0 ]; then
1225	[ ! -f $( basename $1 ) ] && IGCM_sys_Get $1 .
1226	fi
1227	IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1228	}
1229
1230	#D-#==================================================
1231	#D-function IGCM_sys_Put_Out
1232	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1233	#D-* Examples:
1234	#D-
1235	function IGCM_sys_Put_Out {
1236	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1237	if ( $DEBUG_sys ) ; then
1238	echo "IGCM_sys_Put_Out :" $@
1239	fi
1240
1241	typeset NB_ESSAI DELAI status i exist skip
1242
1243	# number of tentative
1244	NB_ESSAI=3
1245	# time delay between tentative
1246	DELAI=2
1247
1248	if [ $DRYRUN = 0 ]; then
1249	if [ ! -f ${1} ] ; then
1250	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1251	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1252	return 1
1253	fi
1254	#
1255	IGCM_sys_MkdirArchive $( dirname $2 )
1256	#
1257
1258	#=====================================================
1259	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1260	#=====================================================
1261
1262	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
1263	#if [ $? -eq 0 ] ; then
1264	# typeset WORKPATH FILEPATH
1265	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_BUF}\|" )
1266	# IGCM_sys_MkdirWork ${WORKPATH}
1267	# FILEPATH=${WORKPATH}/$( basename $2 )
1268	# #
1269	# IGCM_sys_Cp ${1} ${FILEPATH}
1270	#fi
1271
1272	exist=false
1273	skip=false
1274	if [ -f $2 ] ; then
1275	IGCM_debug_Print 1 "$2 already exist"
1276	#dmget $2
1277	ccc_hsm get $2
1278	exist=true
1279	if [ "X$( diff $1 $2 )" = X ] ; then
1280	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1281	skip=true
1282	else
1283	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1284	skip=false
1285	fi
1286	fi
1287	#
1288	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1289	IGCM_sys_Chmod u+w $2
1290	fi
1291
1292	if [ X${skip} = Xfalse ] ; then
1293	i=0
1294	while [ $i -lt $NB_ESSAI ] ; do
1295	# USUAL WAY
1296	\cp $1 $2 > out_rsync 2>&1
1297	status=$?
1298	if [ ${status} -gt 0 ]; then
1299	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1300	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
1301	[ -f ${1} ] && ls -l ${1}
1302	[ -f ${2} ] && ls -l ${2}
1303	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1304	sleep $DELAI
1305	else
1306	break
1307	fi
1308	(( i = i + 1 ))
1309	done
1310	fi
1311
1312	# #RSYNC WITH NETWORK SSH CALL
1313	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1314	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1315
1316	# #RSYNC WITH NFS USE
1317	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1318	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1319
1320	# status=$?
1321	# IGCM_sys_Rsync_out $status
1322
1323	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1324	# (( status=status+$? ))
1325
1326	if [ ${status} -gt 0 ] ; then
1327	echo "IGCM_sys_Put_Out : error."
1328	[ -f ${1} ] && ls -l ${1}
1329	[ -f ${2} ] && ls -l ${2}
1330	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1331	cat out_rsync
1332	IGCM_debug_Exit "IGCM_sys_Put_Out"
1333	else
1334
1335	if [ X${JobType} = XRUN ] ; then
1336	if [ X${3} = X ] ; then
1337	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1338	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1339	fi
1340	fi
1341
1342	\rm out_rsync
1343	fi
1344	fi
1345	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1346	return 0
1347	}
1348
1349	#D-#==================================================
1350	#D-function IGCM_sys_PutBuffer_Out
1351	#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1352	#D-* Examples:
1353	#D-
1354	function IGCM_sys_PutBuffer_Out {
1355	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1356	if ( $DEBUG_sys ) ; then
1357	echo "IGCM_sys_PutBuffer_Out :" $@
1358	fi
1359
1360	typeset NB_ESSAI DELAI status i exist skip
1361
1362	# number of tentative
1363	NB_ESSAI=3
1364	# time delay between tentative
1365	DELAI=2
1366
1367	if [ $DRYRUN = 0 ]; then
1368	if [ ! -f ${1} ] ; then
1369	echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1370	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1371	return 1
1372	fi
1373	#
1374	IGCM_sys_Mkdir $( dirname $2 )
1375	#
1376
1377	exist=false
1378	skip=false
1379	if [ -f $2 ] ; then
1380	IGCM_debug_Print 1 "$2 already exist"
1381	exist=true
1382	if [ "X$( diff $1 $2 )" = X ] ; then
1383	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1384	status=0
1385	skip=true
1386	else
1387	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1388	skip=false
1389	fi
1390	fi
1391	#
1392	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1393	IGCM_sys_Chmod u+w $2
1394	fi
1395
1396	if [ X${skip} = Xfalse ] ; then
1397	i=0
1398	while [ $i -lt $NB_ESSAI ] ; do
1399	# USUAL WAY
1400	\cp $1 $2 > out_rsync 2>&1
1401	status=$?
1402	if [ ${status} -gt 0 ]; then
1403	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1404	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1405	[ -f ${2} ] && ls -l ${2}
1406	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1407	sleep $DELAI
1408	else
1409	break
1410	fi
1411	(( i = i + 1 ))
1412	done
1413	fi
1414
1415	if [ ${status} -gt 0 ] ; then
1416	echo "IGCM_sys_PutBuffer_Out : error."
1417	[ -f ${2} ] && ls -l ${2}
1418	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1419	cat out_rsync
1420	IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1421	else
1422
1423	if [ X${JobType} = XRUN ] ; then
1424	if [ X${3} = X ] ; then
1425	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1426	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1427	fi
1428	fi
1429
1430	\rm out_rsync
1431	fi
1432	fi
1433	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1434	return 0
1435	}
1436
1437	#D-#==================================================
1438	#D-function IGCM_sys_Get
1439	#D-* Purpose: Get a file from ${ARCHIVE}
1440	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1441	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
1442	function IGCM_sys_Get {
1443	IGCM_debug_PushStack "IGCM_sys_Get" $@
1444
1445	typeset DEST dm_liste target file_work
1446	typeset NB_ESSAI DELAI status i
1447
1448	if ( $DEBUG_sys ) ; then
1449	echo "IGCM_sys_Get :" $@
1450	fi
1451
1452	# number of tentative
1453	NB_ESSAI=3
1454	# time delay between tentative
1455	DELAI=2
1456
1457	if [ $DRYRUN -le 2 ]; then
1458	if [ X${1} = X'/l' ] ; then
1459	eval set +A dm_liste \${${2}}
1460	else
1461	eval set +A dm_liste ${1}
1462	fi
1463	eval DEST=\${${#}}
1464
1465	#=====================================================
1466	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1467	#=====================================================
1468
1469	# Is it an R_OUT file (not R_IN) ?
1470	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
1471	#if [ $? -eq 0 ] ; then
1472	# # Yes ? then we try to get it in SCRATCHDIR
1473	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_BUF}\|g" )
1474	# if [ -f ${file_work[0]} ] ; then
1475	# IGCM_sys_Cp ${file_work[*]} ${DEST}
1476	# IGCM_debug_PopStack "IGCM_sys_Get"
1477	# return
1478	# fi
1479	#fi
1480
1481	ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1482	status=$?
1483	if [ ${status} -gt 0 ] ; then
1484	echo "WARNING IGCM_sys_Get : error code ${status}"
1485	cat out_rsync
1486	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1487	fi
1488
1489	#if [ ${status} -gt 0 ] ; then
1490	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1491	# cat out_rsync
1492	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1493	# sleep 30
1494	# echo "We try another time"
1495	## dmget ${dm_liste[*]} > out_rsync 2>&1
1496	# ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1497	# status=$?
1498	# if [ ${status} -gt 0 ] ; then
1499	# echo "ERROR IGCM_sys_Get : again demigration error :"
1500	# cat out_rsync
1501	# IGCM_debug_Exit "IGCM_sys_Get"
1502	# fi
1503	# else
1504	# echo "ERROR IGCM_sys_Get : demigration error :"
1505	# cat out_rsync
1506	# IGCM_debug_Exit "IGCM_sys_Get"
1507	# fi
1508	#fi
1509
1510	# #RSYNC WITH NETWORK SSH CALL
1511	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1512	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1513
1514	# #RSYNC WITH NFS USE
1515	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1516	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1517
1518	# status=$?
1519	# IGCM_sys_Rsync_out $status
1520
1521	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1522	# (( status=status+$? ))
1523
1524	#USUAL WAY
1525	if [ X${1} = X'/l' ] ; then
1526	for target in ${dm_liste[*]} ; do
1527	local_file=$( basename ${target} )
1528	# test if the target file is present before the loop
1529	IGCM_sys_TestFileArchive ${target}
1530	status=$?
1531	if [ ${status} -gt 0 ] ; then
1532	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
1533	IGCM_debug_Exit "IGCM_sys_Get"
1534	else
1535	i=0
1536	while [ $i -lt $NB_ESSAI ] ; do
1537	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1538	status=$?
1539	if [ ${status} -gt 0 ]; then
1540	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1541	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1542	sleep $DELAI
1543	else
1544	break
1545	fi
1546	(( i = i + 1 ))
1547	done
1548	if [ ${status} -gt 0 ] ; then
1549	echo "IGCM_sys_Get : error"
1550	cat out_rsync
1551	\rm out_rsync
1552	IGCM_debug_Exit "IGCM_sys_Get"
1553	else
1554	\rm out_rsync
1555	fi
1556	fi
1557	done
1558	else
1559	i=0
1560	while [ $i -lt $NB_ESSAI ] ; do
1561	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1562	status=$?
1563	if [ ${status} -gt 0 ]; then
1564	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1565	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1566	sleep $DELAI
1567	else
1568	break
1569	fi
1570	(( i = i + 1 ))
1571	done
1572	if [ ${status} -gt 0 ] ; then
1573	echo "IGCM_sys_Get : error"
1574	cat out_rsync
1575	\rm out_rsync
1576	IGCM_debug_Exit "IGCM_sys_Get"
1577	else
1578	\rm out_rsync
1579	fi
1580	fi
1581	fi
1582	IGCM_debug_PopStack "IGCM_sys_Get"
1583	}
1584
1585	#D-#==================================================
1586	#D-function IGCM_sys_GetBuffer
1587	#D-* Purpose: Get a file from ${SCRATCHDIR}
1588	#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1589	#D- IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1590	function IGCM_sys_GetBuffer {
1591	IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1592
1593	typeset DEST buf_liste target file_work
1594	typeset NB_ESSAI DELAI status i
1595
1596	if ( $DEBUG_sys ) ; then
1597	echo "IGCM_sys_GetBuffer :" $@
1598	fi
1599
1600	# number of tentative
1601	NB_ESSAI=3
1602	# time delay between tentative
1603	DELAI=2
1604
1605	if [ $DRYRUN -le 2 ]; then
1606	if [ X${1} = X'/l' ] ; then
1607	eval set +A buf_liste \${${2}}
1608	else
1609	eval set +A buf_liste ${1}
1610	fi
1611	eval DEST=\${${#}}
1612
1613	#USUAL WAY
1614	if [ X${1} = X'/l' ] ; then
1615	for target in ${buf_liste[*]} ; do
1616	local_file=$( basename ${target} )
1617	i=0
1618	while [ $i -lt $NB_ESSAI ] ; do
1619	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1620	status=$?
1621	if [ ${status} -gt 0 ]; then
1622	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1623	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1624	sleep $DELAI
1625	else
1626	break
1627	fi
1628	(( i = i + 1 ))
1629	done
1630	if [ ${status} -gt 0 ] ; then
1631	echo "IGCM_sys_Get : error"
1632	cat out_rsync
1633	\rm out_rsync
1634	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1635	else
1636	\rm out_rsync
1637	fi
1638	done
1639	else
1640	i=0
1641	while [ $i -lt $NB_ESSAI ] ; do
1642	\cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1643	status=$?
1644	if [ ${status} -gt 0 ]; then
1645	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1646	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1647	sleep $DELAI
1648	else
1649	break
1650	fi
1651	(( i = i + 1 ))
1652	done
1653	if [ ${status} -gt 0 ] ; then
1654	echo "IGCM_sys_Get : error"
1655	cat out_rsync
1656	\rm out_rsync
1657	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1658	else
1659	\rm out_rsync
1660	fi
1661	fi
1662	fi
1663	IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1664	}
1665
1666	#D-#==================================================
1667	#D-function IGCM_sys_GetDate_FichWork
1668	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1669	#D-* Examples:
1670	#D-
1671	function IGCM_sys_GetDate_FichWork {
1672	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1673	if ( $DEBUG_sys ) ; then
1674	echo "IGCM_sys_GetDate_FichWork :" $@
1675	fi
1676	typeset dateF
1677	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1678	eval ${2}=${dateF[5]}
1679
1680	# donne la date filesys d'un fichier sur la machine work
1681	IGCM_debug_PopStack "IGCM_sys_FichWork"
1682	}
1683
1684	#D-#==================================================
1685	#D-function IGCM_sys_GetDate_FichArchive
1686	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1687	#D-* Examples:
1688	#D-
1689	function IGCM_sys_GetDate_FichArchive {
1690	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1691	if ( $DEBUG_sys ) ; then
1692	echo "IGCM_sys_GetDate_FichArchive :" $@
1693	fi
1694	typeset dateF
1695	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1696	eval ${2}=${dateF[5]}
1697
1698	IGCM_debug_PopStack "IGCM_sys_FichArchive"
1699	}
1700
1701	#D-#==================================================
1702	#D-function IGCM_sys_GetDate_Monitoring
1703	#D-* Purpose: get the last year for which the monitoring has been computed
1704	#D-* Examples:
1705	#D-
1706	function IGCM_sys_GetDate_Monitoring {
1707	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
1708	if ( $DEBUG_sys ) ; then
1709	echo "IGCM_sys_GetDate_Monitoring :" $@
1710	fi
1711
1712	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
1713
1714	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
1715	}
1716
1717	#D-#==================================================
1718	#D-function IGCM_sys_Dods_Rm
1719	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1720	#D-* Examples:
1721	#D-
1722	function IGCM_sys_Dods_Rm {
1723	if ( $DEBUG_sys ) ; then
1724	echo "IGCM_sys_Dods_Rm :" $@
1725	fi
1726	typeset status
1727	if [ $DRYRUN = 0 ]; then
1728
1729	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1730	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1731	# echo "Nothing has been done."
1732	# return
1733	# fi
1734
1735	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
1736	status=$?
1737
1738	# if [ ${status} -gt 0 ] ; then
1739	# echo "IGCM_sys_Dods_Rm : error."
1740	# cat out_dods_rm
1741	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1742	# else
1743	# rm out_dods_rm
1744	# fi
1745
1746	fi
1747	return $status
1748	}
1749
1750	#D-#==================================================
1751	#D-function IGCM_sys_Dods_Cp
1752	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1753	#D-* Examples:
1754	#D-
1755	function IGCM_sys_Dods_Cp {
1756	if ( $DEBUG_sys ) ; then
1757	echo "IGCM_sys_Dods_Cp :" $@
1758	fi
1759	typeset status
1760	if [ $DRYRUN = 0 ]; then
1761
1762	# if [ ! -d ${R_SAVE}/${1} ] ; then
1763	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1764	# echo "Nothing has been done."
1765	# return
1766	# fi
1767
1768	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1769	status=$?
1770
1771	# if [ ${status} -gt 0 ] ; then
1772	# echo "IGCM_sys_Dods_Cp : error."
1773	# cat out_dods_cp
1774	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1775	# else
1776	# rm out_dods_cp
1777	# fi
1778
1779	fi
1780	return $status
1781	}
1782
1783	#D-#==================================================
1784	#D-function IGCM_sys_Put_Dods
1785	#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1786	#D-* Examples:
1787	#D-
1788	function IGCM_sys_Put_Dods {
1789	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1790	if ( $DEBUG_sys ) ; then
1791	echo "IGCM_sys_Put_Dods :" $@
1792	fi
1793	#set -vx
1794	typeset status
1795	if [ $DRYRUN = 0 ]; then
1796	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1797	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1798	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1799	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1800	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1801	return
1802	fi
1803
1804	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1805	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1806	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1807	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1808	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1809	return
1810	fi
1811	#
1812	if [ -d ${R_SAVE}/${1} ] ; then
1813	cd ${R_SAVE}
1814	elif [ -d ${R_FIGR}/${1} ] ; then
1815	cd ${R_FIGR}
1816	fi
1817
1818	IGCM_sys_Dods_Rm ${1}
1819	IGCM_sys_Dods_Cp ${1}
1820	status=0
1821
1822	if [ ${status} -gt 0 ] ; then
1823	echo "IGCM_sys_Put_Dods : error."
1824	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1825	fi
1826	fi
1827	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1828	}
1829
1830	##############################################################
1831	# REBUILD OPERATOR
1832
1833	#D-#==================================================
1834	#D-function IGCM_sys_rebuild
1835	#D-* Purpose: rebuild parallel files
1836	#D-* Examples:
1837	#D-
1838	function IGCM_sys_rebuild {
1839	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1840	if ( $DEBUG_sys ) ; then
1841	echo "IGCM_sys_rebuild :" $@
1842	fi
1843
1844	typeset NB_ESSAI DELAI status i firstArg
1845	# number of tentative
1846	NB_ESSAI=3
1847	# time delay between tentative
1848	DELAI=2
1849
1850	i=0
1851	while [ $i -lt $NB_ESSAI ] ; do
1852	~p86ipsl/X64_CURIE/bin/rebuild -f -o $@ > out_rsync 2>&1
1853	status=$?
1854	if [ ${status} -gt 0 ] ; then
1855	IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1856	cat out_rsync
1857	\rm out_rsync
1858	IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1859	firstArg=${1}
1860	\rm ${firstArg}
1861	sleep $DELAI
1862	else
1863	\rm out_rsync
1864	break
1865	fi
1866	(( i = i + 1 ))
1867	done
1868
1869	if [ ${status} -gt 0 ] ; then
1870	echo "IGCM_sys_rebuild : rebuild error code is ${status}"
1871	IGCM_debug_Exit "rebuild"
1872	fi
1873
1874	IGCM_debug_PopStack "IGCM_sys_rebuild"
1875	}
1876
1877	#D-#==================================================
1878	#D-function IGCM_sys_rebuild_station
1879	#D-* Purpose: rebuild parallel files describing station
1880	#D-* Examples:
1881	#D-
1882	function IGCM_sys_rebuild_station {
1883	IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1884	typeset i list_opt file_in file_out prefix_invert list_invert
1885	if ( $DEBUG_sys ) ; then
1886	echo "IGCM_sys_rebuild_station :" $@
1887	fi
1888	list_opt=$@
1889
1890	# Invert Axis : t,x -> x,t
1891	# t,pres,x -> x,t,pres
1892	# So that we can concatenate along x
1893	i=0
1894	for file_in in ${list_opt} ; do
1895	(( i = i + 1))
1896	[ ${i} = 1 ] && file_out=${file_in} && continue
1897	prefix_invert=$( basename ${file_in} .nc )
1898	IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1899	list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1900	done
1901
1902	# Concatenate
1903	IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1904
1905	# Re-ivert file
1906	IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1907
1908	# Station re-ordering is too expansive to be run within libIGCM
1909	# This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1910	# This re-ordering must be done "in memory" by the cmorization process
1911	# Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1912	# BEGIN reordering
1913
1914	# Only LMDZ text output contains the exact ordering of the station.
1915	# We isolate this in the code below:
1916	# 0 38 -157.5000000000000 70.98591549295774
1917	# 0 54 27.49999999999999 67.18309859154928
1918	# 0 56 -62.50000000000001 82.39436619718309
1919	# 0 79 12.49999999999999 78.59154929577466
1920	# 0 116 -165.0000000000000 76.05633802816901
1921	# 0 117 130.0000000000000 70.98591549295774
1922	# 0 118 110.0000000000000 87.46478873239437
1923	# 1 40 4.999999999999995 51.97183098591550
1924	# typeset iStation iProc list_opt file_in file_out prefix_invert
1925	# typeset -Z4 j4
1926	# typeset -Z3 j3
1927
1928	# unset list_opt
1929	# set +A list_opt $@
1930
1931	# Filename after rebuild
1932	# file_out=${list_opt[0]}
1933	# Prefix of output files
1934	# prefix_invert=$( basename ${file_out} .nc )
1935	# Number of procs
1936	# num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} \| wc -l )
1937
1938	# iProc=0
1939	# while [ ${iProc} -lt ${num_proc} ] ; do
1940	# Array containing Station as a number
1941	# unset proc_stn
1942	# set +A proc_stn $( grep "iophy_mpi rank ip lon lat $iProc" ${PREFIX}_${Exe_Output} \| sed -e "s/iophy_mpi rank ip lon lat //g" \| gawk ' {print $2}' )
1943	# Number of stations produced by processor proc
1944	# stationLast=${#proc_stn[*]}
1945	# Proc number on 4 digits
1946	# j4=${iProc}
1947	# Init
1948	# iStation=0
1949	# while [ ${iStation} -lt ${stationLast} ] ; do
1950	# Station number on 3 digits
1951	# j3=${proc_stn[${iStation}]}
1952	# Extract station
1953	# Invert Axis : t,x -> x,t
1954	# t,pres,x -> x,t,pres
1955	# So that we can concatenate along x
1956	# IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1957	# (( iStation = iStation + 1 ))
1958	# done
1959	# (( iProc = iProc + 1 ))
1960	# done
1961
1962	# Concatenate all station along x
1963	# IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1964
1965	# Re-invert file
1966	# IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1967
1968	# END reordering
1969
1970	IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1971	}
1972
1973	############################################################
1974	# Activate Running Environnment Variables
1975
1976	#D-#==================================================
1977	#D-function IGCM_sys_desactiv_variables
1978	#D-* Purpose: set environement variables prior to execution
1979	#D-* Examples:
1980	#D-
1981	function IGCM_sys_activ_variables {
1982	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1983	if ( $DEBUG_sys ) ; then
1984	echo "IGCM_sys_activ_variables"
1985	fi
1986
1987	# --------------------------------------------------------------------
1988	#D- MPI specifications
1989	# --------------------------------------------------------------------
1990
1991	# --------------------------------------------------------------------
1992	#D- Other specifications
1993	# --------------------------------------------------------------------
1994
1995	ulimit -s unlimited
1996
1997	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1998	}
1999
2000	############################################################
2001	# Desactivate Running Environnment Variables
2002
2003	#D-#==================================================
2004	#D-function IGCM_sys_desactiv_variables
2005	#D-* Purpose: unset environement variables after execution
2006	#D-* Examples:
2007	#D-
2008	function IGCM_sys_desactiv_variables {
2009	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
2010	if ( $DEBUG_sys ) ; then
2011	echo "IGCM_sys_desactiv_variables"
2012	fi
2013	# --------------------------------------------------------------------
2014	#D- MPI specifications
2015	# --------------------------------------------------------------------
2016
2017	# --------------------------------------------------------------------
2018	#D- Other specifications
2019	# --------------------------------------------------------------------
2020
2021	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
2022	}
2023
2024	############################################################
2025	# Build MPI/OMP scripts run file (dummy function)
2026
2027	#D-#==================================================
2028	#D-function IGCM_sys_build_run_file
2029	#D-* Purpose: build run file (deprecated)
2030	#D-* Examples:
2031	#D-
2032	function IGCM_sys_build_run_file {
2033
2034	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
2035
2036	}
2037
2038	############################################################
2039	# Build MPI/OMP scripts
2040
2041	#D-#==================================================
2042	#D-function IGCM_sys_build_execution_scripts
2043	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
2044	#D-* Examples:
2045	#D-
2046	function IGCM_sys_build_execution_scripts
2047	{
2048	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
2049	if ( $DEBUG_sys ) ; then
2050	echo "IGCM_sys_build_execution_scripts " $@
2051	fi
2052
2053	EXECUTION=${HOST_MPIRUN_COMMAND}
2054
2055	if ( ${OK_PARA_MPMD} ) ; then
2056
2057	if [ -f run_file ] ; then
2058	IGCM_sys_Rm -f run_file
2059	fi
2060	touch run_file
2061
2062	# run_file construction
2063
2064	# Then first loop on the components for the coupler ie oasis
2065
2066	## the coupler ie oasis must be the first one
2067	for comp in ${config_ListOfComponents[*]} ; do
2068
2069	eval ExeNameIn=\${config_Executable_${comp}[0]}
2070	eval ExeNameOut=\${config_Executable_${comp}[1]}
2071
2072	# for CPL component only
2073	if [ "X${comp}" = "XCPL" ] ; then
2074
2075	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2076	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2077	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
2078	fi
2079	done
2080
2081	# Then second loop on the components
2082
2083	for comp in ${config_ListOfComponents[*]} ; do
2084
2085	eval ExeNameIn=\${config_Executable_${comp}[0]}
2086	eval ExeNameOut=\${config_Executable_${comp}[1]}
2087
2088	# Only if we really have an executable for the component and not the coupler ie oasis:
2089	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
2090
2091	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2092	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2093	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
2094	fi
2095	done
2096
2097	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
2098
2099	IGCM_sys_Chmod u+x run_file
2100	if ( $DEBUG_sys ) ; then
2101	echo "run_file contains : "
2102	cat run_file
2103	fi
2104
2105	else # Only one executable. launch it.
2106
2107	for comp in ${config_ListOfComponents[*]} ; do
2108
2109	# Only if we really have an executable for the component :
2110	eval ExeNameOut=\${config_Executable_${comp}[1]}
2111	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
2112
2113	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2114	echo "" >> script_${ExeNameOut}.ksh
2115	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2116
2117	if ( ${OK_PARA_OMP} ) ; then
2118	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2119	echo "" >> script_${ExeNameOut}.ksh
2120	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
2121	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
2122	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
2123	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
2124	fi
2125
2126	if ( ${OK_PARA_MPI} ) ; then
2127	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2128	# Default : ccc_mprun used if nb_proc gt 1
2129	# to have out/err per process on different files
2130	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
2131	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
2132	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
2133	else
2134	# Default : ccc_mprun is NOT used if nb_proc eq 1
2135	# to have out/err per process on different files
2136	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2137	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
2138	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
2139	fi
2140
2141	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
2142	cat script_${ExeNameOut}.ksh
2143
2144	fi
2145
2146	done
2147
2148	fi
2149
2150	IGCM_debug_Print 1 "sys Curie : execution command is "
2151	IGCM_debug_Print 1 "$EXECUTION"
2152
2153	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
2154	}
2155
2156	#D-#==================================================
2157	#D-function IGCM_sys_check_path
2158	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
2159	#D-* do not point to an important use directory. Stop immediately in that case.
2160	#D-* Examples:
2161	#D-
2162	function IGCM_sys_check_path {
2163	IGCM_debug_PushStack "IGCM_sys_check_path"
2164	if ( $DEBUG_sys ) ; then
2165	echo "IGCM_sys_check_path"
2166	fi
2167
2168	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
2169	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
2170	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
2171	IGCM_debug_Exit "This will stop the job"
2172	fi
2173	IGCM_debug_PopStack "IGCM_sys_check_path"
2174	}
2175
2176	#D-#==================================================
2177	#D-function IGCM_sys_check_quota
2178	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
2179	#D-* Examples:
2180	#D-
2181	function IGCM_sys_check_quota {
2182	IGCM_debug_PushStack "IGCM_sys_check_quota"
2183	if ( $DEBUG_sys ) ; then
2184	echo "IGCM_sys_check_quota"
2185	fi
2186	# Limit of quota (in %)
2187	limit_quota=90
2188
2189	# Check of the volume
2190	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
2191	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
2192
2193	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
2194
2195	unit_avail=${volume_avail: -1}
2196	unit_quota=${volume_quota: -1}
2197
2198	if [ "${unit_quota}" = "*" ] ; then
2199	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2200	IGCM_debug_Print 1 "More than 100% of your quota is used"
2201	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2202	IGCM_debug_Print 1 "You must have more than 10% available to run"
2203	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2204	IGCM_debug_Verif_Exit
2205	fi
2206
2207	temp_avail=${volume_avail%%${volume_avail: -1}*}
2208	temp_quota=${volume_quota%%${volume_quota: -1}*}
2209
2210	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2211
2212	# Convertion
2213	if [ ${volume_avail: -1} = "T" ] ; then
2214	(( temp_avail = temp_avail * 1000000000000 ))
2215	elif [ ${volume_avail: -1} = "G" ] ; then
2216	(( temp_avail = temp_avail * 1000000000 ))
2217	elif [ ${volume_avail: -1} = "M" ] ; then
2218	(( temp_avail = temp_avail * 1000000 ))
2219	elif [ ${volume_avail: -1} = "k" ] ; then
2220	(( temp_avail = temp_avail * 1000 ))
2221	else
2222	(( temp_avail = volume_avail ))
2223	fi
2224	if [ ${volume_quota: -1} = "T" ] ; then
2225	(( temp_quota = temp_quota * 1000000000000 ))
2226	elif [ ${volume_quota: -1} = "G" ] ; then
2227	(( temp_quota = temp_quota * 1000000000 ))
2228	elif [ ${volume_quota: -1} = "M" ] ; then
2229	(( temp_quota = temp_quota * 1000000 ))
2230	elif [ ${volume_quota: -1} = "k" ] ; then
2231	(( temp_quota = temp_quota * 1000 ))
2232	else
2233	(( temp_quota = volume_quota ))
2234	fi
2235	fi
2236
2237	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2238	# echo "volume ratio is " $quota_volume
2239
2240	if [ ${quota_volume} -ge ${limit_quota} ] ; then
2241	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2242	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
2243	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2244	IGCM_debug_Print 1 "You must have more than 10% available to run"
2245	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2246	IGCM_debug_Verif_Exit
2247	fi
2248
2249	fi
2250
2251	# Check of the number of inodes
2252
2253	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
2254	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
2255
2256	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
2257
2258	unit_avail=${inode_avail: -1}
2259	unit_quota=${inode_quota: -1}
2260
2261	if [ "${unit_quota}" = "*" ] ; then
2262	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2263	IGCM_debug_Print 1 "More than 100% of your quota is used"
2264	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2265	IGCM_debug_Print 1 "You must have more than 10% available to run"
2266	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2267	IGCM_debug_Verif_Exit
2268	fi
2269
2270	temp_avail=${inode_avail%%${inode_avail: -1}*}
2271	temp_quota=${inode_quota%%${inode_quota: -1}*}
2272
2273	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2274
2275	# Convertion
2276	if [ ${inode_avail: -1} = "T" ] ; then
2277	(( temp_avail = temp_avail * 1000000000000 ))
2278	elif [ ${inode_avail: -1} = "G" ] ; then
2279	(( temp_avail = temp_avail * 1000000000 ))
2280	elif [ ${inode_avail: -1} = "M" ] ; then
2281	(( temp_avail = temp_avail * 1000000 ))
2282	elif [ ${inode_avail: -1} = "k" ] ; then
2283	(( temp_avail = temp_avail * 1000 ))
2284	else
2285	(( temp_avail = inode_avail ))
2286	fi
2287
2288	if [ ${inode_quota: -1} = "T" ] ; then
2289	(( temp_quota = temp_quota * 1000000000000 ))
2290	elif [ ${inode_quota: -1} = "G" ] ; then
2291	(( temp_quota = temp_quota * 1000000000 ))
2292	elif [ ${inode_quota: -1} = "M" ] ; then
2293	(( temp_quota = temp_quota * 1000000 ))
2294	elif [ ${inode_quota: -1} = "k" ] ; then
2295	(( temp_quota = temp_quota * 1000 ))
2296	else
2297	(( temp_quota = inode_quota ))
2298	fi
2299	fi
2300	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2301	# echo "inode ratio is " $quota_inode
2302
2303	if [ ${quota_inode} -ge ${limit_quota} ] ; then
2304	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2305	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
2306	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2307	IGCM_debug_Print 1 "You must have more than 10% available to run"
2308	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2309	IGCM_debug_Verif_Exit
2310	fi
2311	fi
2312	IGCM_debug_PopStack "IGCM_sys_check_quota"
2313	}
2314
2315	#D-#==================================================
2316	#D-function IGCM_sys_CountJobInQueue
2317	#D-* Purpose: Check if job_name is currently
2318	#D- running or in queue
2319	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
2320	#D-
2321	function IGCM_sys_CountJobInQueue {
2322	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
2323	if ( $DEBUG_sys ) ; then
2324	echo "IGCM_sys_CountJobInQueue"
2325	fi
2326
2327	#NbRun=$( ccc_mstat -f \| grep -c ${JobName} )
2328
2329	# With -f option, the full job name is given in the last column
2330	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
2331
2332	eval ${2}=${NbRun}
2333
2334	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
2335	}
2336
2337	##############################################################
2338	# NCO OPERATOR
2339
2340	#D-#==================================================
2341	#D-function IGCM_sys_ncap2
2342	#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
2343	#D-* Examples:
2344	#D-
2345	function IGCM_sys_ncap2 {
2346	IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
2347	if ( $DEBUG_sys ) ; then
2348	echo "IGCM_sys_ncap2 :" $@
2349	fi
2350
2351	typeset NB_ESSAI DELAI status i
2352	# number of tentative
2353	NB_ESSAI=3
2354	# time delay between tentative
2355	DELAI=2
2356
2357	i=0
2358	while [ $i -lt $NB_ESSAI ] ; do
2359	ncap2 "$@" > out_rsync 2>&1
2360	status=$?
2361	if [ ${status} -gt 0 ] ; then
2362	IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
2363	cat out_rsync
2364	\rm out_rsync
2365	IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2366	sleep $DELAI
2367	else
2368	\rm out_rsync
2369	break
2370	fi
2371	(( i = i + 1 ))
2372	done
2373
2374	if [ ${status} -gt 0 ] ; then
2375	echo "IGCM_sys_ncap2 : ncap2 error"
2376	IGCM_debug_Exit "ncap2"
2377	fi
2378
2379	IGCM_debug_PopStack "IGCM_sys_ncap2"
2380	}
2381
2382	#D-#==================================================
2383	#D-function IGCM_sys_ncatted
2384	#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
2385	#D-* Examples:
2386	#D-
2387	function IGCM_sys_ncatted {
2388	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
2389	if ( $DEBUG_sys ) ; then
2390	echo "IGCM_sys_ncatted :" $@
2391	fi
2392
2393	typeset NB_ESSAI DELAI status i
2394	# number of tentative
2395	NB_ESSAI=3
2396	# time delay between tentative
2397	DELAI=2
2398
2399	i=0
2400	while [ $i -lt $NB_ESSAI ] ; do
2401	ncatted "$@" > out_rsync 2>&1
2402	status=$?
2403	if [ ${status} -gt 0 ] ; then
2404	IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
2405	cat out_rsync
2406	\rm out_rsync
2407	IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2408	sleep $DELAI
2409	else
2410	\rm out_rsync
2411	break
2412	fi
2413	(( i = i + 1 ))
2414	done
2415
2416	if [ ${status} -gt 0 ] ; then
2417	echo "IGCM_sys_ncatted : ncatted error"
2418	IGCM_debug_Exit "ncatted"
2419	fi
2420
2421	IGCM_debug_PopStack "IGCM_sys_ncatted"
2422	}
2423
2424	#D-#==================================================
2425	#D-function IGCM_sys_ncbo
2426	#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
2427	#D-* Examples:
2428	#D-
2429	function IGCM_sys_ncbo {
2430	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2431	if ( $DEBUG_sys ) ; then
2432	echo "IGCM_sys_ncbo :" $@
2433	fi
2434
2435	typeset NB_ESSAI DELAI status i
2436	# number of tentative
2437	NB_ESSAI=3
2438	# time delay between tentative
2439	DELAI=2
2440
2441	i=0
2442	while [ $i -lt $NB_ESSAI ] ; do
2443	ncbo $@ > out_rsync 2>&1
2444	status=$?
2445	if [ ${status} -gt 0 ] ; then
2446	IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
2447	cat out_rsync
2448	\rm out_rsync
2449	IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2450	sleep $DELAI
2451	else
2452	\rm out_rsync
2453	break
2454	fi
2455	(( i = i + 1 ))
2456	done
2457
2458	if [ ${status} -gt 0 ] ; then
2459	echo "IGCM_sys_ncbo : ncbo error"
2460	IGCM_debug_Exit "ncbo"
2461	fi
2462
2463	IGCM_debug_PopStack "IGCM_sys_ncbo"
2464	}
2465
2466	#D-#==================================================
2467	#D-function IGCM_sys_ncdif
2468	#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
2469	#D-* Examples:
2470	#D-
2471	function IGCM_sys_ncdiff {
2472	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2473	if ( $DEBUG_sys ) ; then
2474	echo "IGCM_sys_ncdiff :" $@
2475	fi
2476
2477	typeset NB_ESSAI DELAI status i
2478	# number of tentative
2479	NB_ESSAI=3
2480	# time delay between tentative
2481	DELAI=2
2482
2483	i=0
2484	while [ $i -lt $NB_ESSAI ] ; do
2485	ncdiff $@ > out_rsync 2>&1
2486	status=$?
2487	if [ ${status} -gt 0 ] ; then
2488	IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2489	cat out_rsync
2490	\rm out_rsync
2491	IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2492	sleep $DELAI
2493	else
2494	\rm out_rsync
2495	break
2496	fi
2497	(( i = i + 1 ))
2498	done
2499
2500	if [ ${status} -gt 0 ] ; then
2501	echo "IGCM_sys_ncdiff : ncdiff error"
2502	IGCM_debug_Exit "ncdiff"
2503	fi
2504
2505	IGCM_debug_PopStack "IGCM_sys_ncdiff"
2506	}
2507
2508	#D-#==================================================
2509	#D-function IGCM_sys_ncea
2510	#D-* Purpose: encapsulate ncea call so as to manage error code and retry
2511	#D-* Examples:
2512	#D-
2513	function IGCM_sys_ncea {
2514	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2515	if ( $DEBUG_sys ) ; then
2516	echo "IGCM_sys_ncea :" $@
2517	fi
2518
2519	typeset NB_ESSAI DELAI status i
2520	# number of tentative
2521	NB_ESSAI=3
2522	# time delay between tentative
2523	DELAI=2
2524
2525	i=0
2526	while [ $i -lt $NB_ESSAI ] ; do
2527	ncea $@ > out_rsync 2>&1
2528	status=$?
2529	if [ ${status} -gt 0 ] ; then
2530	IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2531	cat out_rsync
2532	\rm out_rsync
2533	IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2534	sleep $DELAI
2535	else
2536	\rm out_rsync
2537	break
2538	fi
2539	(( i = i + 1 ))
2540	done
2541
2542	if [ ${status} -gt 0 ] ; then
2543	echo "IGCM_sys_ncea : ncea error"
2544	IGCM_debug_Exit "ncea"
2545	fi
2546
2547	IGCM_debug_PopStack "IGCM_sys_ncea"
2548	}
2549
2550	#D-#==================================================
2551	#D-function IGCM_sys_ncecat
2552	#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
2553	#D-* Examples:
2554	#D-
2555	function IGCM_sys_ncecat {
2556	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2557	if ( $DEBUG_sys ) ; then
2558	echo "IGCM_sys_ncecat :" $@
2559	fi
2560
2561	typeset NB_ESSAI DELAI status i
2562	# number of tentative
2563	NB_ESSAI=3
2564	# time delay between tentative
2565	DELAI=2
2566
2567	i=0
2568	while [ $i -lt $NB_ESSAI ] ; do
2569	ncecat $@ > out_rsync 2>&1
2570	status=$?
2571	if [ ${status} -gt 0 ] ; then
2572	IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2573	cat out_rsync
2574	\rm out_rsync
2575	IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2576	sleep $DELAI
2577	else
2578	\rm out_rsync
2579	break
2580	fi
2581	(( i = i + 1 ))
2582	done
2583
2584	if [ ${status} -gt 0 ] ; then
2585	echo "IGCM_sys_ncecat : ncecat error"
2586	IGCM_debug_Exit "ncecat"
2587	fi
2588
2589	IGCM_debug_PopStack "IGCM_sys_ncecat"
2590	}
2591
2592	#D-#==================================================
2593	#D-function IGCM_sys_ncflint
2594	#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
2595	#D-* Examples:
2596	#D-
2597	function IGCM_sys_ncflint {
2598	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2599	if ( $DEBUG_sys ) ; then
2600	echo "IGCM_sys_ncflint :" $@
2601	fi
2602
2603	typeset NB_ESSAI DELAI status i
2604	# number of tentative
2605	NB_ESSAI=3
2606	# time delay between tentative
2607	DELAI=2
2608
2609	i=0
2610	while [ $i -lt $NB_ESSAI ] ; do
2611	ncflint $@ > out_rsync 2>&1
2612	status=$?
2613	if [ ${status} -gt 0 ] ; then
2614	IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2615	cat out_rsync
2616	\rm out_rsync
2617	IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2618	sleep $DELAI
2619	else
2620	\rm out_rsync
2621	break
2622	fi
2623	(( i = i + 1 ))
2624	done
2625
2626	if [ ${status} -gt 0 ] ; then
2627	echo "IGCM_sys_ncflint : ncflint error"
2628	IGCM_debug_Exit "ncflint"
2629	fi
2630
2631	IGCM_debug_PopStack "IGCM_sys_ncflint"
2632	}
2633
2634	#D-#==================================================
2635	#D-function IGCM_sys_ncks
2636	#D-* Purpose: encapsulate ncks call so as to manage error code and retry
2637	#D-* Examples:
2638	#D-
2639	function IGCM_sys_ncks {
2640	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2641	if ( $DEBUG_sys ) ; then
2642	echo "IGCM_sys_ncks :" $@
2643	fi
2644
2645	typeset NB_ESSAI DELAI status i
2646	# number of tentative
2647	NB_ESSAI=3
2648	# time delay between tentative
2649	DELAI=2
2650
2651	i=0
2652	while [ $i -lt $NB_ESSAI ] ; do
2653	ncks $@ > out_rsync 2>&1
2654	status=$?
2655	if [ ${status} -gt 0 ] ; then
2656	IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2657	cat out_rsync
2658	\rm out_rsync
2659	IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2660	sleep $DELAI
2661	else
2662	\rm out_rsync
2663	break
2664	fi
2665	(( i = i + 1 ))
2666	done
2667
2668	if [ ${status} -gt 0 ] ; then
2669	echo "IGCM_sys_ncks : ncks error"
2670	IGCM_debug_Exit "ncks"
2671	fi
2672
2673	IGCM_debug_PopStack "IGCM_sys_ncks"
2674	}
2675
2676	#D-#==================================================
2677	#D-function IGCM_sys_ncpdq
2678	#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
2679	#D-* Examples:
2680	#D-
2681	function IGCM_sys_ncpdq {
2682	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2683	if ( $DEBUG_sys ) ; then
2684	echo "IGCM_sys_ncpdq :" $@
2685	fi
2686
2687	typeset NB_ESSAI DELAI status i
2688	# number of tentative
2689	NB_ESSAI=3
2690	# time delay between tentative
2691	DELAI=2
2692
2693	i=0
2694	while [ $i -lt $NB_ESSAI ] ; do
2695	ncpdq $@ > out_rsync 2>&1
2696	status=$?
2697	if [ ${status} -gt 0 ] ; then
2698	IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2699	cat out_rsync
2700	\rm out_rsync
2701	IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2702	sleep $DELAI
2703	else
2704	\rm out_rsync
2705	break
2706	fi
2707	(( i = i + 1 ))
2708	done
2709
2710	if [ ${status} -gt 0 ] ; then
2711	echo "IGCM_sys_ncpdq : ncpdq error"
2712	IGCM_debug_Exit "ncpdq"
2713	fi
2714
2715	IGCM_debug_PopStack "IGCM_sys_ncpdq"
2716	}
2717
2718	#D-#==================================================
2719	#D-function IGCM_sys_ncra
2720	#D-* Purpose: encapsulate ncra call so as to manage error code and retry
2721	#D-* Examples:
2722	#D-
2723	function IGCM_sys_ncra {
2724	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2725	if ( $DEBUG_sys ) ; then
2726	echo "IGCM_sys_ncra :" $@
2727	fi
2728
2729	typeset NB_ESSAI DELAI status i
2730	# number of tentative
2731	NB_ESSAI=3
2732	# time delay between tentative
2733	DELAI=2
2734
2735	i=0
2736	while [ $i -lt $NB_ESSAI ] ; do
2737	ncra $@ > out_rsync 2>&1
2738	status=$?
2739	if [ ${status} -gt 0 ] ; then
2740	IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2741	cat out_rsync
2742	\rm out_rsync
2743	IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2744	sleep $DELAI
2745	else
2746	\rm out_rsync
2747	break
2748	fi
2749	(( i = i + 1 ))
2750	done
2751
2752	if [ ${status} -gt 0 ] ; then
2753	echo "IGCM_sys_ncra : ncra error"
2754	IGCM_debug_Exit "ncra"
2755	fi
2756
2757	IGCM_debug_PopStack "IGCM_sys_ncra"
2758	}
2759
2760	#D-#==================================================
2761	#D-function IGCM_sys_ncrcat
2762	#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
2763	#D-* Examples:
2764	#D-
2765	function IGCM_sys_ncrcat {
2766	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2767	if ( $DEBUG_sys ) ; then
2768	echo "IGCM_sys_ncrcat :" $@
2769	fi
2770
2771	typeset NB_ESSAI DELAI status i lastArg
2772	# number of tentative
2773	NB_ESSAI=3
2774	# time delay between tentative
2775	DELAI=2
2776
2777	i=0
2778	while [ $i -lt $NB_ESSAI ] ; do
2779	ncrcat $@ > out_rsync 2>&1
2780	status=$?
2781	if [ ${status} -gt 0 ] ; then
2782	IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2783	cat out_rsync
2784	\rm out_rsync
2785	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2786	sleep $DELAI
2787	elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" out_rsync )" = "X" ] ; then
2788	IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
2789	cat out_rsync
2790	# remove files having corrupted time axis
2791	eval lastArg=\${$#}
2792	IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
2793	\rm ${lastArg}
2794	\rm out_rsync
2795	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2796	sleep $DELAI
2797	else
2798	\rm out_rsync
2799	break
2800	fi
2801	(( i = i + 1 ))
2802	done
2803
2804	if [ ${status} -gt 0 ] ; then
2805	echo "IGCM_sys_ncrcat : ncrcat error"
2806	#IGCM_debug_Exit "ncrcat"
2807	fi
2808
2809	IGCM_debug_PopStack "IGCM_sys_ncrcat"
2810	}
2811
2812	#D-#==================================================
2813	#D-function IGCM_sys_ncrename
2814	#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
2815	#D-* Examples:
2816	#D-
2817	function IGCM_sys_ncrename {
2818	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2819	if ( $DEBUG_sys ) ; then
2820	echo "IGCM_sys_ncrename :" $@
2821	fi
2822
2823	typeset NB_ESSAI DELAI status i
2824	# number of tentative
2825	NB_ESSAI=3
2826	# time delay between tentative
2827	DELAI=2
2828
2829	i=0
2830	while [ $i -lt $NB_ESSAI ] ; do
2831	ncrename $@ > out_rsync 2>&1
2832	status=$?
2833	if [ ${status} -gt 0 ] ; then
2834	IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2835	cat out_rsync
2836	\rm out_rsync
2837	IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2838	sleep $DELAI
2839	else
2840	\rm out_rsync
2841	break
2842	fi
2843	(( i = i + 1 ))
2844	done
2845
2846	if [ ${status} -gt 0 ] ; then
2847	echo "IGCM_sys_ncrename : ncrename error"
2848	IGCM_debug_Exit "ncrename"
2849	fi
2850
2851	IGCM_debug_PopStack "IGCM_sys_ncrename"
2852	}
2853
2854	#D-#==================================================
2855	#D-function IGCM_sys_ncwa
2856	#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
2857	#D-* Examples:
2858	#D-
2859	function IGCM_sys_ncwa {
2860	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2861	if ( $DEBUG_sys ) ; then
2862	echo "IGCM_sys_ncwa :" $@
2863	fi
2864
2865	typeset NB_ESSAI DELAI status i
2866	# number of tentative
2867	NB_ESSAI=3
2868	# time delay between tentative
2869	DELAI=2
2870
2871	i=0
2872	while [ $i -lt $NB_ESSAI ] ; do
2873	ncwa $@ > out_rsync 2>&1
2874	status=$?
2875	if [ ${status} -gt 0 ] ; then
2876	IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2877	cat out_rsync
2878	\rm out_rsync
2879	IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2880	sleep $DELAI
2881	else
2882	\rm out_rsync
2883	break
2884	fi
2885	(( i = i + 1 ))
2886	done
2887
2888	if [ ${status} -gt 0 ] ; then
2889	echo "IGCM_sys_ncwa : ncwa error"
2890	IGCM_debug_Exit "ncwa"
2891	fi
2892
2893	IGCM_debug_PopStack "IGCM_sys_ncwa"
2894	}
2895
2896	##############################################################
2897	# CDO OPERATOR
2898
2899	#D-#==================================================
2900	#D-function IGCM_sys_cdo
2901	#D-* Purpose: encapsulate cdo call so as to manage error code and retry
2902	#D-* Examples:
2903	#D-
2904	function IGCM_sys_cdo {
2905	IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2906	if ( $DEBUG_sys ) ; then
2907	echo "IGCM_sys_cdo :" $@
2908	fi
2909
2910	typeset status
2911
2912	\cdo $@ > out_rsync 2>&1
2913	status=$?
2914	if [ ${status} -gt 0 ] ; then
2915	echo "IGCM_sys_cdo : error code ${status}"
2916	cat out_rsync
2917	\rm out_rsync
2918	IGCM_debug_PopStack "IGCM_sys_cdo"
2919	return 1
2920	else
2921	IGCM_debug_PopStack "IGCM_sys_cdo"
2922	return 0
2923	fi
2924
2925	IGCM_debug_PopStack "IGCM_sys_cdo"
2926	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: