Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 818

Last change on this file since 818 was 818, checked in by sdipsl, 11 years ago

Side effect of the SpaceName?=TEST behaviour implementation at IDRIS related to IGCM_sys_IsFileArchived function.

Affect only Ada and is in fact the root cause of #110

Property svn:keywords set to Revision Author Date

File size: 82.7 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Curie
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=curie
71
72	#D-
73	#D-#==================================================
74	#D-Program used in libIGCM
75	#D-#==================================================
76
77	# Submit command
78	typeset SUBMIT=${SUBMIT:=ccc_msub}
79	# rsync with path
80	typeset -r RSYNC=/usr/bin/rsync
81	# RSYNC_opt args to rsync
82	typeset -r RSYNC_opt="-va"
83	# ie storage filesystem
84	typeset -r STOREHOST=${MASTER}
85
86	#====================================================
87	# Source default environment
88	#====================================================
89	##. /etc/profile
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo)
93	#====================================================
94	if [ X${TaskType} = Xcomputing ] ; then
95	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_calcul_curie_ksh
96	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64/bin
97	else
98	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_post_curie_ksh
99	fi
100
101	#====================================================
102	# Specific for ocean additionnal diagnostic
103	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
104	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
105
106	#====================================================
107	# Host specific DIRECTORIES
108	#====================================================
109
110	# ============ CESIUM START ============ #
111
112	#====================================================
113	#- Mirror libIGCM from titane to cesium if needed
114	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
115	#if [ ! ${ROOTSYS} = "home" ] ; then
116	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
117	#else
118	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
119	#fi
120
121	#====================================================
122	#- libIGCM_POST
123	#if ( ${MirrorlibIGCM} ) ; then
124	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
125	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
126	#else
127	# typeset -r libIGCM_POST=${libIGCM}
128	#fi
129
130	# ============ CESIUM END ============ #
131
132	#====================================================
133	#- MirrorlibIGCM for frontend
134	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
135
136	#====================================================
137	#- libIGCM_POST for frontend
138	typeset -r libIGCM_POST=${libIGCM}
139
140	#====================================================
141	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
142	typeset -r R_EXE="${MODIPSL}/bin"
143
144	#====================================================
145	#- SUBMIT_DIR : submission dir
146	typeset SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
147
148	#====================================================
149	#- IN
150	typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
151	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
152
153	#====================================================
154	#- ARCHIVE (dedicated to large files)
155	typeset ARCHIVE=${CCCSTOREDIR}
156
157	#====================================================
158	#- STORAGE (dedicated to small/medium files)
159	typeset STORAGE=${CCCWORKDIR}
160
161	#====================================================
162	#- R_OUT
163	typeset R_OUT=${ARCHIVE}/IGCM_OUT
164
165	#====================================================
166	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
167	typeset R_FIG=${STORAGE}/IGCM_OUT
168
169	#====================================================
170	#- R_BUF (ONLY FOR double copy an scratch)
171	typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
172
173	#====================================================
174	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
175	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}}
176
177	#====================================================
178	#- HOST_MPIRUN_COMMAND
179	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}
180
181	#====================================================
182	#- Max number of arguments passed to nco operator or demigration command
183	UNIX_MAX_LIMIT=360
184
185	#====================================================
186	#- set PackDefault true on curie
187	PackDefault=true
188
189	#====================================================
190	#- Number of core per node (max number of OpenMP task)
191	NUM_COREPERNODE=8
192
193	#====================================================
194	#- Default number of MPI task for IPSL coupled model
195	#- required for backward compatibility
196	#-
197	DEFAULT_NUM_PROC_OCE=5
198	DEFAULT_NUM_PROC_CPL=1
199	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
200	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
201
202	#D-#==================================================
203	#D-function IGCM_sys_ChangeArchive
204	#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
205	#D-* Examples:
206	#D-
207	function IGCM_sys_ChangeArchive {
208	IGCM_debug_PushStack "IGCM_sys_ChangeArchive"
209
210	R_OUT=${SCRATCHDIR}/IGCM_OUT
211	R_FIG=${SCRATCHDIR}/IGCM_OUT
212
213	IGCM_debug_Print 1 "R_OUT has been redefined = ${R_OUT}"
214	IGCM_debug_Print 1 "R_FIG has been redefined = ${R_FIG}"
215
216	IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
217	}
218
219	#D-#==================================================
220	#D-function IGCM_sys_RshMaster
221	#D-* Purpose: Connection to frontend machine.
222	#D-* Examples:
223	#D-
224	function IGCM_sys_RshMaster {
225	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
226	/bin/ksh <<-EOF
227	export libIGCM=${libIGCM}
228	export DEBUG_debug=${DEBUG_debug}
229	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
230	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
231	${@}
232	EOF
233	if [ $? -gt 0 ] ; then
234	echo "IGCM_sys_RshMaster : erreur."
235	IGCM_debug_Exit "IGCM_sys_RshMaster"
236	fi
237	IGCM_debug_PopStack "IGCM_sys_RshMaster"
238	}
239
240	#D-#==================================================
241	#D-function IGCM_sys_RshArchive
242	#D-* Purpose: Archive rsh command
243	#D-* Examples:
244	#D-
245	function IGCM_sys_RshArchive {
246	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
247	/bin/ksh <<-EOF
248	${@}
249	EOF
250	status=$?
251	if [ ${status} -gt 0 ] ; then
252	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
253	IGCM_debug_Exit "IGCM_sys_RshArchive"
254	fi
255	IGCM_debug_PopStack "IGCM_sys_RshArchive"
256	}
257
258	#D-#==================================================
259	#D-function IGCM_sys_RshPost
260	#D-* Purpose: Post-process rsh command
261	#D-* Examples:
262	#D-
263	function IGCM_sys_RshPost {
264	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
265	if ( $DEBUG_sys ) ; then
266	echo "IGCM_sys_RshPost :" $@
267	fi
268	# keep standard input (stdin) for the loop onto temporary file
269	cat >/tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
270
271	/bin/ksh </tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
272	if [ $? -gt 0 ] ; then
273	echo "IGCM_sys_RshPost : erreur."
274	IGCM_debug_Exit "IGCM_sys_RshPost"
275	fi
276	# delete temporary file
277	\rm /tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
278
279	# ============ FRONTEND END ============ #
280
281	# ============ CESIUM START ============ #
282	# typeset NB_ESSAI DELAI status i
283	# if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
284	# #little hack so that rebuild submission is done on titane not an cesium
285	#
286	# libIGCM_POST_sed=$( echo $libIGCM_POST \| sed 's/\//\\\//g' )
287	# POST_DIR_sed=$( echo ${POST_DIR} \| sed 's/\//\\\//g' )
288	# sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
289	# sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
290	# sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
291	# sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
292	# \mv tmp.txt tmp_IGCM_sys_RshPost_$$
293	#
294	# echo cat tmp_IGCM_sys_RshPost_$$ AFTER
295	# cat tmp_IGCM_sys_RshPost_$$
296	#
297	# /bin/ksh <tmp_IGCM_sys_RshPost_$$
298	# if [ $? -gt 0 ] ; then
299	# echo "IGCM_sys_RshPost : erreur."
300	# IGCM_debug_Exit "IGCM_sys_RshPost"
301	# fi
302	# # delete temporary file
303	# \rm tmp_IGCM_sys_RshPost_$$
304	#
305	# else
306	# # number of tentative
307	# NB_ESSAI=10
308	# # time delay between tentative
309	# DELAI=10
310	# i=0
311	# while [ $i -ne $NB_ESSAI ] ; do
312	# ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
313	# status=$?
314	# if [ ${status} -ne 0 ];
315	# then
316	# sleep $DELAI
317	# else
318	# break
319	# fi
320	# let i=$i+1
321	# done
322	# # delete temporary file
323	# \rm tmp_IGCM_sys_RshPost_$$
324	#
325	# if [ ${status} -gt 0 ] ; then
326	# echo "IGCM_sys_RshPost : erreur."
327	# IGCM_debug_Exit "IGCM_sys_RshPost"
328	# fi
329	# fi
330
331	# ============ CESIUM END ============ #
332
333	IGCM_debug_PopStack "IGCM_sys_RshPost"
334	}
335
336	#D-#==================================================
337	#D-function IGCM_sys_SendMail
338	#D-* Purpose: Send mail when simulation is over
339	#D-* Examples:
340	#D-
341	function IGCM_sys_SendMail {
342	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
343	if ( $DEBUG_sys ) ; then
344	echo "IGCM_sys_SendMail :" $@
345	fi
346
347	if ( ${ExitFlag} ) ; then
348	status=failed
349	else
350	status=completed
351	fi
352
353	cat << END_MAIL > job_end.mail
354	Dear ${LOGIN},
355
356	Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
357	Job started : ${DateBegin}
358	Job ended : ${DateEnd}
359	Output files are available in ${R_SAVE}
360	Files to be rebuild are temporarily available in ${REBUILD_DIR}
361	Pre-packed files are temporarily available in ${R_BUFR}
362	Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
363	END_MAIL
364
365	if [ ! -z ${config_UserChoices_MailName} ] ; then
366	mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
367	elif [ -f ~/.forward ] ; then
368	mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
369	fi
370
371	sleep 10
372	rm -f job_end.mail
373
374	if [ $? -gt 0 ] ; then
375	echo "IGCM_sys_SendMail : erreur."
376	IGCM_debug_Exit "IGCM_sys_SendMail"
377	fi
378	IGCM_debug_PopStack "IGCM_sys_SendMail"
379	}
380
381	#D-#==================================================
382	#D-function IGCM_sys_Mkdir
383	#D-* Purpose: Master locale mkdir command
384	#D-* Examples:
385	#D-
386	function IGCM_sys_Mkdir {
387	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
388	if ( $DEBUG_sys ) ; then
389	echo "IGCM_sys_Mkdir :" $@
390	fi
391	if [ ! -d ${1} ]; then
392	\mkdir -p $1
393	if [ $? -gt 0 ] ; then
394	echo "IGCM_sys_Mkdir : erreur."
395	IGCM_debug_Exit "IGCM_sys_Mkdir"
396	fi
397	fi
398	# vérification :
399	if [ ! -d ${1} ] ; then
400	echo "IGCM_sys_Mkdir : erreur."
401	IGCM_debug_Exit "IGCM_sys_Mkdir"
402	fi
403	IGCM_debug_PopStack "IGCM_sys_Mkdir"
404	}
405
406	#D-#==================================================
407	#D-function IGCM_sys_MkdirArchive
408	#D-* Purpose: Mkdir on Archive
409	#D-* Examples:
410	#D-
411	function IGCM_sys_MkdirArchive {
412	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
413	if ( $DEBUG_sys ) ; then
414	echo "IGCM_sys_MkdirArchive :" $@
415	fi
416	#- creation de repertoire sur le serveur fichier
417	if [ ! -d ${1} ]; then
418	\mkdir -p $1
419	status=$?
420
421	if [ ${status} -gt 0 ] ; then
422	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
423	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
424	fi
425	fi
426	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
427	}
428
429	#D-#==================================================
430	#D-function IGCM_sys_MkdirWork
431	#D-* Purpose: Mkdir on Work
432	#D-* Examples:
433	#D-
434	function IGCM_sys_MkdirWork {
435	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
436	if ( $DEBUG_sys ) ; then
437	echo "IGCM_sys_MkdirWork :" $@
438	fi
439	#- creation de repertoire sur le serveur fichier
440	if [ ! -d ${1} ]; then
441	\mkdir -p $1
442	if [ $? -gt 0 ] ; then
443	echo "IGCM_sys_MkdirWork : erreur."
444	IGCM_debug_Exit "IGCM_sys_MkdirWork"
445	fi
446	fi
447	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
448	}
449
450	#D-#==================================================
451	#D-function IGCM_sys_Cd
452	#D-* Purpose: master cd command
453	#D-* Examples:
454	#D-
455	function IGCM_sys_Cd {
456	IGCM_debug_PushStack "IGCM_sys_Cd" $@
457	if ( $DEBUG_sys ) ; then
458	echo "IGCM_sys_Cd :" $@
459	fi
460	\cd $1
461	if [ $? -gt 0 ] ; then
462	echo "IGCM_sys_Cd : erreur."
463	IGCM_debug_Exit "IGCM_sys_Cd"
464	fi
465	IGCM_debug_PopStack "IGCM_sys_Cd"
466	}
467
468	#D-#==================================================
469	#D-function IGCM_sys_Chmod
470	#D-* Purpose: Chmod
471	#D-* Examples:
472	#D-
473	function IGCM_sys_Chmod {
474	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
475	if ( $DEBUG_sys ) ; then
476	echo "IGCM_sys_Chmod :" $@
477	fi
478	\chmod $@
479	if [ $? -gt 0 ] ; then
480	echo "IGCM_sys_Chmod : erreur."
481	IGCM_debug_Exit "IGCM_sys_Chmod"
482	fi
483	IGCM_debug_PopStack "IGCM_sys_Chmod"
484	}
485
486	#D-#==================================================
487	#D-function IGCM_sys_FileSize
488	#D-* Purpose: Filesize
489	#D-* Examples:
490	#D-
491	function IGCM_sys_FileSize {
492	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
493
494	typeset sizeF
495	set +A sizeF -- $( ls -la ${1} )
496	if [ $? -gt 0 ] ; then
497	IGCM_debug_Exit "IGCM_sys_FileSize"
498	fi
499	eval ${2}=${sizeF[4]}
500
501	IGCM_debug_PopStack "IGCM_sys_FileSize"
502	}
503
504	#D-#==================================================
505	#D-function IGCM_sys_TestDir
506	#D-* Purpose: Test Directory that must exists
507	#D-* Examples:
508	#D-
509	function IGCM_sys_TestDir {
510	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
511	if ( $DEBUG_sys ) ; then
512	echo "IGCM_sys_TestDir :" $@
513	fi
514	typeset ExistFlag
515	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
516	IGCM_debug_PopStack "IGCM_sys_TestDir"
517
518	return ${ExistFlag}
519	}
520
521	#D-#==================================================
522	#D-function IGCM_sys_TestDirArchive
523	#D-* Purpose: Test Directory that must exists on Archive
524	#D-* Examples:
525	#D-
526	function IGCM_sys_TestDirArchive {
527	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
528	if ( $DEBUG_sys ) ; then
529	echo "IGCM_sys_TestDirArchive :" $@
530	fi
531	typeset ExistFlag
532	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
533	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
534	return ${ExistFlag}
535	}
536
537	#D-#==================================================
538	#D-function IGCM_sys_IsFileArchived
539	#D-* Purpose: Test file that must NOT EXISTS on Archive
540	#D-* Examples:
541	#D-
542	function IGCM_sys_IsFileArchived {
543	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
544	if ( $DEBUG_sys ) ; then
545	echo "IGCM_sys_IsFileArchived :" $@
546	fi
547	typeset IsArchivedFlag
548	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
549	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
550
551	return ${IsArchivedFlag}
552	}
553
554	#D-#==================================================
555	#D-function IGCM_sys_TestFileArchive
556	#D-* Purpose: Test file that must NOT EXISTS on Archive
557	#D-* Examples:
558	#D-
559	function IGCM_sys_TestFileArchive {
560	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
561	typeset ExistFlag
562	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
563	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
564
565	return ${ExistFlag}
566	}
567
568	#D-#==================================================
569	#D-function IGCM_sys_TestFileBuffer
570	#D-* Purpose: Test file that must NOT EXISTS on Buffer
571	#D-* Examples:
572	#D-
573	function IGCM_sys_TestFileBuffer {
574	IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
575	typeset ExistFlag
576	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
577	IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
578
579	return ${ExistFlag}
580	}
581
582	#D-#==================================================
583	#D-function IGCM_sys_CountFileArchive
584	#D-* Purpose: Count files on Archive filesystem
585	#D-* Examples:
586	#D-
587	function IGCM_sys_CountFileArchive {
588	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
589	ls ${@} 2>/dev/null \| wc -l
590	if [ $? -gt 0 ] ; then
591	echo "IGCM_sys_CountFileArchive : erreur."
592	fi
593	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
594	}
595
596	#D-#==================================================
597	#D-function IGCM_sys_CountFileBuffer
598	#D-* Purpose: Count files on Scratch filesystem
599	#D-* Examples:
600	#D-
601	function IGCM_sys_CountFileBuffer {
602	IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
603	ls ${@} 2>/dev/null \| wc -l
604	if [ $? -gt 0 ] ; then
605	echo "IGCM_sys_CountFileBuffer : erreur."
606	fi
607	IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
608	}
609
610	#D-#==================================================
611	#D-function IGCM_sys_Tree
612	#D-* Purpose: Tree directories with files on ${ARCHIVE}
613	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
614	#D-
615	function IGCM_sys_Tree {
616	IGCM_debug_PushStack "IGCM_sys_Tree" $@
617	if ( $DEBUG_sys ) ; then
618	echo "IGCM_sys_Tree :" $@
619	fi
620
621	\ls -lR ${@}
622
623	IGCM_debug_PopStack "IGCM_sys_Tree"
624	}
625
626	#D-#==================================================
627	#D-function IGCM_sys_Tar
628	#D-* Purpose: master tar command
629	#D-* Examples:
630	#D-
631	function IGCM_sys_Tar {
632	IGCM_debug_PushStack "IGCM_sys_Tar" $@
633	if ( $DEBUG_sys ) ; then
634	echo "IGCM_sys_Tar :" $@
635	fi
636	\tar cf $@
637	if [ $? -gt 0 ] ; then
638	echo "IGCM_sys_Tar : erreur."
639	IGCM_debug_Exit "IGCM_sys_Tar"
640	fi
641	IGCM_debug_PopStack "IGCM_sys_Tar"
642	}
643
644	#D-#==================================================
645	#D-function IGCM_sys_UnTar
646	#D-* Purpose: master un-tar command
647	#D-* Examples:
648	#D-
649	function IGCM_sys_UnTar {
650	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
651	if ( $DEBUG_sys ) ; then
652	echo "IGCM_sys_UnTar :" $@
653	fi
654	\tar xvf $1
655	if [ $? -gt 0 ] ; then
656	echo "IGCM_sys_UnTar : erreur."
657	IGCM_debug_Exit "IGCM_sys_UnTar"
658	fi
659	IGCM_debug_PopStack "IGCM_sys_UnTar"
660	}
661
662	#D-#==================================================
663	#D-function IGCM_sys_Qsub
664	#D-* Purpose: Qsub new job
665	#D-* Examples:
666	#D-
667	function IGCM_sys_Qsub {
668	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
669	if ( $DEBUG_sys ) ; then
670	echo "IGCM_sys_Qsub :" $@
671	fi
672	typeset options status
673	if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
674	options=" -o ${Script_Output} -e ${Script_Output}"
675	else
676	options=" -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod}"
677	fi
678
679	/usr/bin/ccc_msub ${options} < $1
680	status=$?
681	if [ ${status} -gt 0 ] ; then
682	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
683	IGCM_debug_Exit "IGCM_sys_Qsub"
684	fi
685	IGCM_debug_PopStack "IGCM_sys_Qsub"
686	}
687
688	#D-#==================================================
689	#D-function IGCM_sys_QsubPost
690	#D-* Purpose: Qsub new job on scalaire
691	#D-* Examples:
692	#D-
693	function IGCM_sys_QsubPost {
694	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
695	if ( $DEBUG_sys ) ; then
696	echo "IGCM_sys_QsubPost :" $@
697	fi
698	typeset status
699	MY_PROJECT=$(echo ${BRIDGE_MSUB_PROJECT} \| cut -d@ -f1 )
700	/usr/bin/ccc_msub -Q normal -A ${MY_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job
701	status=$?
702	if [ ${status} -gt 0 ] ; then
703	IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
704	IGCM_debug_Exit "IGCM_sys_QsubPost"
705	fi
706	IGCM_debug_PopStack "IGCM_sys_QsubPost"
707	}
708
709	#D-*************************
710	#D- File transfer functions
711	#D-*************************
712	#D-
713
714	#D-#==================================================
715	#D-function IGCM_sys_Rsync_out
716	#D-* Purpose: treat return val of rsync
717	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
718	#D- Error values and explanations can depend on your system version.
719	function IGCM_sys_Rsync_out {
720	status=$1
721	if [ ! $status ] ; then
722	echo "rsync error !"
723	fi
724
725	if [ $MYLANG = "fr" ]; then
726	case $status in
727	0) return ;;
728	1) echo "Erreur de rsync ; RERR_SYNTAX : "
729	echo "Erreur de syntaxe ou d'utilisation."
730	return;;
731	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
732	echo "Incompatibilité de protocole."
733	return;;
734	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
735	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
736	echo "répertoires"
737	return;;
738	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
739	echo "Action demandée non supportée : une tentative de manipulation de"
740	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
741	echo "été faite ; ou une option qui est supportée par le client mais"
742	echo "pas par le serveur a été spécifiée."
743	return;;
744	10) echo "Erreur de rsync ; RERR_SOCKETIO"
745	echo "Erreur dans le socket d'entrée sortie"
746	return;;
747	11) echo "Erreur de rsync ; RERR_FILEIO"
748	echo "Erreur d'entrée sortie fichier"
749	return;;
750	12) echo "Erreur de rsync ; RERR_STREAMIO"
751	echo "Erreur dans flux de donnée du protocole rsync"
752	return;;
753	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
754	echo "Erreur avec les diagnostics du programme"
755	return;;
756	14) echo "Erreur de rsync ; RERR_IPC"
757	echo "Erreur dans le code IPC"
758	return;;
759	20) echo "Erreur de rsync ; RERR_SIGNAL"
760	echo "SIGUSR1 ou SIGINT reçu"
761	return;;
762	21) echo "Erreur de rsync ; RERR_WAITCHILD"
763	echo "Une erreur retournée par waitpid()"
764	return;;
765	22) echo "Erreur de rsync ; RERR_MALLOC"
766	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
767	return;;
768	23) echo ""
769	echo "Erreur fichier inexistant"
770	return;;
771	30) echo "Erreur de rsync ; RERR_TIMEOUT"
772	echo "Temps d'attente écoulé dans l'envoi/réception de données"
773	return;;
774	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $status
775	return;;
776	esac
777	elif [ $MYLANG = "en" ] ; then
778	case $status in
779	0) return;;
780	1) echo "rsync error : Syntax or usage error "
781	return;;
782	2) echo "rsync error : Protocol incompatibility "
783	return;;
784	3) echo "rsync error : Errors selecting input/output files, dirs"
785	return;;
786	4) echo "rsync error : Requested action not supported: an attempt"
787	echo "was made to manipulate 64-bit files on a platform that cannot support"
788	echo "them; or an option was specified that is supported by the client and"
789	echo "not by the server."
790	return;;
791	5) echo "rsync error : Error starting client-server protocol"
792	return;;
793	10) echo "rsync error : Error in socket I/O "
794	return;;
795	11) echo "rsync error : Error in file I/O "
796	return;;
797	12) echo "rsync error : Error in rsync protocol data stream "
798	return;;
799	13) echo "rsync error : Errors with program diagnostics "
800	return;;
801	14) echo "rsync error : Error in IPC code "
802	return;;
803	20) echo "rsync error : Received SIGUSR1 or SIGINT "
804	return;;
805	21) echo "rsync error : Some error returned by waitpid() "
806	return;;
807	22) echo "rsync error : Error allocating core memory buffers "
808	return;;
809	23) echo "rsync error : Partial transfer due to error"
810	return;;
811	24) echo "rsync error : Partial transfer due to vanished source files"
812	return;;
813	30) echo "rsync error : Timeout in data send/receive "
814	return;;
815	*) echo "rsync error : return code of rsync unknown :" $status
816	return;;
817	esac
818	else
819	echo "unknown language $MYLANG."
820	return
821	fi
822	}
823
824	#D-#==================================================
825	#D-function IGCM_sys_Miror_libIGCM
826	#D-* Purpose: Mirror libIGCM PATH and lib to frontend
827	#D-* Examples:
828	#D-
829	function IGCM_sys_Mirror_libIGCM {
830	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
831	if ( $DEBUG_sys ) ; then
832	echo "IGCM_sys_Mirror_libIGCM"
833	fi
834
835	typeset status
836
837	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
838
839	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
840	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
841	status=$?
842
843	if [ ${status} -gt 0 ] ; then
844	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
845	cat out_rsync
846	fi
847	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
848	}
849
850	#D-#==================================================
851	#D-function IGCM_sys_Cp
852	#D-* Purpose: generic cp
853	#D-* Examples:
854	#D-
855	function IGCM_sys_Cp {
856	IGCM_debug_PushStack "IGCM_sys_Cp" $@
857	if ( $DEBUG_sys ) ; then
858	echo "IGCM_sys_Cp :" $@
859	fi
860
861	typeset status
862
863	echo cp $@ > out_rsync 2>&1
864	\cp $@ >> out_rsync 2>&1
865	status=$?
866
867	if [ ${status} -gt 0 ] ; then
868	echo "IGCM_sys_Cp : error code ${status}"
869	cat out_rsync
870	IGCM_debug_Exit "IGCM_sys_Cp"
871	else
872	\rm out_rsync
873	fi
874	IGCM_debug_PopStack "IGCM_sys_Cp"
875	}
876
877	#D-#==================================================
878	#D-function IGCM_sys_Rm
879	#D-* Purpose: generic rm
880	#D-* Examples:
881	#D-
882	function IGCM_sys_Rm {
883	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
884	if ( $DEBUG_sys ) ; then
885	echo "IGCM_sys_Rm :" $@
886	fi
887
888	typeset status
889
890	echo rm $@ > out_rsync 2>&1
891	\rm $@ >> out_rsync 2>&1
892	status=$?
893
894	if [ ${status} -gt 0 ] ; then
895	echo "IGCM_sys_Rm : error code ${status}"
896	cat out_rsync
897	IGCM_debug_Exit "IGCM_sys_Rm"
898	else
899	\rm out_rsync
900	fi
901	IGCM_debug_PopStack "IGCM_sys_Rm"
902	}
903
904	#D-#==================================================
905	#D-function IGCM_sys_RmRunDir
906	#D-* Purpose: rm tmpdir (dummy function most of the time batch
907	#D- scheduler will do the job)
908	#D-* Examples:
909	#D-
910	function IGCM_sys_RmRunDir {
911	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
912	if ( $DEBUG_sys ) ; then
913	echo "IGCM_sys_RmRunDir :" $@
914	fi
915
916	typeset status
917
918	echo rm $@ > out_rsync 2>&1
919	\rm $@ >> out_rsync 2>&1
920	status=$?
921
922	if [ ${status} -gt 0 ] ; then
923	echo "IGCM_sys_RmRunDir : rm error code is ${status}."
924	cat out_rsync
925	IGCM_debug_Exit "IGCM_sys_RmRunDir"
926	else
927	\rm out_rsync
928	fi
929	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
930	}
931
932	#D-#==================================================
933	#D-function IGCM_sys_Mv
934	#D-* Purpose: generic move
935	#D-* Examples:
936	#D-
937	function IGCM_sys_Mv {
938	IGCM_debug_PushStack "IGCM_sys_Mv" $@
939	if ( $DEBUG_sys ) ; then
940	echo "IGCM_sys_Mv :" $@
941	fi
942
943	if [ $DRYRUN = 0 ]; then
944
945	typeset status
946
947	echo mv $@ > out_rsync 2>&1
948	\mv $@ >> out_rsync 2>&1
949	status=$?
950
951	if [ ${status} -gt 0 ] ; then
952	echo "IGCM_sys_Mv : error code ${status}"
953	cat out_rsync
954	IGCM_debug_Exit "IGCM_sys_Mv"
955	else
956	\rm out_rsync
957	fi
958	else
959	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
960	fi
961
962	IGCM_debug_PopStack "IGCM_sys_Mv"
963	}
964
965	#D-#==================================================
966	#D-function IGCM_sys_Put_Dir
967	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
968	#D-* Examples:
969	#D-
970	function IGCM_sys_Put_Dir {
971	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
972	if ( $DEBUG_sys ) ; then
973	echo "IGCM_sys_Put_Dir :" $@
974	fi
975	if [ $DRYRUN = 0 ]; then
976	if [ ! -d ${1} ] ; then
977	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
978	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
979	return
980	fi
981
982	typeset status
983
984	# Only if we use rsync
985	#IGCM_sys_TestDirArchive $( dirname $2 )
986	#
987	#USUAL WAY
988	\cp -r $1 $2 > out_rsync 2>&1
989	status=$?
990
991	if [ ${status} -gt 0 ] ; then
992	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
993	cat out_rsync
994	IGCM_debug_Exit "IGCM_sys_Put_Dir"
995	else
996	\rm out_rsync
997	fi
998	else
999	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1000	fi
1001	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1002	}
1003
1004	#D-#==================================================
1005	#D-function IGCM_sys_Get_Dir
1006	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
1007	#D-* Examples:
1008	#D-
1009	function IGCM_sys_Get_Dir {
1010	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
1011	if ( $DEBUG_sys ) ; then
1012	echo "IGCM_sys_Get_Dir :" $@
1013	fi
1014	if [ $DRYRUN = 0 ]; then
1015	# if [ ! -d ${1} ] ; then
1016	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
1017	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1018	# return
1019	# fi
1020
1021	typeset NB_ESSAI DELAI status i
1022	# number of tentative
1023	NB_ESSAI=3
1024	# time delay between tentative
1025	DELAI=2
1026
1027	# Only if we use rsync
1028	#IGCM_sys_TestDirArchive $( dirname $2 )
1029	#
1030	# USUAL WAY
1031	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
1032	ccc_hsm get -r $1
1033
1034	i=0
1035	while [ $i -lt $NB_ESSAI ] ; do
1036	\cp -ur $1 $2 >> out_rsync 2>&1
1037	status=$?
1038	if [ ${status} -gt 0 ] ; then
1039	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
1040	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
1041	sleep $DELAI
1042	else
1043	break
1044	fi
1045	(( i = i + 1 ))
1046	done
1047
1048	if [ ${status} -gt 0 ] ; then
1049	echo "IGCM_sys_Get_Dir : error."
1050	cat out_rsync
1051	IGCM_debug_Exit "IGCM_sys_Get_Dir"
1052	else
1053	\rm out_rsync
1054	fi
1055	else
1056	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1057	fi
1058	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1059	}
1060
1061	#D-#==================================================
1062	#D-function IGCM_sys_Get_Master
1063	#D-* Purpose: Copy a complete directory from MASTER filesystem
1064	#D-* Examples:
1065	#D-
1066	function IGCM_sys_Get_Master {
1067	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
1068	if ( $DEBUG_sys ) ; then
1069	echo "IGCM_sys_Get_Master :" $@
1070	fi
1071	if [ $DRYRUN = 0 ]; then
1072	if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
1073	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
1074	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1075	return
1076	fi
1077
1078	typeset NB_ESSAI DELAI status i
1079	# number of tentative
1080	NB_ESSAI=3
1081	# time delay between tentative
1082	DELAI=2
1083
1084	i=0
1085	while [ $i -lt $NB_ESSAI ] ; do
1086	\cp -urL $1 $2 > out_rsync 2>&1
1087	status=$?
1088	if [ ${status} -gt 0 ]; then
1089	IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
1090	IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
1091	sleep $DELAI
1092	else
1093	break
1094	fi
1095	(( i = i + 1 ))
1096	done
1097
1098	if [ ${status} -gt 0 ] ; then
1099	echo "IGCM_sys_Get_Master : error."
1100	cat out_rsync
1101	IGCM_debug_Exit "IGCM_sys_Get_Master"
1102	else
1103	\rm out_rsync
1104	fi
1105	else
1106	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1107	fi
1108	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1109	}
1110
1111	#====================================================
1112	#- Call IGCM_sys_Mirror_libIGCM now !
1113	if ( $MirrorlibIGCM ) ; then
1114	IGCM_sys_Mirror_libIGCM
1115	fi
1116
1117	#D-#==================================================
1118	#D-function IGCM_sys_Put_Rest
1119	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1120	#D- File and target directory must exist.
1121	#D-* Examples:
1122	#D-
1123	function IGCM_sys_Put_Rest {
1124	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1125	if ( $DEBUG_sys ) ; then
1126	echo "IGCM_sys_Put_Rest :" $@
1127	fi
1128	if [ $DRYRUN = 0 ]; then
1129	if [ ! -f ${1} ] ; then
1130	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1131	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1132	fi
1133
1134	typeset status
1135	#
1136	# USUAL WAY
1137	\cp $1 $2 > out_rsync 2>&1
1138	status=$?
1139
1140	# #RSYNC WITH NETWORK SSH CALL
1141	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1142	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1143
1144	# #RSYNC WITH NFS USE
1145	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1146	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1147
1148	# status=$?
1149	# IGCM_sys_Rsync_out $status
1150
1151	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1152	# (( status=status+$? ))
1153
1154	if [ ${status} -gt 0 ] ; then
1155	echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1156	[ -f ${1} ] && ls -l ${1}
1157	[ -f ${2} ] && ls -l ${2}
1158	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1159	cat out_rsync
1160	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1161	else
1162
1163	if [ X${JobType} = XRUN ] ; then
1164	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1165	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1166	fi
1167
1168	\rm out_rsync
1169	fi
1170	else
1171	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1172	fi
1173	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1174	}
1175
1176	#D-#==================================================
1177	#D-function IGCM_sys_PutBuffer_Rest
1178	#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1179	#D- File and target directory must exist.
1180	#D-* Examples:
1181	#D-
1182	function IGCM_sys_PutBuffer_Rest {
1183	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1184	if ( $DEBUG_sys ) ; then
1185	echo "IGCM_sys_PutBuffer_Rest :" $@
1186	fi
1187	if [ $DRYRUN = 0 ]; then
1188	if [ ! -f ${1} ] ; then
1189	echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1190	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1191	fi
1192
1193	typeset status
1194	#
1195	# USUAL WAY
1196	\cp $1 $2 > out_rsync 2>&1
1197	status=$?
1198
1199	if [ ${status} -gt 0 ] ; then
1200	echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1201	[ -f ${2} ] && ls -l ${2}
1202	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1203	cat out_rsync
1204	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1205	else
1206
1207	if [ X${JobType} = XRUN ] ; then
1208	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1209	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1210	fi
1211
1212	\rm out_rsync
1213	fi
1214	else
1215	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1216	fi
1217	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1218	}
1219
1220	#D-#==================================================
1221	#D-function IGCM_sys_PrepareTaredRestart
1222	#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
1223	#D-* Examples:
1224	#D-
1225	function IGCM_sys_PrepareTaredRestart {
1226	IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
1227	if [ $DRYRUN = 0 ]; then
1228	[ ! -f $( basename $1 ) ] && DEBUG_sys=false IGCM_sys_Get $1 .
1229	else
1230	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1231	fi
1232	echo $( basename $1 )
1233	IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1234	}
1235
1236	#D-#==================================================
1237	#D-function IGCM_sys_Put_Out
1238	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1239	#D-* Examples:
1240	#D-
1241	function IGCM_sys_Put_Out {
1242	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1243	if ( $DEBUG_sys ) ; then
1244	echo "IGCM_sys_Put_Out :" $@
1245	fi
1246
1247	typeset NB_ESSAI DELAI status i exist skip
1248
1249	# number of tentative
1250	NB_ESSAI=3
1251	# time delay between tentative
1252	DELAI=2
1253
1254	if [ $DRYRUN = 0 ]; then
1255	if [ ! -f ${1} ] ; then
1256	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1257	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1258	return 1
1259	fi
1260	#
1261	IGCM_sys_MkdirArchive $( dirname $2 )
1262	#
1263
1264	#=====================================================
1265	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1266	#=====================================================
1267
1268	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
1269	#if [ $? -eq 0 ] ; then
1270	# typeset WORKPATH FILEPATH
1271	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_BUF}\|" )
1272	# IGCM_sys_MkdirWork ${WORKPATH}
1273	# FILEPATH=${WORKPATH}/$( basename $2 )
1274	# #
1275	# IGCM_sys_Cp ${1} ${FILEPATH}
1276	#fi
1277
1278	exist=false
1279	skip=false
1280	if [ -f $2 ] ; then
1281	IGCM_debug_Print 1 "$2 already exist"
1282	#dmget $2
1283	ccc_hsm get $2
1284	exist=true
1285	if [ "X$( diff $1 $2 )" = X ] ; then
1286	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1287	skip=true
1288	else
1289	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1290	skip=false
1291	fi
1292	fi
1293	#
1294	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1295	IGCM_sys_Chmod u+w $2
1296	fi
1297
1298	if [ X${skip} = Xfalse ] ; then
1299	i=0
1300	while [ $i -lt $NB_ESSAI ] ; do
1301	# USUAL WAY
1302	\cp $1 $2 > out_rsync 2>&1
1303	status=$?
1304	if [ ${status} -gt 0 ]; then
1305	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1306	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
1307	[ -f ${1} ] && ls -l ${1}
1308	[ -f ${2} ] && ls -l ${2}
1309	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1310	sleep $DELAI
1311	else
1312	break
1313	fi
1314	(( i = i + 1 ))
1315	done
1316	fi
1317
1318	# #RSYNC WITH NETWORK SSH CALL
1319	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1320	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1321
1322	# #RSYNC WITH NFS USE
1323	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1324	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1325
1326	# status=$?
1327	# IGCM_sys_Rsync_out $status
1328
1329	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1330	# (( status=status+$? ))
1331
1332	if [ ${status} -gt 0 ] ; then
1333	echo "IGCM_sys_Put_Out : error."
1334	[ -f ${1} ] && ls -l ${1}
1335	[ -f ${2} ] && ls -l ${2}
1336	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1337	cat out_rsync
1338	IGCM_debug_Exit "IGCM_sys_Put_Out"
1339	else
1340
1341	if [ X${JobType} = XRUN ] ; then
1342	if [ X${3} = X ] ; then
1343	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1344	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1345	fi
1346	fi
1347
1348	\rm out_rsync
1349	fi
1350	else
1351	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1352	fi
1353	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1354	return 0
1355	}
1356
1357	#D-#==================================================
1358	#D-function IGCM_sys_PutBuffer_Out
1359	#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1360	#D-* Examples:
1361	#D-
1362	function IGCM_sys_PutBuffer_Out {
1363	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1364	if ( $DEBUG_sys ) ; then
1365	echo "IGCM_sys_PutBuffer_Out :" $@
1366	fi
1367
1368	typeset NB_ESSAI DELAI status i exist skip
1369
1370	# number of tentative
1371	NB_ESSAI=3
1372	# time delay between tentative
1373	DELAI=2
1374
1375	if [ $DRYRUN = 0 ]; then
1376	if [ ! -f ${1} ] ; then
1377	echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1378	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1379	return 1
1380	fi
1381	#
1382	IGCM_sys_Mkdir $( dirname $2 )
1383	#
1384
1385	exist=false
1386	skip=false
1387	if [ -f $2 ] ; then
1388	IGCM_debug_Print 1 "$2 already exist"
1389	exist=true
1390	if [ "X$( diff $1 $2 )" = X ] ; then
1391	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1392	status=0
1393	skip=true
1394	else
1395	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1396	skip=false
1397	fi
1398	fi
1399	#
1400	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1401	IGCM_sys_Chmod u+w $2
1402	fi
1403
1404	if [ X${skip} = Xfalse ] ; then
1405	i=0
1406	while [ $i -lt $NB_ESSAI ] ; do
1407	# USUAL WAY
1408	\cp $1 $2 > out_rsync 2>&1
1409	status=$?
1410	if [ ${status} -gt 0 ]; then
1411	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1412	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1413	[ -f ${2} ] && ls -l ${2}
1414	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1415	sleep $DELAI
1416	else
1417	break
1418	fi
1419	(( i = i + 1 ))
1420	done
1421	fi
1422
1423	if [ ${status} -gt 0 ] ; then
1424	echo "IGCM_sys_PutBuffer_Out : error."
1425	[ -f ${2} ] && ls -l ${2}
1426	[ -f ${2}/${1} ] && ls -l ${2}/${1}
1427	cat out_rsync
1428	IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1429	else
1430
1431	if [ X${JobType} = XRUN ] ; then
1432	if [ X${3} = X ] ; then
1433	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1434	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1435	fi
1436	fi
1437
1438	\rm out_rsync
1439	fi
1440	else
1441	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1442	fi
1443	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1444	return 0
1445	}
1446
1447	#D-#==================================================
1448	#D-function IGCM_sys_Get
1449	#D-* Purpose: Get a file from ${ARCHIVE}
1450	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1451	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
1452	function IGCM_sys_Get {
1453	IGCM_debug_PushStack "IGCM_sys_Get" $@
1454
1455	typeset DEST dm_liste target file_work
1456	typeset NB_ESSAI DELAI status i
1457
1458	if ( $DEBUG_sys ) ; then
1459	echo "IGCM_sys_Get :" $@
1460	fi
1461
1462	# number of tentative
1463	NB_ESSAI=3
1464	# time delay between tentative
1465	DELAI=2
1466
1467	if [ $DRYRUN -le 2 ]; then
1468	if [ X${1} = X'/l' ] ; then
1469	# test if the first file is present in the old computation :
1470	eval set +A dm_liste \${${2}}
1471	else
1472	eval set +A dm_liste ${1}
1473	fi
1474	eval DEST=\${${#}}
1475
1476	#=====================================================
1477	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1478	#=====================================================
1479
1480	# Is it an R_OUT file (not R_IN) ?
1481	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
1482	#if [ $? -eq 0 ] ; then
1483	# # Yes ? then we try to get it in SCRATCHDIR
1484	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_BUF}\|g" )
1485	# if [ -f ${file_work[0]} ] ; then
1486	# IGCM_sys_Cp ${file_work[*]} ${DEST}
1487	# IGCM_debug_PopStack "IGCM_sys_Get"
1488	# return
1489	# fi
1490	#fi
1491
1492	ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1493	status=$?
1494	if [ ${status} -gt 0 ] ; then
1495	echo "WARNING IGCM_sys_Get : error code ${status}"
1496	cat out_rsync
1497	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1498	fi
1499
1500	#if [ ${status} -gt 0 ] ; then
1501	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1502	# cat out_rsync
1503	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1504	# sleep 30
1505	# echo "We try another time"
1506	## dmget ${dm_liste[*]} > out_rsync 2>&1
1507	# ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1508	# status=$?
1509	# if [ ${status} -gt 0 ] ; then
1510	# echo "ERROR IGCM_sys_Get : again demigration error :"
1511	# cat out_rsync
1512	# IGCM_debug_Exit "IGCM_sys_Get"
1513	# fi
1514	# else
1515	# echo "ERROR IGCM_sys_Get : demigration error :"
1516	# cat out_rsync
1517	# IGCM_debug_Exit "IGCM_sys_Get"
1518	# fi
1519	#fi
1520
1521	# #RSYNC WITH NETWORK SSH CALL
1522	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1523	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1524
1525	# #RSYNC WITH NFS USE
1526	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1527	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1528
1529	# status=$?
1530	# IGCM_sys_Rsync_out $status
1531
1532	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1533	# (( status=status+$? ))
1534
1535	#USUAL WAY
1536	if [ X${1} = X'/l' ] ; then
1537	for target in ${dm_liste[*]} ; do
1538	local_file=$( basename ${target} )
1539	# test if the target file is present before the loop
1540	IGCM_sys_TestFileArchive ${target}
1541	status=$?
1542	if [ ${status} -gt 0 ] ; then
1543	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
1544	IGCM_debug_Exit "IGCM_sys_Get"
1545	else
1546	i=0
1547	while [ $i -lt $NB_ESSAI ] ; do
1548	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1549	status=$?
1550	if [ ${status} -gt 0 ]; then
1551	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1552	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1553	sleep $DELAI
1554	else
1555	break
1556	fi
1557	(( i = i + 1 ))
1558	done
1559	if [ ${status} -gt 0 ] ; then
1560	echo "IGCM_sys_Get : error"
1561	cat out_rsync
1562	\rm out_rsync
1563	IGCM_debug_Exit "IGCM_sys_Get"
1564	else
1565	\rm out_rsync
1566	fi
1567	fi
1568	done
1569	else
1570	i=0
1571	while [ $i -lt $NB_ESSAI ] ; do
1572	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1573	status=$?
1574	if [ ${status} -gt 0 ]; then
1575	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1576	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1577	sleep $DELAI
1578	else
1579	break
1580	fi
1581	(( i = i + 1 ))
1582	done
1583	if [ ${status} -gt 0 ] ; then
1584	echo "IGCM_sys_Get : error"
1585	cat out_rsync
1586	\rm out_rsync
1587	IGCM_debug_Exit "IGCM_sys_Get"
1588	else
1589	\rm out_rsync
1590	fi
1591	fi
1592	else
1593	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1594	fi
1595	IGCM_debug_PopStack "IGCM_sys_Get"
1596	}
1597
1598	#D-#==================================================
1599	#D-function IGCM_sys_GetBuffer
1600	#D-* Purpose: Get a file from ${SCRATCHDIR}
1601	#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1602	#D- IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1603	function IGCM_sys_GetBuffer {
1604	IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1605
1606	typeset DEST buf_liste target file_work
1607	typeset NB_ESSAI DELAI status i
1608
1609	if ( $DEBUG_sys ) ; then
1610	echo "IGCM_sys_GetBuffer :" $@
1611	fi
1612
1613	# number of tentative
1614	NB_ESSAI=3
1615	# time delay between tentative
1616	DELAI=2
1617
1618	if [ $DRYRUN -le 2 ]; then
1619	if [ X${1} = X'/l' ] ; then
1620	# test if the first file is present in the old computation :
1621	eval set +A buf_liste \${${2}}
1622	else
1623	eval set +A buf_liste ${1}
1624	fi
1625	eval DEST=\${${#}}
1626
1627	#USUAL WAY
1628	if [ X${1} = X'/l' ] ; then
1629	for target in ${buf_liste[*]} ; do
1630	local_file=$( basename ${target} )
1631	i=0
1632	while [ $i -lt $NB_ESSAI ] ; do
1633	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1634	status=$?
1635	if [ ${status} -gt 0 ]; then
1636	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1637	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1638	sleep $DELAI
1639	else
1640	break
1641	fi
1642	(( i = i + 1 ))
1643	done
1644	if [ ${status} -gt 0 ] ; then
1645	echo "IGCM_sys_Get : error"
1646	cat out_rsync
1647	\rm out_rsync
1648	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1649	else
1650	\rm out_rsync
1651	fi
1652	done
1653	else
1654	i=0
1655	while [ $i -lt $NB_ESSAI ] ; do
1656	\cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1657	status=$?
1658	if [ ${status} -gt 0 ]; then
1659	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1660	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1661	sleep $DELAI
1662	else
1663	break
1664	fi
1665	(( i = i + 1 ))
1666	done
1667	if [ ${status} -gt 0 ] ; then
1668	echo "IGCM_sys_Get : error"
1669	cat out_rsync
1670	\rm out_rsync
1671	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1672	else
1673	\rm out_rsync
1674	fi
1675	fi
1676	else
1677	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1678	fi
1679	IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1680	}
1681
1682	#D-#==================================================
1683	#D-function IGCM_sys_GetDate_FichWork
1684	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1685	#D-* Examples:
1686	#D-
1687	function IGCM_sys_GetDate_FichWork {
1688	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1689	if ( $DEBUG_sys ) ; then
1690	echo "IGCM_sys_GetDate_FichWork :" $@
1691	fi
1692	typeset dateF
1693	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1694	eval ${2}=${dateF[5]}
1695
1696	# donne la date filesys d'un fichier sur la machine work
1697	IGCM_debug_PopStack "IGCM_sys_FichWork"
1698	}
1699
1700	#D-#==================================================
1701	#D-function IGCM_sys_GetDate_FichArchive
1702	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1703	#D-* Examples:
1704	#D-
1705	function IGCM_sys_GetDate_FichArchive {
1706	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1707	if ( $DEBUG_sys ) ; then
1708	echo "IGCM_sys_GetDate_FichArchive :" $@
1709	fi
1710	typeset dateF
1711	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1712	eval ${2}=${dateF[5]}
1713
1714	IGCM_debug_PopStack "IGCM_sys_FichArchive"
1715	}
1716
1717	#D-#==================================================
1718	#D-function IGCM_sys_Dods_Rm
1719	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1720	#D-* Examples:
1721	#D-
1722	function IGCM_sys_Dods_Rm {
1723	if ( $DEBUG_sys ) ; then
1724	echo "IGCM_sys_Dods_Rm :" $@
1725	fi
1726	typeset status
1727	if [ $DRYRUN = 0 ]; then
1728
1729	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1730	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1731	# echo "Nothing has been done."
1732	# return
1733	# fi
1734
1735	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
1736	status=$?
1737
1738	# if [ ${status} -gt 0 ] ; then
1739	# echo "IGCM_sys_Dods_Rm : error."
1740	# cat out_dods_rm
1741	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1742	# else
1743	# rm out_dods_rm
1744	# fi
1745
1746	else
1747	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1748	fi
1749	return $status
1750	}
1751
1752	#D-#==================================================
1753	#D-function IGCM_sys_Dods_Cp
1754	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1755	#D-* Examples:
1756	#D-
1757	function IGCM_sys_Dods_Cp {
1758	if ( $DEBUG_sys ) ; then
1759	echo "IGCM_sys_Dods_Cp :" $@
1760	fi
1761	typeset status
1762	if [ $DRYRUN = 0 ]; then
1763
1764	# if [ ! -d ${R_SAVE}/${1} ] ; then
1765	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1766	# echo "Nothing has been done."
1767	# return
1768	# fi
1769
1770	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1771	status=$?
1772
1773	# if [ ${status} -gt 0 ] ; then
1774	# echo "IGCM_sys_Dods_Cp : error."
1775	# cat out_dods_cp
1776	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1777	# else
1778	# rm out_dods_cp
1779	# fi
1780
1781	else
1782	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1783	fi
1784	return $status
1785	}
1786
1787	#D-#==================================================
1788	#D-function IGCM_sys_Put_Dods
1789	#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1790	#D-* Examples:
1791	#D-
1792	function IGCM_sys_Put_Dods {
1793	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1794	if ( $DEBUG_sys ) ; then
1795	echo "IGCM_sys_Put_Dods :" $@
1796	fi
1797	#set -vx
1798	typeset status
1799	if [ $DRYRUN = 0 ]; then
1800	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1801	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1802	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1803	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1804	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1805	return
1806	fi
1807
1808	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1809	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1810	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1811	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1812	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1813	return
1814	fi
1815	#
1816	if [ -d ${R_SAVE}/${1} ] ; then
1817	cd ${R_SAVE}
1818	elif [ -d ${R_FIGR}/${1} ] ; then
1819	cd ${R_FIGR}
1820	fi
1821
1822	IGCM_sys_Dods_Rm ${1}
1823	IGCM_sys_Dods_Cp ${1}
1824	status=0
1825
1826	if [ ${status} -gt 0 ] ; then
1827	echo "IGCM_sys_Put_Dods : error."
1828	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1829	fi
1830	else
1831	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1832	fi
1833	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1834	}
1835
1836	##############################################################
1837	# REBUILD OPERATOR
1838
1839	#D-#==================================================
1840	#D-function IGCM_sys_rebuild
1841	#D-* Purpose: rebuild parallel files
1842	#D-* Examples:
1843	#D-
1844	function IGCM_sys_rebuild {
1845	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1846	if ( $DEBUG_sys ) ; then
1847	echo "IGCM_sys_rebuild :" $@
1848	fi
1849
1850	typeset NB_ESSAI DELAI status i firstArg
1851	# number of tentative
1852	NB_ESSAI=3
1853	# time delay between tentative
1854	DELAI=2
1855
1856	i=0
1857	while [ $i -lt $NB_ESSAI ] ; do
1858	~p86ipsl/CESIUM/bin/rebuild -f -o $@ > out_rsync 2>&1
1859	status=$?
1860	if [ ${status} -gt 0 ] ; then
1861	IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1862	cat out_rsync
1863	\rm out_rsync
1864	IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1865	firstArg=${1}
1866	\rm ${firstArg}
1867	sleep $DELAI
1868	else
1869	\rm out_rsync
1870	break
1871	fi
1872	(( i = i + 1 ))
1873	done
1874
1875	if [ ${status} -gt 0 ] ; then
1876	echo "IGCM_sys_rebuild : rebuild error code is ${status}"
1877	IGCM_debug_Exit "rebuild"
1878	fi
1879
1880	IGCM_debug_PopStack "IGCM_sys_rebuild"
1881	}
1882
1883	#D-#==================================================
1884	#D-function IGCM_sys_rebuild_station
1885	#D-* Purpose: rebuild parallel files describing station
1886	#D-* Examples:
1887	#D-
1888	function IGCM_sys_rebuild_station {
1889	IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1890	typeset i list_opt file_in file_out prefix_invert list_invert
1891	if ( $DEBUG_sys ) ; then
1892	echo "IGCM_sys_rebuild_station :" $@
1893	fi
1894	list_opt=$@
1895
1896	# Invert Axis : t,x -> x,t
1897	# t,pres,x -> x,t,pres
1898	# So that we can concatenate along x
1899	i=0
1900	for file_in in ${list_opt} ; do
1901	(( i = i + 1))
1902	[ ${i} = 1 ] && file_out=${file_in} && continue
1903	prefix_invert=$( basename ${file_in} .nc )
1904	IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1905	list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1906	done
1907
1908	# Concatenate
1909	IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1910
1911	# Re-ivert file
1912	IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1913
1914	# Station re-ordering is too expansive to be run within libICGM
1915	# This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1916	# This re-ordering must be done "in memory" by the cmorization process
1917	# Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1918	# BEGIN reordering
1919
1920	# Only LMDZ text output contains the exact ordering of the station.
1921	# We isolate this in the code below:
1922	# 0 38 -157.5000000000000 70.98591549295774
1923	# 0 54 27.49999999999999 67.18309859154928
1924	# 0 56 -62.50000000000001 82.39436619718309
1925	# 0 79 12.49999999999999 78.59154929577466
1926	# 0 116 -165.0000000000000 76.05633802816901
1927	# 0 117 130.0000000000000 70.98591549295774
1928	# 0 118 110.0000000000000 87.46478873239437
1929	# 1 40 4.999999999999995 51.97183098591550
1930	# typeset iStation iProc list_opt file_in file_out prefix_invert
1931	# typeset -Z4 j4
1932	# typeset -Z3 j3
1933
1934	# unset list_opt
1935	# set +A list_opt $@
1936
1937	# Filename after rebuild
1938	# file_out=${list_opt[0]}
1939	# Prefix of output files
1940	# prefix_invert=$( basename ${file_out} .nc )
1941	# Number of procs
1942	# num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} \| wc -l )
1943
1944	# iProc=0
1945	# while [ ${iProc} -lt ${num_proc} ] ; do
1946	# Array containing Station as a number
1947	# unset proc_stn
1948	# set +A proc_stn $( grep "iophy_mpi rank ip lon lat $iProc" ${PREFIX}_${Exe_Output} \| sed -e "s/iophy_mpi rank ip lon lat //g" \| awk ' {print $2}' )
1949	# Number of stations produced by processor proc
1950	# stationLast=${#proc_stn[*]}
1951	# Proc number on 4 digits
1952	# j4=${iProc}
1953	# Init
1954	# iStation=0
1955	# while [ ${iStation} -lt ${stationLast} ] ; do
1956	# Station number on 3 digits
1957	# j3=${proc_stn[${iStation}]}
1958	# Extract station
1959	# Invert Axis : t,x -> x,t
1960	# t,pres,x -> x,t,pres
1961	# So that we can concatenate along x
1962	# IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1963	# (( iStation = iStation + 1 ))
1964	# done
1965	# (( iProc = iProc + 1 ))
1966	# done
1967
1968	# Concatenate all station along x
1969	# IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1970
1971	# Re-invert file
1972	# IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1973
1974	# END reordering
1975
1976	IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1977	}
1978
1979	############################################################
1980	# Activate Running Environnment Variables
1981
1982	#D-#==================================================
1983	#D-function IGCM_sys_desactiv_variables
1984	#D-* Purpose: set environement variables prior to execution
1985	#D-* Examples:
1986	#D-
1987	function IGCM_sys_activ_variables {
1988	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1989	if ( $DEBUG_sys ) ; then
1990	echo "IGCM_sys_activ_variables"
1991	fi
1992
1993	# --------------------------------------------------------------------
1994	#D- MPI specifications
1995	# --------------------------------------------------------------------
1996
1997	# --------------------------------------------------------------------
1998	#D- Other specifications
1999	# --------------------------------------------------------------------
2000
2001	ulimit -s unlimited
2002
2003	IGCM_debug_PopStack "IGCM_sys_activ_variables"
2004	}
2005
2006	############################################################
2007	# Desactivate Running Environnment Variables
2008
2009	#D-#==================================================
2010	#D-function IGCM_sys_desactiv_variables
2011	#D-* Purpose: unset environement variables after execution
2012	#D-* Examples:
2013	#D-
2014	function IGCM_sys_desactiv_variables {
2015	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
2016	if ( $DEBUG_sys ) ; then
2017	echo "IGCM_sys_desactiv_variables"
2018	fi
2019	# --------------------------------------------------------------------
2020	#D- MPI specifications
2021	# --------------------------------------------------------------------
2022
2023	# --------------------------------------------------------------------
2024	#D- Other specifications
2025	# --------------------------------------------------------------------
2026
2027	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
2028	}
2029
2030	############################################################
2031	# Build MPI/OMP scripts run file (dummy function)
2032
2033	#D-#==================================================
2034	#D-function IGCM_sys_build_run_file
2035	#D-* Purpose: build run file (deprecated)
2036	#D-* Examples:
2037	#D-
2038	function IGCM_sys_build_run_file {
2039
2040	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
2041
2042	}
2043
2044	############################################################
2045	# Build MPI/OMP scripts
2046
2047	#D-#==================================================
2048	#D-function IGCM_sys_build_execution_scripts
2049	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
2050	#D-* Examples:
2051	#D-
2052	function IGCM_sys_build_execution_scripts
2053	{
2054	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
2055	if ( $DEBUG_sys ) ; then
2056	echo "IGCM_sys_build_execution_scripts " $@
2057	fi
2058
2059	EXECUTION=${HOST_MPIRUN_COMMAND}
2060
2061	if ( ${OK_PARA_MPMD} ) ; then
2062
2063	if [ -f run_file ] ; then
2064	IGCM_sys_Rm -f run_file
2065	fi
2066	touch run_file
2067
2068	# run_file construction
2069
2070	# Then first loop on the components for the coupler ie oasis
2071
2072	## the coupler ie oasis must be the first one
2073	for comp in ${config_ListOfComponents[*]} ; do
2074
2075	eval ExeNameIn=\${config_Executable_${comp}[0]}
2076	eval ExeNameOut=\${config_Executable_${comp}[1]}
2077
2078	# for CPL component only
2079	if [ "X${comp}" = "XCPL" ] ; then
2080
2081	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2082	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2083	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
2084	fi
2085	done
2086
2087	# Then second loop on the components
2088
2089	for comp in ${config_ListOfComponents[*]} ; do
2090
2091	eval ExeNameIn=\${config_Executable_${comp}[0]}
2092	eval ExeNameOut=\${config_Executable_${comp}[1]}
2093
2094	# Only if we really have an executable for the component and not the coupler ie oasis:
2095	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
2096
2097	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2098	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2099	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
2100	fi
2101	done
2102
2103	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
2104
2105	IGCM_sys_Chmod u+x run_file
2106	if ( $DEBUG_sys ) ; then
2107	echo "run_file contains : "
2108	cat run_file
2109	fi
2110
2111	else # Only one executable. launch it.
2112
2113	for comp in ${config_ListOfComponents[*]} ; do
2114
2115	# Only if we really have an executable for the component :
2116	eval ExeNameOut=\${config_Executable_${comp}[1]}
2117	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
2118
2119	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2120	echo "" >> script_${ExeNameOut}.ksh
2121	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2122
2123	if ( ${OK_PARA_OMP} ) ; then
2124	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2125	echo "" >> script_${ExeNameOut}.ksh
2126	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
2127	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
2128	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
2129	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
2130	fi
2131
2132	if ( ${OK_PARA_MPI} ) ; then
2133	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2134	# Default : ccc_mprun used if nb_proc gt 1
2135	# to have out/err per process on different files
2136	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
2137	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
2138	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
2139	else
2140	# Default : ccc_mprun is NOT used if nb_proc eq 1
2141	# to have out/err per process on different files
2142	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2143	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
2144	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
2145	fi
2146
2147	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
2148	cat script_${ExeNameOut}.ksh
2149
2150	fi
2151
2152	done
2153
2154	fi
2155
2156	IGCM_debug_Print 1 "sys Curie : execution command is "
2157	IGCM_debug_Print 1 "$EXECUTION"
2158
2159	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
2160	}
2161
2162	############################################################
2163	# Check of space available on temporary filesytems.
2164
2165	#D-#==================================================
2166	#D-function IGCM_sys_check_quota
2167	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
2168	#D-* Examples:
2169	#D-
2170	function IGCM_sys_check_quota {
2171	IGCM_debug_PushStack "IGCM_sys_check_quota"
2172	if ( $DEBUG_sys ) ; then
2173	echo "IGCM_sys_check_quota"
2174	fi
2175	# Limit of quota (in %)
2176	limit_quota=90
2177
2178	# Check of the volume
2179	volume_quota=$(ccc_quota \| grep ' scratch' \| awk '{print $2}')
2180	volume_avail=$(ccc_quota \| grep ' scratch' \| awk '{print $3}')
2181
2182	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
2183
2184	unit_avail=${volume_avail: -1}
2185	unit_quota=${volume_quota: -1}
2186
2187	if [ "${unit_quota}" = "*" ] ; then
2188	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2189	IGCM_debug_Print 1 "More than 100% of your quota is used"
2190	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2191	IGCM_debug_Print 1 "You must have more than 10% available to run"
2192	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2193	IGCM_debug_Verif_Exit
2194	fi
2195
2196	temp_avail=${volume_avail%%${volume_avail: -1}*}
2197	temp_quota=${volume_quota%%${volume_quota: -1}*}
2198
2199	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2200
2201	# Convertion
2202	if [ ${volume_avail: -1} = "T" ] ; then
2203	(( temp_avail = temp_avail * 1000000000000 ))
2204	elif [ ${volume_avail: -1} = "G" ] ; then
2205	(( temp_avail = temp_avail * 1000000000 ))
2206	elif [ ${volume_avail: -1} = "M" ] ; then
2207	(( temp_avail = temp_avail * 1000000 ))
2208	elif [ ${volume_avail: -1} = "k" ] ; then
2209	(( temp_avail = temp_avail * 1000 ))
2210	else
2211	(( temp_avail = volume_avail ))
2212	fi
2213	if [ ${volume_quota: -1} = "T" ] ; then
2214	(( temp_quota = temp_quota * 1000000000000 ))
2215	elif [ ${volume_quota: -1} = "G" ] ; then
2216	(( temp_quota = temp_quota * 1000000000 ))
2217	elif [ ${volume_quota: -1} = "M" ] ; then
2218	(( temp_quota = temp_quota * 1000000 ))
2219	elif [ ${volume_quota: -1} = "k" ] ; then
2220	(( temp_quota = temp_quota * 1000 ))
2221	else
2222	(( temp_quota = volume_quota ))
2223	fi
2224	fi
2225
2226	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2227	# echo "volume ratio is " $quota_volume
2228
2229	if [ ${quota_volume} -ge ${limit_quota} ] ; then
2230	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2231	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
2232	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2233	IGCM_debug_Print 1 "You must have more than 10% available to run"
2234	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2235	IGCM_debug_Verif_Exit
2236	fi
2237
2238	fi
2239
2240	# Check of the number of inodes
2241
2242	inode_quota=$(ccc_quota \| grep ' scratch' \| awk '{print $6}')
2243	inode_avail=$(ccc_quota \| grep ' scratch' \| awk '{print $7}')
2244
2245	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
2246
2247	unit_avail=${inode_avail: -1}
2248	unit_quota=${inode_quota: -1}
2249
2250	if [ "${unit_quota}" = "*" ] ; then
2251	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2252	IGCM_debug_Print 1 "More than 100% of your quota is used"
2253	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2254	IGCM_debug_Print 1 "You must have more than 10% available to run"
2255	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2256	IGCM_debug_Verif_Exit
2257	fi
2258
2259	temp_avail=${inode_avail%%${inode_avail: -1}*}
2260	temp_quota=${inode_quota%%${inode_quota: -1}*}
2261
2262	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2263
2264	# Convertion
2265	if [ ${inode_avail: -1} = "T" ] ; then
2266	(( temp_avail = temp_avail * 1000000000000 ))
2267	elif [ ${inode_avail: -1} = "G" ] ; then
2268	(( temp_avail = temp_avail * 1000000000 ))
2269	elif [ ${inode_avail: -1} = "M" ] ; then
2270	(( temp_avail = temp_avail * 1000000 ))
2271	elif [ ${inode_avail: -1} = "k" ] ; then
2272	(( temp_avail = temp_avail * 1000 ))
2273	else
2274	(( temp_avail = inode_avail ))
2275	fi
2276
2277	if [ ${inode_quota: -1} = "T" ] ; then
2278	(( temp_quota = temp_quota * 1000000000000 ))
2279	elif [ ${inode_quota: -1} = "G" ] ; then
2280	(( temp_quota = temp_quota * 1000000000 ))
2281	elif [ ${inode_quota: -1} = "M" ] ; then
2282	(( temp_quota = temp_quota * 1000000 ))
2283	elif [ ${inode_quota: -1} = "k" ] ; then
2284	(( temp_quota = temp_quota * 1000 ))
2285	else
2286	(( temp_quota = inode_quota ))
2287	fi
2288	fi
2289	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2290	# echo "inode ratio is " $quota_inode
2291
2292	if [ ${quota_inode} -ge ${limit_quota} ] ; then
2293	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2294	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
2295	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2296	IGCM_debug_Print 1 "You must have more than 10% available to run"
2297	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2298	IGCM_debug_Verif_Exit
2299	fi
2300	fi
2301	IGCM_debug_PopStack "IGCM_sys_check_quota"
2302	}
2303
2304	#D-#==================================================
2305	#D-function IGCM_sys_CountJobInQueue
2306	#D-* Purpose: Check if job_name is currently
2307	#D- running or in queue
2308	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
2309	#D-
2310	function IGCM_sys_CountJobInQueue {
2311	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
2312	if ( $DEBUG_sys ) ; then
2313	echo "IGCM_sys_CountJobInQueue"
2314	fi
2315
2316	#NbRun=$( ccc_mstat -f \| grep -c ${JobName} )
2317
2318	# With -f option, the full job name is given in the last column
2319	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ($NF~String) { x=x+1 } END { print x }' )
2320
2321	eval ${2}=${NbRun}
2322
2323	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
2324	}
2325
2326	##############################################################
2327	# NCO OPERATOR
2328
2329	#D-#==================================================
2330	#D-function IGCM_sys_ncap2
2331	#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
2332	#D-* Examples:
2333	#D-
2334	function IGCM_sys_ncap2 {
2335	IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
2336	if ( $DEBUG_sys ) ; then
2337	echo "IGCM_sys_ncap2 :" $@
2338	fi
2339
2340	typeset NB_ESSAI DELAI status i
2341	# number of tentative
2342	NB_ESSAI=3
2343	# time delay between tentative
2344	DELAI=2
2345
2346	i=0
2347	while [ $i -lt $NB_ESSAI ] ; do
2348	ncap2 "$@" > out_rsync 2>&1
2349	status=$?
2350	if [ ${status} -gt 0 ] ; then
2351	IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
2352	cat out_rsync
2353	\rm out_rsync
2354	IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2355	sleep $DELAI
2356	else
2357	\rm out_rsync
2358	break
2359	fi
2360	(( i = i + 1 ))
2361	done
2362
2363	if [ ${status} -gt 0 ] ; then
2364	echo "IGCM_sys_ncap2 : ncap2 error"
2365	IGCM_debug_Exit "ncap2"
2366	fi
2367
2368	IGCM_debug_PopStack "IGCM_sys_ncap2"
2369	}
2370
2371	#D-#==================================================
2372	#D-function IGCM_sys_ncatted
2373	#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
2374	#D-* Examples:
2375	#D-
2376	function IGCM_sys_ncatted {
2377	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
2378	if ( $DEBUG_sys ) ; then
2379	echo "IGCM_sys_ncatted :" $@
2380	fi
2381
2382	typeset NB_ESSAI DELAI status i
2383	# number of tentative
2384	NB_ESSAI=3
2385	# time delay between tentative
2386	DELAI=2
2387
2388	i=0
2389	while [ $i -lt $NB_ESSAI ] ; do
2390	ncatted "$@" > out_rsync 2>&1
2391	status=$?
2392	if [ ${status} -gt 0 ] ; then
2393	IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
2394	cat out_rsync
2395	\rm out_rsync
2396	IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2397	sleep $DELAI
2398	else
2399	\rm out_rsync
2400	break
2401	fi
2402	(( i = i + 1 ))
2403	done
2404
2405	if [ ${status} -gt 0 ] ; then
2406	echo "IGCM_sys_ncatted : ncatted error"
2407	IGCM_debug_Exit "ncatted"
2408	fi
2409
2410	IGCM_debug_PopStack "IGCM_sys_ncatted"
2411	}
2412
2413	#D-#==================================================
2414	#D-function IGCM_sys_ncbo
2415	#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
2416	#D-* Examples:
2417	#D-
2418	function IGCM_sys_ncbo {
2419	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2420	if ( $DEBUG_sys ) ; then
2421	echo "IGCM_sys_ncbo :" $@
2422	fi
2423
2424	typeset NB_ESSAI DELAI status i
2425	# number of tentative
2426	NB_ESSAI=3
2427	# time delay between tentative
2428	DELAI=2
2429
2430	i=0
2431	while [ $i -lt $NB_ESSAI ] ; do
2432	ncbo $@ > out_rsync 2>&1
2433	status=$?
2434	if [ ${status} -gt 0 ] ; then
2435	IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
2436	cat out_rsync
2437	\rm out_rsync
2438	IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2439	sleep $DELAI
2440	else
2441	\rm out_rsync
2442	break
2443	fi
2444	(( i = i + 1 ))
2445	done
2446
2447	if [ ${status} -gt 0 ] ; then
2448	echo "IGCM_sys_ncbo : ncbo error"
2449	IGCM_debug_Exit "ncbo"
2450	fi
2451
2452	IGCM_debug_PopStack "IGCM_sys_ncbo"
2453	}
2454
2455	#D-#==================================================
2456	#D-function IGCM_sys_ncdif
2457	#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
2458	#D-* Examples:
2459	#D-
2460	function IGCM_sys_ncdiff {
2461	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2462	if ( $DEBUG_sys ) ; then
2463	echo "IGCM_sys_ncdiff :" $@
2464	fi
2465
2466	typeset NB_ESSAI DELAI status i
2467	# number of tentative
2468	NB_ESSAI=3
2469	# time delay between tentative
2470	DELAI=2
2471
2472	i=0
2473	while [ $i -lt $NB_ESSAI ] ; do
2474	ncdiff $@ > out_rsync 2>&1
2475	status=$?
2476	if [ ${status} -gt 0 ] ; then
2477	IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2478	cat out_rsync
2479	\rm out_rsync
2480	IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2481	sleep $DELAI
2482	else
2483	\rm out_rsync
2484	break
2485	fi
2486	(( i = i + 1 ))
2487	done
2488
2489	if [ ${status} -gt 0 ] ; then
2490	echo "IGCM_sys_ncdiff : ncdiff error"
2491	IGCM_debug_Exit "ncdiff"
2492	fi
2493
2494	IGCM_debug_PopStack "IGCM_sys_ncdiff"
2495	}
2496
2497	#D-#==================================================
2498	#D-function IGCM_sys_ncea
2499	#D-* Purpose: encapsulate ncea call so as to manage error code and retry
2500	#D-* Examples:
2501	#D-
2502	function IGCM_sys_ncea {
2503	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2504	if ( $DEBUG_sys ) ; then
2505	echo "IGCM_sys_ncea :" $@
2506	fi
2507
2508	typeset NB_ESSAI DELAI status i
2509	# number of tentative
2510	NB_ESSAI=3
2511	# time delay between tentative
2512	DELAI=2
2513
2514	i=0
2515	while [ $i -lt $NB_ESSAI ] ; do
2516	ncea $@ > out_rsync 2>&1
2517	status=$?
2518	if [ ${status} -gt 0 ] ; then
2519	IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2520	cat out_rsync
2521	\rm out_rsync
2522	IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2523	sleep $DELAI
2524	else
2525	\rm out_rsync
2526	break
2527	fi
2528	(( i = i + 1 ))
2529	done
2530
2531	if [ ${status} -gt 0 ] ; then
2532	echo "IGCM_sys_ncea : ncea error"
2533	IGCM_debug_Exit "ncea"
2534	fi
2535
2536	IGCM_debug_PopStack "IGCM_sys_ncea"
2537	}
2538
2539	#D-#==================================================
2540	#D-function IGCM_sys_ncecat
2541	#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
2542	#D-* Examples:
2543	#D-
2544	function IGCM_sys_ncecat {
2545	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2546	if ( $DEBUG_sys ) ; then
2547	echo "IGCM_sys_ncecat :" $@
2548	fi
2549
2550	typeset NB_ESSAI DELAI status i
2551	# number of tentative
2552	NB_ESSAI=3
2553	# time delay between tentative
2554	DELAI=2
2555
2556	i=0
2557	while [ $i -lt $NB_ESSAI ] ; do
2558	ncecat $@ > out_rsync 2>&1
2559	status=$?
2560	if [ ${status} -gt 0 ] ; then
2561	IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2562	cat out_rsync
2563	\rm out_rsync
2564	IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2565	sleep $DELAI
2566	else
2567	\rm out_rsync
2568	break
2569	fi
2570	(( i = i + 1 ))
2571	done
2572
2573	if [ ${status} -gt 0 ] ; then
2574	echo "IGCM_sys_ncecat : ncecat error"
2575	IGCM_debug_Exit "ncecat"
2576	fi
2577
2578	IGCM_debug_PopStack "IGCM_sys_ncecat"
2579	}
2580
2581	#D-#==================================================
2582	#D-function IGCM_sys_ncflint
2583	#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
2584	#D-* Examples:
2585	#D-
2586	function IGCM_sys_ncflint {
2587	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2588	if ( $DEBUG_sys ) ; then
2589	echo "IGCM_sys_ncflint :" $@
2590	fi
2591
2592	typeset NB_ESSAI DELAI status i
2593	# number of tentative
2594	NB_ESSAI=3
2595	# time delay between tentative
2596	DELAI=2
2597
2598	i=0
2599	while [ $i -lt $NB_ESSAI ] ; do
2600	ncflint $@ > out_rsync 2>&1
2601	status=$?
2602	if [ ${status} -gt 0 ] ; then
2603	IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2604	cat out_rsync
2605	\rm out_rsync
2606	IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2607	sleep $DELAI
2608	else
2609	\rm out_rsync
2610	break
2611	fi
2612	(( i = i + 1 ))
2613	done
2614
2615	if [ ${status} -gt 0 ] ; then
2616	echo "IGCM_sys_ncflint : ncflint error"
2617	IGCM_debug_Exit "ncflint"
2618	fi
2619
2620	IGCM_debug_PopStack "IGCM_sys_ncflint"
2621	}
2622
2623	#D-#==================================================
2624	#D-function IGCM_sys_ncks
2625	#D-* Purpose: encapsulate ncks call so as to manage error code and retry
2626	#D-* Examples:
2627	#D-
2628	function IGCM_sys_ncks {
2629	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2630	if ( $DEBUG_sys ) ; then
2631	echo "IGCM_sys_ncks :" $@
2632	fi
2633
2634	typeset NB_ESSAI DELAI status i
2635	# number of tentative
2636	NB_ESSAI=3
2637	# time delay between tentative
2638	DELAI=2
2639
2640	i=0
2641	while [ $i -lt $NB_ESSAI ] ; do
2642	ncks $@ > out_rsync 2>&1
2643	status=$?
2644	if [ ${status} -gt 0 ] ; then
2645	IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2646	cat out_rsync
2647	\rm out_rsync
2648	IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2649	sleep $DELAI
2650	else
2651	\rm out_rsync
2652	break
2653	fi
2654	(( i = i + 1 ))
2655	done
2656
2657	if [ ${status} -gt 0 ] ; then
2658	echo "IGCM_sys_ncks : ncks error"
2659	IGCM_debug_Exit "ncks"
2660	fi
2661
2662	IGCM_debug_PopStack "IGCM_sys_ncks"
2663	}
2664
2665	#D-#==================================================
2666	#D-function IGCM_sys_ncpdq
2667	#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
2668	#D-* Examples:
2669	#D-
2670	function IGCM_sys_ncpdq {
2671	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2672	if ( $DEBUG_sys ) ; then
2673	echo "IGCM_sys_ncpdq :" $@
2674	fi
2675
2676	typeset NB_ESSAI DELAI status i
2677	# number of tentative
2678	NB_ESSAI=3
2679	# time delay between tentative
2680	DELAI=2
2681
2682	i=0
2683	while [ $i -lt $NB_ESSAI ] ; do
2684	ncpdq $@ > out_rsync 2>&1
2685	status=$?
2686	if [ ${status} -gt 0 ] ; then
2687	IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2688	cat out_rsync
2689	\rm out_rsync
2690	IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2691	sleep $DELAI
2692	else
2693	\rm out_rsync
2694	break
2695	fi
2696	(( i = i + 1 ))
2697	done
2698
2699	if [ ${status} -gt 0 ] ; then
2700	echo "IGCM_sys_ncpdq : ncpdq error"
2701	IGCM_debug_Exit "ncpdq"
2702	fi
2703
2704	IGCM_debug_PopStack "IGCM_sys_ncpdq"
2705	}
2706
2707	#D-#==================================================
2708	#D-function IGCM_sys_ncra
2709	#D-* Purpose: encapsulate ncra call so as to manage error code and retry
2710	#D-* Examples:
2711	#D-
2712	function IGCM_sys_ncra {
2713	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2714	if ( $DEBUG_sys ) ; then
2715	echo "IGCM_sys_ncra :" $@
2716	fi
2717
2718	typeset NB_ESSAI DELAI status i
2719	# number of tentative
2720	NB_ESSAI=3
2721	# time delay between tentative
2722	DELAI=2
2723
2724	i=0
2725	while [ $i -lt $NB_ESSAI ] ; do
2726	ncra $@ > out_rsync 2>&1
2727	status=$?
2728	if [ ${status} -gt 0 ] ; then
2729	IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2730	cat out_rsync
2731	\rm out_rsync
2732	IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2733	sleep $DELAI
2734	else
2735	\rm out_rsync
2736	break
2737	fi
2738	(( i = i + 1 ))
2739	done
2740
2741	if [ ${status} -gt 0 ] ; then
2742	echo "IGCM_sys_ncra : ncra error"
2743	IGCM_debug_Exit "ncra"
2744	fi
2745
2746	IGCM_debug_PopStack "IGCM_sys_ncra"
2747	}
2748
2749	#D-#==================================================
2750	#D-function IGCM_sys_ncrcat
2751	#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
2752	#D-* Examples:
2753	#D-
2754	function IGCM_sys_ncrcat {
2755	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2756	if ( $DEBUG_sys ) ; then
2757	echo "IGCM_sys_ncrcat :" $@
2758	fi
2759
2760	typeset NB_ESSAI DELAI status i lastArg
2761	# number of tentative
2762	NB_ESSAI=3
2763	# time delay between tentative
2764	DELAI=2
2765
2766	i=0
2767	while [ $i -lt $NB_ESSAI ] ; do
2768	ncrcat $@ > out_rsync 2>&1
2769	status=$?
2770	if [ ${status} -gt 0 ] ; then
2771	IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2772	cat out_rsync
2773	\rm out_rsync
2774	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2775	sleep $DELAI
2776	elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" out_rsync )" = "X" ] ; then
2777	IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
2778	cat out_rsync
2779	# remove files having corrupted time axis
2780	eval lastArg=\${$#}
2781	IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
2782	\rm ${lastArg}
2783	\rm out_rsync
2784	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2785	sleep $DELAI
2786	else
2787	\rm out_rsync
2788	break
2789	fi
2790	(( i = i + 1 ))
2791	done
2792
2793	if [ ${status} -gt 0 ] ; then
2794	echo "IGCM_sys_ncrcat : ncrcat error"
2795	#IGCM_debug_Exit "ncrcat"
2796	fi
2797
2798	IGCM_debug_PopStack "IGCM_sys_ncrcat"
2799	}
2800
2801	#D-#==================================================
2802	#D-function IGCM_sys_ncrename
2803	#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
2804	#D-* Examples:
2805	#D-
2806	function IGCM_sys_ncrename {
2807	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2808	if ( $DEBUG_sys ) ; then
2809	echo "IGCM_sys_ncrename :" $@
2810	fi
2811
2812	typeset NB_ESSAI DELAI status i
2813	# number of tentative
2814	NB_ESSAI=3
2815	# time delay between tentative
2816	DELAI=2
2817
2818	i=0
2819	while [ $i -lt $NB_ESSAI ] ; do
2820	ncrename $@ > out_rsync 2>&1
2821	status=$?
2822	if [ ${status} -gt 0 ] ; then
2823	IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2824	cat out_rsync
2825	\rm out_rsync
2826	IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2827	sleep $DELAI
2828	else
2829	\rm out_rsync
2830	break
2831	fi
2832	(( i = i + 1 ))
2833	done
2834
2835	if [ ${status} -gt 0 ] ; then
2836	echo "IGCM_sys_ncrename : ncrename error"
2837	IGCM_debug_Exit "ncrename"
2838	fi
2839
2840	IGCM_debug_PopStack "IGCM_sys_ncrename"
2841	}
2842
2843	#D-#==================================================
2844	#D-function IGCM_sys_ncwa
2845	#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
2846	#D-* Examples:
2847	#D-
2848	function IGCM_sys_ncwa {
2849	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2850	if ( $DEBUG_sys ) ; then
2851	echo "IGCM_sys_ncwa :" $@
2852	fi
2853
2854	typeset NB_ESSAI DELAI status i
2855	# number of tentative
2856	NB_ESSAI=3
2857	# time delay between tentative
2858	DELAI=2
2859
2860	i=0
2861	while [ $i -lt $NB_ESSAI ] ; do
2862	ncwa $@ > out_rsync 2>&1
2863	status=$?
2864	if [ ${status} -gt 0 ] ; then
2865	IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2866	cat out_rsync
2867	\rm out_rsync
2868	IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2869	sleep $DELAI
2870	else
2871	\rm out_rsync
2872	break
2873	fi
2874	(( i = i + 1 ))
2875	done
2876
2877	if [ ${status} -gt 0 ] ; then
2878	echo "IGCM_sys_ncwa : ncwa error"
2879	IGCM_debug_Exit "ncwa"
2880	fi
2881
2882	IGCM_debug_PopStack "IGCM_sys_ncwa"
2883	}
2884
2885	##############################################################
2886	# CDO OPERATOR
2887
2888	#D-#==================================================
2889	#D-function IGCM_sys_cdo
2890	#D-* Purpose: encapsulate cdo call so as to manage error code and retry
2891	#D-* Examples:
2892	#D-
2893	function IGCM_sys_cdo {
2894	IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2895	if ( $DEBUG_sys ) ; then
2896	echo "IGCM_sys_cdo :" $@
2897	fi
2898
2899	typeset status
2900
2901	\cdo $@ > out_rsync 2>&1
2902	status=$?
2903	if [ ${status} -gt 0 ] ; then
2904	echo "IGCM_sys_cdo : error code ${status}"
2905	cat out_rsync
2906	\rm out_rsync
2907	IGCM_debug_PopStack "IGCM_sys_cdo"
2908	return 1
2909	else
2910	IGCM_debug_PopStack "IGCM_sys_cdo"
2911	return 0
2912	fi
2913
2914	IGCM_debug_PopStack "IGCM_sys_cdo"
2915	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: