Context Navigation

source: branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh @ 559

Last change on this file since 559 was 534, checked in by mmaipsl, 12 years ago
Correct SUBMIT_DIR on titane.
Property svn:keywords set to `Revision Author Date`
File size: 47.4 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Titane
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe param files \| \| Chmod \| \|
46	# \| \| Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=titane
71
72	#D-
73	#D-#==================================================
74	#D-Program used in libIGCM
75	#D-#==================================================
76
77	# rsync with path
78	typeset -r RSYNC=/usr/bin/rsync
79	# RSYNC_opt args to rsync
80	typeset -r RSYNC_opt="-va"
81	# ie storage filesystem
82	typeset -r RHOST=titane
83
84	#====================================================
85	# Source default environment
86	#====================================================
87	. /etc/profile
88
89	#====================================================
90	# Set environment tools (ferret, nco, cdo)
91	#====================================================
92	. /home/cont003/p86ipsl/.atlas_env_titane_ksh
93
94	#====================================================
95	# Specific for ocean additionnal diagnostic
96	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
97	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
98
99	#====================================================
100	# Host specific DIRECTORIES
101	#====================================================
102
103	#====================================================
104	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
105	typeset -r R_EXE="${MODIPSL}/bin"
106
107	#====================================================
108	#- SUBMIT_DIR : submission dir
109	typeset SUBMIT_DIR=${SUBMIT_DIR:=${LS_SUBCWD}}
110
111	#====================================================
112	#- ARCHIVE
113	typeset -r ARCHIVE=${DMFDIR}
114
115	#====================================================
116	#- Mirror libIGCM from titane to cesium if needed
117	ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $2}' )
118	if [ ! ${ROOTSYS} = "home" ] ; then
119	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
120	else
121	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
122	fi
123
124	#====================================================
125	#- libIGCM_POST
126	if ( ${MirrorlibIGCM} ) ; then
127	PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
128	typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
129	else
130	typeset -r libIGCM_POST=${libIGCM}
131	fi
132
133	#====================================================
134	#- IN
135	typeset -r R_IN=${R_IN:=/dmnfs/cont003/p86ipsl/IGCM}
136	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/dmnfs/cont003/p24data}
137
138	#====================================================
139	#- OUT
140	typeset -r R_OUT=${ARCHIVE}/IGCM_OUT
141
142	#====================================================
143	#- OUT_SCR (ONLY FOR double copy an scratch)
144	typeset -r R_OUT_SCR=${SCRATCHDIR}/IGCM_OUT
145
146	#====================================================
147	#- OUT_POST
148	typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
149
150	#====================================================
151	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
152	if [ ! X${LSB_QUEUE} = Xmono ] ; then
153	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${LSB_JOBID}}
154	else
155	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${LSB_JOBID}}
156	fi
157
158	#====================================================
159	#- BIG_DIR : BIG_DIR to store files waiting for rebuild
160	typeset -r BIG_DIR=${BIG_DIR:=${SCRATCHDIR}/REBUILD}
161
162	#====================================================
163	#- HOST_MPIRUN_COMMAND
164	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
165
166	#====================================================
167	#- Max number of arguments passed to nco operator or demigration command
168	UNIX_MAX_LIMIT=360
169
170	NUM_COREPERNODE=8
171
172
173	#D-#==================================================
174	#D-function IGCM_sys_RshMaster
175	#D-* Purpose: Just a fake command to wrapp
176	#D- IGCM_card call in post-treatment
177	#D- Ulam do not see brodie filesystem
178	#D- Cesium do not see all mercure filesystem
179	#D- That's why we need this hack.
180	#D-* Examples:
181	#D-
182	function IGCM_sys_RshMaster {
183	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
184	/bin/ksh <<-EOF
185	export libIGCM=${libIGCM}
186	export DEBUG_debug=${DEBUG_debug}
187	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
188	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
189	${@}
190	EOF
191	if [ $? -gt 0 ] ; then
192	echo "IGCM_sys_RshMaster : erreur."
193	IGCM_debug_Exit "IGCM_sys_RshMaster"
194	fi
195	IGCM_debug_PopStack "IGCM_sys_RshMaster"
196	}
197
198	#D-#==================================================
199	#D-function IGCM_sys_RshArchive
200	#D-* Purpose: Archive rsh command
201	#D-* Examples:
202	#D-
203	function IGCM_sys_RshArchive {
204	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
205	/bin/ksh <<-EOF
206	${@}
207	EOF
208	if [ $? -gt 0 ] ; then
209	echo "IGCM_sys_RshArchive : erreur."
210	IGCM_debug_Exit "IGCM_sys_RshArchive"
211	fi
212	IGCM_debug_PopStack "IGCM_sys_RshArchive"
213	}
214
215	#D-#==================================================
216	#D-function IGCM_sys_RshPost
217	#D-* Purpose: Post-process rsh command
218	#D-* Examples:
219	#D-
220	function IGCM_sys_RshPost {
221	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
222	if ( $DEBUG_sys ) ; then
223	echo "IGCM_sys_RshPost :" $@
224	fi
225
226	typeset NB_ESSAI DELAI status i
227	# keep standard input for the loop onto temporary file
228	cat >tmp_IGCM_sys_RshPost_$$
229
230	#echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
231	#cat tmp_IGCM_sys_RshPost_$$
232
233	if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
234	# little hack so that rebuild submission is done on titane not an cesium
235	#
236	libIGCM_POST_sed=$( echo $libIGCM_POST \| sed 's/\//\\\//g' )
237	POST_DIR_sed=$( echo ${POST_DIR} \| sed 's/\//\\\//g' )
238	sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
239	sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
240	sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
241	sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
242	#\mv tmp.txt tmp_IGCM_sys_RshPost_$$
243	#
244	#echo cat tmp_IGCM_sys_RshPost_$$ AFTER
245	#cat tmp_IGCM_sys_RshPost_$$
246	/bin/ksh <tmp_IGCM_sys_RshPost_$$
247	if [ $? -gt 0 ] ; then
248	echo "IGCM_sys_RshPost : erreur."
249	IGCM_debug_Exit "IGCM_sys_RshPost"
250	fi
251	# delete temporary file
252	\rm tmp_IGCM_sys_RshPost_$$
253	else
254	# number of tentative
255	NB_ESSAI=10
256	# time delay between tentative
257	DELAI=10
258	i=0
259	while [ $i -ne $NB_ESSAI ] ; do
260	ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
261	status=$?
262	if [ ${status} -ne 0 ];
263	then
264	sleep $DELAI
265	else
266	break
267	fi
268	let i=$i+1
269	done
270	# delete temporary file
271	\rm tmp_IGCM_sys_RshPost_$$
272
273	if [ ${status} -gt 0 ] ; then
274	echo "IGCM_sys_RshPost : erreur."
275	IGCM_debug_Exit "IGCM_sys_RshPost"
276	fi
277	fi
278	IGCM_debug_PopStack "IGCM_sys_RshPost"
279	}
280
281	#D-#==================================================
282	#D-function IGCM_sys_SendMail
283	#D-* Purpose: Send mail when simulation is over
284	#D-* Examples:
285	#D-
286	function IGCM_sys_SendMail {
287	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
288	if ( $DEBUG_sys ) ; then
289	echo "IGCM_sys_SendMail :" $@
290	fi
291
292	if ( ${ExitFlag} ) ; then
293	status=failed
294	else
295	status=completed
296	fi
297
298	ssh -t titane996 ssh platine /bin/ksh <<-EOF
299	export LOGIN=${LOGIN}
300	export config_UserChoices_JobName=${config_UserChoices_JobName}
301	export config_UserChoices_MailName=${config_UserChoices_MailName}
302	export DateBegin=${DateBegin}
303	export DateEnd=${DateEnd}
304	export R_SAVE=${R_SAVE}
305	export SUBMIT_DIR=${SUBMIT_DIR}
306	export status=${status}
307
308	cat << END_MAIL > job_end.mail
309	Dear ${LOGIN},
310
311	Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
312	Job started : ${DateBegin}
313	Job ended : ${DateEnd}
314	Output files are available in ${R_SAVE}
315	Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
316	END_MAIL
317
318	if [ ! -z ${config_UserChoices_MailName} ] ; then
319	mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
320	elif [ -f ~/.forward ] ; then
321	mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
322	fi
323
324	sleep 10
325	rm -f job_end.mail
326	EOF
327
328	if [ $? -gt 0 ] ; then
329	echo "IGCM_sys_SendMail : erreur."
330	IGCM_debug_Exit "IGCM_sys_SendMail"
331	fi
332	IGCM_debug_PopStack "IGCM_sys_SendMail"
333	}
334
335	#D-#==================================================
336	#D-function IGCM_sys_Mkdir
337	#D-* Purpose: Master locale mkdir command
338	#D-* Examples:
339	#D-
340	function IGCM_sys_Mkdir {
341	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
342	if ( $DEBUG_sys ) ; then
343	echo "IGCM_sys_Mkdir :" $@
344	fi
345	if [ ! -d ${1} ]; then
346	\mkdir -p $1
347	if [ $? -gt 0 ] ; then
348	echo "IGCM_sys_Mkdir : erreur."
349	IGCM_debug_Exit "IGCM_sys_Mkdir"
350	fi
351	fi
352	# vérification :
353	if [ ! -d ${1} ] ; then
354	echo "IGCM_sys_Mkdir : erreur."
355	IGCM_debug_Exit "IGCM_sys_Mkdir"
356	fi
357	IGCM_debug_PopStack "IGCM_sys_Mkdir"
358	}
359
360	#D-#==================================================
361	#D-function IGCM_sys_MkdirArchive
362	#D-* Purpose: Mkdir on Archive
363	#D-* Examples:
364	#D-
365	function IGCM_sys_MkdirArchive {
366	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
367	if ( $DEBUG_sys ) ; then
368	echo "IGCM_sys_MkdirArchive :" $@
369	fi
370	#- creation de repertoire sur le serveur fichier
371	if [ ! -d ${1} ]; then
372	\mkdir -p $1
373	if [ $? -gt 0 ] ; then
374	echo "IGCM_sys_MkdirArchive : erreur."
375	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
376	fi
377	fi
378	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
379	}
380
381	#D-#==================================================
382	#D-function IGCM_sys_MkdirWork
383	#D-* Purpose: Mkdir on Work
384	#D-* Examples:
385	#D-
386	function IGCM_sys_MkdirWork {
387	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
388	if ( $DEBUG_sys ) ; then
389	echo "IGCM_sys_MkdirWork :" $@
390	fi
391	#- creation de repertoire sur le serveur fichier
392	if [ ! -d ${1} ]; then
393	\mkdir -p $1
394	if [ $? -gt 0 ] ; then
395	echo "IGCM_sys_MkdirWork : erreur."
396	IGCM_debug_Exit "IGCM_sys_MkdirWork"
397	fi
398	fi
399	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
400	}
401
402	#D-#==================================================
403	#D-function IGCM_sys_Cd
404	#D-* Purpose: master cd command
405	#D-* Examples:
406	#D-
407	function IGCM_sys_Cd {
408	IGCM_debug_PushStack "IGCM_sys_Cd" $@
409	if ( $DEBUG_sys ) ; then
410	echo "IGCM_sys_Cd :" $@
411	fi
412	\cd $1
413	if [ $? -gt 0 ] ; then
414	echo "IGCM_sys_Cd : erreur."
415	IGCM_debug_Exit "IGCM_sys_Cd"
416	fi
417	IGCM_debug_PopStack "IGCM_sys_Cd"
418	}
419
420	#D-#==================================================
421	#D-function IGCM_sys_Chmod
422	#D-* Purpose: Chmod
423	#D-* Examples:
424	#D-
425	function IGCM_sys_Chmod {
426	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
427	if ( $DEBUG_sys ) ; then
428	echo "IGCM_sys_Chmod :" $@
429	fi
430	if [ $DRYRUN -le 1 ]; then
431	\chmod $@
432	if [ $? -gt 0 ] ; then
433	echo "IGCM_sys_Chmod : erreur."
434	IGCM_debug_Exit "IGCM_sys_Chmod"
435	fi
436	else
437	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
438	fi
439	IGCM_debug_PopStack "IGCM_sys_Chmod"
440	}
441
442	#D-#==================================================
443	#D-function IGCM_sys_FileSize
444	#D-* Purpose: Filesize
445	#D-* Examples:
446	#D-
447	function IGCM_sys_FileSize {
448	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
449
450	typeset sizeF
451	set +A sizeF -- $( ls -la ${1} )
452	if [ $? -gt 0 ] ; then
453	IGCM_debug_Exit "IGCM_sys_FileSize"
454	fi
455	eval ${2}=${sizeF[4]}
456
457	IGCM_debug_PopStack "IGCM_sys_FileSize"
458	}
459
460	#D-#==================================================
461	#D-function IGCM_sys_TestDir
462	#D-* Purpose: Test Directory that must exists
463	#D-* Examples:
464	#D-
465	function IGCM_sys_TestDir {
466	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
467	if ( $DEBUG_sys ) ; then
468	echo "IGCM_sys_TestDir :" $@
469	fi
470	typeset ExistFlag
471	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
472	IGCM_debug_PopStack "IGCM_sys_TestDir"
473
474	return ${ExistFlag}
475	}
476
477	#D-#==================================================
478	#D-function IGCM_sys_TestDirArchive
479	#D-* Purpose: Test Directory that must exists on Archive
480	#D-* Examples:
481	#D-
482	function IGCM_sys_TestDirArchive {
483	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
484	if ( $DEBUG_sys ) ; then
485	echo "IGCM_sys_TestDirArchive :" $@
486	fi
487	typeset ExistFlag
488	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
489	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
490
491	return ${ExistFlag}
492	}
493
494	#D-#==================================================
495	#D-function IGCM_sys_TestFileArchive
496	#D-* Purpose: Test file that must NOT EXISTS on Archive
497	#D-* Examples:
498	#D-
499	function IGCM_sys_TestFileArchive {
500	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
501	typeset ExistFlag
502	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
503	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
504
505	return ${ExistFlag}
506	}
507
508	#D-#==================================================
509	#D-function IGCM_sys_CountFileArchive
510	#D-* Purpose: Count files on Archive filesystem
511	#D-* Examples:
512	#D-
513	function IGCM_sys_CountFileArchive {
514	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
515	ls ${@} 2>/dev/null \| wc -l
516	if [ $? -gt 0 ] ; then
517	echo "IGCM_sys_CountFileArchive : erreur."
518	fi
519	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
520	}
521
522	#D-#==================================================
523	#D-function IGCM_sys_Tree
524	#D-* Purpose: Tree directories with files on ${ARCHIVE}
525	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
526	#D-
527	function IGCM_sys_Tree {
528	IGCM_debug_PushStack "IGCM_sys_Tree" $@
529	if ( $DEBUG_sys ) ; then
530	echo "IGCM_sys_Tree :" $@
531	fi
532
533	\ls -lR ${@}
534
535	IGCM_debug_PopStack "IGCM_sys_Tree"
536	}
537
538	#D-#==================================================
539	#D-function IGCM_sys_Tar
540	#D-* Purpose: master un-tar command
541	#D-* Examples:
542	#D-
543	function IGCM_sys_Tar {
544	IGCM_debug_PushStack "IGCM_sys_Tar" $@
545	if ( $DEBUG_sys ) ; then
546	echo "IGCM_sys_Tar :" $@
547	fi
548	\tar xvf $1
549	if [ $? -gt 0 ] ; then
550	echo "IGCM_sys_Tar : erreur."
551	IGCM_debug_Exit "IGCM_sys_Tar"
552	fi
553	IGCM_debug_PopStack "IGCM_sys_Tar"
554	}
555
556	#D-#==================================================
557	#D-function IGCM_sys_UnTar
558	#D-* Purpose: master un-tar command
559	#D-* Examples:
560	#D-
561	function IGCM_sys_UnTar {
562	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
563	if ( $DEBUG_sys ) ; then
564	echo "IGCM_sys_UnTar :" $@
565	fi
566	\tar xvf $1
567	if [ $? -gt 0 ] ; then
568	echo "IGCM_sys_UnTar : erreur."
569	IGCM_debug_Exit "IGCM_sys_UnTar"
570	fi
571	IGCM_debug_PopStack "IGCM_sys_UnTar"
572	}
573
574	#D-#==================================================
575	#D-function IGCM_sys_Qsub
576	#D-* Purpose: Qsub new job
577	#D-* Examples:
578	#D-
579	function IGCM_sys_Qsub {
580	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
581	if ( $DEBUG_sys ) ; then
582	echo "IGCM_sys_Qsub :" $@
583	fi
584
585	if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
586	/usr/local/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} < $1
587	else
588	/usr/local/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod} < $1
589	fi
590
591	if [ $? -gt 0 ] ; then
592	echo "IGCM_sys_Qsub : erreur " $@
593	IGCM_debug_Exit "IGCM_sys_Qsub"
594	fi
595	IGCM_debug_PopStack "IGCM_sys_Qsub"
596	}
597
598	#D-#==================================================
599	#D-function IGCM_sys_QsubPost
600	#D-* Purpose: Qsub new job on scalaire
601	#D-* Examples:
602	#D-
603	function IGCM_sys_QsubPost {
604	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
605	if ( $DEBUG_sys ) ; then
606	echo "IGCM_sys_QsubPost :" $@
607	fi
608	/usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
609	if [ $? -gt 0 ] ; then
610	echo "IGCM_sys_QsubPost : erreur " $@
611	IGCM_debug_Exit "IGCM_sys_QsubPost"
612	fi
613	IGCM_debug_PopStack "IGCM_sys_QsubPost"
614	}
615
616	#D-*************************
617	#D- File transfer functions
618	#D-*************************
619	#D-
620
621	#D-#==================================================
622	#D-function IGCM_sys_Rsync_out
623	#D-* Purpose: treat return val of rsync
624	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
625	#D- Error values and explanations can depend on your system version.
626	function IGCM_sys_Rsync_out {
627	RET=$1
628	if [ ! $RET ] ; then
629	echo "rsync error !"
630	fi
631
632	if [ $MYLANG = "fr" ]; then
633	case $RET in
634	0) return ;;
635	1) echo "Erreur de rsync ; RERR_SYNTAX : "
636	echo "Erreur de syntaxe ou d'utilisation."
637	return;;
638	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
639	echo "Incompatibilité de protocole."
640	return;;
641	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
642	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
643	echo "répertoires"
644	return;;
645	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
646	echo "Action demandée non supportée : une tentative de manipulation de"
647	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
648	echo "été faite ; ou une option qui est supportée par le client mais"
649	echo "pas par le serveur a été spécifiée."
650	return;;
651	10) echo "Erreur de rsync ; RERR_SOCKETIO"
652	echo "Erreur dans le socket d'entrée sortie"
653	return;;
654	11) echo "Erreur de rsync ; RERR_FILEIO"
655	echo "Erreur d'entrée sortie fichier"
656	return;;
657	12) echo "Erreur de rsync ; RERR_STREAMIO"
658	echo "Erreur dans flux de donnée du protocole rsync"
659	return;;
660	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
661	echo "Erreur avec les diagnostics du programme"
662	return;;
663	14) echo "Erreur de rsync ; RERR_IPC"
664	echo "Erreur dans le code IPC"
665	return;;
666	20) echo "Erreur de rsync ; RERR_SIGNAL"
667	echo "SIGUSR1 ou SIGINT reçu"
668	return;;
669	21) echo "Erreur de rsync ; RERR_WAITCHILD"
670	echo "Une erreur retournée par waitpid()"
671	return;;
672	22) echo "Erreur de rsync ; RERR_MALLOC"
673	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
674	return;;
675	23) echo ""
676	echo "Erreur fichier inexistant"
677	return;;
678	30) echo "Erreur de rsync ; RERR_TIMEOUT"
679	echo "Temps d'attente écoulé dans l'envoi/réception de données"
680	return;;
681	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $RET
682	return;;
683	esac
684	elif [ $MYLANG = "en" ] ; then
685	case $RET in
686	0) return;;
687	1) echo "rsync error : Syntax or usage error "
688	return;;
689	2) echo "rsync error : Protocol incompatibility "
690	return;;
691	3) echo "rsync error : Errors selecting input/output files, dirs"
692	return;;
693	4) echo "rsync error : Requested action not supported: an attempt"
694	echo "was made to manipulate 64-bit files on a platform that cannot support"
695	echo "them; or an option was specified that is supported by the client and"
696	echo "not by the server."
697	return;;
698	5) echo "rsync error : Error starting client-server protocol"
699	return;;
700	10) echo "rsync error : Error in socket I/O "
701	return;;
702	11) echo "rsync error : Error in file I/O "
703	return;;
704	12) echo "rsync error : Error in rsync protocol data stream "
705	return;;
706	13) echo "rsync error : Errors with program diagnostics "
707	return;;
708	14) echo "rsync error : Error in IPC code "
709	return;;
710	20) echo "rsync error : Received SIGUSR1 or SIGINT "
711	return;;
712	21) echo "rsync error : Some error returned by waitpid() "
713	return;;
714	22) echo "rsync error : Error allocating core memory buffers "
715	return;;
716	23) echo "rsync error : Partial transfer due to error"
717	return;;
718	24) echo "rsync error : Partial transfer due to vanished source files"
719	return;;
720	30) echo "rsync error : Timeout in data send/receive "
721	return;;
722	*) echo "rsync error : return code of rsync unknown :" $RET
723	return;;
724	esac
725	else
726	echo "unknown language $MYLANG."
727	return
728	fi
729	}
730
731
732	#D-#==================================================
733	#D-function IGCM_sys_Miror_libIGCM
734	#D-* Purpose: Mirror libIGCM PATH and lib to cesium
735	#D-* Examples:
736	#D-
737	function IGCM_sys_Mirror_libIGCM {
738	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
739	if ( $DEBUG_sys ) ; then
740	echo "IGCM_sys_Mirror_libIGCM"
741	fi
742
743	typeset RET DEST
744
745	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
746
747	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
748	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
749	RET=$?
750
751	if [ ${RET} -gt 0 ] ; then
752	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
753	cat out_rsync
754	fi
755	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
756	}
757
758	#====================================================
759	#- Call IGCM_sys_Mirror_libIGCM now !
760	if ( $MirrorlibIGCM ) ; then
761	IGCM_sys_Mirror_libIGCM
762	fi
763
764	#D-#==================================================
765	#D-function IGCM_sys_Cp
766	#D-* Purpose: generic cp
767	#D-* Examples:
768	#D-
769	function IGCM_sys_Cp {
770	IGCM_debug_PushStack "IGCM_sys_Cp" $@
771	if ( $DEBUG_sys ) ; then
772	echo "IGCM_sys_Cp :" $@
773	fi
774
775	typeset RET
776
777	echo cp $@ > out_rsync 2>&1
778	\cp $@ >> out_rsync 2>&1
779	RET=$?
780
781	if [ ${RET} -gt 0 ] ; then
782	echo "IGCM_sys_Cp : error."
783	cat out_rsync
784	IGCM_debug_Exit "IGCM_sys_Cp"
785	else
786	\rm out_rsync
787	fi
788	IGCM_debug_PopStack "IGCM_sys_Cp"
789	}
790
791	#D-#==================================================
792	#D-function IGCM_sys_Rm
793	#D-* Purpose: generic rm
794	#D-* Examples:
795	#D-
796	function IGCM_sys_Rm {
797	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
798	if ( $DEBUG_sys ) ; then
799	echo "IGCM_sys_Rm :" $@
800	fi
801
802	typeset RET
803
804	echo rm $@ > out_rsync 2>&1
805	\rm $@ >> out_rsync 2>&1
806	RET=$?
807
808	if [ ${RET} -gt 0 ] ; then
809	echo "IGCM_sys_Rm : error."
810	cat out_rsync
811	IGCM_debug_Exit "IGCM_sys_Rm"
812	else
813	\rm out_rsync
814	fi
815	IGCM_debug_PopStack "IGCM_sys_Rm"
816	}
817
818	#D-#==================================================
819	#D-function IGCM_sys_RmRunDir
820	#D-* Purpose: rm tmpdir (dummy function most of the time batch
821	#D- scheduler will do the job)
822	#D-* Examples:
823	#D-
824	function IGCM_sys_RmRunDir {
825	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
826	if ( $DEBUG_sys ) ; then
827	echo "IGCM_sys_RmRunDir :" $@
828	fi
829
830	typeset RET
831
832	echo rm $@ > out_rsync 2>&1
833	\rm $@ >> out_rsync 2>&1
834	RET=$?
835
836	if [ ${RET} -gt 0 ] ; then
837	echo "IGCM_sys_RmRunDir : error."
838	cat out_rsync
839	IGCM_debug_Exit "IGCM_sys_RmRunDir"
840	else
841	\rm out_rsync
842	fi
843
844	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
845	}
846
847	#D-#==================================================
848	#D-function IGCM_sys_Mv
849	#D-* Purpose: generic move
850	#D-* Examples:
851	#D-
852	function IGCM_sys_Mv {
853	IGCM_debug_PushStack "IGCM_sys_Mv" $@
854	if ( $DEBUG_sys ) ; then
855	echo "IGCM_sys_Mv :" $@
856	fi
857
858	if [ $DRYRUN = 0 ]; then
859
860	typeset RET
861
862	echo mv $@ > out_rsync 2>&1
863	\mv $@ >> out_rsync 2>&1
864	RET=$?
865
866	if [ ${RET} -gt 0 ] ; then
867	echo "IGCM_sys_Mv : error in mv."
868	cat out_rsync
869	IGCM_debug_Exit "IGCM_sys_Mv"
870	else
871	\rm out_rsync
872	fi
873	else
874	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
875	fi
876
877	IGCM_debug_PopStack "IGCM_sys_Mv"
878	}
879
880	#D-#==================================================
881	#D-function IGCM_sys_Put_Dir
882	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
883	#D-* Examples:
884	#D-
885	function IGCM_sys_Put_Dir {
886	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
887	if ( $DEBUG_sys ) ; then
888	echo "IGCM_sys_Put_Dir :" $@
889	fi
890	if [ $DRYRUN = 0 ]; then
891	if [ ! -d ${1} ] ; then
892	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
893	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
894	return
895	fi
896
897	typeset RET
898
899	# Only if we use rsync
900	#IGCM_sys_TestDirArchive $( dirname $2 )
901	#
902	#USUAL WAY
903	\cp -r $1 $2 > out_rsync 2>&1
904	RET=$?
905
906	if [ ${RET} -gt 0 ] ; then
907	echo "IGCM_sys_Put_Dir : error."
908	cat out_rsync
909	IGCM_debug_Exit "IGCM_sys_Put_Dir"
910	else
911	\rm out_rsync
912	fi
913	else
914	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
915	fi
916	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
917	}
918
919	#D-#==================================================
920	#D-function IGCM_sys_Get_Dir
921	#D-* Purpose: Copy a complete directory from $(ARCHIVE)
922	#D-* Examples:
923	#D-
924	function IGCM_sys_Get_Dir {
925	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
926	if ( $DEBUG_sys ) ; then
927	echo "IGCM_sys_Get_Dir :" $@
928	fi
929	if [ $DRYRUN = 0 ]; then
930	# if [ ! -d ${1} ] ; then
931	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
932	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
933	# return
934	# fi
935
936	typeset RET
937
938	# Only if we use rsync
939	#IGCM_sys_TestDirArchive $( dirname $2 )
940	#
941	#USUAL WAY
942	# add dmget (to demigrate all offline files) to reduce time of this command :
943	dmget $1/*
944	\cp -r $1 $2 > out_rsync 2>&1
945	RET=$?
946
947	if [ ${RET} -gt 0 ] ; then
948	echo "IGCM_sys_Get_Dir : error."
949	cat out_rsync
950	IGCM_debug_Exit "IGCM_sys_Get_Dir"
951	else
952	\rm out_rsync
953	fi
954	else
955	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
956	fi
957	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
958	}
959
960	#D-#==================================================
961	#D-function IGCM_sys_Get_Master
962	#D-* Purpose: Copy a complete directory from MASTER filesystem
963	#D-* Examples:
964	#D-
965	function IGCM_sys_Get_Master {
966	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
967	if ( $DEBUG_sys ) ; then
968	echo "IGCM_sys_Get_Master :" $@
969	fi
970	if [ $DRYRUN = 0 ]; then
971	if [ ! -d ${1} ] ; then
972	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
973	IGCM_debug_PopStack "IGCM_sys_Get_Master"
974	return
975	fi
976
977	typeset RET
978
979	#USUAL WAY
980	\cp -r $1 $2 > out_rsync 2>&1
981	RET=$?
982
983	if [ ${RET} -gt 0 ] ; then
984	echo "IGCM_sys_Get_Master : error."
985	cat out_rsync
986	IGCM_debug_Exit "IGCM_sys_Get_Master"
987	fi
988	else
989	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
990	fi
991	IGCM_debug_PopStack "IGCM_sys_Get_Master"
992	}
993
994	#D-#==================================================
995	#D-function IGCM_sys_Put_Rest
996	#D-* Purpose: Put computied restarts on $(ARCHIVE).
997	#D- File and target directory must exist.
998	#D-* Examples:
999	#D-
1000	function IGCM_sys_Put_Rest {
1001	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1002	if ( $DEBUG_sys ) ; then
1003	echo "IGCM_sys_Put_Rest :" $@
1004	fi
1005	if [ $DRYRUN = 0 ]; then
1006	if [ ! -f ${1} ] ; then
1007	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1008	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1009	fi
1010
1011	typeset RET
1012	#
1013	if [ X${JobType} = XRUN ] ; then
1014	IGCM_sys_Chmod 444 ${1}
1015	fi
1016	#
1017	IGCM_sys_TestDirArchive $( dirname $2 )
1018	#
1019	# USUAL WAY
1020	\cp $1 $2 > out_rsync 2>&1
1021	RET=$?
1022
1023	# #RSYNC WITH NETWORK SSH CALL
1024	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} > out_rsync 2>&1
1025	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} >> out_rsync 2>&1
1026
1027	# #RSYNC WITH NFS USE
1028	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1029	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1030
1031	# RET=$?
1032	# IGCM_sys_Rsync_out $RET
1033
1034	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1035	# (( RET=RET+$? ))
1036
1037	if [ ${RET} -gt 0 ] ; then
1038	echo "IGCM_sys_Put_Rest : error."
1039	cat out_rsync
1040	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1041	else
1042	\rm out_rsync
1043	fi
1044	else
1045	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1046	fi
1047	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1048	}
1049
1050	#D-#==================================================
1051	#D-function IGCM_sys_Put_Out
1052	#D-* Purpose: Copy a file on $(ARCHIVE) after have chmod it in readonly
1053	#D-* Examples:
1054	#D-
1055	function IGCM_sys_Put_Out {
1056	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1057	if ( $DEBUG_sys ) ; then
1058	echo "IGCM_sys_Put_Out :" $@
1059	fi
1060	if [ $DRYRUN = 0 ]; then
1061	if [ ! -f ${1} ] ; then
1062	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1063	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1064	return 1
1065	fi
1066	#
1067	IGCM_sys_MkdirArchive $( dirname $2 )
1068	#
1069	typeset RET
1070
1071	#=====================================================
1072	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1073	#=====================================================
1074
1075	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
1076	#if [ $? -eq 0 ] ; then
1077	# typeset WORKPATH FILEPATH
1078	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_OUT_SCR}\|" )
1079	# IGCM_sys_MkdirWork ${WORKPATH}
1080	# FILEPATH=${WORKPATH}/$( basename $2 )
1081	# #
1082	# IGCM_sys_Cp ${1} ${FILEPATH}
1083	#fi
1084
1085	if [ X${JobType} = XRUN ] ; then
1086	if [ X${3} = X ] ; then
1087	IGCM_sys_Chmod 444 ${1}
1088	fi
1089	fi
1090	#
1091	# USUAL WAY
1092	\cp $1 $2 > out_rsync 2>&1
1093	RET=$?
1094
1095	# #RSYNC WITH NETWORK SSH CALL
1096	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} > out_rsync 2>&1
1097	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} >> out_rsync 2>&1
1098
1099	# #RSYNC WITH NFS USE
1100	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1101	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1102
1103	# RET=$?
1104	# IGCM_sys_Rsync_out $RET
1105
1106	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1107	# (( RET=RET+$? ))
1108
1109	if [ ${RET} -gt 0 ] ; then
1110	echo "IGCM_sys_Put_Out : error."
1111	cat out_rsync
1112	IGCM_debug_Exit "IGCM_sys_Put_Out"
1113	else
1114	\rm out_rsync
1115	fi
1116	else
1117	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1118	fi
1119	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1120	return 0
1121	}
1122
1123	#D-#==================================================
1124	#D-function IGCM_sys_Get
1125	#D-* Purpose: Get a file from ${ARCHIVE}
1126	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1127	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
1128	function IGCM_sys_Get {
1129	IGCM_debug_PushStack "IGCM_sys_Get" $@
1130
1131	typeset DEST RET dm_liste ifile target file_work
1132
1133	if ( $DEBUG_sys ) ; then
1134	echo "IGCM_sys_Get :" $@
1135	fi
1136	if [ $DRYRUN -le 2 ]; then
1137	if [ X${1} = X'/l' ] ; then
1138	# test if the first file is present in the old computation :
1139	eval set +A dm_liste \${${2}}
1140	else
1141	eval set +A dm_liste ${1}
1142	fi
1143	eval DEST=\${${#}}
1144
1145	#=====================================================
1146	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1147	#=====================================================
1148
1149	# Is it an R_OUT file (not R_IN) ?
1150	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
1151	#if [ $? -eq 0 ] ; then
1152	# # Yes ? then we try to get it in SCRATCHDIR
1153	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_OUT_SCR}\|g" )
1154	# if [ -f ${file_work[0]} ] ; then
1155	# IGCM_sys_Cp ${file_work[*]} ${DEST}
1156	# IGCM_debug_PopStack "IGCM_sys_Get"
1157	# return
1158	# fi
1159	#fi
1160
1161	# test if the (first) file is present in the old computation :
1162	IGCM_sys_TestFileArchive ${dm_liste[0]}
1163	RET=$?
1164	if [ ${RET} -gt 0 ] ; then
1165	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1166	IGCM_debug_Exit "IGCM_sys_Get"
1167	fi
1168
1169	dmget ${dm_liste[*]} > out_rsync 2>&1
1170	RET=$?
1171
1172	if [ ${RET} -gt 0 ] ; then
1173	echo "WARNING IGCM_sys_Get : demigration error."
1174	cat out_rsync
1175	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1176	fi
1177
1178	#if [ ${RET} -gt 0 ] ; then
1179	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1180	# cat out_rsync
1181	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1182	# sleep 30
1183	# echo "We try another time"
1184	# dmget ${dm_liste[*]} > out_rsync 2>&1
1185	# RET=$?
1186	# if [ ${RET} -gt 0 ] ; then
1187	# echo "ERROR IGCM_sys_Get : again demigration error :"
1188	# cat out_rsync
1189	# IGCM_debug_Exit "IGCM_sys_Get"
1190	# fi
1191	# else
1192	# echo "ERROR IGCM_sys_Get : demigration error :"
1193	# cat out_rsync
1194	# IGCM_debug_Exit "IGCM_sys_Get"
1195	# fi
1196	#fi
1197
1198	#USUAL WAY
1199	if [ X${1} = X'/l' ] ; then
1200	(( RET=0 ))
1201	for target in ${dm_liste[*]} ; do
1202	local_file=$( basename ${target} )
1203	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1204	(( RET = RET + $? ))
1205	done
1206	else
1207	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1208	RET=$?
1209	fi
1210
1211	# #RSYNC WITH NETWORK SSH CALL
1212	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RHOST}:"${dm_liste}" ${RHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1213	# ${RSYNC} ${RSYNC_opt} -e ssh ${RHOST}:"${dm_liste}" ${RHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1214
1215	# #RSYNC WITH NFS USE
1216	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1217	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1218
1219	# RET=$?
1220	# IGCM_sys_Rsync_out $RET
1221
1222	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1223	# (( RET=RET+$? ))
1224
1225	if [ ${RET} -gt 0 ] ; then
1226	echo "IGCM_sys_Get : copy error."
1227	cat out_rsync
1228	IGCM_debug_Exit "IGCM_sys_Get"
1229	else
1230	\rm out_rsync
1231	fi
1232	else
1233	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1234	fi
1235	IGCM_debug_PopStack "IGCM_sys_Get"
1236	}
1237
1238	############################################################## A FINIR !!
1239
1240	#D-#==================================================
1241	#D-function IGCM_sys_GetDate_FichWork
1242	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1243	#D-* Examples:
1244	#D-
1245	function IGCM_sys_GetDate_FichWork {
1246	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1247	if ( $DEBUG_sys ) ; then
1248	echo "IGCM_sys_GetDate_FichWork :" $@
1249	fi
1250	# donne la date filesys d'un fichier sur la machine work
1251	IGCM_debug_PopStack "IGCM_sys_FichWork"
1252	}
1253
1254	#D-#==================================================
1255	#D-function IGCM_sys_GetDate_FichArchive
1256	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1257	#D-* Examples:
1258	#D-
1259	function IGCM_sys_GetDate_FichArchive {
1260	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1261	if ( $DEBUG_sys ) ; then
1262	echo "IGCM_sys_GetDate_FichArchive :" $@
1263	fi
1264	IGCM_debug_PopStack "IGCM_sys_FichArchive"
1265	}
1266
1267	##############################################################
1268	# REBUILD OPERATOR
1269
1270	function IGCM_sys_rebuild {
1271	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1272	if ( $DEBUG_sys ) ; then
1273	echo "IGCM_sys_rebuild :" $@
1274	fi
1275	/home/cont003/p86ipsl/X64/bin/rebuild -f -o $@
1276	if [ $? -gt 0 ] ; then
1277	echo "IGCM_sys_rebuild : erreur ${@}."
1278	IGCM_debug_Exit "rebuild"
1279	fi
1280
1281	IGCM_debug_PopStack "IGCM_sys_rebuild"
1282	}
1283
1284	############################################################
1285	# Activate Running Environnment Variables
1286
1287	function IGCM_sys_activ_variables {
1288	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1289	if ( $DEBUG_sys ) ; then
1290	echo "IGCM_sys_activ_variables"
1291	fi
1292	# ulimit -s 2097152
1293
1294	typeset max_omp
1295
1296	ulimit -s unlimited
1297
1298	(( max_omp = 0 ))
1299	for comp in ${config_ListOfComponents[*]} ; do
1300
1301	eval ExeNameIn=\${config_Executable_${comp}[0]}
1302	eval ExeNameOut=\${config_Executable_${comp}[1]}
1303
1304
1305	# Only if we really have an executable for the component :
1306	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1307	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1308
1309	if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
1310
1311	(( max_omp = comp_proc_omp_loc ))
1312	fi
1313	fi
1314	done
1315	if [ ${max_omp} -gt 1 ] ; then
1316	module load openmp/${max_omp}thds
1317	fi
1318
1319	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1320	}
1321
1322	############################################################
1323	# Desactivate Running Environnment Variables
1324
1325	function IGCM_sys_desactiv_variables {
1326	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1327	if ( $DEBUG_sys ) ; then
1328	echo "IGCM_sys_desactiv_variables"
1329	fi
1330	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1331	}
1332
1333
1334	# function IGCM_sys_build_run_file {
1335	############################################################
1336	# Build run file
1337
1338	# IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
1339	# if ( $DEBUG_sys ) ; then
1340	# echo "IGCM_sys_build_run_file " $@
1341	# fi
1342
1343	# # set Number of processors for OCE here
1344	# NUM_PROC_OCE=5
1345
1346	# (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1))
1347	# (( nb_tot_m1 = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE ))
1348	# if [ $1 = MPI1 ]; then
1349	# cat <<EOF > run_file
1350	# -np 1 ./oasis
1351	# -np ${NUM_PROC_ATM} ./lmdz.x
1352	# -np ${NUM_PROC_OCE} ./opa.xx
1353	# EOF
1354	# config_UserChoices_JobRunOptions='"--app"'
1355	# IGCM_sys_Chmod u+x run_file
1356	# fi
1357	# IGCM_debug_PopStack "IGCM_sys_build_run_file"
1358
1359	# }
1360
1361	############################################################
1362	# Build MPI/OMP scripts
1363	function IGCM_sys_build_execution_scripts
1364	{
1365	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1366	if ( $DEBUG_sys ) ; then
1367	echo "IGCM_sys_build_execution_scripts " $@
1368	fi
1369
1370	typeset nodes listnodes init_node node_num_current start_num init_exec comp ExeNameIn ExeNameOut
1371	typeset nombre_restant_node node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
1372	typeset num_corempi nombre_restant_node nombre_restant_comp
1373
1374	# Verification with PBS parameter
1375	if [ X${BATCH_NUM_PROC_TOT} != X ] ; then
1376
1377	if ( ${OK_PARA_MPI} ) ; then
1378	EXECUTION=${HOST_MPIRUN_COMMAND}
1379	else
1380	if [ ${BATCH_NUM_PROC_TOT} -eq 1 ] ; then
1381	EXECUTION=${MPIRUN_COMMAND:="time "}
1382	fi
1383	fi
1384	else
1385	if ( ! ${OK_PARA_MPI} ) ; then
1386	EXECUTION=${MPIRUN_COMMAND:="time "}
1387	fi
1388	fi
1389
1390	if [ ${PROCESSUS_NUMBER} -gt 1 ] ; then
1391
1392	# Hosts treatment
1393
1394	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
1395
1396	i=0
1397	rm -f hosts
1398	IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
1399	for nodes in `cat hosts.tmp`
1400	do
1401	host[$i]=$nodes
1402	echo "${host[$i]} slots=8 max_slots=8" >> hosts
1403	IGCM_debug_Print 1 ${host[$i]}
1404	i=$((i+1))
1405	done
1406	rm -f hosts.tmp
1407
1408	listnodes=${host[*]}
1409
1410	EXECUTION="${EXECUTION} -hostfile hosts"
1411	fi
1412
1413	# Initialisation
1414
1415	init_node=y
1416	node_num_current=0
1417	start_num=0
1418	init_exec=n
1419
1420
1421	# Test : if oasis is there, we put it at the first position
1422
1423	for comp in ${config_ListOfComponents[*]} ; do
1424
1425	if [ "X${comp}" = "XCPL" ] ; then
1426
1427	eval ExeNameIn=\${config_Executable_${comp}[0]}
1428	eval ExeNameOut=\${config_Executable_${comp}[1]}
1429
1430	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1431	echo "" >> script_${ExeNameOut}.ksh
1432	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1433	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1434	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1435	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1436	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1437
1438	init_node=n
1439
1440	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
1441	node_num_current=0
1442	node_current=${host[${node_num_current}]}
1443
1444	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
1445
1446	init_exec=y
1447	start_num=1
1448
1449	fi
1450
1451	done
1452
1453	# Then loop on the components (except for oasis)
1454
1455	for comp in ${config_ListOfComponents[*]} ; do
1456
1457	eval ExeNameIn=\${config_Executable_${comp}[0]}
1458	eval ExeNameOut=\${config_Executable_${comp}[1]}
1459
1460
1461	# Only if we really have an executable for the component :
1462	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1463
1464	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1465	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1466
1467	if ( ${OK_PARA_MPI} ) ; then
1468
1469	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1470	# echo "set -vx" >> script_${ExeNameOut}.ksh
1471	echo "" >> script_${ExeNameOut}.ksh
1472	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1473	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1474	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1475	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1476	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
1477	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1478	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1479
1480	node_num=0
1481
1482	# We define the number of MPI process to be assigned for the component
1483
1484	nombre_restant_comp=${comp_proc_mpi_loc}
1485
1486	# Loop on the allocated nodes
1487
1488	for node in ${listnodes} ; do
1489
1490	# We go to the current node
1491
1492	if [ ${node_num} = ${node_num_current} ] ; then
1493
1494	node_current=${host[${node_num_current}]}
1495
1496	# If first time on the node : initialisation
1497
1498	if [ ${init_node} = y ] ; then
1499	nombre_restant_node=${NUM_COREPERNODE}
1500	fi
1501
1502	# Test on the number of OMP threads
1503
1504	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
1505	(( node_num = node_num + 1 ))
1506	node_num_current=${node_num}
1507	init_node=y
1508	continue
1509	fi
1510
1511	# Number of MPI process to assign
1512
1513	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
1514
1515	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
1516	num_corempi=${nombre_restant_comp}
1517	fi
1518
1519	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
1520	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
1521
1522	if [ ${init_exec} = y ] ; then
1523	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1524	else
1525	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1526	init_exec=y
1527	fi
1528
1529	(( start_num = num_corempi + start_num ))
1530
1531	else
1532
1533	(( node_num = node_num + 1 ))
1534	continue
1535	fi
1536
1537	# Test on the number of core/process remaining on the node/component
1538
1539	if [ ${nombre_restant_node} = 0 ] ; then
1540	(( node_num = node_num + 1 ))
1541	node_num_current=${node_num}
1542	init_node=y
1543
1544	if [ ${nombre_restant_comp} = 0 ] ; then
1545	break 1
1546	fi
1547	else
1548
1549	node_num_current=${node_num}
1550	init_node=n
1551
1552	if [ ${nombre_restant_comp} = 0 ] ; then
1553	break 1
1554	fi
1555	fi
1556	done
1557	else
1558	EXECUTION="${EXECUTION} ./${ExeNameOut}"
1559	fi
1560	fi
1561	done
1562
1563	IGCM_debug_Print 1 "sys Titane : La commande d execution est "
1564	IGCM_debug_Print 1 $EXECUTION
1565
1566	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1567	}
1568
1569
1570
1571	##############################################################
1572	# NCO OPERATOR
1573
1574	function IGCM_sys_ncatted {
1575	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
1576	if ( $DEBUG_sys ) ; then
1577	echo "IGCM_sys_ncatted :" $@
1578	fi
1579	ncatted "$@"
1580	if [ $? -gt 0 ] ; then
1581	echo "IGCM_sys_ncatted : erreur ${@}."
1582	IGCM_debug_Exit "ncatted"
1583	fi
1584
1585	IGCM_debug_PopStack "IGCM_sys_ncatted"
1586	}
1587
1588	function IGCM_sys_ncbo {
1589	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
1590	if ( $DEBUG_sys ) ; then
1591	echo "IGCM_sys_ncbo :" $@
1592	fi
1593	ncbo $@
1594	if [ $? -gt 0 ] ; then
1595	echo "IGCM_sys_ncbo : erreur ${@}."
1596	IGCM_debug_Exit "ncbo"
1597	fi
1598
1599	IGCM_debug_PopStack "IGCM_sys_ncbo"
1600	}
1601
1602	function IGCM_sys_ncdiff {
1603	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
1604	if ( $DEBUG_sys ) ; then
1605	echo "IGCM_sys_ncdiff :" $@
1606	fi
1607	ncdiff $@
1608	if [ $? -gt 0 ] ; then
1609	echo "IGCM_sys_ncdiff : erreur ${@}."
1610	IGCM_debug_Exit "ncdiff"
1611	fi
1612
1613	IGCM_debug_PopStack "IGCM_sys_ncdiff"
1614	}
1615
1616	function IGCM_sys_ncea {
1617	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
1618	if ( $DEBUG_sys ) ; then
1619	echo "IGCM_sys_ncea :" $@
1620	fi
1621	ncea $@
1622	if [ $? -gt 0 ] ; then
1623	echo "IGCM_sys_ncea : erreur ${@}."
1624	IGCM_debug_Exit "ncea"
1625	fi
1626
1627	IGCM_debug_PopStack "IGCM_sys_ncea"
1628	}
1629
1630	function IGCM_sys_ncecat {
1631	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
1632	if ( $DEBUG_sys ) ; then
1633	echo "IGCM_sys_ncecat :" $@
1634	fi
1635	ncecat $@
1636	if [ $? -gt 0 ] ; then
1637	echo "IGCM_sys_ncecat : erreur ${@}."
1638	IGCM_debug_Exit "ncecat"
1639	fi
1640
1641	IGCM_debug_PopStack "IGCM_sys_ncecat"
1642	}
1643
1644	function IGCM_sys_ncflint {
1645	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
1646	if ( $DEBUG_sys ) ; then
1647	echo "IGCM_sys_ncflint :" $@
1648	fi
1649	ncflint $@
1650	if [ $? -gt 0 ] ; then
1651	echo "IGCM_sys_ncflint : erreur ${@}."
1652	IGCM_debug_Exit "ncflint"
1653	fi
1654
1655	IGCM_debug_PopStack "IGCM_sys_ncflint"
1656	}
1657
1658	function IGCM_sys_ncks {
1659	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
1660	if ( $DEBUG_sys ) ; then
1661	echo "IGCM_sys_ncks :" $@
1662	fi
1663	ncks $@
1664	if [ $? -gt 0 ] ; then
1665	echo "IGCM_sys_ncks : erreur ${@}."
1666	IGCM_debug_Exit "ncks"
1667	fi
1668
1669	IGCM_debug_PopStack "IGCM_sys_ncks"
1670	}
1671
1672	function IGCM_sys_ncpdq {
1673	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
1674	if ( $DEBUG_sys ) ; then
1675	echo "IGCM_sys_ncpdq :" $@
1676	fi
1677	ncpdq $@
1678	if [ $? -gt 0 ] ; then
1679	echo "IGCM_sys_ncpdq : erreur ${@}."
1680	IGCM_debug_Exit "ncpdq"
1681	fi
1682
1683	IGCM_debug_PopStack "IGCM_sys_ncpdq"
1684	}
1685
1686	function IGCM_sys_ncra {
1687	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
1688	if ( $DEBUG_sys ) ; then
1689	echo "IGCM_sys_ncra :" $@
1690	fi
1691	ncra $@
1692	if [ $? -gt 0 ] ; then
1693	echo "IGCM_sys_ncra : erreur ${@}."
1694	IGCM_debug_Exit "ncra"
1695	fi
1696
1697	IGCM_debug_PopStack "IGCM_sys_ncra"
1698	}
1699
1700	function IGCM_sys_ncrcat {
1701	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
1702	if ( $DEBUG_sys ) ; then
1703	echo "IGCM_sys_ncrcat :" $@
1704	fi
1705	ncrcat $@
1706	if [ $? -gt 0 ] ; then
1707	echo "IGCM_sys_ncrcat : erreur ${@}."
1708	# IGCM_debug_Exit "ncrcat"
1709	fi
1710
1711	IGCM_debug_PopStack "IGCM_sys_ncrcat"
1712	}
1713
1714	function IGCM_sys_ncrename {
1715	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
1716	if ( $DEBUG_sys ) ; then
1717	echo "IGCM_sys_ncrename :" $@
1718	fi
1719	ncrename $@
1720	if [ $? -gt 0 ] ; then
1721	echo "IGCM_sys_ncrename : erreur ${@}."
1722	IGCM_debug_Exit "ncrename"
1723	fi
1724
1725	IGCM_debug_PopStack "IGCM_sys_ncrename"
1726	}
1727
1728	function IGCM_sys_ncwa {
1729	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
1730	if ( $DEBUG_sys ) ; then
1731	echo "IGCM_sys_ncwa :" $@
1732	fi
1733	ncwa $@
1734	if [ $? -gt 0 ] ; then
1735	echo "IGCM_sys_ncwa : erreur ${@}."
1736	IGCM_debug_Exit "ncwa"
1737	fi
1738
1739	IGCM_debug_PopStack "IGCM_sys_ncwa"
1740	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: