Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ciclad.ksh @ 1204

Last change on this file since 1204 was 1201, checked in by sdipsl, 9 years ago
add jobWarningDelay in seconds to supervision messages see #253 (threshold for heartbeat) add accountingProject to supervision messages see #255 (may ease gencmip6 data production tracking)
Property svn:keywords set to `Revision Date Author`
File size: 35.7 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil
5	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for ciclad
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=${MASTER:=$( hostname )}
71	# project name
72	typeset PROJECT=NONE
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=${PBS_WALLTIME}
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=qsub}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=${MASTER}
89
90	#====================================================
91	# Access to module command
92	#====================================================
93	. /usr/share/Modules/init/ksh
94
95	#====================================================
96	# Set environment tools (ferret, nco, cdo, rebuild, ...)
97	#====================================================
98	. /home/igcmg/.atlas_env_ciclad_ksh
99	export PATH=${PATH}:/home/igcmg/rebuild/src_X64_CICLAD/modipsl_v2_2_3_netcdf4.2/bin/
100
101	#====================================================
102	# Host specific DIRECTORIES
103	#====================================================
104
105	#====================================================
106	#- MirrorlibIGCM for frontend
107	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
108
109	#====================================================
110	#- libIGCM_POST for frontend
111	typeset -r libIGCM_POST=${libIGCM}
112
113	#====================================================
114	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
115	typeset -r R_EXE="${MODIPSL}/bin"
116
117	#====================================================
118	#- SUBMIT_DIR : submission dir
119	if [ X${PBS_O_WORKDIR} != X ] ; then
120	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
121	else
122	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
123	fi
124
125	#====================================================
126	#- IN
127	typeset -r R_IN=${R_IN:=/prodigfs/ipslfs/igcmg/IGCM}
128	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/prodigfs/ipslfs/igcmg/IGCM}
129
130	#====================================================
131	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
132	if [ X${PBS_JOBID} != X ] ; then
133	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/data/${LOGIN}/RUN_DIR/${PBS_JOBID}_${$}}
134	else
135	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/tmp/tmp$$}
136	fi
137
138	#====================================================
139	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
140	typeset -r OUTCOMMAND_PATH=/tmp
141
142	#====================================================
143	#- HOST_MPIRUN_COMMAND
144	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
145
146	#====================================================
147	#- Max number of arguments passed to nco operator or demigration command
148	UNIX_MAX_LIMIT=360
149
150	#====================================================
151	#- set PackDefault to false on ciclad
152	PackDefault=false
153
154	#====================================================
155	#- Number of core per node (max number of OpenMP task)
156	NUM_COREPERNODE=1
157
158	#====================================================
159	#- Default number of MPI task for IPSL coupled model
160	#- required for backward compatibility
161	#-
162	DEFAULT_NUM_PROC_OCE=1
163	DEFAULT_NUM_PROC_CPL=1
164	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
165	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
166
167	#D-#==================================================
168	#D-function IGCM_sys_defineArchives
169	#D-* Purpose:
170	#D-* Define ARCHIVE : Dedicated to large files
171	#D-* Define STORAGE : Dedicated to small/medium files
172	#D-* Define R_OUT : Output tree located on ARCHIVE
173	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
174	#D-* Define R_BUF : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
175	#D-* if SpaceName=TEST nothing special will hapen
176	#D-* Examples:
177	#D-
178	function IGCM_sys_defineArchives {
179	IGCM_debug_PushStack "IGCM_sys_defineArchives"
180
181	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
182	#====================================================
183	#- ARCHIVE (dedicated to large files)
184	ARCHIVE=${config_UserChoices_ARCHIVE}
185	else
186	#====================================================
187	#- ARCHIVE (dedicated to large files)
188	ARCHIVE=${ARCHIVE:=/data/${LOGIN}}
189	fi
190
191	if [ ! X${config_UserChoices_STORAGE} = X ]; then
192	#====================================================
193	#- STORAGE (dedicated to small/medium files)
194	STORAGE=${config_UserChoices_STORAGE}
195	else
196	#====================================================
197	#- STORAGE (dedicated to small/medium files)
198	STORAGE=${ARCHIVE}
199	fi
200
201	# ON CICLAD NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
202
203	#====================================================
204	#- R_OUT
205	R_OUT=${ARCHIVE}/IGCM_OUT
206
207	#====================================================
208	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
209	R_FIG=${STORAGE}/IGCM_OUT
210
211	#====================================================
212	#- R_BUF (ONLY FOR double copy an scratch)
213	R_BUF=${STORAGE}/IGCM_OUT
214
215	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
216	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
217	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
218
219	IGCM_debug_PopStack "IGCM_sys_defineArchives"
220	}
221
222	#D-#==================================================
223	#D-function IGCM_sys_RshArchive
224	#D-* Purpose: Archive rsh command
225	#D-* Examples:
226	#D-
227	function IGCM_sys_RshArchive {
228	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
229	/bin/ksh <<-EOF
230	${@}
231	EOF
232	status=$?
233	if [ ${status} -gt 0 ] ; then
234	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
235	IGCM_debug_Exit "IGCM_sys_RshArchive"
236	fi
237	IGCM_debug_PopStack "IGCM_sys_RshArchive"
238	}
239
240	#D-#==================================================
241	#D-function IGCM_sys_RshArchive_NoError
242	#D-* Purpose: Archive rsh command, without error
243	#D-* used only in monitoring.job
244	#D-* Examples:
245	#D-
246	function IGCM_sys_RshArchive_NoError {
247	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
248	/bin/ksh <<-EOF
249	${@} 2> /dev/null
250	EOF
251	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
252	}
253
254	#D-#==================================================
255	#D-function IGCM_sys_MkdirArchive
256	#D-* Purpose: Mkdir on Archive
257	#D-* Examples:
258	#D-
259	function IGCM_sys_MkdirArchive {
260	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
261	if ( $DEBUG_sys ) ; then
262	echo "IGCM_sys_MkdirArchive :" $@
263	fi
264	#- creation de repertoire sur le serveur fichier
265	if [ ! -d ${1} ]; then
266	\mkdir -p $1
267	status=$?
268
269	if [ ${status} -gt 0 ] ; then
270	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
271	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
272	fi
273	fi
274	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
275	}
276
277	#D-#==================================================
278	#D-function IGCM_sys_TestDirArchive
279	#D-* Purpose: Test Directory that must exists on Archive
280	#D-* Examples:
281	#D-
282	function IGCM_sys_TestDirArchive {
283	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
284	if ( $DEBUG_sys ) ; then
285	echo "IGCM_sys_TestDirArchive :" $@
286	fi
287	typeset ExistFlag
288	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
289	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
290	return ${ExistFlag}
291	}
292
293	#D-#==================================================
294	#D-function IGCM_sys_IsFileArchived
295	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
296	#D-* Examples:
297	#D-
298	function IGCM_sys_IsFileArchived {
299	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
300	if ( $DEBUG_sys ) ; then
301	echo "IGCM_sys_IsFileArchived :" $@
302	fi
303	typeset IsArchivedFlag
304	# Never archived for this system
305	IsArchivedFlag=1
306	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
307
308	return ${IsArchivedFlag}
309	}
310
311	#D-#==================================================
312	#D-function IGCM_sys_TestFileArchive
313	#D-* Purpose: Test file that must NOT EXISTS on Archive
314	#D-* Examples:
315	#D-
316	function IGCM_sys_TestFileArchive {
317	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
318	typeset ExistFlag
319	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
320	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
321
322	return ${ExistFlag}
323	}
324
325	#D-#==================================================
326	#D-function IGCM_sys_CountFileArchive
327	#D-* Purpose: Count files on Archive filesystem
328	#D-* Examples:
329	#D-
330	function IGCM_sys_CountFileArchive {
331	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
332	ls ${@} 2>/dev/null \| wc -l
333	if [ $? -gt 0 ] ; then
334	echo "IGCM_sys_CountFileArchive : erreur."
335	fi
336	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
337	}
338
339	#D-#==================================================
340	#D-function IGCM_sys_Tree
341	#D-* Purpose: Tree directories with files on ${ARCHIVE}
342	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
343	#D-
344	function IGCM_sys_Tree {
345	IGCM_debug_PushStack "IGCM_sys_Tree" $@
346	if ( $DEBUG_sys ) ; then
347	echo "IGCM_sys_Tree :" $@
348	fi
349
350	\tree -f $@
351
352	IGCM_debug_PopStack "IGCM_sys_Tree"
353	}
354
355	#D-#==================================================
356	#D-function IGCM_sys_Qsub
357	#D-* Purpose: Qsub new job
358	#D-* Examples:
359	#D-
360	function IGCM_sys_Qsub {
361	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
362	if ( $DEBUG_sys ) ; then
363	echo "IGCM_sys_Qsub :" $@
364	fi
365	typeset options status
366	options="-o ${SUBMIT_DIR}/${Script_Output}"
367	/usr/bin/qsub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
368	status=$?
369
370	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
371	if [ ${status} -gt 0 ] ; then
372	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
373	IGCM_debug_Exit "IGCM_sys_Qsub"
374	else
375	JobID=$( gawk {'print $1'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
376	fi
377	IGCM_debug_PopStack "IGCM_sys_Qsub"
378	}
379
380	#D-#==================================================
381	#D-function IGCM_sys_QsubPost
382	#D-* Purpose: Qsub new job on scalaire
383	#D-* Examples:
384	#D-
385	function IGCM_sys_QsubPost {
386	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
387	if ( $DEBUG_sys ) ; then
388	echo "IGCM_sys_QsubPost :" $@
389	fi
390	typeset options status
391	options="-o ${POST_DIR}/${Script_Post_Output}.out -v ${listVarEnv}"
392	/usr/bin/qsub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
393	status=$?
394
395	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
396	if [ ${status} -gt 0 ] ; then
397	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
398	IGCM_debug_Exit "IGCM_sys_QsubPost"
399	else
400	JobID=$( gawk {'print $1'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
401	fi
402	IGCM_debug_PopStack "IGCM_sys_QsubPost"
403	}
404
405	#D-*************************
406	#D- File transfer functions
407	#D-*************************
408	#D-
409
410	#D-#==================================================
411	#D-function IGCM_sys_RmRunDir
412	#D-* Purpose: rm tmpdir (dummy function most of the time batch
413	#D- scheduler will do the job)
414	#D-* Examples:
415	#D-
416	function IGCM_sys_RmRunDir {
417	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
418	if ( $DEBUG_sys ) ; then
419	echo "IGCM_sys_RmRunDir :" $@
420	fi
421
422	typeset status
423
424	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
425	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
426	status=$?
427
428	if [ ${status} -gt 0 ] ; then
429	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
430	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
431	IGCM_debug_Exit "IGCM_sys_RmRunDir"
432	else
433	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
434	fi
435	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
436	}
437
438	#D-#==================================================
439	#D-function IGCM_sys_Put_Dir
440	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
441	#D-* Examples:
442	#D-
443	function IGCM_sys_Put_Dir {
444	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
445	if ( $DEBUG_sys ) ; then
446	echo "IGCM_sys_Put_Dir :" $@
447	fi
448	if [ $DRYRUN = 0 ]; then
449	if [ ! -d ${1} ] ; then
450	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
451	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
452	return
453	fi
454
455	typeset status
456
457	# Only if we use rsync
458	#IGCM_sys_TestDirArchive $( dirname $2 )
459	#
460	#USUAL WAY
461	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
462	status=$?
463
464	if [ ${status} -gt 0 ] ; then
465	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
466	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
467	IGCM_debug_Exit "IGCM_sys_Put_Dir"
468	else
469	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
470	fi
471	fi
472	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
473	}
474
475	#D-#==================================================
476	#D-function IGCM_sys_Get_Dir
477	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
478	#D-* Examples:
479	#D-
480	function IGCM_sys_Get_Dir {
481	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
482	if ( $DEBUG_sys ) ; then
483	echo "IGCM_sys_Get_Dir :" $@
484	fi
485	if [ $DRYRUN = 0 ]; then
486	typeset NB_ESSAI DELAI status i
487	# number of tentative
488	NB_ESSAI=3
489	# time delay between tentative
490	DELAI=2
491
492	#
493	# USUAL WAY
494	i=0
495	while [ $i -lt $NB_ESSAI ] ; do
496	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
497	status=$?
498	if [ ${status} -gt 0 ] ; then
499	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
500	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
501	sleep $DELAI
502	else
503	break
504	fi
505	(( i = i + 1 ))
506	done
507
508	if [ ${status} -gt 0 ] ; then
509	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
510	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
511	IGCM_debug_Exit "IGCM_sys_Get_Dir"
512	else
513	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
514	fi
515	fi
516	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
517	}
518
519	#D-#==================================================
520	#D-function IGCM_sys_Put_Rest
521	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
522	#D- File and target directory must exist.
523	#D-* Examples:
524	#D-
525	function IGCM_sys_Put_Rest {
526	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
527	if ( $DEBUG_sys ) ; then
528	echo "IGCM_sys_Put_Rest :" $@
529	fi
530	if [ $DRYRUN = 0 ]; then
531	if [ ! -f ${1} ] ; then
532	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
533	IGCM_debug_Exit "IGCM_sys_Put_Rest"
534	fi
535
536	typeset status
537	#
538	# USUAL WAY
539	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
540	status=$?
541
542	# #RSYNC WITH NETWORK SSH CALL
543	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
544	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
545
546	# #RSYNC WITH NFS USE
547	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
548	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
549
550	# status=$?
551	# IGCM_sys_Rsync_out $status
552
553	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
554	# (( status=status+$? ))
555
556	if [ ${status} -gt 0 ] ; then
557	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
558	[ -f ${1} ] && ls -l ${1}
559	[ -f ${2} ] && ls -l ${2}
560	[ -f ${2}/${1} ] && ls -l ${2}/${1}
561	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
562	IGCM_debug_Exit "IGCM_sys_Put_Rest"
563	else
564
565	if [ X${JobType} = XRUN ] ; then
566	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
567	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
568	fi
569
570	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
571	fi
572	fi
573	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
574	}
575
576	#D-#==================================================
577	#D-function IGCM_sys_Put_Out
578	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
579	#D-* Examples:
580	#D-
581	function IGCM_sys_Put_Out {
582	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
583	if ( $DEBUG_sys ) ; then
584	echo "IGCM_sys_Put_Out :" $@
585	fi
586
587	typeset status
588
589	if [ $DRYRUN = 0 ]; then
590	if [ ! -f ${1} ] ; then
591	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
592	IGCM_debug_PopStack "IGCM_sys_Put_Out"
593	return 1
594	fi
595	#
596	IGCM_sys_MkdirArchive $( dirname $2 )
597	#
598	if [ X${JobType} = XRUN ] ; then
599	if [ X${3} = X ] ; then
600	IGCM_sys_Chmod 444 ${1}
601	fi
602	fi
603	#
604
605	echo ${RSYNC} ${RSYNC_opt} $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
606	${RSYNC} ${RSYNC_opt} $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
607	status=$?
608	IGCM_sys_Rsync_out $status
609
610	${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
611	(( status=status+$? ))
612
613	if [ ${status} -gt 0 ] ; then
614	IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
615	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
616	IGCM_debug_Exit "IGCM_sys_Put_Out"
617	else
618	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
619	fi
620	fi
621	IGCM_debug_PopStack "IGCM_sys_Put_Out"
622	return 0
623	}
624
625	#D-#==================================================
626	#D-function IGCM_sys_Get
627	#D-* Purpose: Get a file from ${ARCHIVE}
628	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
629	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
630	function IGCM_sys_Get {
631	IGCM_debug_PushStack "IGCM_sys_Get" $@
632
633	typeset DEST status dm_liste
634
635	if ( $DEBUG_sys ) ; then
636	echo "IGCM_sys_Get :" $@
637	fi
638	if [ $DRYRUN -le 2 ]; then
639	if [ X${1} = X'/l' ] ; then
640	eval set +A dm_liste \${${2}}
641	else
642	dm_liste=${1}
643	fi
644	eval DEST=\${${#}}
645
646	# test if the (first) file is present in the old computation :
647	IGCM_sys_TestFileArchive ${dm_liste[0]}
648	status=$?
649	if [ ${status} -gt 0 ] ; then
650	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
651	IGCM_debug_Exit "IGCM_sys_Get"
652	return
653	fi
654
655	#USUAL WAY
656	\cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
657	status=$?
658
659	if [ ${status} -gt 0 ] ; then
660	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
661	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
662	IGCM_debug_Exit "IGCM_sys_Get"
663	else
664	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
665	fi
666	fi
667	IGCM_debug_PopStack "IGCM_sys_Get"
668	}
669
670	#D-#==================================================
671	#D-function IGCM_sys_GetDate_Monitoring
672	#D-* Purpose: get the last year for which the monitoring has been computed
673	#D-* Examples:
674	#D-
675	function IGCM_sys_GetDate_Monitoring {
676	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
677	if ( $DEBUG_sys ) ; then
678	echo "IGCM_sys_GetDate_Monitoring :" $@
679	fi
680
681	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
682
683	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
684	}
685
686	#D-#==================================================
687	#D-function IGCM_sys_Dods_Rm
688	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
689	#D-* Examples:
690	#D-
691	function IGCM_sys_Dods_Rm {
692	if ( $DEBUG_sys ) ; then
693	echo "IGCM_sys_Dods_Rm :" $@
694	fi
695	return 0
696	}
697
698	#D-#==================================================
699	#D-function IGCM_sys_Dods_Cp
700	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
701	#D-* Examples:
702	#D-
703	function IGCM_sys_Dods_Cp {
704	if ( $DEBUG_sys ) ; then
705	echo "IGCM_sys_Dods_Cp :" $@
706	fi
707	return 0
708	}
709
710	#D-#==================================================
711	#D-function IGCM_sys_Put_Dods
712	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
713	#D-* Examples:
714	#D-
715	function IGCM_sys_Put_Dods {
716	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
717	if ( $DEBUG_sys ) ; then
718	echo "IGCM_sys_Put_Dods :" $@
719	fi
720	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
721	}
722
723	##############################################################
724	# REBUILD OPERATOR
725
726	#D-#==================================================
727	#D-function IGCM_sys_sync
728	#D-* Purpose: flush buffer on disk (dummy function on Ada)
729	#D-* Examples:
730	#D-
731	function IGCM_sys_sync {
732	IGCM_debug_PushStack "IGCM_sys_sync" $@
733	if ( $DEBUG_sys ) ; then
734	echo "IGCM_sys_sync :" $@
735	echo "Dummy call, let the system do that."
736	fi
737	IGCM_debug_PopStack "IGCM_sys_sync"
738	}
739
740	############################################################
741	# Activate Running Environnment Variables
742
743	#D-#==================================================
744	#D-function IGCM_sys_desactiv_variables
745	#D-* Purpose: set environement variables prior to execution
746	#D-* Examples:
747	#D-
748	function IGCM_sys_activ_variables {
749	IGCM_debug_PushStack "IGCM_sys_activ_variables"
750	if ( $DEBUG_sys ) ; then
751	echo "IGCM_sys_activ_variables"
752	fi
753
754	# --------------------------------------------------------------------
755	#D- MPI specifications
756	# --------------------------------------------------------------------
757
758	# --------------------------------------------------------------------
759	#D- Other specifications
760	# --------------------------------------------------------------------
761
762	ulimit -s unlimited
763
764	IGCM_debug_PopStack "IGCM_sys_activ_variables"
765	}
766
767	############################################################
768	# Desactivate Running Environnment Variables
769
770	#D-#==================================================
771	#D-function IGCM_sys_desactiv_variables
772	#D-* Purpose: unset environement variables after execution
773	#D-* Examples:
774	#D-
775	function IGCM_sys_desactiv_variables {
776	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
777	if ( $DEBUG_sys ) ; then
778	echo "IGCM_sys_desactiv_variables"
779	fi
780	# --------------------------------------------------------------------
781	#D- MPI specifications
782	# --------------------------------------------------------------------
783
784	# --------------------------------------------------------------------
785	#D- Other specifications
786	# --------------------------------------------------------------------
787
788	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
789	}
790
791	############################################################
792	# Build MPI/OMP scripts run file (dummy function)
793
794	#D-#==================================================
795	#D-function IGCM_sys_build_run_file
796	#D-* Purpose: build run file (deprecated)
797	#D-* Examples:
798	#D-
799	function IGCM_sys_build_run_file {
800
801	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
802
803	}
804
805	############################################################
806	# Build MPI/OMP scripts
807
808	#D-#==================================================
809	#D-function IGCM_sys_build_execution_scripts
810	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
811	#D-* Examples:
812	#D-
813	function IGCM_sys_build_execution_scripts
814	{
815	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
816	if ( $DEBUG_sys ) ; then
817	echo "IGCM_sys_build_execution_scripts " $@
818	fi
819
820	typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
821	typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
822	typeset num_corempi nombre_restant_node nombre_restant_comp
823
824	if ( ${OK_PARA_MPMD} ) ; then
825
826	if [ -f run_file ] ; then
827	IGCM_sys_Rm -f run_file
828	fi
829	touch run_file
830
831	if ( ${OK_PARA_OMP} ) ; then
832
833	# Hosts treatment
834
835	${HOST_MPIRUN_COMMAND} hostname \| sort \| uniq > hosts.tmp
836
837	i=0
838	rm -f hosts
839	IGCM_debug_Print 1 "sys Ciclad, Hosts avaible :"
840	for nodes in `cat hosts.tmp` ; do
841	host[$i]=$nodes
842	echo "${host[$i]} slots=1 max_slots=1" >> hosts
843	IGCM_debug_Print 1 ${host[$i]}
844	i=$((i+1))
845	done
846	rm -f hosts.tmp
847
848	listnodes=${host[*]}
849
850	EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
851
852	# Initialisation
853
854	init_node=y
855	node_num_current=0
856	start_num=0
857	init_exec=n
858
859	# Build run_file
860
861	# First loop on the components for the coupler ie oasis (only if oasis3)
862	# the coupler ie oasis3 must be the first one
863	for comp in ${config_ListOfComponents[*]} ; do
864
865	if [ "X${comp}" = "XCPL" ] ; then
866
867	eval ExeNameIn=\${config_Executable_${comp}[0]}
868	eval ExeNameOut=\${config_Executable_${comp}[1]}
869
870	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
871	echo "" >> script_${ExeNameOut}.ksh
872	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
873	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
874	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
875	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
876	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
877
878	init_node=n
879
880	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
881	node_num_current=0
882	node_current=${host[${node_num_current}]}
883
884	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
885
886	init_exec=y
887	start_num=1
888
889	fi
890
891	done
892
893	# Then loop on the components (except for oasis)
894
895	for comp in ${config_ListOfComponents[*]} ; do
896
897	eval ExeNameIn=\${config_Executable_${comp}[0]}
898	eval ExeNameOut=\${config_Executable_${comp}[1]}
899
900	# Only if we really have an executable for the component :
901	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
902
903	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
904	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
905
906	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
907	# echo "set -vx" >> script_${ExeNameOut}.ksh
908	echo "" >> script_${ExeNameOut}.ksh
909	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
910	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
911	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
912	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
913	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
914	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
915	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
916
917	node_num=0
918
919	# We define the number of MPI process to be assigned for the component
920
921	nombre_restant_comp=${comp_proc_mpi_loc}
922
923	# Loop on the allocated nodes
924
925	for node in ${listnodes} ; do
926
927	# We go to the current node
928	if [ ${node_num} = ${node_num_current} ] ; then
929
930	node_current=${host[${node_num_current}]}
931
932	# If first time on the node : initialisation
933
934	if [ ${init_node} = y ] ; then
935	nombre_restant_node=${NUM_COREPERNODE}
936	fi
937
938	# Test on the number of OMP threads
939
940	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
941	(( node_num = node_num + 1 ))
942	node_num_current=${node_num}
943	init_node=y
944	continue
945	fi
946
947	# Number of MPI process to assign
948
949	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
950
951	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
952	num_corempi=${nombre_restant_comp}
953	fi
954
955	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
956	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
957
958	if [ ${init_exec} = y ] ; then
959	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
960	else
961	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
962	init_exec=y
963	fi
964
965	(( start_num = num_corempi + start_num ))
966
967	else
968
969	(( node_num = node_num + 1 ))
970	continue
971	fi
972
973	# Test on the number of core/process remaining on the node/component
974
975	if [ ${nombre_restant_node} = 0 ] ; then
976	(( node_num = node_num + 1 ))
977	node_num_current=${node_num}
978	init_node=y
979
980	if [ ${nombre_restant_comp} = 0 ] ; then
981	break 1
982	fi
983	else
984
985	node_num_current=${node_num}
986	init_node=n
987
988	if [ ${nombre_restant_comp} = 0 ] ; then
989	break 1
990	fi
991	fi
992	done
993	fi
994	done
995
996	else
997
998	# Then first loop on the components for the coupler ie oasis
999
1000	## the coupler ie oasis must be the first one
1001	for comp in ${config_ListOfComponents[*]} ; do
1002
1003	eval ExeNameOut=\${config_Executable_${comp}[1]}
1004
1005	# for CPL component only
1006	if [ "X${comp}" = "XCPL" ] ; then
1007	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1008	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1009	fi
1010	done
1011
1012	# Then second loop on the components
1013
1014	for comp in ${config_ListOfComponents[*]} ; do
1015
1016	eval ExeNameOut=\${config_Executable_${comp}[1]}
1017
1018	# Only if we really have an executable for the component and not the coupler ie oasis:
1019	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1020	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1021	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1022	fi
1023	done
1024	IGCM_sys_Chmod u+x run_file
1025
1026	EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1027
1028	fi
1029
1030	else # Only one executable. launch it.
1031
1032	for comp in ${config_ListOfComponents[*]} ; do
1033
1034	# Only if we really have an executable for the component :
1035	eval ExeNameOut=\${config_Executable_${comp}[1]}
1036	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1037
1038	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1039	echo "" >> script_${ExeNameOut}.ksh
1040	if ( ${OK_PARA_OMP} ) ; then
1041	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1042	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1043	fi
1044	if ( ${OK_PARA_MPI} ) ; then
1045	# Default : mpirun used if nb_proc gt 1
1046	# pour sortie out/err par process
1047	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}" >> script_${ExeNameOut}.ksh
1048	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1049	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1050	EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1051	else
1052	# Default : mpirun is NOT used if nb_proc eq 1
1053	# pour sortie out/err par process
1054	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1055	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1056	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1057	EXECUTION="time ./script_${ExeNameOut}.ksh"
1058	fi
1059
1060	IGCM_debug_Print 1 "sys Ciclad : script_${ExeNameOut}.ksh contains"
1061	cat script_${ExeNameOut}.ksh
1062
1063	fi
1064
1065	done
1066
1067	fi
1068
1069	IGCM_debug_Print 1 "sys Ciclad : execution command is"
1070	IGCM_debug_Print 1 "$EXECUTION"
1071
1072	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1073	}
1074
1075	#D-#==================================================
1076	#D-function IGCM_sys_check_path
1077	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1078	#D-* do not point to an important use directory. Stop immediately in that case.
1079	#D-* Examples:
1080	#D-
1081	function IGCM_sys_check_path {
1082	IGCM_debug_PushStack "IGCM_sys_check_path"
1083	if ( $DEBUG_sys ) ; then
1084	echo "IGCM_sys_check_path"
1085	fi
1086
1087	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1088	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1089	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1090	IGCM_debug_Exit "This will stop the job"
1091	fi
1092	IGCM_debug_PopStack "IGCM_sys_check_path"
1093	}
1094
1095	#D-#==================================================
1096	#D-function IGCM_sys_check_quota. Dummy call here
1097	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1098	#D-* Examples:
1099	#D-
1100	function IGCM_sys_check_quota {
1101	IGCM_debug_PushStack "IGCM_sys_check_quota"
1102	if ( $DEBUG_sys ) ; then
1103	echo "IGCM_sys_check_quota"
1104	fi
1105
1106	# TO BE IMPLEMENTED
1107
1108	IGCM_debug_PopStack "IGCM_sys_check_quota"
1109	}
1110
1111	#D-#==================================================
1112	#D-function IGCM_sys_GetJobID
1113	#D-* Purpose: Check if job_name is currently
1114	#D- running or in queue
1115	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1116	#D-
1117	function IGCM_sys_GetJobID {
1118	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1119	if ( $DEBUG_sys ) ; then
1120	echo "IGCM_sys_GetJobID"
1121	fi
1122
1123	# With -f option, the full job name is given in the last column
1124	ID="$( qstat -u $2 \| grep -w $1 \| gawk '-F ' '{print $10}' )"
1125
1126	eval ${3}=${ID}
1127	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1128	}
1129
1130	#D-#==================================================
1131	#D-function IGCM_sys_CountJobInQueue
1132	#D-* Purpose: Check if job_name is currently
1133	#D- running or in queue
1134	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1135	#D-
1136	function IGCM_sys_CountJobInQueue {
1137	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1138	if ( $DEBUG_sys ) ; then
1139	echo "IGCM_sys_CountJobInQueue"
1140	fi
1141	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1142	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: