Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_default.ksh @ 1204

Last change on this file since 1204 was 1201, checked in by sdipsl, 9 years ago
add jobWarningDelay in seconds to supervision messages see #253 (threshold for heartbeat) add accountingProject to supervision messages see #255 (may ease gencmip6 data production tracking)
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 34.0 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip
5	# Contact: Martial.Mancip__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys Default host
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=${MASTER:=$( hostname )}
71	# project name
72	typeset PROJECT=NONE
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=NONE
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=qsub}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=${MASTER}
89
90	#====================================================
91	# Set environment tools (ferret, nco, cdo, rebuild, ...)
92	#====================================================
93	#. /home/${LOGIN}/.atlas_env_${HOST}_ksh
94	#export PATH=${PATH}:/home/${LOGIN}/rebuild/bin/
95
96	#====================================================
97	# Host specific DIRECTORIES
98	#====================================================
99
100	#====================================================
101	#- MirrorlibIGCM for frontend
102	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
103
104	#====================================================
105	#- libIGCM_POST for frontend
106	typeset -r libIGCM_POST=${libIGCM}
107
108	#====================================================
109	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
110	typeset -r R_EXE="${MODIPSL}/bin"
111
112	#====================================================
113	#- SUBMIT_DIR : submission dir
114	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
115
116	#====================================================
117	#- IN
118	typeset -r R_IN=${R_IN:=/home/${LOGIN}/IGCM}
119	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/home/${LOGIN}/IGCM}
120
121	#====================================================
122	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
123	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/home/${LOGIN}/RUN_DIR/tmp$$}
124
125	#====================================================
126	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
127	typeset -r OUTCOMMAND_PATH=/tmp
128
129	#====================================================
130	#- HOST_MPIRUN_COMMAND
131	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
132
133	#====================================================
134	#- Max number of arguments passed to nco operator or demigration command
135	UNIX_MAX_LIMIT=360
136
137	#====================================================
138	#- set PackDefault to false on obelix
139	PackDefault=false
140
141	#====================================================
142	#- Number of core per node (max number of OpenMP task)
143	NUM_COREPERNODE=1
144
145	#====================================================
146	#- Default number of MPI task for IPSL coupled model
147	#- required for backward compatibility
148	#-
149	DEFAULT_NUM_PROC_OCE=1
150	DEFAULT_NUM_PROC_CPL=1
151	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
152	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
153
154	#D-#==================================================
155	#D-function IGCM_sys_defineArchives
156	#D-* Purpose:
157	#D-* Define ARCHIVE : Dedicated to large files
158	#D-* Define STORAGE : Dedicated to small/medium files
159	#D-* Define R_OUT : Output tree located on ARCHIVE
160	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
161	#D-* Define R_BUF : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
162	#D-* if SpaceName=TEST nothing special will hapen
163	#D-* Examples:
164	#D-
165	function IGCM_sys_defineArchives {
166	IGCM_debug_PushStack "IGCM_sys_defineArchives"
167
168	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
169	#====================================================
170	#- ARCHIVE (dedicated to large files)
171	ARCHIVE=${config_UserChoices_ARCHIVE}
172	else
173	#====================================================
174	#- ARCHIVE (dedicated to large files)
175	ARCHIVE=${ARCHIVE:=/home/${LOGIN}}
176	fi
177
178	if [ ! X${config_UserChoices_STORAGE} = X ]; then
179	#====================================================
180	#- STORAGE (dedicated to small/medium files)
181	STORAGE=${config_UserChoices_STORAGE}
182	else
183	#====================================================
184	#- STORAGE (dedicated to small/medium files)
185	STORAGE=${ARCHIVE}
186	fi
187
188	# ON DEFAULT MACHINE NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
189
190	#====================================================
191	#- R_OUT
192	R_OUT=${ARCHIVE}/IGCM_OUT
193
194	#====================================================
195	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
196	R_FIG=${STORAGE}/IGCM_OUT
197
198	#====================================================
199	#- R_BUF (ONLY FOR double copy an scratch)
200	R_BUF=${STORAGE}/IGCM_OUT
201
202	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
203	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
204	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
205
206	IGCM_debug_PopStack "IGCM_sys_defineArchives"
207	}
208
209	#D-#==================================================
210	#D-function IGCM_sys_RshArchive
211	#D-* Purpose: Archive rsh command
212	#D-* Examples:
213	#D-
214	function IGCM_sys_RshArchive {
215	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
216	/bin/ksh <<-EOF
217	${@}
218	EOF
219	status=$?
220	if [ ${status} -gt 0 ] ; then
221	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
222	IGCM_debug_Exit "IGCM_sys_RshArchive"
223	fi
224	IGCM_debug_PopStack "IGCM_sys_RshArchive"
225	}
226
227	#D-#==================================================
228	#D-function IGCM_sys_RshArchive_NoError
229	#D-* Purpose: Archive rsh command, without error
230	#D-* used only in monitoring.job
231	#D-* Examples:
232	#D-
233	function IGCM_sys_RshArchive_NoError {
234	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
235	/bin/ksh <<-EOF
236	${@} 2> /dev/null
237	EOF
238	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
239	}
240
241	#D-#==================================================
242	#D-function IGCM_sys_MkdirArchive
243	#D-* Purpose: Mkdir on Archive
244	#D-* Examples:
245	#D-
246	function IGCM_sys_MkdirArchive {
247	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
248	if ( $DEBUG_sys ) ; then
249	echo "IGCM_sys_MkdirArchive :" $@
250	fi
251	#- creation de repertoire sur le serveur fichier
252	if [ ! -d ${1} ]; then
253	\mkdir -p $1
254	status=$?
255
256	if [ ${status} -gt 0 ] ; then
257	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
258	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
259	fi
260	fi
261	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
262	}
263
264	#D-#==================================================
265	#D-function IGCM_sys_TestDirArchive
266	#D-* Purpose: Test Directory that must exists on Archive
267	#D-* Examples:
268	#D-
269	function IGCM_sys_TestDirArchive {
270	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
271	if ( $DEBUG_sys ) ; then
272	echo "IGCM_sys_TestDirArchive :" $@
273	fi
274	typeset ExistFlag
275	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
276	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
277	return ${ExistFlag}
278	}
279
280	#D-#==================================================
281	#D-function IGCM_sys_IsFileArchived
282	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
283	#D-* Examples:
284	#D-
285	function IGCM_sys_IsFileArchived {
286	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
287	if ( $DEBUG_sys ) ; then
288	echo "IGCM_sys_IsFileArchived :" $@
289	fi
290	typeset IsArchivedFlag
291	# Never archived for this system
292	IsArchivedFlag=1
293	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
294
295	return ${IsArchivedFlag}
296	}
297
298	#D-#==================================================
299	#D-function IGCM_sys_TestFileArchive
300	#D-* Purpose: Test file that must NOT EXISTS on Archive
301	#D-* Examples:
302	#D-
303	function IGCM_sys_TestFileArchive {
304	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
305	typeset ExistFlag
306	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
307	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
308
309	return ${ExistFlag}
310	}
311
312	#D-#==================================================
313	#D-function IGCM_sys_CountFileArchive
314	#D-* Purpose: Count files on Archive filesystem
315	#D-* Examples:
316	#D-
317	function IGCM_sys_CountFileArchive {
318	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
319	ls ${@} 2>/dev/null \| wc -l
320	if [ $? -gt 0 ] ; then
321	echo "IGCM_sys_CountFileArchive : erreur."
322	fi
323	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
324	}
325
326	#D-#==================================================
327	#D-function IGCM_sys_Tree
328	#D-* Purpose: Tree directories with files on ${ARCHIVE}
329	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
330	#D-
331	function IGCM_sys_Tree {
332	IGCM_debug_PushStack "IGCM_sys_Tree" $@
333	if ( $DEBUG_sys ) ; then
334	echo "IGCM_sys_Tree :" $@
335	fi
336
337	\tree -f $@
338
339	IGCM_debug_PopStack "IGCM_sys_Tree"
340	}
341
342	#D-#==================================================
343	#D-function IGCM_sys_Qsub
344	#D-* Purpose: Qsub new job
345	#D-* Examples:
346	#D-
347	function IGCM_sys_Qsub {
348	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
349	if ( $DEBUG_sys ) ; then
350	echo "IGCM_sys_Qsub :" $@
351	fi
352	echo "${@} > ${SUBMIT_DIR}/${Script_Output} 2>&1 &"
353	${@} > ${SUBMIT_DIR}/${Script_Output} 2>&1 &
354	if [ $? -gt 0 ] ; then
355	echo "IGCM_sys_Qsub : erreur ${options} $1"
356	IGCM_debug_Exit "IGCM_sys_Qsub"
357	fi
358	IGCM_debug_PopStack "IGCM_sys_Qsub"
359	}
360
361	#D-#==================================================
362	#D-function IGCM_sys_QsubPost
363	#D-* Purpose: Qsub new job on scalaire
364	#D-* Examples:
365	#D-
366	function IGCM_sys_QsubPost {
367	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
368	if ( $DEBUG_sys ) ; then
369	echo "IGCM_sys_QsubPost :" $@
370	fi
371	echo "${libIGCM_POST}/$1.job > ${POST_DIR}/$1.${PeriodDateEnd}.out 2>&1 &"
372	${libIGCM_POST}/$1.job > ${POST_DIR}/${Script_Post_Output}.out 2>&1 &
373	if [ $? -gt 0 ] ; then
374	echo "IGCM_sys_QsubPost : erreur " $@
375	IGCM_debug_Exit "IGCM_sys_QsubPost"
376	fi
377	IGCM_debug_PopStack "IGCM_sys_QsubPost"
378	}
379
380	#D-*************************
381	#D- File transfer functions
382	#D-*************************
383	#D-
384
385	#D-#==================================================
386	#D-function IGCM_sys_RmRunDir
387	#D-* Purpose: rm tmpdir (dummy function most of the time batch
388	#D- scheduler will do the job)
389	#D-* Examples:
390	#D-
391	function IGCM_sys_RmRunDir {
392	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
393	if ( $DEBUG_sys ) ; then
394	echo "IGCM_sys_RmRunDir :" $@
395	IGCM_debug_Print 1 "Dummy call, let the user do that."
396	# IGCM_sys_Rm -rf ${RUN_DIR}
397	fi
398	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
399	}
400
401	#D-#==================================================
402	#D-function IGCM_sys_Put_Dir
403	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
404	#D-* Examples:
405	#D-
406	function IGCM_sys_Put_Dir {
407	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
408	if ( $DEBUG_sys ) ; then
409	echo "IGCM_sys_Put_Dir :" $@
410	fi
411	if [ $DRYRUN = 0 ]; then
412	if [ ! -d ${1} ] ; then
413	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
414	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
415	return
416	fi
417
418	typeset status
419
420	# Only if we use rsync
421	#IGCM_sys_TestDirArchive $( dirname $2 )
422	#
423	#USUAL WAY
424	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
425	status=$?
426
427	if [ ${status} -gt 0 ] ; then
428	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
429	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
430	IGCM_debug_Exit "IGCM_sys_Put_Dir"
431	else
432	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
433	fi
434	fi
435	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
436	}
437
438	#D-#==================================================
439	#D-function IGCM_sys_Get_Dir
440	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
441	#D-* Examples:
442	#D-
443	function IGCM_sys_Get_Dir {
444	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
445	if ( $DEBUG_sys ) ; then
446	echo "IGCM_sys_Get_Dir :" $@
447	fi
448	if [ $DRYRUN = 0 ]; then
449	typeset NB_ESSAI DELAI status i
450	# number of tentative
451	NB_ESSAI=3
452	# time delay between tentative
453	DELAI=2
454
455	#
456	# USUAL WAY
457	i=0
458	while [ $i -lt $NB_ESSAI ] ; do
459	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
460	status=$?
461	if [ ${status} -gt 0 ] ; then
462	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
463	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
464	sleep $DELAI
465	else
466	break
467	fi
468	(( i = i + 1 ))
469	done
470
471	if [ ${status} -gt 0 ] ; then
472	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
473	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
474	IGCM_debug_Exit "IGCM_sys_Get_Dir"
475	else
476	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
477	fi
478	fi
479	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
480	}
481
482	#D-#==================================================
483	#D-function IGCM_sys_Put_Rest
484	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
485	#D- File and target directory must exist.
486	#D-* Examples:
487	#D-
488	function IGCM_sys_Put_Rest {
489	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
490	if ( $DEBUG_sys ) ; then
491	echo "IGCM_sys_Put_Rest :" $@
492	fi
493	if [ $DRYRUN = 0 ]; then
494	if [ ! -f ${1} ] ; then
495	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
496	IGCM_debug_Exit "IGCM_sys_Put_Rest"
497	fi
498
499	typeset status
500	#
501	# USUAL WAY
502	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
503	status=$?
504
505	# #RSYNC WITH NETWORK SSH CALL
506	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
507	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
508
509	# #RSYNC WITH NFS USE
510	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
511	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
512
513	# status=$?
514	# IGCM_sys_Rsync_out $status
515
516	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
517	# (( status=status+$? ))
518
519	if [ ${status} -gt 0 ] ; then
520	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
521	[ -f ${1} ] && ls -l ${1}
522	[ -f ${2} ] && ls -l ${2}
523	[ -f ${2}/${1} ] && ls -l ${2}/${1}
524	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
525	IGCM_debug_Exit "IGCM_sys_Put_Rest"
526	else
527
528	if [ X${JobType} = XRUN ] ; then
529	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
530	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
531	fi
532
533	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
534	fi
535	fi
536	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
537	}
538
539	#D-#==================================================
540	#D-function IGCM_sys_Put_Out
541	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
542	#D-* Examples:
543	#D-
544	function IGCM_sys_Put_Out {
545	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
546	if ( $DEBUG_sys ) ; then
547	echo "IGCM_sys_Put_Out :" $@
548	fi
549
550	typeset status
551
552	if [ $DRYRUN = 0 ]; then
553	if [ ! -f ${1} ] ; then
554	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
555	IGCM_debug_PopStack "IGCM_sys_Put_Out"
556	return 1
557	fi
558	#
559	IGCM_sys_MkdirArchive $( dirname $2 )
560	#
561	if [ X${JobType} = XRUN ] ; then
562	if [ X${3} = X ] ; then
563	IGCM_sys_Chmod 444 ${1}
564	fi
565	fi
566	#
567
568	echo ${RSYNC} ${RSYNC_opt} $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
569	${RSYNC} ${RSYNC_opt} $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
570	status=$?
571	IGCM_sys_Rsync_out $status
572
573	${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
574	(( status=status+$? ))
575
576	if [ ${status} -gt 0 ] ; then
577	IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
578	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
579	IGCM_debug_Exit "IGCM_sys_Put_Out"
580	else
581	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
582	fi
583	fi
584	IGCM_debug_PopStack "IGCM_sys_Put_Out"
585	return 0
586	}
587
588	#D-#==================================================
589	#D-function IGCM_sys_Get
590	#D-* Purpose: Get a file from ${ARCHIVE}
591	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
592	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
593	function IGCM_sys_Get {
594	IGCM_debug_PushStack "IGCM_sys_Get" $@
595
596	typeset DEST status dm_liste
597
598	if ( $DEBUG_sys ) ; then
599	echo "IGCM_sys_Get :" $@
600	fi
601	if [ $DRYRUN -le 2 ]; then
602	if [ X${1} = X'/l' ] ; then
603	eval set +A dm_liste \${${2}}
604	else
605	dm_liste=${1}
606	fi
607	eval DEST=\${${#}}
608
609	# test if the (first) file is present in the old computation :
610	IGCM_sys_TestFileArchive ${dm_liste[0]}
611	status=$?
612	if [ ${status} -gt 0 ] ; then
613	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
614	IGCM_debug_Exit "IGCM_sys_Get"
615	return
616	fi
617
618	#USUAL WAY
619	\cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
620	status=$?
621
622	if [ ${status} -gt 0 ] ; then
623	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
624	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
625	IGCM_debug_Exit "IGCM_sys_Get"
626	else
627	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
628	fi
629	fi
630	IGCM_debug_PopStack "IGCM_sys_Get"
631	}
632
633	#D-#==================================================
634	#D-function IGCM_sys_GetDate_Monitoring
635	#D-* Purpose: get the last year for which the monitoring has been computed
636	#D-* Examples:
637	#D-
638	function IGCM_sys_GetDate_Monitoring {
639	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
640	if ( $DEBUG_sys ) ; then
641	echo "IGCM_sys_GetDate_Monitoring :" $@
642	fi
643
644	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
645
646	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
647	}
648
649	#D-#==================================================
650	#D-function IGCM_sys_Dods_Rm
651	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
652	#D-* Examples:
653	#D-
654	function IGCM_sys_Dods_Rm {
655	if ( $DEBUG_sys ) ; then
656	echo "IGCM_sys_Dods_Rm :" $@
657	fi
658	return 0
659	}
660
661	#D-#==================================================
662	#D-function IGCM_sys_Dods_Cp
663	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
664	#D-* Examples:
665	#D-
666	function IGCM_sys_Dods_Cp {
667	if ( $DEBUG_sys ) ; then
668	echo "IGCM_sys_Dods_Cp :" $@
669	fi
670	return 0
671	}
672
673	#D-#==================================================
674	#D-function IGCM_sys_Put_Dods
675	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
676	#D-* Examples:
677	#D-
678	function IGCM_sys_Put_Dods {
679	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
680	if ( $DEBUG_sys ) ; then
681	echo "IGCM_sys_Put_Dods :" $@
682	fi
683	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
684	}
685
686	##############################################################
687	# REBUILD OPERATOR
688
689	#D-#==================================================
690	#D-function IGCM_sys_sync
691	#D-* Purpose: flush buffer on disk (dummy function on Ada)
692	#D-* Examples:
693	#D-
694	function IGCM_sys_sync {
695	IGCM_debug_PushStack "IGCM_sys_sync" $@
696	if ( $DEBUG_sys ) ; then
697	echo "IGCM_sys_sync :" $@
698	echo "Dummy call, let the system do that."
699	fi
700	IGCM_debug_PopStack "IGCM_sys_sync"
701	}
702
703	############################################################
704	# Activate Running Environnment Variables
705
706	#D-#==================================================
707	#D-function IGCM_sys_desactiv_variables
708	#D-* Purpose: set environement variables prior to execution
709	#D-* Examples:
710	#D-
711	function IGCM_sys_activ_variables {
712	IGCM_debug_PushStack "IGCM_sys_activ_variables"
713	if ( $DEBUG_sys ) ; then
714	echo "IGCM_sys_activ_variables"
715	fi
716
717	# --------------------------------------------------------------------
718	#D- MPI specifications
719	# --------------------------------------------------------------------
720
721	# --------------------------------------------------------------------
722	#D- Other specifications
723	# --------------------------------------------------------------------
724
725	IGCM_debug_PopStack "IGCM_sys_activ_variables"
726	}
727
728	############################################################
729	# Desactivate Running Environnment Variables
730
731	#D-#==================================================
732	#D-function IGCM_sys_desactiv_variables
733	#D-* Purpose: unset environement variables after execution
734	#D-* Examples:
735	#D-
736	function IGCM_sys_desactiv_variables {
737	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
738	if ( $DEBUG_sys ) ; then
739	echo "IGCM_sys_desactiv_variables"
740	fi
741	# --------------------------------------------------------------------
742	#D- MPI specifications
743	# --------------------------------------------------------------------
744
745	# --------------------------------------------------------------------
746	#D- Other specifications
747	# --------------------------------------------------------------------
748
749	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
750	}
751
752	############################################################
753	# Build MPI/OMP scripts run file (dummy function)
754
755	#D-#==================================================
756	#D-function IGCM_sys_build_run_file
757	#D-* Purpose: build run file (deprecated)
758	#D-* Examples:
759	#D-
760	function IGCM_sys_build_run_file {
761
762	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
763
764	}
765
766	############################################################
767	# Build MPI/OMP scripts
768
769	#D-#==================================================
770	#D-function IGCM_sys_build_execution_scripts
771	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
772	#D-* Examples:
773	#D-
774	function IGCM_sys_build_execution_scripts
775	{
776	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
777	if ( $DEBUG_sys ) ; then
778	echo "IGCM_sys_build_execution_scripts " $@
779	fi
780
781	typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
782	typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
783	typeset num_corempi nombre_restant_node nombre_restant_comp
784
785	if ( ${OK_PARA_MPMD} ) ; then
786
787	if [ -f run_file ] ; then
788	IGCM_sys_Rm -f run_file
789	fi
790	touch run_file
791
792	if ( ${OK_PARA_OMP} ) ; then
793
794	# Hosts treatment
795
796	${HOST_MPIRUN_COMMAND} hostname \| sort \| uniq > hosts.tmp
797
798	i=0
799	rm -f hosts
800	IGCM_debug_Print 1 "sys $( hostname ), Hosts avaible :"
801	for nodes in `cat hosts.tmp` ; do
802	host[$i]=$nodes
803	echo "${host[$i]} slots=1 max_slots=1" >> hosts
804	IGCM_debug_Print 1 ${host[$i]}
805	i=$((i+1))
806	done
807	rm -f hosts.tmp
808
809	listnodes=${host[*]}
810
811	EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
812
813	# Initialisation
814
815	init_node=y
816	node_num_current=0
817	start_num=0
818	init_exec=n
819
820	# Build run_file
821
822	# First loop on the components for the coupler ie oasis (only if oasis3)
823	# the coupler ie oasis3 must be the first one
824	for comp in ${config_ListOfComponents[*]} ; do
825
826	if [ "X${comp}" = "XCPL" ] ; then
827
828	eval ExeNameIn=\${config_Executable_${comp}[0]}
829	eval ExeNameOut=\${config_Executable_${comp}[1]}
830
831	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
832	echo "" >> script_${ExeNameOut}.ksh
833	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
834	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
835	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
836	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
837	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
838
839	init_node=n
840
841	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
842	node_num_current=0
843	node_current=${host[${node_num_current}]}
844
845	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
846
847	init_exec=y
848	start_num=1
849
850	fi
851
852	done
853
854	# Then loop on the components (except for oasis)
855
856	for comp in ${config_ListOfComponents[*]} ; do
857
858	eval ExeNameIn=\${config_Executable_${comp}[0]}
859	eval ExeNameOut=\${config_Executable_${comp}[1]}
860
861	# Only if we really have an executable for the component :
862	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
863
864	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
865	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
866
867	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
868	# echo "set -vx" >> script_${ExeNameOut}.ksh
869	echo "" >> script_${ExeNameOut}.ksh
870	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
871	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
872	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
873	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
874	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
875	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
876	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
877
878	node_num=0
879
880	# We define the number of MPI process to be assigned for the component
881
882	nombre_restant_comp=${comp_proc_mpi_loc}
883
884	# Loop on the allocated nodes
885
886	for node in ${listnodes} ; do
887
888	# We go to the current node
889	if [ ${node_num} = ${node_num_current} ] ; then
890
891	node_current=${host[${node_num_current}]}
892
893	# If first time on the node : initialisation
894
895	if [ ${init_node} = y ] ; then
896	nombre_restant_node=${NUM_COREPERNODE}
897	fi
898
899	# Test on the number of OMP threads
900
901	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
902	(( node_num = node_num + 1 ))
903	node_num_current=${node_num}
904	init_node=y
905	continue
906	fi
907
908	# Number of MPI process to assign
909
910	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
911
912	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
913	num_corempi=${nombre_restant_comp}
914	fi
915
916	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
917	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
918
919	if [ ${init_exec} = y ] ; then
920	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
921	else
922	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
923	init_exec=y
924	fi
925
926	(( start_num = num_corempi + start_num ))
927
928	else
929
930	(( node_num = node_num + 1 ))
931	continue
932	fi
933
934	# Test on the number of core/process remaining on the node/component
935
936	if [ ${nombre_restant_node} = 0 ] ; then
937	(( node_num = node_num + 1 ))
938	node_num_current=${node_num}
939	init_node=y
940
941	if [ ${nombre_restant_comp} = 0 ] ; then
942	break 1
943	fi
944	else
945
946	node_num_current=${node_num}
947	init_node=n
948
949	if [ ${nombre_restant_comp} = 0 ] ; then
950	break 1
951	fi
952	fi
953	done
954	fi
955	done
956
957	else
958
959	# Then first loop on the components for the coupler ie oasis
960
961	## the coupler ie oasis must be the first one
962	for comp in ${config_ListOfComponents[*]} ; do
963
964	eval ExeNameOut=\${config_Executable_${comp}[1]}
965
966	# for CPL component only
967	if [ "X${comp}" = "XCPL" ] ; then
968	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
969	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
970	fi
971	done
972
973	# Then second loop on the components
974
975	for comp in ${config_ListOfComponents[*]} ; do
976
977	eval ExeNameOut=\${config_Executable_${comp}[1]}
978
979	# Only if we really have an executable for the component and not the coupler ie oasis:
980	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
981	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
982	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
983	fi
984	done
985	IGCM_sys_Chmod u+x run_file
986
987	EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
988
989	fi
990
991	else # Only one executable. launch it.
992
993	for comp in ${config_ListOfComponents[*]} ; do
994
995	# Only if we really have an executable for the component :
996	eval ExeNameOut=\${config_Executable_${comp}[1]}
997	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
998
999	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1000	echo "" >> script_${ExeNameOut}.ksh
1001	if ( ${OK_PARA_OMP} ) ; then
1002	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1003	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1004	fi
1005	if ( ${OK_PARA_MPI} ) ; then
1006	# Default : mpirun used if nb_proc gt 1
1007	# pour sortie out/err par process
1008	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}" >> script_${ExeNameOut}.ksh
1009	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1010	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1011	EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1012	else
1013	# Default : mpirun is NOT used if nb_proc eq 1
1014	# pour sortie out/err par process
1015	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1016	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1017	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1018	EXECUTION="time ./script_${ExeNameOut}.ksh"
1019	fi
1020
1021	IGCM_debug_Print 1 "sys $( hostname ) : script_${ExeNameOut}.ksh contains"
1022	cat script_${ExeNameOut}.ksh
1023
1024	fi
1025
1026	done
1027
1028	fi
1029
1030	IGCM_debug_Print 1 "sys $( hostname ) : execution command is"
1031	IGCM_debug_Print 1 "$EXECUTION"
1032
1033	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1034	}
1035
1036	#D-#==================================================
1037	#D-function IGCM_sys_check_path
1038	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1039	#D-* do not point to an important use directory. Stop immediately in that case.
1040	#D-* Examples:
1041	#D-
1042	function IGCM_sys_check_path {
1043	IGCM_debug_PushStack "IGCM_sys_check_path"
1044	if ( $DEBUG_sys ) ; then
1045	echo "IGCM_sys_check_path"
1046	fi
1047
1048	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1049	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1050	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1051	IGCM_debug_Exit "This will stop the job"
1052	fi
1053	IGCM_debug_PopStack "IGCM_sys_check_path"
1054	}
1055
1056	#D-#==================================================
1057	#D-function IGCM_sys_check_quota. Dummy call here
1058	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1059	#D-* Examples:
1060	#D-
1061	function IGCM_sys_check_quota {
1062	IGCM_debug_PushStack "IGCM_sys_check_quota"
1063	if ( $DEBUG_sys ) ; then
1064	echo "IGCM_sys_check_quota"
1065	fi
1066	IGCM_debug_PopStack "IGCM_sys_check_quota"
1067	}
1068
1069	#D-#==================================================
1070	#D-function IGCM_sys_CountJobInQueue
1071	#D-* Purpose: Check if job_name is currently
1072	#D- running or in queue
1073	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1074	#D-
1075	function IGCM_sys_CountJobInQueue {
1076	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1077	if ( $DEBUG_sys ) ; then
1078	echo "IGCM_sys_CountJobInQueue"
1079	fi
1080	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1081	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: