Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ciclad.ksh @ 1232

Last change on this file since 1232 was 1230, checked in by sdipsl, 9 years ago
MPI/OMP handling refactoring IGCM_config_ConfigureExexution will define the MPMD/SPMDO/MPI/OMP context 6 execution types are introduced and will be documented later prerequisite to have ins_job editing the headers IGCM_sys_build_execution_scripts will be adapted later
Property svn:keywords set to `Revision Date Author`
File size: 35.8 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil
5	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for ciclad
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=${MASTER:=$( hostname )}
71	# project name
72	typeset PROJECT=NONE
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=${PBS_WALLTIME}
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=qsub}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=${MASTER}
89
90	#====================================================
91	# Access to module command
92	#====================================================
93	. /usr/share/Modules/init/ksh
94
95	#====================================================
96	# Set environment tools (ferret, nco, cdo, rebuild, ...)
97	#====================================================
98	. /home/igcmg/.atlas_env_ciclad_ksh
99	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
100	[ ! X${TaskType} = Xchecking ] && module list
101
102	export PATH=${PATH}:/home/igcmg/rebuild/src_X64_CICLAD/modipsl_v2_2_3_netcdf4.2/bin/
103
104	#====================================================
105	# Host specific DIRECTORIES
106	#====================================================
107
108	#====================================================
109	#- MirrorlibIGCM for frontend
110	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
111
112	#====================================================
113	#- libIGCM_POST for frontend
114	typeset -r libIGCM_POST=${libIGCM}
115
116	#====================================================
117	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
118	typeset -r R_EXE="${MODIPSL}/bin"
119
120	#====================================================
121	#- SUBMIT_DIR : submission dir
122	if [ X${PBS_O_WORKDIR} != X ] ; then
123	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
124	else
125	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
126	fi
127
128	#====================================================
129	#- IN
130	typeset -r R_IN=${R_IN:=/prodigfs/ipslfs/igcmg/IGCM}
131	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/prodigfs/ipslfs/igcmg/IGCM}
132
133	#====================================================
134	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
135	if [ X${PBS_JOBID} != X ] ; then
136	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/data/${LOGIN}/RUN_DIR/${PBS_JOBID}_${$}}
137	else
138	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/tmp/tmp$$}
139	fi
140
141	#====================================================
142	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
143	typeset -r OUTCOMMAND_PATH=/tmp
144
145	#====================================================
146	#- HOST_MPIRUN_COMMAND
147	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
148
149	#====================================================
150	#- Max number of arguments passed to nco operator or demigration command
151	UNIX_MAX_LIMIT=360
152
153	#====================================================
154	#- set PackDefault to false on ciclad
155	PackDefault=false
156
157	#====================================================
158	#- Number of core per node (max number of OpenMP task)
159	NUM_COREPERNODE=1
160
161	#====================================================
162	#- Default number of MPI task for IPSL coupled model
163	#- required for backward compatibility
164	#-
165	DEFAULT_NUM_PROC_OCE=1
166	DEFAULT_NUM_PROC_CPL=1
167	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
168
169	#D-#==================================================
170	#D-function IGCM_sys_defineArchives
171	#D-* Purpose:
172	#D-* Define ARCHIVE : Dedicated to large files
173	#D-* Define STORAGE : Dedicated to small/medium files
174	#D-* Define R_OUT : Output tree located on ARCHIVE
175	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
176	#D-* Define R_BUF : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
177	#D-* if SpaceName=TEST nothing special will hapen
178	#D-* Examples:
179	#D-
180	function IGCM_sys_defineArchives {
181	IGCM_debug_PushStack "IGCM_sys_defineArchives"
182
183	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
184	#====================================================
185	#- ARCHIVE (dedicated to large files)
186	ARCHIVE=${config_UserChoices_ARCHIVE}
187	else
188	#====================================================
189	#- ARCHIVE (dedicated to large files)
190	ARCHIVE=${ARCHIVE:=/data/${LOGIN}}
191	fi
192
193	if [ ! X${config_UserChoices_STORAGE} = X ]; then
194	#====================================================
195	#- STORAGE (dedicated to small/medium files)
196	STORAGE=${config_UserChoices_STORAGE}
197	else
198	#====================================================
199	#- STORAGE (dedicated to small/medium files)
200	STORAGE=${ARCHIVE}
201	fi
202
203	# ON CICLAD NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
204
205	#====================================================
206	#- R_OUT
207	R_OUT=${ARCHIVE}/IGCM_OUT
208
209	#====================================================
210	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
211	R_FIG=${STORAGE}/IGCM_OUT
212
213	#====================================================
214	#- R_BUF (ONLY FOR double copy an scratch)
215	R_BUF=${STORAGE}/IGCM_OUT
216
217	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
218	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
219	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
220
221	IGCM_debug_PopStack "IGCM_sys_defineArchives"
222	}
223
224	#D-#==================================================
225	#D-function IGCM_sys_RshArchive
226	#D-* Purpose: Archive rsh command
227	#D-* Examples:
228	#D-
229	function IGCM_sys_RshArchive {
230	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
231	/bin/ksh <<-EOF
232	${@}
233	EOF
234	status=$?
235	if [ ${status} -gt 0 ] ; then
236	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
237	IGCM_debug_Exit "IGCM_sys_RshArchive"
238	fi
239	IGCM_debug_PopStack "IGCM_sys_RshArchive"
240	}
241
242	#D-#==================================================
243	#D-function IGCM_sys_RshArchive_NoError
244	#D-* Purpose: Archive rsh command, without error
245	#D-* used only in monitoring.job
246	#D-* Examples:
247	#D-
248	function IGCM_sys_RshArchive_NoError {
249	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
250	/bin/ksh <<-EOF
251	${@} 2> /dev/null
252	EOF
253	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
254	}
255
256	#D-#==================================================
257	#D-function IGCM_sys_MkdirArchive
258	#D-* Purpose: Mkdir on Archive
259	#D-* Examples:
260	#D-
261	function IGCM_sys_MkdirArchive {
262	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
263	if ( $DEBUG_sys ) ; then
264	echo "IGCM_sys_MkdirArchive :" $@
265	fi
266	#- creation de repertoire sur le serveur fichier
267	if [ ! -d ${1} ]; then
268	\mkdir -p $1
269	status=$?
270
271	if [ ${status} -gt 0 ] ; then
272	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
273	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
274	fi
275	fi
276	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
277	}
278
279	#D-#==================================================
280	#D-function IGCM_sys_TestDirArchive
281	#D-* Purpose: Test Directory that must exists on Archive
282	#D-* Examples:
283	#D-
284	function IGCM_sys_TestDirArchive {
285	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
286	if ( $DEBUG_sys ) ; then
287	echo "IGCM_sys_TestDirArchive :" $@
288	fi
289	typeset ExistFlag
290	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
291	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
292	return ${ExistFlag}
293	}
294
295	#D-#==================================================
296	#D-function IGCM_sys_IsFileArchived
297	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
298	#D-* Examples:
299	#D-
300	function IGCM_sys_IsFileArchived {
301	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
302	if ( $DEBUG_sys ) ; then
303	echo "IGCM_sys_IsFileArchived :" $@
304	fi
305	typeset IsArchivedFlag
306	# Never archived for this system
307	IsArchivedFlag=1
308	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
309
310	return ${IsArchivedFlag}
311	}
312
313	#D-#==================================================
314	#D-function IGCM_sys_TestFileArchive
315	#D-* Purpose: Test file that must NOT EXISTS on Archive
316	#D-* Examples:
317	#D-
318	function IGCM_sys_TestFileArchive {
319	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
320	typeset ExistFlag
321	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
322	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
323
324	return ${ExistFlag}
325	}
326
327	#D-#==================================================
328	#D-function IGCM_sys_CountFileArchive
329	#D-* Purpose: Count files on Archive filesystem
330	#D-* Examples:
331	#D-
332	function IGCM_sys_CountFileArchive {
333	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
334	ls ${@} 2>/dev/null \| wc -l
335	if [ $? -gt 0 ] ; then
336	echo "IGCM_sys_CountFileArchive : erreur."
337	fi
338	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
339	}
340
341	#D-#==================================================
342	#D-function IGCM_sys_Tree
343	#D-* Purpose: Tree directories with files on ${ARCHIVE}
344	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
345	#D-
346	function IGCM_sys_Tree {
347	IGCM_debug_PushStack "IGCM_sys_Tree" $@
348	if ( $DEBUG_sys ) ; then
349	echo "IGCM_sys_Tree :" $@
350	fi
351
352	\tree -f $@
353
354	IGCM_debug_PopStack "IGCM_sys_Tree"
355	}
356
357	#D-#==================================================
358	#D-function IGCM_sys_Qsub
359	#D-* Purpose: Qsub new job
360	#D-* Examples:
361	#D-
362	function IGCM_sys_Qsub {
363	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
364	if ( $DEBUG_sys ) ; then
365	echo "IGCM_sys_Qsub :" $@
366	fi
367	typeset options status
368	options="-o ${SUBMIT_DIR}/${Script_Output}"
369	/usr/bin/qsub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
370	status=$?
371
372	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
373	if [ ${status} -gt 0 ] ; then
374	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
375	IGCM_debug_Exit "IGCM_sys_Qsub"
376	else
377	JobID=$( gawk {'print $1'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
378	fi
379	IGCM_debug_PopStack "IGCM_sys_Qsub"
380	}
381
382	#D-#==================================================
383	#D-function IGCM_sys_QsubPost
384	#D-* Purpose: Qsub new job on scalaire
385	#D-* Examples:
386	#D-
387	function IGCM_sys_QsubPost {
388	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
389	if ( $DEBUG_sys ) ; then
390	echo "IGCM_sys_QsubPost :" $@
391	fi
392	typeset options status
393	options="-o ${POST_DIR}/${Script_Post_Output}.out -v ${listVarEnv}"
394	/usr/bin/qsub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
395	status=$?
396
397	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
398	if [ ${status} -gt 0 ] ; then
399	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
400	IGCM_debug_Exit "IGCM_sys_QsubPost"
401	else
402	JobID=$( gawk {'print $1'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
403	fi
404	IGCM_debug_PopStack "IGCM_sys_QsubPost"
405	}
406
407	#D-*************************
408	#D- File transfer functions
409	#D-*************************
410	#D-
411
412	#D-#==================================================
413	#D-function IGCM_sys_RmRunDir
414	#D-* Purpose: rm tmpdir (dummy function most of the time batch
415	#D- scheduler will do the job)
416	#D-* Examples:
417	#D-
418	function IGCM_sys_RmRunDir {
419	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
420	if ( $DEBUG_sys ) ; then
421	echo "IGCM_sys_RmRunDir :" $@
422	fi
423
424	typeset status
425
426	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
427	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
428	status=$?
429
430	if [ ${status} -gt 0 ] ; then
431	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
432	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
433	IGCM_debug_Exit "IGCM_sys_RmRunDir"
434	else
435	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
436	fi
437	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
438	}
439
440	#D-#==================================================
441	#D-function IGCM_sys_Put_Dir
442	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
443	#D-* Examples:
444	#D-
445	function IGCM_sys_Put_Dir {
446	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
447	if ( $DEBUG_sys ) ; then
448	echo "IGCM_sys_Put_Dir :" $@
449	fi
450	if [ $DRYRUN = 0 ]; then
451	if [ ! -d ${1} ] ; then
452	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
453	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
454	return
455	fi
456
457	typeset status
458
459	# Only if we use rsync
460	#IGCM_sys_TestDirArchive $( dirname $2 )
461	#
462	#USUAL WAY
463	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
464	status=$?
465
466	if [ ${status} -gt 0 ] ; then
467	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
468	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
469	IGCM_debug_Exit "IGCM_sys_Put_Dir"
470	else
471	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
472	fi
473	fi
474	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
475	}
476
477	#D-#==================================================
478	#D-function IGCM_sys_Get_Dir
479	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
480	#D-* Examples:
481	#D-
482	function IGCM_sys_Get_Dir {
483	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
484	if ( $DEBUG_sys ) ; then
485	echo "IGCM_sys_Get_Dir :" $@
486	fi
487	if [ $DRYRUN = 0 ]; then
488	typeset NB_ESSAI DELAI status i
489	# number of tentative
490	NB_ESSAI=3
491	# time delay between tentative
492	DELAI=2
493
494	#
495	# USUAL WAY
496	i=0
497	while [ $i -lt $NB_ESSAI ] ; do
498	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
499	status=$?
500	if [ ${status} -gt 0 ] ; then
501	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
502	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
503	sleep $DELAI
504	else
505	break
506	fi
507	(( i = i + 1 ))
508	done
509
510	if [ ${status} -gt 0 ] ; then
511	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
512	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
513	IGCM_debug_Exit "IGCM_sys_Get_Dir"
514	else
515	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
516	fi
517	fi
518	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
519	}
520
521	#D-#==================================================
522	#D-function IGCM_sys_Put_Rest
523	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
524	#D- File and target directory must exist.
525	#D-* Examples:
526	#D-
527	function IGCM_sys_Put_Rest {
528	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
529	if ( $DEBUG_sys ) ; then
530	echo "IGCM_sys_Put_Rest :" $@
531	fi
532	if [ $DRYRUN = 0 ]; then
533	if [ ! -f ${1} ] ; then
534	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
535	IGCM_debug_Exit "IGCM_sys_Put_Rest"
536	fi
537
538	typeset status
539	#
540	# USUAL WAY
541	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
542	status=$?
543
544	# #RSYNC WITH NETWORK SSH CALL
545	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
546	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
547
548	# #RSYNC WITH NFS USE
549	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
550	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
551
552	# status=$?
553	# IGCM_sys_Rsync_out $status
554
555	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
556	# (( status=status+$? ))
557
558	if [ ${status} -gt 0 ] ; then
559	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
560	[ -f ${1} ] && ls -l ${1}
561	[ -f ${2} ] && ls -l ${2}
562	[ -f ${2}/${1} ] && ls -l ${2}/${1}
563	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
564	IGCM_debug_Exit "IGCM_sys_Put_Rest"
565	else
566
567	if [ X${JobType} = XRUN ] ; then
568	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
569	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
570	fi
571
572	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
573	fi
574	fi
575	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
576	}
577
578	#D-#==================================================
579	#D-function IGCM_sys_Put_Out
580	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
581	#D-* Examples:
582	#D-
583	function IGCM_sys_Put_Out {
584	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
585	if ( $DEBUG_sys ) ; then
586	echo "IGCM_sys_Put_Out :" $@
587	fi
588
589	typeset status
590
591	if [ $DRYRUN = 0 ]; then
592	if [ ! -f ${1} ] ; then
593	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
594	IGCM_debug_PopStack "IGCM_sys_Put_Out"
595	return 1
596	fi
597	#
598	IGCM_sys_MkdirArchive $( dirname $2 )
599	#
600	if [ X${JobType} = XRUN ] ; then
601	if [ X${3} = X ] ; then
602	IGCM_sys_Chmod 444 ${1}
603	fi
604	fi
605	#
606
607	echo ${RSYNC} ${RSYNC_opt} $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
608	${RSYNC} ${RSYNC_opt} $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
609	status=$?
610	IGCM_sys_Rsync_out $status
611
612	${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
613	(( status=status+$? ))
614
615	if [ ${status} -gt 0 ] ; then
616	IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
617	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
618	IGCM_debug_Exit "IGCM_sys_Put_Out"
619	else
620	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
621	fi
622	fi
623	IGCM_debug_PopStack "IGCM_sys_Put_Out"
624	return 0
625	}
626
627	#D-#==================================================
628	#D-function IGCM_sys_Get
629	#D-* Purpose: Get a file from ${ARCHIVE}
630	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
631	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
632	function IGCM_sys_Get {
633	IGCM_debug_PushStack "IGCM_sys_Get" $@
634
635	typeset DEST status dm_liste
636
637	if ( $DEBUG_sys ) ; then
638	echo "IGCM_sys_Get :" $@
639	fi
640	if [ $DRYRUN -le 2 ]; then
641	if [ X${1} = X'/l' ] ; then
642	eval set +A dm_liste \${${2}}
643	else
644	dm_liste=${1}
645	fi
646	eval DEST=\${${#}}
647
648	# test if the (first) file is present in the old computation :
649	IGCM_sys_TestFileArchive ${dm_liste[0]}
650	status=$?
651	if [ ${status} -gt 0 ] ; then
652	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
653	IGCM_debug_Exit "IGCM_sys_Get"
654	return
655	fi
656
657	#USUAL WAY
658	\cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
659	status=$?
660
661	if [ ${status} -gt 0 ] ; then
662	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
663	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
664	IGCM_debug_Exit "IGCM_sys_Get"
665	else
666	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
667	fi
668	fi
669	IGCM_debug_PopStack "IGCM_sys_Get"
670	}
671
672	#D-#==================================================
673	#D-function IGCM_sys_GetDate_Monitoring
674	#D-* Purpose: get the last year for which the monitoring has been computed
675	#D-* Examples:
676	#D-
677	function IGCM_sys_GetDate_Monitoring {
678	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
679	if ( $DEBUG_sys ) ; then
680	echo "IGCM_sys_GetDate_Monitoring :" $@
681	fi
682
683	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
684
685	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
686	}
687
688	#D-#==================================================
689	#D-function IGCM_sys_Dods_Rm
690	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
691	#D-* Examples:
692	#D-
693	function IGCM_sys_Dods_Rm {
694	if ( $DEBUG_sys ) ; then
695	echo "IGCM_sys_Dods_Rm :" $@
696	fi
697	return 0
698	}
699
700	#D-#==================================================
701	#D-function IGCM_sys_Dods_Cp
702	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
703	#D-* Examples:
704	#D-
705	function IGCM_sys_Dods_Cp {
706	if ( $DEBUG_sys ) ; then
707	echo "IGCM_sys_Dods_Cp :" $@
708	fi
709	return 0
710	}
711
712	#D-#==================================================
713	#D-function IGCM_sys_Put_Dods
714	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
715	#D-* Examples:
716	#D-
717	function IGCM_sys_Put_Dods {
718	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
719	if ( $DEBUG_sys ) ; then
720	echo "IGCM_sys_Put_Dods :" $@
721	fi
722	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
723	}
724
725	##############################################################
726	# REBUILD OPERATOR
727
728	#D-#==================================================
729	#D-function IGCM_sys_sync
730	#D-* Purpose: flush buffer on disk (dummy function on Ada)
731	#D-* Examples:
732	#D-
733	function IGCM_sys_sync {
734	IGCM_debug_PushStack "IGCM_sys_sync" $@
735	if ( $DEBUG_sys ) ; then
736	echo "IGCM_sys_sync :" $@
737	echo "Dummy call, let the system do that."
738	fi
739	IGCM_debug_PopStack "IGCM_sys_sync"
740	}
741
742	############################################################
743	# Activate Running Environnment Variables
744
745	#D-#==================================================
746	#D-function IGCM_sys_desactiv_variables
747	#D-* Purpose: set environement variables prior to execution
748	#D-* Examples:
749	#D-
750	function IGCM_sys_activ_variables {
751	IGCM_debug_PushStack "IGCM_sys_activ_variables"
752	if ( $DEBUG_sys ) ; then
753	echo "IGCM_sys_activ_variables"
754	fi
755
756	# --------------------------------------------------------------------
757	#D- MPI specifications
758	# --------------------------------------------------------------------
759
760	# --------------------------------------------------------------------
761	#D- Other specifications
762	# --------------------------------------------------------------------
763
764	ulimit -s unlimited
765
766	IGCM_debug_PopStack "IGCM_sys_activ_variables"
767	}
768
769	############################################################
770	# Desactivate Running Environnment Variables
771
772	#D-#==================================================
773	#D-function IGCM_sys_desactiv_variables
774	#D-* Purpose: unset environement variables after execution
775	#D-* Examples:
776	#D-
777	function IGCM_sys_desactiv_variables {
778	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
779	if ( $DEBUG_sys ) ; then
780	echo "IGCM_sys_desactiv_variables"
781	fi
782	# --------------------------------------------------------------------
783	#D- MPI specifications
784	# --------------------------------------------------------------------
785
786	# --------------------------------------------------------------------
787	#D- Other specifications
788	# --------------------------------------------------------------------
789
790	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
791	}
792
793	############################################################
794	# Build MPI/OMP scripts run file (dummy function)
795
796	#D-#==================================================
797	#D-function IGCM_sys_build_run_file
798	#D-* Purpose: build run file (deprecated)
799	#D-* Examples:
800	#D-
801	function IGCM_sys_build_run_file {
802
803	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
804
805	}
806
807	############################################################
808	# Build MPI/OMP scripts
809
810	#D-#==================================================
811	#D-function IGCM_sys_build_execution_scripts
812	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
813	#D-* Examples:
814	#D-
815	function IGCM_sys_build_execution_scripts
816	{
817	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
818	if ( $DEBUG_sys ) ; then
819	echo "IGCM_sys_build_execution_scripts " $@
820	fi
821
822	typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
823	typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
824	typeset num_corempi nombre_restant_node nombre_restant_comp
825
826	if ( ${OK_PARA_MPMD} ) ; then
827
828	if [ -f run_file ] ; then
829	IGCM_sys_Rm -f run_file
830	fi
831	touch run_file
832
833	if ( ${OK_PARA_OMP} ) ; then
834
835	# Hosts treatment
836
837	${HOST_MPIRUN_COMMAND} hostname \| sort \| uniq > hosts.tmp
838
839	i=0
840	rm -f hosts
841	IGCM_debug_Print 1 "sys Ciclad, Hosts avaible :"
842	for nodes in `cat hosts.tmp` ; do
843	host[$i]=$nodes
844	echo "${host[$i]} slots=1 max_slots=1" >> hosts
845	IGCM_debug_Print 1 ${host[$i]}
846	i=$((i+1))
847	done
848	rm -f hosts.tmp
849
850	listnodes=${host[*]}
851
852	EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
853
854	# Initialisation
855
856	init_node=y
857	node_num_current=0
858	start_num=0
859	init_exec=n
860
861	# Build run_file
862
863	# First loop on the components for the coupler ie oasis (only if oasis3)
864	# the coupler ie oasis3 must be the first one
865	for comp in ${config_ListOfComponents[*]} ; do
866
867	if [ "X${comp}" = "XCPL" ] ; then
868
869	eval ExeNameIn=\${config_Executable_${comp}[0]}
870	eval ExeNameOut=\${config_Executable_${comp}[1]}
871
872	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
873	echo "" >> script_${ExeNameOut}.ksh
874	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
875	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
876	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
877	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
878	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
879
880	init_node=n
881
882	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
883	node_num_current=0
884	node_current=${host[${node_num_current}]}
885
886	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
887
888	init_exec=y
889	start_num=1
890
891	fi
892
893	done
894
895	# Then loop on the components (except for oasis)
896
897	for comp in ${config_ListOfComponents[*]} ; do
898
899	eval ExeNameIn=\${config_Executable_${comp}[0]}
900	eval ExeNameOut=\${config_Executable_${comp}[1]}
901
902	# Only if we really have an executable for the component :
903	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
904
905	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
906	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
907
908	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
909	# echo "set -vx" >> script_${ExeNameOut}.ksh
910	echo "" >> script_${ExeNameOut}.ksh
911	#echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
912	#echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
913	#echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
914	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
915	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
916	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
917	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
918
919	node_num=0
920
921	# We define the number of MPI process to be assigned for the component
922
923	nombre_restant_comp=${comp_proc_mpi_loc}
924
925	# Loop on the allocated nodes
926
927	for node in ${listnodes} ; do
928
929	# We go to the current node
930	if [ ${node_num} = ${node_num_current} ] ; then
931
932	node_current=${host[${node_num_current}]}
933
934	# If first time on the node : initialisation
935
936	if [ ${init_node} = y ] ; then
937	nombre_restant_node=${NUM_COREPERNODE}
938	fi
939
940	# Test on the number of OMP threads
941
942	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
943	(( node_num = node_num + 1 ))
944	node_num_current=${node_num}
945	init_node=y
946	continue
947	fi
948
949	# Number of MPI process to assign
950
951	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
952
953	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
954	num_corempi=${nombre_restant_comp}
955	fi
956
957	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
958	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
959
960	if [ ${init_exec} = y ] ; then
961	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
962	else
963	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
964	init_exec=y
965	fi
966
967	(( start_num = num_corempi + start_num ))
968
969	else
970
971	(( node_num = node_num + 1 ))
972	continue
973	fi
974
975	# Test on the number of core/process remaining on the node/component
976
977	if [ ${nombre_restant_node} = 0 ] ; then
978	(( node_num = node_num + 1 ))
979	node_num_current=${node_num}
980	init_node=y
981
982	if [ ${nombre_restant_comp} = 0 ] ; then
983	break 1
984	fi
985	else
986
987	node_num_current=${node_num}
988	init_node=n
989
990	if [ ${nombre_restant_comp} = 0 ] ; then
991	break 1
992	fi
993	fi
994	done
995	fi
996	done
997
998	else
999
1000	# Then first loop on the components for the coupler ie oasis
1001
1002	## the coupler ie oasis must be the first one
1003	for comp in ${config_ListOfComponents[*]} ; do
1004
1005	eval ExeNameOut=\${config_Executable_${comp}[1]}
1006
1007	# for CPL component only
1008	if [ "X${comp}" = "XCPL" ] ; then
1009	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1010	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1011	fi
1012	done
1013
1014	# Then second loop on the components
1015
1016	for comp in ${config_ListOfComponents[*]} ; do
1017
1018	eval ExeNameOut=\${config_Executable_${comp}[1]}
1019
1020	# Only if we really have an executable for the component and not the coupler ie oasis:
1021	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1022	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1023	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1024	fi
1025	done
1026	IGCM_sys_Chmod u+x run_file
1027
1028	EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1029
1030	fi
1031
1032	else # Only one executable. launch it.
1033
1034	for comp in ${config_ListOfComponents[*]} ; do
1035
1036	# Only if we really have an executable for the component :
1037	eval ExeNameOut=\${config_Executable_${comp}[1]}
1038	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1039
1040	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1041	echo "" >> script_${ExeNameOut}.ksh
1042	if ( ${OK_PARA_OMP} ) ; then
1043	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1044	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1045	fi
1046	if ( ${OK_PARA_MPI} ) ; then
1047	# Default : mpirun used if nb_proc gt 1
1048	# pour sortie out/err par process
1049	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}" >> script_${ExeNameOut}.ksh
1050	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1051	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1052	EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1053	else
1054	# Default : mpirun is NOT used if nb_proc eq 1
1055	# pour sortie out/err par process
1056	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1057	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1058	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1059	EXECUTION="time ./script_${ExeNameOut}.ksh"
1060	fi
1061
1062	IGCM_debug_Print 1 "sys Ciclad : script_${ExeNameOut}.ksh contains"
1063	cat script_${ExeNameOut}.ksh
1064
1065	fi
1066
1067	done
1068
1069	fi
1070
1071	IGCM_debug_Print 1 "sys Ciclad : execution command is"
1072	IGCM_debug_Print 1 "$EXECUTION"
1073
1074	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1075	}
1076
1077	#D-#==================================================
1078	#D-function IGCM_sys_check_path
1079	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1080	#D-* do not point to an important use directory. Stop immediately in that case.
1081	#D-* Examples:
1082	#D-
1083	function IGCM_sys_check_path {
1084	IGCM_debug_PushStack "IGCM_sys_check_path"
1085	if ( $DEBUG_sys ) ; then
1086	echo "IGCM_sys_check_path"
1087	fi
1088
1089	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1090	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1091	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1092	IGCM_debug_Exit "This will stop the job"
1093	fi
1094	IGCM_debug_PopStack "IGCM_sys_check_path"
1095	}
1096
1097	#D-#==================================================
1098	#D-function IGCM_sys_check_quota. Dummy call here
1099	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1100	#D-* Examples:
1101	#D-
1102	function IGCM_sys_check_quota {
1103	IGCM_debug_PushStack "IGCM_sys_check_quota"
1104	if ( $DEBUG_sys ) ; then
1105	echo "IGCM_sys_check_quota"
1106	fi
1107
1108	# TO BE IMPLEMENTED
1109
1110	IGCM_debug_PopStack "IGCM_sys_check_quota"
1111	}
1112
1113	#D-#==================================================
1114	#D-function IGCM_sys_GetJobID
1115	#D-* Purpose: Check if job_name is currently
1116	#D- running or in queue
1117	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1118	#D-
1119	function IGCM_sys_GetJobID {
1120	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1121	if ( $DEBUG_sys ) ; then
1122	echo "IGCM_sys_GetJobID"
1123	fi
1124
1125	# With -f option, the full job name is given in the last column
1126	ID="$( qstat -u $2 \| grep -w $1 \| gawk '-F ' '{print $10}' )"
1127
1128	eval ${3}=${ID}
1129	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1130	}
1131
1132	#D-#==================================================
1133	#D-function IGCM_sys_CountJobInQueue
1134	#D-* Purpose: Check if job_name is currently
1135	#D- running or in queue
1136	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1137	#D-
1138	function IGCM_sys_CountJobInQueue {
1139	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1140	if ( $DEBUG_sys ) ; then
1141	echo "IGCM_sys_CountJobInQueue"
1142	fi
1143	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1144	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: