source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_iitm.ksh @ 1201

Last change on this file since 1201 was 1201, checked in by sdipsl, 9 years ago
  • add jobWarningDelay in seconds to supervision messages see #253 (threshold for heartbeat)
  • add accountingProject to supervision messages see #255 (may ease gencmip6 data production tracking)
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Date Author Rev
File size: 30.9 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sébastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for IITM IBM machine
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=iitm01
71# project name
72typeset PROJECT=NONE
73# jobWarningDelay in seconds
74typeset jobWarningDelay=NONE
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=/usr/lpp/LoadL/full/bin/llsubmit}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=iitm01
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94# Not applicable here. Only rebuild path
95export export PATH=${PATH}:/gpfs1/home/sebastien/REBUILD/
96
97#====================================================
98# Host specific DIRECTORIES
99#====================================================
100
101#====================================================
102#- MirrorlibIGCM for frontend
103typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
104
105#====================================================
106#- libIGCM_POST for frontend
107typeset -r libIGCM_POST=${libIGCM}
108
109#====================================================
110#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
111typeset -r R_EXE="${MODIPSL}/bin"
112
113#====================================================
114#- SUBMIT_DIR : submission dir
115typeset SUBMIT_DIR=${SUBMIT_DIR:=${LOADL_STEP_INITDIR}}
116
117#====================================================
118#- IN
119typeset -r R_IN=${R_IN:=/gpfs1/home/sabin}
120typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/gpfs1/home/sabin}
121
122#====================================================
123#- RUN_DIR_PATH : Temporary working directory (=> TMP)
124typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=$( echo ${HOME} | sed -e "s/gpfs1/gpfs3/" )/RUN/${LOADL_STEP_ID}}
125
126#====================================================
127#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
128typeset -r OUTCOMMAND_PATH=/tmp
129
130#====================================================
131#- HOST_MPIRUN_COMMAND
132typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
133
134#====================================================
135#- Max number of arguments passed to nco operator or demigration command
136UNIX_MAX_LIMIT=360
137
138#====================================================
139#- set PackDefault to false on iitm
140PackDefault=false
141
142#====================================================
143#- Number of core per node (max number of OpenMP task)
144NUM_COREPERNODE=2
145
146#====================================================
147#- Default number of MPI task for IPSL coupled model
148#- required for backward compatibility
149#-
150DEFAULT_NUM_PROC_OCE=5
151DEFAULT_NUM_PROC_CPL=1
152(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
153DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
154
155#D-#==================================================
156#D-function IGCM_sys_defineArchives
157#D-* Purpose:
158#D-* Define ARCHIVE : Dedicated to large files
159#D-* Define STORAGE : Dedicated to small/medium files
160#D-* Define R_OUT   : Output tree located on ARCHIVE
161#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
162#D-* Define R_BUF   : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
163#D-* if SpaceName=TEST nothing special will hapen
164#D-* Examples:
165#D-
166function IGCM_sys_defineArchives {
167  IGCM_debug_PushStack "IGCM_sys_defineArchives"
168
169  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
170    #====================================================
171    #- ARCHIVE (dedicated to large files)
172    ARCHIVE=${config_UserChoices_ARCHIVE}
173  else
174    #====================================================
175    #- ARCHIVE (dedicated to large files)
176    ARCHIVE=$( echo ${HOME} | sed -e "s/gpfs1/gpfs3/" )
177  fi
178
179  if [ ! X${config_UserChoices_STORAGE} = X ]; then
180    #====================================================
181    #- STORAGE (dedicated to small/medium files)
182    STORAGE=${config_UserChoices_STORAGE}
183  else
184    #====================================================
185    #- STORAGE (dedicated to small/medium files)
186    STORAGE=${ARCHIVE}
187  fi
188
189  # ON OBELIX NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
190
191  #====================================================
192  #- R_OUT
193  R_OUT=${ARCHIVE}/IGCM_OUT
194
195  #====================================================
196  #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
197  R_FIG=${WORKDIR}/IGCM_OUT
198
199  #====================================================
200  #- R_BUF (ONLY FOR double copy an scratch)
201  R_BUF=${WORKDIR}/IGCM_OUT
202
203  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
204  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
205  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
206
207  IGCM_debug_PopStack "IGCM_sys_defineArchives"
208}
209
210#D-#==================================================
211#D-function IGCM_sys_RshArchive
212#D-* Purpose: Archive rsh command
213#D-* Examples:
214#D-
215function IGCM_sys_RshArchive {
216  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
217  /bin/ksh <<-EOF
218    ${@}
219EOF
220  status=$?
221  if [ ${status} -gt 0 ] ; then
222    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
223    IGCM_debug_Exit "IGCM_sys_RshArchive"
224  fi
225  IGCM_debug_PopStack "IGCM_sys_RshArchive"
226}
227
228#D-#==================================================
229#D-function IGCM_sys_RshArchive_NoError
230#D-* Purpose: Archive rsh command, without error
231#D-*          used only in monitoring.job
232#D-* Examples:
233#D-
234function IGCM_sys_RshArchive_NoError {
235  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
236  /bin/ksh <<-EOF
237    ${@} 2> /dev/null
238EOF
239  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
240}
241
242#D-#==================================================
243#D-function IGCM_sys_MkdirArchive
244#D-* Purpose: Mkdir on Archive
245#D-* Examples:
246#D-
247function IGCM_sys_MkdirArchive {
248  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
249  if ( $DEBUG_sys ) ; then
250    echo "IGCM_sys_MkdirArchive :" $@
251  fi
252  #- creation de repertoire sur le serveur fichier
253  if [ ! -d ${1} ]; then
254    \mkdir -p $1
255    status=$?
256
257    if [ ${status} -gt 0 ] ; then
258      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
259      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
260    fi
261  fi
262  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
263}
264
265#D-#==================================================
266#D-function IGCM_sys_TestDirArchive
267#D-* Purpose: Test Directory that must exists on Archive
268#D-* Examples:
269#D-
270function IGCM_sys_TestDirArchive {
271  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
272  if ( $DEBUG_sys ) ; then
273    echo "IGCM_sys_TestDirArchive :" $@
274  fi
275  typeset ExistFlag
276  ExistFlag=$( IGCM_sys_RshArchive "[ -d $1 ] && echo 0 || echo 1" )
277  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
278  return ${ExistFlag}
279}
280
281#D-#==================================================
282#D-function IGCM_sys_IsFileArchived
283#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
284#D-* Examples:
285#D-
286function IGCM_sys_IsFileArchived {
287  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
288  if ( $DEBUG_sys ) ; then
289    echo "IGCM_sys_IsFileArchived :" $@
290  fi
291  typeset IsArchivedFlag
292  IsArchivedFlag=$( [ "X$( echo $@ | grep \/gpfs3 )" != "X" ] && echo 0 || echo 1 )
293  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
294
295  return ${IsArchivedFlag}
296}
297
298#D-#==================================================
299#D-function IGCM_sys_TestFileArchive
300#D-* Purpose: Test file that must NOT EXISTS on Archive
301#D-* Examples:
302#D-
303function IGCM_sys_TestFileArchive {
304  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
305  typeset ExistFlag
306  ExistFlag=$( IGCM_sys_RshArchive "[ -f $1 ] && echo 0 || echo 1" )
307  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
308
309  return ${ExistFlag}
310}
311
312#D-#==================================================
313#D-function IGCM_sys_CountFileArchive
314#D-* Purpose: Count files on Archive filesystem
315#D-* Examples:
316#D-
317function IGCM_sys_CountFileArchive {
318  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
319  #Command depends on targeted file system
320  DEBUG_sys=false IGCM_sys_IsFileArchived $1
321  if [ $? = 0 ] ; then
322    IGCM_sys_RshArchive "ls ${@} 2>/dev/null | wc -l"
323    status=$?
324  else
325    ls ${@} 2>/dev/null | wc -l
326    status=$?
327  fi
328  if [ ${status} -gt 0 ] ; then
329    echo "IGCM_sys_CountFileArchive : erreur."
330  fi
331  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
332}
333
334#D-#==================================================
335#D-function IGCM_sys_Qsub
336#D-* Purpose: Qsub new job
337#D-* Examples:
338#D-
339function IGCM_sys_Qsub {
340  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
341
342  typeset status
343
344  if ( $DEBUG_sys ) ; then
345    echo "IGCM_sys_Qsub :" $@
346  fi
347  # We have to change output/error file
348  [ ${#@} = 1 ] &&  REP_FOR_JOB=${SUBMIT_DIR}
349  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
350  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Output}:" \
351    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
352    $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
353  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
354
355  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
356  if [ ${status} -gt 0 ] ; then
357    IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
358    IGCM_debug_Exit "IGCM_sys_Qsub"
359  else
360    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ | tail -1 | sed -e s/\"//g )
361    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
362  fi
363  IGCM_debug_PopStack "IGCM_sys_Qsub"
364}
365
366#D-#==================================================
367#D-function IGCM_sys_QsubPost
368#D-* Purpose: Qsub new job on scalaire
369#D-* Examples:
370#D-
371function IGCM_sys_QsubPost {
372  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
373
374  typeset status
375
376  if ( $DEBUG_sys ) ; then
377    echo "IGCM_sys_QsubPost :" $@
378  fi
379  # We have to change output/error file
380  [ ${#@} = 1 ] &&  REP_FOR_JOB=${POST_DIR}
381  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
382
383  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Post_Output}.out:" \
384    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Post_Output}.out:"   \
385    ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM
386
387  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
388
389  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
390  if [ ${status} -gt 0 ] ; then
391    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
392    IGCM_debug_Exit "IGCM_sys_QsubPost"
393  else
394    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ | tail -1 | sed -e s/\"//g )
395    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
396  fi
397  IGCM_debug_PopStack "IGCM_sys_QsubPost"
398}
399
400#D-*************************
401#D- File transfer functions
402#D-*************************
403#D-
404
405#D-#==================================================
406#D-function IGCM_sys_RmRunDir
407#D-* Purpose: rm tmpdir (dummy function most of the time batch
408#D-                      scheduler will do the job)
409#D-* Examples:
410#D-
411function IGCM_sys_RmRunDir {
412  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
413  if ( $DEBUG_sys ) ; then
414    echo "IGCM_sys_RmRunDir :" $@
415  fi
416
417  typeset status
418
419  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
420  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
421  status=$?
422
423  if [ ${status} -gt 0 ] ; then
424    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
425    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
426    IGCM_debug_Exit "IGCM_sys_RmRunDir"
427  else
428    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
429  fi
430  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
431}
432
433#D-#==================================================
434#D-function IGCM_sys_Put_Dir
435#D-* Purpose: Copy a complete directory on $(ARCHIVE)
436#D-* Examples:
437#D-
438function IGCM_sys_Put_Dir {
439  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
440  if ( $DEBUG_sys ) ; then
441    echo "IGCM_sys_Put_Dir :" $@
442  fi
443  if [ $DRYRUN = 0 ]; then
444    if [ ! -d ${1} ] ; then
445      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
446      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
447      return
448    fi
449
450    typeset status
451
452    # Only if we use rsync
453    #IGCM_sys_TestDirArchive $( dirname $2 )
454    #
455    #USUAL WAY
456    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
457    status=$?
458
459    if [ ${status} -gt 0 ] ; then
460      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
461      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
462      IGCM_debug_Exit "IGCM_sys_Put_Dir"
463    else
464      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
465    fi
466  fi
467  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
468}
469
470#D-#==================================================
471#D-function IGCM_sys_Get_Dir
472#D-* Purpose: Copy a complete directory from ${ARCHIVE}
473#D-* Examples:
474#D-
475function IGCM_sys_Get_Dir {
476  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
477  if ( $DEBUG_sys ) ; then
478    echo "IGCM_sys_Get_Dir :" $@
479  fi
480  if [ $DRYRUN = 0 ]; then
481    if [ ! -d ${1} ] ; then
482      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
483      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
484      return
485    fi
486
487    typeset status
488
489    #USUAL WAY
490    \cp -ur $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
491    status=$?
492
493    if [ ${status} -gt 0 ] ; then
494      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
495      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
496      IGCM_debug_Exit "IGCM_sys_Get_Dir"
497    else
498      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
499    fi
500  fi
501  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
502}
503
504#D-#==================================================
505#D-function IGCM_sys_Put_Rest
506#D-* Purpose: Put computied restarts on ${ARCHIVE}.
507#D-           File and target directory must exist.
508#D-* Examples:
509#D-
510function IGCM_sys_Put_Rest {
511  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
512  if ( $DEBUG_sys ) ; then
513    echo "IGCM_sys_Put_Rest :" $@
514  fi
515  if [ $DRYRUN = 0 ]; then
516    if [ ! -f ${1} ] ; then
517      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
518      IGCM_debug_Exit "IGCM_sys_Put_Rest"
519    fi
520
521    typeset status
522    #
523    if [ X${JobType} = XRUN ] ; then
524      IGCM_sys_Chmod 444 ${1}
525    fi
526    #
527    # Only if we use rsync
528    #IGCM_sys_MkdirArchive $( dirname $2 )
529    #
530    #USUAL WAY
531    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
532    status=$?
533
534    if [ ${status} -gt 0 ] ; then
535      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
536      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
537      IGCM_debug_Exit "IGCM_sys_Put_Rest"
538    else
539      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
540    fi
541  fi
542  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
543}
544
545#D-#==================================================
546#D-function IGCM_sys_Put_Out
547#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
548#D-* Examples:
549#D-
550function IGCM_sys_Put_Out {
551  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
552  if ( $DEBUG_sys ) ; then
553    echo "IGCM_sys_Put_Out :" $@
554  fi
555
556  typeset status
557
558  if [ $DRYRUN = 0 ]; then
559    if [ ! -f ${1} ] ; then
560      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
561      IGCM_debug_PopStack "IGCM_sys_Put_Out"
562      return 1
563    fi
564    #
565    IGCM_sys_MkdirArchive $( dirname $2 )
566    #
567    if [ X${JobType} = XRUN ] ; then
568      if [ X${3} = X ] ; then
569        IGCM_sys_Chmod 444 ${1}
570      fi
571    fi
572    #
573    #USUAL WAY
574    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
575    status=$?
576
577    if [ ${status} -gt 0 ] ; then
578      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
579      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
580      IGCM_debug_Exit "IGCM_sys_Put_Out"
581    else
582      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
583    fi
584  fi
585  IGCM_debug_PopStack "IGCM_sys_Put_Out"
586  return 0
587}
588
589#D-#==================================================
590#D-function IGCM_sys_Get
591#D-* Purpose: Get a file from ${ARCHIVE}
592#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
593#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
594function IGCM_sys_Get {
595  IGCM_debug_PushStack "IGCM_sys_Get" $@
596
597  typeset DEST status dm_liste
598
599  if ( $DEBUG_sys ) ; then
600    echo "IGCM_sys_Get :" $@
601  fi
602  if [ $DRYRUN -le 2 ]; then
603    if [ X${1} = X'/l' ] ; then
604      eval set +A dm_liste \${${2}}
605    else
606      dm_liste=${1}
607    fi
608    eval DEST=\${${#}}
609
610    # test if the (first) file is present in the old computation :
611    DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
612    if [ $? = 0 ] ; then
613      IGCM_sys_TestFileArchive ${dm_liste[0]}
614      status=$?
615    else
616      IGCM_sys_TestFileBuffer ${dm_liste[0]}
617      status=$?
618    fi
619
620    if [ ${status} -gt 0 ] ; then
621      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
622      IGCM_debug_Exit "IGCM_sys_Get"
623      return
624    fi
625
626    #USUAL WAY
627    \cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
628    status=$?
629
630    if [ ${status} -gt 0 ] ; then
631      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
632      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
633      IGCM_debug_Exit "IGCM_sys_Get"
634    else
635      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
636    fi
637  fi
638  IGCM_debug_PopStack "IGCM_sys_Get"
639}
640
641#D-#==================================================
642#D-function IGCM_sys_Dods_Rm
643#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
644#D-* Examples:
645#D-
646function IGCM_sys_Dods_Rm {
647  if ( $DEBUG_sys ) ; then
648    echo "IGCM_sys_Dods_Rm :" $@
649  fi
650  return 0
651}
652
653#D-#==================================================
654#D-function IGCM_sys_Dods_Cp
655#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
656#D-* Examples:
657#D-
658function IGCM_sys_Dods_Cp {
659  if ( $DEBUG_sys ) ; then
660    echo "IGCM_sys_Dods_Cp :" $@
661  fi
662  return 0
663}
664
665#D-#==================================================
666#D-function IGCM_sys_Put_Dods
667#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
668#D-* Examples:
669#D-
670function IGCM_sys_Put_Dods {
671  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
672  if ( $DEBUG_sys ) ; then
673    echo "IGCM_sys_Put_Dods :" $@
674  fi
675  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
676}
677
678##############################################################
679# REBUILD OPERATOR
680
681#D-#==================================================
682#D-function IGCM_sys_sync
683#D-* Purpose: flush buffer on disk (dummy function on Ada)
684#D-* Examples:
685#D-
686function IGCM_sys_sync {
687  IGCM_debug_PushStack "IGCM_sys_sync" $@
688  if ( $DEBUG_sys ) ; then
689    echo "IGCM_sys_sync :" $@
690    echo "Dummy call, let the system do that."
691  fi
692  IGCM_debug_PopStack "IGCM_sys_sync"
693}
694
695############################################################
696# Activate Running Environnment Variables
697
698#D-#==================================================
699#D-function IGCM_sys_desactiv_variables
700#D-* Purpose: set environement variables prior to execution
701#D-* Examples:
702#D-
703function IGCM_sys_activ_variables {
704  IGCM_debug_PushStack "IGCM_sys_activ_variables"
705  if ( $DEBUG_sys ) ; then
706    echo "IGCM_sys_activ_variables"
707  fi
708
709# --------------------------------------------------------------------
710#D- MPI specifications
711# --------------------------------------------------------------------
712
713# --------------------------------------------------------------------
714#D- Other specifications
715# --------------------------------------------------------------------
716
717  IGCM_debug_PopStack "IGCM_sys_activ_variables"
718}
719
720############################################################
721# Desactivate Running Environnment Variables
722
723#D-#==================================================
724#D-function IGCM_sys_desactiv_variables
725#D-* Purpose: unset environement variables after execution
726#D-* Examples:
727#D-
728function IGCM_sys_desactiv_variables {
729  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
730  if ( $DEBUG_sys ) ; then
731    echo "IGCM_sys_desactiv_variables"
732  fi
733# --------------------------------------------------------------------
734#D- MPI specifications
735# --------------------------------------------------------------------
736
737# --------------------------------------------------------------------
738#D- Other specifications
739# --------------------------------------------------------------------
740
741  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
742}
743
744############################################################
745# Build MPI/OMP scripts run file (dummy function)
746
747#D-#==================================================
748#D-function IGCM_sys_build_run_file
749#D-* Purpose: build run file (deprecated)
750#D-* Examples:
751#D-
752function IGCM_sys_build_run_file {
753
754  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
755
756}
757
758############################################################
759# Build MPI/OMP scripts
760
761#D-#==================================================
762#D-function IGCM_sys_build_execution_scripts
763#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
764#D-* Examples:
765#D-
766function IGCM_sys_build_execution_scripts
767{
768  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
769  if ( $DEBUG_sys ) ; then
770    echo "IGCM_sys_build_execution_scripts " $@
771  fi
772
773  typeset NbNodes_Job NbProc_Job comp_proc_mpi_loc comp_proc_omp_loc mpi_count
774
775  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
776    IGCM_debug_Exit "IGCM_sys_iitm build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
777  fi
778
779  if ( ${OK_PARA_MPMD} ) ; then
780
781    if [ -f run_file ] ; then
782      IGCM_sys_Rm -f run_file
783    fi
784    touch run_file
785
786    if ( ${OK_PARA_OMP} ) ; then
787# NEW : 2 Noeuds
788# @ task_geometry={(0)(1,2,3)}
789# Nombre de processus demandes
790      echo "Job_${config_UserChoices_JobName} includes task_geometry = \c"
791      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g"
792      echo "Job_${config_UserChoices_JobName} includes task_geometry  with NbNodes = \c"
793      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g" | wc -c
794      NbNodes_Job=$(( $( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g" | wc -c ) - 1 ))
795
796      if [ ${NbNodes_Job} -eq 0 ] ; then
797        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no task_geometry defined with OpenMP run."
798        exit 1
799      fi
800    else
801
802# OLD :
803# @ total_tasks = 3
804# @ environment = "BATCH_NUM_PROC_TOT=3"
805# Pas d OpenMP
806# @ resources = ConsumableCpus(1)
807
808      echo "Job_${config_UserChoices_JobName} includes total_tasks = \c"
809      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "total_tasks" | sed -e "s/.*total_tasks = //"
810      NbProc_Job=$( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "total_tasks" | sed -e "s/.*total_tasks = //" )
811      NbProc_Job=${NbProc_Job:=0}
812      if [ ${NbProc_Job} -eq 0 ] ; then
813        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no total_tasks defined with MPI only run."
814        exit 1
815      fi
816      if ( $( egrep '^# *@ *resources *= *ConsumableCpus\(1\)' ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} >/dev/null 2>&1 ) ) ; then
817        IGCM_debug_Print 2 "ressources =  ConsumableCpus(1) line found into Job_${config_UserChoices_JobName}"
818      else
819        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources = line not found. Job should include resources = ConsumableCpus(1) "
820        exit 1
821      fi
822    fi
823
824# run_file construction
825
826# Then first loop on the components for the coupler ie oasis
827
828### the coupler ie oasis must be the first one
829    for comp in ${config_ListOfComponents[*]} ; do
830
831      eval ExeNameIn=\${config_Executable_${comp}[0]}
832      eval ExeNameOut=\${config_Executable_${comp}[1]}
833
834      # for CPL component only
835      if [ "X${comp}" = "XCPL" ] ; then
836
837        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
838        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
839
840        if ( ${OK_PARA_MPI} ) ; then
841          (( mpi_count = 1 ))
842          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
843            if ( ${OK_PARA_OMP} ) ; then
844              echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
845            else
846              echo "./${ExeNameOut}" >> run_file
847            fi
848            (( mpi_count = mpi_count + 1 ))
849          done
850        else
851          if ( ${OK_PARA_OMP} ) ; then
852            echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
853          else
854            echo "./${ExeNameOut} " >> run_file
855          fi
856        fi
857      fi
858    done
859
860# Then second loop on the components
861
862    for comp in ${config_ListOfComponents[*]} ; do
863
864      eval ExeNameIn=\${config_Executable_${comp}[0]}
865      eval ExeNameOut=\${config_Executable_${comp}[1]}
866
867      # Only if we really have an executable for the component and not the coupler ie oasis:
868      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
869
870        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
871        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
872
873        if ( ${OK_PARA_MPI} ) ; then
874          (( mpi_count = 1 ))
875          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
876            if ( ${OK_PARA_OMP} ) ; then
877              echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
878            else
879              echo "./${ExeNameOut}" >> run_file
880            fi
881            (( mpi_count = mpi_count + 1 ))
882          done
883        else
884          if ( ${OK_PARA_OMP} ) ; then
885            echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
886          else
887                    # to be tested : no MPI only OpenMP into MPMD mode
888            echo "./${ExeNameOut} " >> run_file
889          fi
890        fi
891      fi
892    done
893
894    EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
895
896    IGCM_sys_Chmod u+x run_file
897    if ( $DEBUG_sys ) ; then
898      echo "run_file contains : "
899      cat run_file
900    fi
901
902  # Only one executable (SPMD mode).
903  else
904
905    for comp in ${config_ListOfComponents[*]} ; do
906
907      # Only if we really have an executable for the component :
908      eval ExeNameOut=\${config_Executable_${comp}[1]}
909      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
910        EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
911      fi
912
913    done
914
915  fi
916
917  IGCM_debug_Print 1 "sys iitm : execution command is"
918  IGCM_debug_Print 1 "$EXECUTION"
919
920  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
921}
922
923#D-#==================================================
924#D-function IGCM_sys_check_path
925#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
926#D-* do not point to an important use directory. Stop immediately in that case.
927#D-* Examples:
928#D-
929function IGCM_sys_check_path {
930  IGCM_debug_PushStack "IGCM_sys_check_path"
931  if ( $DEBUG_sys ) ; then
932    echo "IGCM_sys_check_path"
933  fi
934
935  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
936    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
937    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
938    IGCM_debug_Exit "This will stop the job"
939  fi
940  IGCM_debug_PopStack "IGCM_sys_check_path"
941}
942
943#D-#==================================================
944#D-function IGCM_sys_check_quota. Dummy call here
945#D-* Purpose: check user quota. Stop the simulation if quota above 90%
946#D-* Examples:
947#D-
948function IGCM_sys_check_quota {
949  IGCM_debug_PushStack "IGCM_sys_check_quota"
950  if ( $DEBUG_sys ) ; then
951    echo "IGCM_sys_check_quota"
952  fi
953  IGCM_debug_PopStack "IGCM_sys_check_quota"
954}
955
956#D-#==================================================
957#D-function IGCM_sys_CountJobInQueue
958#D-* Purpose: Check if job_name is currently
959#D-  running or in queue
960#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
961#D-
962function IGCM_sys_CountJobInQueue {
963  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
964  if ( $DEBUG_sys ) ; then
965    echo "IGCM_sys_CountJobInQueue"
966  fi
967
968  # Print only the full (-W) JobName (%jn)
969  NbRun=$( llq -W -f %jn | grep -c "$1" )
970
971  eval ${2}=${NbRun}
972
973  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
974}
Note: See TracBrowser for help on using the repository browser.