source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_iitm.ksh @ 1368

Last change on this file since 1368 was 1244, checked in by sdipsl, 9 years ago
  • rationalyze somehow message code logic
  • extend the information gathered by big brother
  • clean up
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Date Author Rev
File size: 32.5 KB
RevLine 
[428]1#!/bin/ksh
2
3#**************************************************************
4# Author: Sébastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
[1061]6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
[428]9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
[809]16# if you use the prefix #D- for comments to be extracted.
[428]17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for IITM IBM machine
22#D-#==================================================
23#D-
[809]24#D- This ksh library if a layer under some usefull
[428]25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
[809]45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
[428]47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
[809]64# Host and user names
[428]65# $hostname ou hostname
[1180]66typeset HOST=${HOST:=$( hostname )}
[428]67# $username ou whoami
[1180]68typeset LOGIN=${LOGIN:=$( whoami )}
[428]69# $hostname of the MASTER job
[1180]70typeset MASTER=iitm01
[1201]71# project name
72typeset PROJECT=NONE
73# jobWarningDelay in seconds
74typeset jobWarningDelay=NONE
[428]75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
[810]81# Submit command
82typeset SUBMIT=${SUBMIT:=/usr/lpp/LoadL/full/bin/llsubmit}
[428]83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
[810]87# ie storage filesystem
88typeset -r STOREHOST=iitm01
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
[428]90
91#====================================================
[1180]92# Set environment tools (ferret, nco, cdo, rebuild, ...)
[810]93#====================================================
[1180]94# Not applicable here. Only rebuild path
95export export PATH=${PATH}:/gpfs1/home/sebastien/REBUILD/
[810]96
97#====================================================
[428]98# Host specific DIRECTORIES
99#====================================================
100
101#====================================================
[1180]102#- MirrorlibIGCM for frontend
[810]103typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
104
105#====================================================
106#- libIGCM_POST for frontend
107typeset -r libIGCM_POST=${libIGCM}
108
109#====================================================
[428]110#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
111typeset -r R_EXE="${MODIPSL}/bin"
112
113#====================================================
114#- SUBMIT_DIR : submission dir
115typeset SUBMIT_DIR=${SUBMIT_DIR:=${LOADL_STEP_INITDIR}}
116
117#====================================================
[725]118#- IN
119typeset -r R_IN=${R_IN:=/gpfs1/home/sabin}
120typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/gpfs1/home/sabin}
121
122#====================================================
[428]123#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[1064]124typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=$( echo ${HOME} | sed -e "s/gpfs1/gpfs3/" )/RUN/${LOADL_STEP_ID}}
[428]125
126#====================================================
[1179]127#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
[1186]128typeset -r OUTCOMMAND_PATH=/tmp
[1179]129
130#====================================================
[428]131#- HOST_MPIRUN_COMMAND
[810]132typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
[428]133
134#====================================================
135#- Max number of arguments passed to nco operator or demigration command
136UNIX_MAX_LIMIT=360
137
[810]138#====================================================
[1180]139#- set PackDefault to false on iitm
[810]140PackDefault=false
141
142#====================================================
143#- Number of core per node (max number of OpenMP task)
144NUM_COREPERNODE=2
145
146#====================================================
147#- Default number of MPI task for IPSL coupled model
148#- required for backward compatibility
149#-
150DEFAULT_NUM_PROC_OCE=5
151DEFAULT_NUM_PROC_CPL=1
152(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
153
[428]154#D-#==================================================
[1057]155#D-function IGCM_sys_defineArchives
156#D-* Purpose:
157#D-* Define ARCHIVE : Dedicated to large files
158#D-* Define STORAGE : Dedicated to small/medium files
159#D-* Define R_OUT   : Output tree located on ARCHIVE
160#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
161#D-* Define R_BUF   : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
162#D-* if SpaceName=TEST nothing special will hapen
[725]163#D-* Examples:
164#D-
[1057]165function IGCM_sys_defineArchives {
166  IGCM_debug_PushStack "IGCM_sys_defineArchives"
167
[1060]168  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
[1057]169    #====================================================
170    #- ARCHIVE (dedicated to large files)
[1063]171    ARCHIVE=${config_UserChoices_ARCHIVE}
[1057]172  else
173    #====================================================
174    #- ARCHIVE (dedicated to large files)
[1063]175    ARCHIVE=$( echo ${HOME} | sed -e "s/gpfs1/gpfs3/" )
[1057]176  fi
177
[1060]178  if [ ! X${config_UserChoices_STORAGE} = X ]; then
[1057]179    #====================================================
180    #- STORAGE (dedicated to small/medium files)
[1063]181    STORAGE=${config_UserChoices_STORAGE}
[1057]182  else
183    #====================================================
184    #- STORAGE (dedicated to small/medium files)
[1063]185    STORAGE=${ARCHIVE}
[1057]186  fi
187
188  # ON OBELIX NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
189
190  #====================================================
191  #- R_OUT
[1063]192  R_OUT=${ARCHIVE}/IGCM_OUT
[1057]193
194  #====================================================
195  #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]196  R_FIG=${WORKDIR}/IGCM_OUT
[1057]197
198  #====================================================
199  #- R_BUF (ONLY FOR double copy an scratch)
[1063]200  R_BUF=${WORKDIR}/IGCM_OUT
[1057]201
202  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
203  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
204  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
205
206  IGCM_debug_PopStack "IGCM_sys_defineArchives"
[725]207}
208
209#D-#==================================================
[428]210#D-function IGCM_sys_RshArchive
211#D-* Purpose: Archive rsh command
212#D-* Examples:
213#D-
214function IGCM_sys_RshArchive {
[810]215  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
[811]216  /bin/ksh <<-EOF
[428]217    ${@}
218EOF
[811]219  status=$?
[810]220  if [ ${status} -gt 0 ] ; then
221    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
222    IGCM_debug_Exit "IGCM_sys_RshArchive"
223  fi
224  IGCM_debug_PopStack "IGCM_sys_RshArchive"
[428]225}
226
227#D-#==================================================
[1180]228#D-function IGCM_sys_RshArchive_NoError
229#D-* Purpose: Archive rsh command, without error
230#D-*          used only in monitoring.job
231#D-* Examples:
232#D-
233function IGCM_sys_RshArchive_NoError {
234  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
235  /bin/ksh <<-EOF
236    ${@} 2> /dev/null
237EOF
238  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
239}
240
241#D-#==================================================
[428]242#D-function IGCM_sys_MkdirArchive
243#D-* Purpose: Mkdir on Archive
244#D-* Examples:
245#D-
246function IGCM_sys_MkdirArchive {
[810]247  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
248  if ( $DEBUG_sys ) ; then
249    echo "IGCM_sys_MkdirArchive :" $@
250  fi
251  #- creation de repertoire sur le serveur fichier
[1180]252  if [ ! -d ${1} ]; then
253    \mkdir -p $1
254    status=$?
[428]255
[1180]256    if [ ${status} -gt 0 ] ; then
257      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
258      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
259    fi
[810]260  fi
261  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[428]262}
263
264#D-#==================================================
265#D-function IGCM_sys_TestDirArchive
266#D-* Purpose: Test Directory that must exists on Archive
267#D-* Examples:
268#D-
269function IGCM_sys_TestDirArchive {
[810]270  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
271  if ( $DEBUG_sys ) ; then
272    echo "IGCM_sys_TestDirArchive :" $@
273  fi
274  typeset ExistFlag
275  ExistFlag=$( IGCM_sys_RshArchive "[ -d $1 ] && echo 0 || echo 1" )
276  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
277  return ${ExistFlag}
278}
[428]279
[810]280#D-#==================================================
281#D-function IGCM_sys_IsFileArchived
[1180]282#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
[810]283#D-* Examples:
284#D-
285function IGCM_sys_IsFileArchived {
286  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
287  if ( $DEBUG_sys ) ; then
288    echo "IGCM_sys_IsFileArchived :" $@
289  fi
290  typeset IsArchivedFlag
[818]291  IsArchivedFlag=$( [ "X$( echo $@ | grep \/gpfs3 )" != "X" ] && echo 0 || echo 1 )
[810]292  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
293
294  return ${IsArchivedFlag}
[428]295}
296
297#D-#==================================================
298#D-function IGCM_sys_TestFileArchive
299#D-* Purpose: Test file that must NOT EXISTS on Archive
300#D-* Examples:
301#D-
302function IGCM_sys_TestFileArchive {
[810]303  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
304  typeset ExistFlag
305  ExistFlag=$( IGCM_sys_RshArchive "[ -f $1 ] && echo 0 || echo 1" )
306  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[428]307
[810]308  return ${ExistFlag}
[428]309}
310
311#D-#==================================================
312#D-function IGCM_sys_CountFileArchive
313#D-* Purpose: Count files on Archive filesystem
314#D-* Examples:
315#D-
316function IGCM_sys_CountFileArchive {
[810]317  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
318  #Command depends on targeted file system
319  DEBUG_sys=false IGCM_sys_IsFileArchived $1
320  if [ $? = 0 ] ; then
[428]321    IGCM_sys_RshArchive "ls ${@} 2>/dev/null | wc -l"
[810]322    status=$?
323  else
324    ls ${@} 2>/dev/null | wc -l
325    status=$?
326  fi
327  if [ ${status} -gt 0 ] ; then
328    echo "IGCM_sys_CountFileArchive : erreur."
329  fi
330  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[428]331}
332
333#D-#==================================================
334#D-function IGCM_sys_Qsub
335#D-* Purpose: Qsub new job
336#D-* Examples:
337#D-
338function IGCM_sys_Qsub {
[810]339  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
340
[811]341  typeset status
[810]342
343  if ( $DEBUG_sys ) ; then
344    echo "IGCM_sys_Qsub :" $@
345  fi
346  # We have to change output/error file
347  [ ${#@} = 1 ] &&  REP_FOR_JOB=${SUBMIT_DIR}
348  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
349  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Output}:" \
[811]350    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
351    $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
[1179]352  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
[810]353
[1179]354  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]355  if [ ${status} -gt 0 ] ; then
356    IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
[810]357    IGCM_debug_Exit "IGCM_sys_Qsub"
358  else
[1179]359    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ | tail -1 | sed -e s/\"//g )
[810]360    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
361  fi
362  IGCM_debug_PopStack "IGCM_sys_Qsub"
[428]363}
364
365#D-#==================================================
366#D-function IGCM_sys_QsubPost
367#D-* Purpose: Qsub new job on scalaire
368#D-* Examples:
369#D-
370function IGCM_sys_QsubPost {
[810]371  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
372
[811]373  typeset status
[810]374
375  if ( $DEBUG_sys ) ; then
376    echo "IGCM_sys_QsubPost :" $@
377  fi
378  # We have to change output/error file
379  [ ${#@} = 1 ] &&  REP_FOR_JOB=${POST_DIR}
380  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
381
382  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Post_Output}.out:" \
[811]383    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Post_Output}.out:"   \
384    ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM
[810]385
[1179]386  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
[948]387
[1179]388  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]389  if [ ${status} -gt 0 ] ; then
390    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
[810]391    IGCM_debug_Exit "IGCM_sys_QsubPost"
392  else
[1179]393    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ | tail -1 | sed -e s/\"//g )
[810]394    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
395  fi
396  IGCM_debug_PopStack "IGCM_sys_QsubPost"
[428]397}
398
399#D-*************************
[809]400#D- File transfer functions
[428]401#D-*************************
402#D-
403
404#D-#==================================================
405#D-function IGCM_sys_RmRunDir
406#D-* Purpose: rm tmpdir (dummy function most of the time batch
407#D-                      scheduler will do the job)
408#D-* Examples:
409#D-
410function IGCM_sys_RmRunDir {
[1057]411  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
[810]412  if ( $DEBUG_sys ) ; then
413    echo "IGCM_sys_RmRunDir :" $@
414  fi
[428]415
[810]416  typeset status
[428]417
[1179]418  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
419  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[810]420  status=$?
[428]421
[810]422  if [ ${status} -gt 0 ] ; then
[1183]423    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
[1179]424    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[810]425    IGCM_debug_Exit "IGCM_sys_RmRunDir"
426  else
[1179]427    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[810]428  fi
429  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[428]430}
431
432#D-#==================================================
433#D-function IGCM_sys_Put_Dir
434#D-* Purpose: Copy a complete directory on $(ARCHIVE)
435#D-* Examples:
436#D-
437function IGCM_sys_Put_Dir {
[810]438  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
439  if ( $DEBUG_sys ) ; then
440    echo "IGCM_sys_Put_Dir :" $@
441  fi
442  if [ $DRYRUN = 0 ]; then
443    if [ ! -d ${1} ] ; then
444      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
445      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
446      return
[428]447    fi
448
[810]449    typeset status
[428]450
[810]451    # Only if we use rsync
452    #IGCM_sys_TestDirArchive $( dirname $2 )
453    #
454    #USUAL WAY
[1179]455    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[810]456    status=$?
[428]457
[810]458    if [ ${status} -gt 0 ] ; then
459      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
[1179]460      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[810]461      IGCM_debug_Exit "IGCM_sys_Put_Dir"
[428]462    else
[1179]463      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[428]464    fi
[810]465  fi
466  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[428]467}
468
469#D-#==================================================
470#D-function IGCM_sys_Get_Dir
[810]471#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[428]472#D-* Examples:
473#D-
474function IGCM_sys_Get_Dir {
[810]475  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
476  if ( $DEBUG_sys ) ; then
477    echo "IGCM_sys_Get_Dir :" $@
478  fi
479  if [ $DRYRUN = 0 ]; then
480    if [ ! -d ${1} ] ; then
481      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
482      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
483      return
[428]484    fi
485
[810]486    typeset status
[428]487
[811]488    #USUAL WAY
[1179]489    \cp -ur $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[810]490    status=$?
[428]491
[810]492    if [ ${status} -gt 0 ] ; then
493      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
[1179]494      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[810]495      IGCM_debug_Exit "IGCM_sys_Get_Dir"
[428]496    else
[1179]497      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[428]498    fi
[810]499  fi
500  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[428]501}
502
503#D-#==================================================
504#D-function IGCM_sys_Put_Rest
[810]505#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[428]506#D-           File and target directory must exist.
507#D-* Examples:
508#D-
509function IGCM_sys_Put_Rest {
[810]510  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
511  if ( $DEBUG_sys ) ; then
512    echo "IGCM_sys_Put_Rest :" $@
513  fi
514  if [ $DRYRUN = 0 ]; then
515    if [ ! -f ${1} ] ; then
516      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
517      IGCM_debug_Exit "IGCM_sys_Put_Rest"
[428]518    fi
519
[810]520    typeset status
521    #
522    if [ X${JobType} = XRUN ] ; then
523      IGCM_sys_Chmod 444 ${1}
524    fi
525    #
526    # Only if we use rsync
527    #IGCM_sys_MkdirArchive $( dirname $2 )
528    #
529    #USUAL WAY
[1179]530    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[810]531    status=$?
[428]532
[810]533    if [ ${status} -gt 0 ] ; then
[1180]534      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
[1179]535      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[810]536      IGCM_debug_Exit "IGCM_sys_Put_Rest"
[428]537    else
[1179]538      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[428]539    fi
[810]540  fi
541  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[428]542}
543
544#D-#==================================================
545#D-function IGCM_sys_Put_Out
[810]546#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[428]547#D-* Examples:
548#D-
549function IGCM_sys_Put_Out {
[810]550  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
551  if ( $DEBUG_sys ) ; then
552    echo "IGCM_sys_Put_Out :" $@
553  fi
554
555  typeset status
556
557  if [ $DRYRUN = 0 ]; then
558    if [ ! -f ${1} ] ; then
559      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
560      IGCM_debug_PopStack "IGCM_sys_Put_Out"
561      return 1
[428]562    fi
[810]563    #
564    IGCM_sys_MkdirArchive $( dirname $2 )
565    #
566    if [ X${JobType} = XRUN ] ; then
567      if [ X${3} = X ] ; then
568        IGCM_sys_Chmod 444 ${1}
[786]569      fi
[810]570    fi
571    #
572    #USUAL WAY
[1179]573    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[810]574    status=$?
575
576    if [ ${status} -gt 0 ] ; then
577      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
[1179]578      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[810]579      IGCM_debug_Exit "IGCM_sys_Put_Out"
580    else
[1179]581      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[810]582    fi
583  fi
584  IGCM_debug_PopStack "IGCM_sys_Put_Out"
585  return 0
586}
587
588#D-#==================================================
[428]589#D-function IGCM_sys_Get
590#D-* Purpose: Get a file from ${ARCHIVE}
591#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
592#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
593function IGCM_sys_Get {
[810]594  IGCM_debug_PushStack "IGCM_sys_Get" $@
[428]595
[810]596  typeset DEST status dm_liste
[428]597
[810]598  if ( $DEBUG_sys ) ; then
599    echo "IGCM_sys_Get :" $@
600  fi
601  if [ $DRYRUN -le 2 ]; then
602    if [ X${1} = X'/l' ] ; then
603      eval set +A dm_liste \${${2}}
604    else
605      dm_liste=${1}
[428]606    fi
[810]607    eval DEST=\${${#}}
[428]608
[810]609    # test if the (first) file is present in the old computation :
610    DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
611    if [ $? = 0 ] ; then
[786]612      IGCM_sys_TestFileArchive ${dm_liste[0]}
[810]613      status=$?
614    else
615      IGCM_sys_TestFileBuffer ${dm_liste[0]}
616      status=$?
617    fi
[428]618
[810]619    if [ ${status} -gt 0 ] ; then
620      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
621      IGCM_debug_Exit "IGCM_sys_Get"
622      return
623    fi
[428]624
[810]625    #USUAL WAY
[1179]626    \cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[810]627    status=$?
628
629    if [ ${status} -gt 0 ] ; then
630      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
[1179]631      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[810]632      IGCM_debug_Exit "IGCM_sys_Get"
633    else
[1179]634      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[810]635    fi
636  fi
637  IGCM_debug_PopStack "IGCM_sys_Get"
638}
639
640#D-#==================================================
[811]641#D-function IGCM_sys_Dods_Rm
642#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
643#D-* Examples:
644#D-
645function IGCM_sys_Dods_Rm {
646  if ( $DEBUG_sys ) ; then
647    echo "IGCM_sys_Dods_Rm :" $@
648  fi
649  return 0
650}
651
652#D-#==================================================
653#D-function IGCM_sys_Dods_Cp
654#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
655#D-* Examples:
656#D-
657function IGCM_sys_Dods_Cp {
658  if ( $DEBUG_sys ) ; then
659    echo "IGCM_sys_Dods_Cp :" $@
660  fi
661  return 0
662}
663
664#D-#==================================================
[810]665#D-function IGCM_sys_Put_Dods
666#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
667#D-* Examples:
668#D-
669function IGCM_sys_Put_Dods {
670  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
671  if ( $DEBUG_sys ) ; then
672    echo "IGCM_sys_Put_Dods :" $@
673  fi
674  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
675}
676
[428]677##############################################################
678# REBUILD OPERATOR
679
[810]680#D-#==================================================
[1163]681#D-function IGCM_sys_sync
682#D-* Purpose: flush buffer on disk (dummy function on Ada)
683#D-* Examples:
684#D-
685function IGCM_sys_sync {
686  IGCM_debug_PushStack "IGCM_sys_sync" $@
687  if ( $DEBUG_sys ) ; then
688    echo "IGCM_sys_sync :" $@
689    echo "Dummy call, let the system do that."
690  fi
691  IGCM_debug_PopStack "IGCM_sys_sync"
692}
693
[428]694############################################################
695# Activate Running Environnment Variables
696
[810]697#D-#==================================================
[1239]698#D-function IGCM_sys_activ_variables
[810]699#D-* Purpose: set environement variables prior to execution
700#D-* Examples:
701#D-
[428]702function IGCM_sys_activ_variables {
[810]703  IGCM_debug_PushStack "IGCM_sys_activ_variables"
704  if ( $DEBUG_sys ) ; then
[811]705    echo "IGCM_sys_activ_variables"
[810]706  fi
[428]707
708# --------------------------------------------------------------------
709#D- MPI specifications
710# --------------------------------------------------------------------
711
712# --------------------------------------------------------------------
713#D- Other specifications
714# --------------------------------------------------------------------
715
[810]716  IGCM_debug_PopStack "IGCM_sys_activ_variables"
[428]717}
718
719############################################################
720# Desactivate Running Environnment Variables
721
[810]722#D-#==================================================
723#D-function IGCM_sys_desactiv_variables
724#D-* Purpose: unset environement variables after execution
725#D-* Examples:
726#D-
[428]727function IGCM_sys_desactiv_variables {
[810]728  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
729  if ( $DEBUG_sys ) ; then
[811]730    echo "IGCM_sys_desactiv_variables"
[810]731  fi
[428]732# --------------------------------------------------------------------
733#D- MPI specifications
734# --------------------------------------------------------------------
735
736# --------------------------------------------------------------------
737#D- Other specifications
738# --------------------------------------------------------------------
739
[810]740  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
741}
[809]742
[810]743############################################################
[1239]744# Update job headers to be used by the scheduler
745
746#D-#==================================================
747#D-function IGCM_sys_updateHeaders
748#D-* Purpose: Update job headers to be used by the scheduler
749#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
750#D-
751function IGCM_sys_updateHeaders {
752  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
753  if ( $DEBUG_sys ) ; then
754    echo "IGCM_sys_updateHeaders"
755  fi
756  typeset file
757  file=$1
758
759  if [ ${executionType} -eq 1 ] ; then
760    # MPMD + MPI
761    sed -e "/::openMPthreads::/d"                  \
762        -e "s/::JobNumProcTot::/${coreNumber}/"    \
763      ${file} > ${file}.tmp
764
765  elif [ ${executionType} -eq 2 ] ; then
766    # MPMD + MPI + OMP
767    sed -e "s/::openMPthreads::/${openMPthreads}/" \
768        -e "s/::JobNumProcTot::/${coreNumber}/"    \
769      ${file} > ${file}.tmp
770
771  elif [ ${executionType} -eq 3 ] ; then
772    # SPMD + MPI/OMP
773    sed -e "s/::openMPthreads::/${openMPthreads}/" \
774        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
775      ${file} > ${file}.tmp
776
777  elif [ ${executionType} -eq 4 ] ; then
778    # SPMD + MPI only
779    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
780        -e "/::openMPthreads::/d"                  \
781      ${file} > ${file}.tmp
782
783  elif [ ${executionType} -eq 5 ] ; then
784    # SPMD + OMP only
785    sed -e "s/::openMPthreads::/${openMPthreads}/" \
786        -e "/::JobNumProcTot::/d"                  \
787      ${file} > ${file}.tmp
788
789  elif [ ${executionType} -eq 6 ] ; then
790    # SEQUENTIAL THEN
791    sed -e "s/::JobNumProcTot::/1/"                \
792        -e "/::openMPthreads::/d"                  \
793      ${file} > ${file}.tmp
794
795  fi
796
797  IGCM_sys_Mv ${file}.tmp ${file}
798
799  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
800}
801
802############################################################
[810]803# Build MPI/OMP scripts run file (dummy function)
804
805#D-#==================================================
806#D-function IGCM_sys_build_run_file
807#D-* Purpose: build run file (deprecated)
808#D-* Examples:
809#D-
810function IGCM_sys_build_run_file {
811
[811]812  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[810]813
[428]814}
815
816############################################################
[810]817# Build MPI/OMP scripts
[428]818
[810]819#D-#==================================================
820#D-function IGCM_sys_build_execution_scripts
821#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
822#D-* Examples:
823#D-
824function IGCM_sys_build_execution_scripts
825{
826  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
827  if ( $DEBUG_sys ) ; then
828    echo "IGCM_sys_build_execution_scripts " $@
829  fi
830
831  typeset NbNodes_Job NbProc_Job comp_proc_mpi_loc comp_proc_omp_loc mpi_count
832
833  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
834    IGCM_debug_Exit "IGCM_sys_iitm build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
835  fi
836
837  if ( ${OK_PARA_MPMD} ) ; then
838
839    if [ -f run_file ] ; then
840      IGCM_sys_Rm -f run_file
841    fi
842    touch run_file
843
844    if ( ${OK_PARA_OMP} ) ; then
845# NEW : 2 Noeuds
846# @ task_geometry={(0)(1,2,3)}
847# Nombre de processus demandes
848      echo "Job_${config_UserChoices_JobName} includes task_geometry = \c"
849      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g"
850      echo "Job_${config_UserChoices_JobName} includes task_geometry  with NbNodes = \c"
851      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g" | wc -c
852      NbNodes_Job=$(( $( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g" | wc -c ) - 1 ))
853
854      if [ ${NbNodes_Job} -eq 0 ] ; then
855        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no task_geometry defined with OpenMP run."
856        exit 1
857      fi
858    else
859
860# OLD :
861# @ total_tasks = 3
862# @ environment = "BATCH_NUM_PROC_TOT=3"
863# Pas d OpenMP
864# @ resources = ConsumableCpus(1)
865
866      echo "Job_${config_UserChoices_JobName} includes total_tasks = \c"
867      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "total_tasks" | sed -e "s/.*total_tasks = //"
868      NbProc_Job=$( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "total_tasks" | sed -e "s/.*total_tasks = //" )
869      NbProc_Job=${NbProc_Job:=0}
870      if [ ${NbProc_Job} -eq 0 ] ; then
871        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no total_tasks defined with MPI only run."
872        exit 1
873      fi
874      if ( $( egrep '^# *@ *resources *= *ConsumableCpus\(1\)' ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} >/dev/null 2>&1 ) ) ; then
875        IGCM_debug_Print 2 "ressources =  ConsumableCpus(1) line found into Job_${config_UserChoices_JobName}"
876      else
877        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources = line not found. Job should include resources = ConsumableCpus(1) "
878        exit 1
879      fi
880    fi
881
882# run_file construction
883
884# Then first loop on the components for the coupler ie oasis
885
886### the coupler ie oasis must be the first one
887    for comp in ${config_ListOfComponents[*]} ; do
888
889      eval ExeNameIn=\${config_Executable_${comp}[0]}
890      eval ExeNameOut=\${config_Executable_${comp}[1]}
891
892      # for CPL component only
893      if [ "X${comp}" = "XCPL" ] ; then
894
895        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
896        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
897
898        if ( ${OK_PARA_MPI} ) ; then
899          (( mpi_count = 1 ))
900          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
901            if ( ${OK_PARA_OMP} ) ; then
902              echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
903            else
904              echo "./${ExeNameOut}" >> run_file
905            fi
906            (( mpi_count = mpi_count + 1 ))
907          done
908        else
909          if ( ${OK_PARA_OMP} ) ; then
910            echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
911          else
912            echo "./${ExeNameOut} " >> run_file
913          fi
914        fi
915      fi
916    done
917
918# Then second loop on the components
919
920    for comp in ${config_ListOfComponents[*]} ; do
921
922      eval ExeNameIn=\${config_Executable_${comp}[0]}
923      eval ExeNameOut=\${config_Executable_${comp}[1]}
924
925      # Only if we really have an executable for the component and not the coupler ie oasis:
926      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
927
928        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
929        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
930
931        if ( ${OK_PARA_MPI} ) ; then
932          (( mpi_count = 1 ))
933          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
934            if ( ${OK_PARA_OMP} ) ; then
935              echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
936            else
937              echo "./${ExeNameOut}" >> run_file
938            fi
939            (( mpi_count = mpi_count + 1 ))
940          done
941        else
942          if ( ${OK_PARA_OMP} ) ; then
943            echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
944          else
945                    # to be tested : no MPI only OpenMP into MPMD mode
946            echo "./${ExeNameOut} " >> run_file
947          fi
948        fi
949      fi
950    done
951
952    EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
953
954    IGCM_sys_Chmod u+x run_file
[428]955    if ( $DEBUG_sys ) ; then
[810]956      echo "run_file contains : "
957      cat run_file
[428]958    fi
959
[1180]960  # Only one executable (SPMD mode).
961  else
[428]962
[810]963    for comp in ${config_ListOfComponents[*]} ; do
[428]964
[810]965      # Only if we really have an executable for the component :
966      eval ExeNameOut=\${config_Executable_${comp}[1]}
967      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
968        EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
969      fi
[786]970
[810]971    done
[428]972
[810]973  fi
[428]974
[811]975  IGCM_debug_Print 1 "sys iitm : execution command is"
[810]976  IGCM_debug_Print 1 "$EXECUTION"
[428]977
[810]978  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[428]979}
[651]980
[933]981#D-#==================================================
982#D-function IGCM_sys_check_path
983#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
984#D-* do not point to an important use directory. Stop immediately in that case.
985#D-* Examples:
986#D-
987function IGCM_sys_check_path {
988  IGCM_debug_PushStack "IGCM_sys_check_path"
989  if ( $DEBUG_sys ) ; then
990    echo "IGCM_sys_check_path"
991  fi
[810]992
[933]993  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
994    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
995    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
[936]996    IGCM_debug_Exit "This will stop the job"
[933]997  fi
998  IGCM_debug_PopStack "IGCM_sys_check_path"
999}
1000
[810]1001#D-#==================================================
1002#D-function IGCM_sys_check_quota. Dummy call here
1003#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1004#D-* Examples:
1005#D-
[651]1006function IGCM_sys_check_quota {
[810]1007  IGCM_debug_PushStack "IGCM_sys_check_quota"
1008  if ( $DEBUG_sys ) ; then
1009    echo "IGCM_sys_check_quota"
1010  fi
1011  IGCM_debug_PopStack "IGCM_sys_check_quota"
1012}
1013
1014#D-#==================================================
1015#D-function IGCM_sys_CountJobInQueue
[1244]1016#D-* Purpose: Count number of users job
[810]1017#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1018#D-
1019function IGCM_sys_CountJobInQueue {
1020  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1021  if ( $DEBUG_sys ) ; then
1022    echo "IGCM_sys_CountJobInQueue"
1023  fi
1024
1025  # Print only the full (-W) JobName (%jn)
1026  NbRun=$( llq -W -f %jn | grep -c "$1" )
1027
1028  eval ${2}=${NbRun}
1029
1030  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1031}
Note: See TracBrowser for help on using the repository browser.