source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh @ 1305

Last change on this file since 1305 was 1305, checked in by aclsce, 8 years ago

First implementation of Intel MPI environment for ADA-IDRIS machine (not fully validated, desactivated by default).

  • insÃ_job : choice of MPI Enironment to use by answering a question or -m option (./ins_job -m Intel). Default is IBM Environment.
  • libIGCM_sys_ada.ksh : added specific part for Intel Environment
    • mpirun command (instead of poe for IBM MPI)
    • build of new configuration "run_file"
    • specific header of the Job script -...
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 53.1 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sébastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Ada
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=ada
71# project name
72typeset PROJECT=$(echo ${LOADL_STEP_GROUP:=NONE})
73# jobWarningDelay in seconds
74typeset jobWarningDelay=$( TZ=utc date -d '1970-01-01 '${wall_clock_limit} +%s )
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=llsubmit}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=ergon
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94source /smplocal/pub/Modules/default/init/ksh
95if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] || [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
96  . /smphome/rech/psl/rpsl035/.atlas_env_ada_intel_2013_0_bash > /dev/null 2>&1
97else
98  module load intel/2016.2 > /dev/null 2>&1
99  . /smphome/rech/psl/rpsl035/.atlas_env_ada_bash > /dev/null 2>&1
100fi
101[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
102[ ! X${TaskType} = Xchecking ] && module list
103
104export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddNoise/src_X64_ADA/bin
105export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddPerturbation/src_X64_ADA/bin
106export PATH=${PATH}:/smphome/rech/psl/rpsl035/bin/
107
108#====================================================
109# Host specific DIRECTORIES
110#====================================================
111
112#====================================================
113#- MirrorlibIGCM for frontend
114typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
115
116#====================================================
117#- libIGCM_POST for frontend
118typeset -r libIGCM_POST=${libIGCM}
119
120#====================================================
121#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
122typeset -r R_EXE="${MODIPSL}/bin"
123
124#====================================================
125#- SUBMIT_DIR : submission dir
126typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${LOADL_STEP_INITDIR}}
127
128#====================================================
129#- IN
130typeset -r R_IN=${R_IN:=/workgpfs/rech/psl/rpsl035/IGCM}
131typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/u/rech/psl/rpsl376}
132
133#====================================================
134#- RUN_DIR_PATH : Temporary working directory (=> TMP)
135typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${TMPDIR}}
136
137#====================================================
138#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
139typeset -r OUTCOMMAND_PATH=${RUN_DIR_PATH:-/tmp}
140
141#====================================================
142#- HOST_MPIRUN_COMMAND
143if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] || [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
144  typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
145else
146  typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time mpirun"}
147fi
148#====================================================
149#- Max number of arguments passed to nco operator or demigration command
150UNIX_MAX_LIMIT=360
151
152#====================================================
153#- set PackDefault to true on ada
154PackDefault=true
155
156#====================================================
157#- Number of cores per node
158core_per_node=32
159
160#====================================================
161#- Default number of MPI task for IPSL coupled model
162#- required for backward compatibility
163#-
164DEFAULT_NUM_PROC_OCE=5
165DEFAULT_NUM_PROC_CPL=1
166(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
167
168#D-#==================================================
169#D-function IGCM_sys_defineArchives
170#D-* Purpose:
171#D-* Define ARCHIVE : Dedicated to large files
172#D-* Define STORAGE : Dedicated to small/medium files
173#D-* Define R_OUT   : Output tree located on ARCHIVE
174#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
175#D-* Define R_BUF   : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
176#D-* if SpaceName=TEST everything is stored on WORKDIR
177#D-* Examples:
178#D-
179function IGCM_sys_defineArchives {
180  IGCM_debug_PushStack "IGCM_sys_defineArchives"
181
182  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
183    #====================================================
184    #- ARCHIVE (dedicated to large files)
185    ARCHIVE=${config_UserChoices_ARCHIVE}
186  else
187    #====================================================
188    #- ARCHIVE (dedicated to large files)
189    ARCHIVE=$(echo ${HOME} | sed -e "s,/.*home/,/arch/home/,")
190  fi
191
192  if [ ! X${config_UserChoices_STORAGE} = X ]; then
193    #====================================================
194    #- STORAGE (dedicated to small/medium files)
195    STORAGE=${config_UserChoices_STORAGE}
196  else
197    #====================================================
198    #- STORAGE (dedicated to small/medium files)
199    STORAGE=${WORKDIR}
200  fi
201
202  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
203    #====================================================
204    #- R_OUT
205    R_OUT=${WORKDIR}/IGCM_OUT
206
207    #====================================================
208    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
209    R_FIG=${WORKDIR}/IGCM_OUT
210
211    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
212
213  else
214    #====================================================
215    #- R_OUT
216    R_OUT=${ARCHIVE}/IGCM_OUT
217
218    #====================================================
219    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
220    R_FIG=${ARCHIVE}/IGCM_OUT
221  fi
222
223  #====================================================
224  #- R_BUF (ONLY FOR double copy an scratch)
225  R_BUF=${WORKDIR}/IGCM_OUT
226
227  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
228  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
229  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
230
231  IGCM_debug_PopStack "IGCM_sys_defineArchives"
232}
233
234#D-#==================================================
235#D-function IGCM_sys_RshArchive
236#D-* Purpose: Archive rsh command
237#D-* Examples:
238#D-
239function IGCM_sys_RshArchive {
240  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
241  DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
242  if [ $? = 0 ] ; then
243    rsh ${STOREHOST} exec /bin/ksh <<-EOF
244    ${@}
245EOF
246    status=$?
247  else
248    /bin/ksh <<-EOF
249    ${@}
250EOF
251    status=$?
252  fi
253  if [ ${status} -gt 0 ] ; then
254    IGCM_debug_Print 2 "IGCM_sys_RshArchive : rsh or command failed error code ${status}"
255    IGCM_debug_Exit "IGCM_sys_RshArchive"
256  fi
257  IGCM_debug_PopStack "IGCM_sys_RshArchive"
258}
259
260#D-#==================================================
261#D-function IGCM_sys_RshArchive_NoError
262#D-* Purpose: Archive rsh command, without error
263#D-*          used only in monitoring.job
264#D-* Examples:
265#D-
266function IGCM_sys_RshArchive_NoError {
267  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
268  DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
269  if [ $? = 0 ] ; then
270    rsh ${STOREHOST} exec /bin/ksh <<-EOF
271    ${@} 2> /dev/null
272EOF
273  else
274    /bin/ksh <<-EOF
275    ${@} 2> /dev/null
276EOF
277  fi
278  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
279}
280
281#D-#==================================================
282#D-function IGCM_sys_MkdirArchive
283#D-* Purpose: Mkdir on Archive
284#D-* Examples:
285#D-
286function IGCM_sys_MkdirArchive {
287  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
288  if ( $DEBUG_sys ) ; then
289    echo "IGCM_sys_MkdirArchive :" $@
290  fi
291  #- creation de repertoire sur le serveur fichier
292  DEBUG_sys=false IGCM_sys_IsFileArchived $1
293  if [ $? = 0 ] ; then
294    rsh ${STOREHOST} -n mkdir -p $1
295    status=$?
296  else
297    mkdir -p $1
298    status=$?
299  fi
300
301  if [ ${status} -gt 0 ] ; then
302    IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : rsh or mkdir failed error code ${status}"
303    IGCM_debug_Exit "IGCM_sys_MkdirArchive"
304  fi
305  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
306}
307
308#D-#==================================================
309#D-function IGCM_sys_TestDirArchive
310#D-* Purpose: Test Directory that must exists on Archive
311#D-* Examples:
312#D-
313function IGCM_sys_TestDirArchive {
314  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
315  if ( $DEBUG_sys ) ; then
316    echo "IGCM_sys_TestDirArchive :" $@
317  fi
318  typeset ExistFlag
319  #Command depends on targeted file system
320  DEBUG_sys=false IGCM_sys_IsFileArchived $1
321  if [ $? = 0 ] ; then
322    ExistFlag=$( IGCM_sys_RshArchive "[ -d $1 ] && echo 0 || echo 1" )
323  else
324    ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
325  fi
326  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
327  return ${ExistFlag}
328}
329
330#D-#==================================================
331#D-function IGCM_sys_IsFileArchived
332#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
333#D-* Examples:
334#D-
335function IGCM_sys_IsFileArchived {
336  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
337  if ( $DEBUG_sys ) ; then
338    echo "IGCM_sys_IsFileArchived :" $@
339  fi
340  typeset IsArchivedFlag
341  IsArchivedFlag=$( [ "X$( echo $@ | grep \/arch\/home )" != "X" ] && echo 0 || echo 1 )
342  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
343
344  return ${IsArchivedFlag}
345}
346
347#D-#==================================================
348#D-function IGCM_sys_TestFileArchive
349#D-* Purpose: Test file that must NOT EXISTS on Archive
350#D-* Examples:
351#D-
352function IGCM_sys_TestFileArchive {
353  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
354  typeset ExistFlag
355  ExistFlag=$( IGCM_sys_RshArchive "[ -f $1 ] && echo 0 || echo 1" )
356  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
357
358  return ${ExistFlag}
359}
360
361#D-#==================================================
362#D-function IGCM_sys_CountFileArchive
363#D-* Purpose: Count files on Archive filesystem
364#D-* Examples:
365#D-
366function IGCM_sys_CountFileArchive {
367  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
368  #Command depends on targeted file system
369  DEBUG_sys=false IGCM_sys_IsFileArchived $1
370  if [ $? = 0 ] ; then
371    IGCM_sys_RshArchive "ls ${@} 2>/dev/null | wc -l"
372    status=$?
373  else
374    ls ${@} 2>/dev/null | wc -l
375    status=$?
376  fi
377  if [ ${status} -gt 0 ] ; then
378    echo "IGCM_sys_CountFileArchive : erreur."
379  fi
380  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
381}
382
383#D-#==================================================
384#D-function IGCM_sys_Tree
385#D-* Purpose: Tree directories with files on ${ARCHIVE}
386#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
387#D-
388function IGCM_sys_Tree {
389  IGCM_debug_PushStack "IGCM_sys_Tree" $@
390  if ( $DEBUG_sys ) ; then
391    echo "IGCM_sys_Tree :" $@
392  fi
393
394  \mfls -R $@
395
396  IGCM_debug_PopStack "IGCM_sys_Tree"
397}
398
399#D-#==================================================
400#D-function IGCM_sys_Qsub
401#D-* Purpose: Qsub new job
402#D-* Examples:
403#D-
404function IGCM_sys_Qsub {
405  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
406  if ( $DEBUG_sys ) ; then
407    echo "IGCM_sys_Qsub :" $@
408  fi
409  typeset status
410
411  # We have to change output/error file
412  [ ${#@} = 1 ] &&  REP_FOR_JOB=${SUBMIT_DIR}
413  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
414  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Output}:" \
415    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
416    $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
417  cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
418
419  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
420  if [ ${status} -gt 0 ] ; then
421    IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
422    IGCM_debug_Exit "IGCM_sys_Qsub"
423  else
424    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ | tail -1 | sed -e s/\"//g )
425    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
426  fi
427  IGCM_debug_PopStack "IGCM_sys_Qsub"
428}
429
430#D-#==================================================
431#D-function IGCM_sys_QsubPost
432#D-* Purpose: Qsub new job on scalaire
433#D-* Examples:
434#D-
435function IGCM_sys_QsubPost {
436  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
437  if ( $DEBUG_sys ) ; then
438    echo "IGCM_sys_QsubPost :" $@
439  fi
440  typeset status
441
442  # We have to change output/error file
443  [ ${#@} = 1 ] &&  REP_FOR_JOB=${POST_DIR}
444  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
445
446  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Post_Output}.out:" \
447    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Post_Output}.out:"   \
448    ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
449
450  cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM_$$ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
451
452  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
453  if [ ${status} -gt 0 ] ; then
454    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
455    IGCM_debug_Exit "IGCM_sys_QsubPost"
456  else
457    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ | tail -1 | sed -e s/\"//g )
458    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
459  fi
460  IGCM_debug_PopStack "IGCM_sys_QsubPost"
461}
462
463#D-*************************
464#D- File transfer functions
465#D-*************************
466#D-
467
468#D-#==================================================
469#D-function IGCM_sys_RmRunDir
470#D-* Purpose: rm tmpdir (dummy function most of the time batch
471#D-                      scheduler will do the job)
472#D-* Examples:
473#D-
474function IGCM_sys_RmRunDir {
475  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
476  if ( $DEBUG_sys ) ; then
477    echo "IGCM_sys_RmRunDir :" $@
478    IGCM_debug_Print 1 "Dummy call, let the scheduler do that."
479  fi
480  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
481}
482
483#D-#==================================================
484#D-function IGCM_sys_Put_Dir
485#D-* Purpose: Copy a complete directory on $(ARCHIVE)
486#D-* Examples:
487#D-
488function IGCM_sys_Put_Dir {
489  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
490  if ( $DEBUG_sys ) ; then
491    echo "IGCM_sys_Put_Dir :" $@
492  fi
493  if [ $DRYRUN = 0 ]; then
494    if [ ! -d ${1} ] ; then
495      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
496      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
497      return
498    fi
499
500    typeset status
501
502    # Only if we use rsync
503    #IGCM_sys_TestDirArchive $( dirname $2 )
504    #
505    #Command depends on targeted file system
506    DEBUG_sys=false IGCM_sys_IsFileArchived $2
507    if [ $? = 0 ] ; then
508      \rcp -r $1 ${STOREHOST}:$2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
509      status=$?
510    else
511      \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
512      status=$?
513    fi
514
515    if [ ${status} -gt 0 ] ; then
516      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : rcp or cp failed error code ${status}"
517      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
518      IGCM_debug_Exit "IGCM_sys_Put_Dir"
519    else
520      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
521    fi
522  fi
523  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
524}
525
526#D-#==================================================
527#D-function IGCM_sys_Get_Dir
528#D-* Purpose: Copy a complete directory from ${ARCHIVE}
529#D-* Examples:
530#D-
531function IGCM_sys_Get_Dir {
532  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
533  if ( $DEBUG_sys ) ; then
534    echo "IGCM_sys_Get_Dir :" $@
535  fi
536  if [ $DRYRUN = 0 ]; then
537    typeset NB_ESSAI DELAI status i
538    # number of tentative
539    NB_ESSAI=3
540    # time delay between tentative
541    DELAI=2
542
543    # Only if we use rsync
544    #IGCM_sys_TestDirArchive $( dirname $2 )
545    #
546    # Command depends on targeted filesystem
547    DEBUG_sys=false IGCM_sys_IsFileArchived $1
548    if [ $? = 0 ] ; then
549      # add dmget (to demigrate all offline files) to reduce time of this command :
550      #IGCM_sys_RshArchive "dmfind $1 -state MIG -o -state OFL -o -state PAR | dmget -q -n"
551      i=0
552      while [ $i -lt $NB_ESSAI ] ; do
553        \rcp -rp ${STOREHOST}:$1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
554        status=$?
555        if [ ${status} -gt 0 ]; then
556          IGCM_debug_Print 2 "IGCM_sys_Get_Dir : rcp failed error code ${status} ${i}/${NB_ESSAI}"
557          IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
558          sleep $DELAI
559        else
560          break
561        fi
562        (( i = i + 1 ))
563      done
564    else
565      \cp -rp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
566      status=$?
567      if [ ${status} -gt 0 ] ; then
568        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
569        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
570        IGCM_debug_Exit "IGCM_sys_Get_Dir"
571      else
572        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
573      fi
574    fi
575    if [ ${status} -gt 0 ] ; then
576      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : (r)cp failed error code ${status}"
577      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
578      IGCM_debug_Exit "IGCM_sys_Get_Dir"
579    else
580      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
581    fi
582  fi
583  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
584}
585
586#D-#==================================================
587#D-function IGCM_sys_Put_Rest
588#D-* Purpose: Put computied restarts on ${ARCHIVE}.
589#D-           File and target directory must exist.
590#D-* Examples:
591#D-
592function IGCM_sys_Put_Rest {
593  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
594  if ( $DEBUG_sys ) ; then
595    echo "IGCM_sys_Put_Rest :" $@
596  fi
597  if [ $DRYRUN = 0 ]; then
598    if [ ! -f ${1} ] ; then
599      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
600      IGCM_debug_Exit "IGCM_sys_Put_Rest"
601    fi
602
603    typeset status
604    #
605    if [ X${JobType} = XRUN ] ; then
606      IGCM_sys_Chmod 444 ${1}
607    fi
608    #
609    #
610    #Command depends on targeted file system
611    DEBUG_sys=false IGCM_sys_IsFileArchived $2
612    if [ $? = 0 ] ; then
613      mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
614      status=$?
615    else
616      IGCM_sys_MkdirArchive $( dirname $2 )
617      \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
618      status=$?
619    fi
620
621#       #RSYNC WITH NETWORK RSH CALL
622#       IGCM_sys_MkdirArchive $( dirname $2 )
623#       echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
624#       ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
625
626#       #RSYNC WITH NFS USE
627#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
628#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
629
630#       status=$?
631#       IGCM_sys_Rsync_out $status
632
633#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
634#       (( status=status+$? ))
635
636    if [ ${status} -gt 0 ] ; then
637      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : mfput or cp failed error code ${status}"
638      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
639      IGCM_debug_Exit "IGCM_sys_Put_Rest"
640    else
641      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
642    fi
643  fi
644  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
645}
646
647#D-#==================================================
648#D-function IGCM_sys_Put_Out
649#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
650#D-* Examples:
651#D-
652function IGCM_sys_Put_Out {
653  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
654  if ( $DEBUG_sys ) ; then
655    echo "IGCM_sys_Put_Out :" $@
656  fi
657
658  typeset status
659
660  if [ $DRYRUN = 0 ]; then
661    if [ ! -f ${1} ] ; then
662      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
663      IGCM_debug_PopStack "IGCM_sys_Put_Out"
664      return 1
665    fi
666    #
667    if [ X${JobType} = XRUN ] ; then
668      if [ X${3} = X ] ; then
669        IGCM_sys_Chmod 444 ${1}
670      fi
671    fi
672    #
673    #
674    #Command depends on targeted file system
675    DEBUG_sys=false IGCM_sys_IsFileArchived $2
676    if [ $? = 0 ] ; then
677      mfput $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
678      status=$?
679    else
680      IGCM_sys_MkdirArchive $( dirname $2 )
681      \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
682      status=$?
683    fi
684
685#       #RSYNC WITH NETWORK RSH CALL
686#       IGCM_sys_MkdirArchive $( dirname $2 )
687#       echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
688#       ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
689
690#       #RSYNC WITH NFS USE
691#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
692#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
693
694#       status=$?
695#       IGCM_sys_Rsync_out $status
696
697#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
698#       (( status=status+$? ))
699
700    if [ ${status} -gt 0 ] ; then
701      IGCM_debug_Print 2 "IGCM_sys_Put_Out : mfput or cp failed error code ${status}"
702      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
703      IGCM_debug_Exit "IGCM_sys_Put_Out"
704    else
705      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
706    fi
707  fi
708  IGCM_debug_PopStack "IGCM_sys_Put_Out"
709  return 0
710}
711
712#D-#==================================================
713#D-function IGCM_sys_Get
714#D-* Purpose: Get a file from ${ARCHIVE}
715#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
716#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
717function IGCM_sys_Get {
718  IGCM_debug_PushStack "IGCM_sys_Get" $@
719
720  typeset DEST status dm_liste
721
722  if ( $DEBUG_sys ) ; then
723    echo "IGCM_sys_Get :" $@
724  fi
725  if [ $DRYRUN -le 2 ]; then
726    if [ X${1} = X'/l' ] ; then
727      eval set +A dm_liste \${${2}}
728    else
729      dm_liste=${1}
730    fi
731    eval DEST=\${${#}}
732
733    # test if the (first) file is present in the old computation :
734    DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
735    if [ $? = 0 ] ; then
736      IGCM_sys_TestFileArchive ${dm_liste[0]}
737      status=$?
738    else
739      IGCM_sys_TestFileBuffer ${dm_liste[0]}
740      status=$?
741    fi
742
743    if [ ${status} -gt 0 ] ; then
744      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
745      IGCM_debug_Exit "IGCM_sys_Get"
746      return
747    fi
748
749    #Command depends on targeted file system
750    DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
751    if [ $? = 0 ] ; then
752      mfget ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
753      status=$?
754    else
755      \cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
756      status=$?
757    fi
758
759#       #RSYNC WITH NETWORK RSH CALL
760#       echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
761#       ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
762
763#       #RSYNC WITH NFS USE
764#       echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
765#       ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
766
767#       status=$?
768#       IGCM_sys_Rsync_out $status
769
770#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
771#       (( status=status+$? ))
772
773    if [ ${status} -gt 0 ] ; then
774      IGCM_debug_Print 2 "IGCM_sys_Get : mfget or cp failed error code ${status}"
775      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
776      IGCM_debug_Exit "IGCM_sys_Get"
777    else
778      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
779    fi
780  fi
781  IGCM_debug_PopStack "IGCM_sys_Get"
782}
783
784#D-#==================================================
785#D-function IGCM_sys_GetDate_Monitoring
786#D-* Purpose: get the last year for which the monitoring has been computed
787#D-* Examples:
788#D-
789function IGCM_sys_GetDate_Monitoring {
790  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
791  if ( $DEBUG_sys ) ; then
792    echo "IGCM_sys_GetDate_Monitoring :" $@
793  fi
794
795  IGCM_sys_Cd /tmp
796  # NASTY HACK. We force /tmp here. This function is called only from the front-end and interactively.
797  RUN_DIR_PATH=/tmp IGCM_sys_Get ${1} .
798  eval ${2}=$( cdo showyear $( basename ${1} ) 2> /dev/null | gawk '{ print $NF }' )
799  # Need to erase this file to avoid collision (permission denied getting file) between users on the front-end
800  IGCM_sys_Rm -f $( basename ${1} )
801  IGCM_sys_Cd -
802
803  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
804}
805
806#D-#==================================================
807#D-function IGCM_sys_Put_Dods
808#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
809#D-* Examples:
810#D-
811function IGCM_sys_Put_Dods {
812  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
813  if ( $DEBUG_sys ) ; then
814    echo "IGCM_sys_Put_Dods :" $@
815  fi
816  typeset status
817  if [ $DRYRUN = 0 ]; then
818    # We take our time on that
819    sleep 10
820    IGCM_sys_TestDirArchive ${R_SAVE}/${1}
821    if [ $? != 0 ] ; then
822      echo "WARNING : IGCM_sys_Put_Dods ${R_SAVE}/${1} DOES NOT EXIST ."
823      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
824      return
825    fi
826
827    rsh ${STOREHOST} exec /bin/ksh <<EOF
828          cd ${R_SAVE}
829          /arch/local/bin/dods_rm DODS/pub/${LOGIN}/${R_DODS}/${1} > /dev/null 2>&1
830          /bin/chmod -R u+w ${R_SAVE}/${1}
831          /arch/local/bin/dods_cp ${1} DODS/pub/${LOGIN}/${R_DODS} > /dev/null 2>&1
832          /bin/chmod -R +rX ${R_SAVE}/${1}
833          /bin/chmod -R u+w ${R_SAVE}/${1}
834EOF
835    status=$?
836
837    if [ ${status} -gt 0 ] ; then
838      echo "IGCM_sys_Put_Dods : error."
839      IGCM_debug_Exit "IGCM_sys_Put_Dods"
840    fi
841  fi
842  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
843}
844
845##############################################################
846# REBUILD OPERATOR
847
848#D-#==================================================
849#D-function IGCM_sys_sync
850#D-* Purpose: flush buffer on disk (dummy function on Ada)
851#D-* Examples:
852#D-
853function IGCM_sys_sync {
854  IGCM_debug_PushStack "IGCM_sys_sync" $@
855  if ( $DEBUG_sys ) ; then
856    echo "IGCM_sys_sync :" $@
857    echo "Dummy call, let the system do that."
858  fi
859  IGCM_debug_PopStack "IGCM_sys_sync"
860}
861
862############################################################
863# Activate Running Environnment Variables
864
865#D-#==================================================
866#D-function IGCM_sys_activ_variables
867#D-* Purpose: set environement variables prior to execution
868#D-* Examples:
869#D-
870function IGCM_sys_activ_variables {
871  IGCM_debug_PushStack "IGCM_sys_activ_variables"
872  if ( [ ! "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] && [ ! "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
873    # Temporary modifications for Intel MPI Environment
874    # To use DAPL Provider instead of OFA for inter-node communications
875    # and shm for intra-node communications
876    export I_MPI_FABRICS=shm:dapl
877    # To use UD protocol instead of RC (default with DAPL)
878    export I_MPI_DAPL_UD=1
879    # Some optimizations
880    export DAPL_UCM_REP_TIME=2000 # REQUEST timer, waiting for REPLY in millisecs
881    export DAPL_UCM_RTU_TIME=2000 # REPLY timer, waiting for RTU in millisecs
882    export DAPL_UCM_CQ_SIZE=2000 # CM completion queue
883    export DAPL_UCM_QP_SIZE=2000 # CM message queue
884    export DAPL_UCM_RETRY=7 # REQUEST and REPLY retries
885    export DAPL_ACK_RETRY=7 # IB RC Ack retry count
886    export DAPL_ACK_TIMER=20 # IB RC Ack retry timer
887    # To enable RDMA
888    export I_MPI_DAPL_UD_RDMA_MIXED=enable
889    # To use DAPL 2.1.7
890    export DAT_OVERRIDE=/smplocal/pub/DAPL/2.1.7/etc/dat.conf
891    export LD_LIBRARY_PATH=/smplocal/pub/DAPL/2.1.7/lib:${LD_LIBRARY_PATH}
892  fi
893  if ( $DEBUG_sys ) ; then
894    echo "IGCM_sys_activ_variables"
895  fi
896
897# --------------------------------------------------------------------
898#D- MPI specifications
899# --------------------------------------------------------------------
900
901# --------------------------------------------------------------------
902#D- Other specifications
903# --------------------------------------------------------------------
904
905  ulimit -s unlimited
906## to be done only one time
907## export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
908##  echo ${LD_LIBRARY_PATH} | grep -i netcdf >/dev/null 2>&1 || export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
909  IGCM_debug_Print 1 "set LD_LIBRARY_PATH=${LD_LIBRARY_PATH}"
910
911  export MP_STDOUTMODE=combined
912  IGCM_debug_Print 1 "set MP_STDOUTMODE=${MP_STDOUTMODE}"
913
914  ## fix to reduce memory usage. Required since 2014/22/04. On ada IDRIS.
915  #export MP_EUILIBPATH=/smplocal/lib/ibmhpc/pe12012/ppe.pami/gnu/lib64/pami64
916  #IGCM_debug_Print 1 "set MP_EUILIBPATH=${MP_EUILIBPATH}"
917
918  IGCM_debug_PopStack "IGCM_sys_activ_variables"
919}
920
921############################################################
922# Desactivate Running Environnment Variables
923
924#D-#==================================================
925#D-function IGCM_sys_desactiv_variables
926#D-* Purpose: unset environement variables after execution
927#D-* Examples:
928#D-
929function IGCM_sys_desactiv_variables {
930  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
931  if ( $DEBUG_sys ) ; then
932    echo "IGCM_sys_desactiv_variables"
933  fi
934# --------------------------------------------------------------------
935#D- MPI specifications
936# --------------------------------------------------------------------
937
938# --------------------------------------------------------------------
939#D- Other specifications
940# --------------------------------------------------------------------
941
942  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
943}
944
945############################################################
946# Update job headers to be used by the scheduler
947
948#D-#==================================================
949#D-function IGCM_sys_updateHeaders
950#D-* Purpose: Update job headers to be used by the scheduler
951#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
952#D-
953function IGCM_sys_updateHeaders {
954  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
955  if ( $DEBUG_sys ) ; then
956    echo "IGCM_sys_updateHeaders"
957  fi
958  typeset file
959  file=$1
960
961  if [ ${executionType} -eq 1 ] ; then
962    # MPMD + MPI
963    if [ X${MPIEnvironment} = XIBM ] ; then
964      sed -e "/::openMPthreads::/d"                  \
965        -e "s/::JobNumProcTot::/${coreNumber}/"    \
966        ${file} > ${file}.tmp
967    else
968      sed -e "/::openMPthreads::/d"                  \
969        -e "s/@ job_type = parallel/@ job_type = mpich/" \
970        -e "s/::JobNumProcTot::/${coreNumber}/"    \
971        ${file} > ${file}.tmp
972    fi
973
974  elif [ ${executionType} -eq 2 ] ; then
975    # MPMD + MPI + OMP
976    if [ X${MPIEnvironment} = XIBM ] ; then
977      sed -e "s/::openMPthreads::/${openMPthreads}/" \
978        -e "s/::JobNumProcTot::/${mpiTasks}/"    \
979        ${file} > ${file}.tmp
980    else
981      (( nodeNumber = coreNumber / core_per_node ))
982      [ $(( ${coreNumber} % ${core_per_node} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
983      sed -e "/::openMPthreads::/d"                  \
984        -e "s/@ job_type = parallel/@ job_type = mpich/" \
985        -e "s/@ total_tasks = ::JobNumProcTot::/@ node = ${nodeNumber} /"    \
986        -e "/@ as_limit = 3.5gb/d"      \
987        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
988        ${file} > ${file}.tmp
989    fi
990
991  elif [ ${executionType} -eq 3 ] ; then
992    # SPMD + MPI/OMP
993    if [ X${MPIEnvironment} = XIBM ] ; then
994      sed -e "s/::openMPthreads::/${openMPthreads}/" \
995        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
996        ${file} > ${file}.tmp
997    else
998      sed -e "s/::openMPthreads::/${openMPthreads}/" \
999        -e "s/@ job_type = parallel/@ job_type = mpich/" \
1000        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1001        ${file} > ${file}.tmp
1002    fi
1003
1004  elif [ ${executionType} -eq 4 ] ; then
1005    # SPMD + MPI only
1006    if [ X${MPIEnvironment} = XIBM ] ; then
1007      sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1008        -e "/::openMPthreads::/d"                  \
1009        ${file} > ${file}.tmp
1010    else
1011      sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1012        -e "s/@ job_type = parallel/@ job_type = mpich/" \
1013        -e "/::openMPthreads::/d"                  \
1014        ${file} > ${file}.tmp
1015    fi
1016
1017  elif [ ${executionType} -eq 5 ] ; then
1018    # SPMD + OMP only
1019    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1020        -e "s/@ job_type = parallel/@ job_type = serial/" \
1021        -e "/::JobNumProcTot::/d"                  \
1022      ${file} > ${file}.tmp
1023
1024  elif [ ${executionType} -eq 6 ] ; then
1025    # SEQUENTIAL THEN
1026    sed -e "s/::JobNumProcTot::/1/"                \
1027        -e "s/@ job_type = parallel/@ job_type = serial/" \
1028        -e "/::openMPthreads::/d"                  \
1029      ${file} > ${file}.tmp
1030
1031  fi
1032
1033  IGCM_sys_Mv ${file}.tmp ${file}
1034
1035  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1036}
1037
1038############################################################
1039# Build MPI/OMP scripts run file (dummy function)
1040
1041#D-#==================================================
1042#D-function IGCM_sys_build_run_file
1043#D-* Purpose: build run file (deprecated)
1044#D-* Examples:
1045#D-
1046function IGCM_sys_build_run_file {
1047
1048  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1049
1050}
1051
1052############################################################
1053# Build MPI/OMP scripts
1054
1055#D-#==================================================
1056#D-function IGCM_sys_build_execution_scripts
1057#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1058#D-* Examples:
1059#D-
1060function IGCM_sys_build_execution_scripts
1061{
1062  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1063  if ( $DEBUG_sys ) ; then
1064    echo "IGCM_sys_build_execution_scripts " $@
1065  fi
1066
1067  EXECUTION=${HOST_MPIRUN_COMMAND}
1068
1069  # MPMD mode
1070  if ( ${OK_PARA_MPMD} ) ; then
1071
1072    # MPI IBM Environment
1073    if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] || [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1074      IGCM_debug_Print 1 "You use IBM MPI environment"
1075      if [ -f run_file ] ; then
1076        IGCM_sys_Rm -f run_file
1077      fi
1078      touch run_file
1079
1080      # Build run_file
1081
1082      # First loop on the components for the coupler ie oasis (only if oasis3)
1083      # the coupler ie oasis3 must be the first one
1084      for comp in ${config_ListOfComponents[*]} ; do
1085
1086        eval ExeNameIn=\${config_Executable_${comp}[0]}
1087        eval ExeNameOut=\${config_Executable_${comp}[1]}
1088
1089        # for CPL component only
1090        if [ "X${comp}" = "XCPL" ] ; then
1091
1092          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1093          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1094
1095          if ( ${OK_PARA_MPI} ) ; then
1096            (( mpi_count = 1 ))
1097            until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1098              echo "./${ExeNameOut}" >> run_file
1099              (( mpi_count = mpi_count + 1 ))
1100            done
1101          else
1102            echo "./${ExeNameOut} " >> run_file
1103          fi
1104        fi
1105
1106      done
1107
1108      # Then second loop on the components
1109      for comp in ${config_ListOfComponents[*]} ; do
1110
1111        eval ExeNameIn=\${config_Executable_${comp}[0]}
1112        eval ExeNameOut=\${config_Executable_${comp}[1]}
1113
1114        # Only if we really have an executable for the component and not the coupler ie oasis:
1115        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1116
1117          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1118          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1119
1120          if ( ${OK_PARA_OMP} ) ; then
1121            # Check if the number of threads is correct
1122            case ${comp_proc_omp_loc} in
1123            1|2|4)
1124              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads on IBM MPI Environment"
1125              IGCM_debug_Print 1 "Beware : it may you use more CPU than needed."
1126              IGCM_debug_Print 1 "Think to the Intel MPI Environment to do what you want to do !"
1127              ;;
1128            8|16)
1129              IGCM_debug_Exit "ERROR with OMP parameters !"
1130              IGCM_debug_Print 2 "Beware : ${comp_proc_omp_loc} is too much for MPMD mode"
1131              IGCM_debug_Print 2 "You will use more CPU than needed : try to use Intel-MPI environment to do such a thing !"
1132              IGCM_debug_Verif_Exit
1133              ;;
1134            *)
1135              IGCM_debug_Exit "ERROR with OMP parameters !"
1136              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1137              IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1138              IGCM_debug_Verif_Exit
1139              ;;
1140            esac
1141
1142          fi
1143
1144          if ( ${OK_PARA_MPI} ) ; then
1145            (( mpi_count = 1 ))
1146            until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1147              if ( ${OK_PARA_OMP} ) ; then
1148                echo "env OMP_NUM_THREADS=$comp_proc_omp_loc ./${ExeNameOut}" >> run_file
1149              else
1150                echo "./${ExeNameOut}" >> run_file
1151              fi
1152              (( mpi_count = mpi_count + 1 ))
1153            done
1154          else
1155            echo "./${ExeNameOut} " >> run_file
1156          fi
1157        fi
1158      done
1159      if ( ${OK_PARA_OMP} ) ; then
1160        export KMP_STACKSIZE=200m
1161      fi
1162
1163      EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
1164
1165      IGCM_sys_Chmod u+x run_file
1166      if ( $DEBUG_sys ) ; then
1167        echo "run_file contains : "
1168        cat run_file
1169      fi
1170
1171
1172    # MPI Intel Environment
1173    else
1174      IGCM_debug_Print 1 "You use Intel MPI environment"
1175
1176      # Only MPI (MPMD)
1177      if  ( ! ${OK_PARA_OMP} ) ; then
1178        init_exec=n
1179 
1180        # First loop on the components for the coupler ie oasis (only if oasis3)
1181        # the coupler ie oasis3 must be the first one
1182        for comp in ${config_ListOfComponents[*]} ; do
1183
1184          eval ExeNameIn=\${config_Executable_${comp}[0]}
1185          eval ExeNameOut=\${config_Executable_${comp}[1]}
1186
1187          # for CPL component only
1188          if [ "X${comp}" = "XCPL"  ]  && [ "X${ExeNameOut}" != X\"\" ] ; then
1189
1190            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1191            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1192
1193            echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1194            echo ""  >> script_${ExeNameOut}.ksh
1195            echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1196            IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1197            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1198            init_exec=y
1199          fi
1200        done
1201        # Then second loop on the components
1202        for comp in ${config_ListOfComponents[*]} ; do
1203
1204          eval ExeNameIn=\${config_Executable_${comp}[0]}
1205          eval ExeNameOut=\${config_Executable_${comp}[1]}
1206
1207          # Only if we really have an executable for the component and not the coupler ie oasis:
1208          if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1209
1210            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1211            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1212
1213            echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1214            echo ""  >> script_${ExeNameOut}.ksh
1215            echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1216            IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1217
1218            if [ ${init_exec} = y ] ; then
1219              EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1220            else
1221              EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1222              init_exec=y
1223            fi
1224          fi
1225        done
1226      # MPI-OpenMP (MPMD)
1227      else
1228
1229        # Execution specifications
1230        EXECUTION="${EXECUTION} -configfile run_file"
1231        export KMP_STACKSIZE=200m
1232        if [ -f run_file ] ; then
1233          IGCM_sys_Rm -f run_file
1234        fi
1235        touch run_file
1236
1237        # Initialisation of variables
1238        string_final=""
1239        string_final_hexa=""
1240        current_core=0
1241        executable_nb=1
1242
1243        #  Hosts treatment
1244        for nodes in `echo $LOADL_PROCESSOR_LIST`
1245        do
1246          host[$i]=$nodes
1247          i=$((i+1))
1248        done
1249
1250        # Loop on the components
1251        for comp in ${config_ListOfComponents[*]} ; do
1252
1253          eval ExeNameIn=\${config_Executable_${comp}[0]}
1254          eval ExeNameOut=\${config_Executable_${comp}[1]}
1255
1256          # Not possible if oasis has an executable (i.e old version of oasis3)
1257          if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1258            IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1259            IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1260            IGCM_debug_Verif_Exit
1261          fi
1262
1263          # Only if we really have an executable for the component :
1264          if [ "X${ExeNameOut}" != X\"\" ] ; then
1265
1266            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1267            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1268
1269            # Check if the number of threads is correct
1270            case ${comp_proc_omp_loc} in
1271            1|2|4|8|16)
1272              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1273              ;;
1274            *)
1275              IGCM_debug_Exit "ERROR with OMP parameters !"
1276              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1277              IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
1278              IGCM_debug_Verif_Exit
1279              ;;
1280            esac
1281
1282
1283            # Build run_file for Ada Intel-MPI environment : method used to assign cores and nodes for the MPI process by using hexadecimal mask
1284            # Example of run_file :
1285            #-host host0 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1286            #-host host1 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
1287            #-host host2 -n 10 -env I_MPI_PIN_DOMAIN=[3,c,30,c0,300,c00,3000,c000,30000,c0000,100000,200000,400000,800000,1000000,2000000,4000000,8000000,10000000,20000000] ./b.out
1288            #-host host2 -n 10 ./c.out
1289            # Example of final command :
1290            # mpirun -configfile run_file
1291
1292            rank=0
1293            # For one specific executable, loop on mpi process
1294            for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1295              (( index_host = current_core / core_per_node ))
1296              host_value=${host[${index_host}]}
1297              (( slot =  current_core % core_per_node ))
1298              # loop on omp threads for each mpi process (set the appropriate bit to 1 and append it to previous one)
1299              for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1300                string_final="1"$string_final
1301              done
1302              # convert binary mask to hexadecimal mask
1303              if [ $rank -ne 0 ] ; then
1304                string_final_hexa=$string_final_hexa","$( printf '%x\n' "$((2#$string_final))" )
1305              else
1306                string_final_hexa=$( printf '%x\n' "$((2#$string_final))" )
1307              fi
1308              # replace bit 1 by bit 0 in order to append next one (next one wil be 1)
1309              string_final=$( echo $string_final | sed "s/1/0/g" )
1310              # mpi rank = mpi_rank + 1
1311              (( rank = rank + 1 ))
1312              # current core takes into account the number of omp threads which was previously append
1313              (( current_core = current_core + comp_proc_omp_loc ))
1314              # We write to the configuration file either we switch to the next node or we switch to the next executable
1315              if ( [ $(( current_core / core_per_node )) -ne $index_host ] || [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ) ; then
1316                # I_MPI_PIN_DOMAIN variable must be given once per node
1317                if [ $executable_nb -eq 1 ] ; then
1318                  echo "-host $host_value -n $rank -env I_MPI_PIN_DOMAIN=[$string_final_hexa] ./$ExeNameOut" >> run_file
1319                else
1320                  sed -i "/$host_value/s/\]/\,$string_final_hexa\]/g" run_file
1321                  echo "-host $host_value -n $rank ./$ExeNameOut" >> run_file
1322                fi
1323                # +1 for the number of executbale on the same node
1324                if [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ; then
1325                  (( executable_nb = executable_nb + 1 ))
1326                fi
1327                # Some initializations if we switch to the next node
1328                if [ $(( current_core / core_per_node )) -ne $index_host ] ; then
1329                  string_final=""
1330                  string_final_hexa=""
1331                  rank=0
1332                  executable_nb=1
1333                fi
1334              fi
1335            done
1336
1337          fi
1338        done
1339        IGCM_sys_Chmod u+x run_file
1340        if ( $DEBUG_sys ) ; then
1341          echo "run_file contains : "
1342          cat run_file
1343        fi
1344      fi
1345    fi
1346
1347  # Only one executable (SPMD mode).
1348  else
1349
1350    for comp in ${config_ListOfComponents[*]} ; do
1351
1352      # Only if we really have an executable for the component :
1353      eval ExeNameOut=\${config_Executable_${comp}[1]}
1354      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1355
1356        if ( ${OK_PARA_OMP} ) ; then
1357          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1358          export KMP_STACKSIZE=200m
1359          #export KMP_LIBRARY=turnaround
1360          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1361          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1362          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1363        fi
1364        if  ( ${OK_PARA_MPI} ) ; then
1365          if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] || [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
1366            EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
1367          else
1368            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1369            EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
1370          fi
1371        else
1372          EXECUTION="/usr/bin/time ./${ExeNameOut}"
1373        fi
1374      fi
1375
1376    done
1377
1378  fi
1379
1380  IGCM_debug_Print 1 "sys ada : execution command is "
1381  IGCM_debug_Print 1 "$EXECUTION"
1382
1383  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1384}
1385
1386#D-#==================================================
1387#D-function IGCM_sys_check_path
1388#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1389#D-* do not point to an important use directory. Stop immediately in that case.
1390#D-* Examples:
1391#D-
1392function IGCM_sys_check_path {
1393  IGCM_debug_PushStack "IGCM_sys_check_path"
1394  if ( $DEBUG_sys ) ; then
1395    echo "IGCM_sys_check_path"
1396  fi
1397
1398  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1399    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1400    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1401    IGCM_debug_Exit "This will stop the job"
1402  fi
1403  IGCM_debug_PopStack "IGCM_sys_check_path"
1404}
1405
1406#D-#==================================================
1407#D-function IGCM_sys_check_quota
1408#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1409#D-* Examples:
1410#D-
1411function IGCM_sys_check_quota {
1412  IGCM_debug_PushStack "IGCM_sys_check_quota"
1413  if ( $DEBUG_sys ) ; then
1414    echo "IGCM_sys_check_quota"
1415  fi
1416  # Limit of quota (in %)
1417  limit_quota=90
1418
1419  # Check of the volume
1420  volume_quota=$(quota_u -w 2>/dev/null| grep 'Quota soft' | gawk '{print $5}')
1421  if [ ! X${volume_quota} = X ] ; then
1422    quota_volume=${volume_quota%%\%}
1423#    echo $quota_volume
1424    if [ $(echo "${quota_volume} > ${limit_quota}" | bc) -eq 1 ] ; then
1425      IGCM_debug_Print 1 "Please, check your quota of volume on workgpfs"
1426      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1427      IGCM_debug_Print 1 "Use the quota_u -w command to check"
1428      IGCM_debug_Print 1 "You must have more than 10% available to run"
1429      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1430      IGCM_debug_Verif_Exit
1431    fi
1432
1433  fi
1434  IGCM_debug_PopStack "IGCM_sys_check_quota"
1435}
1436
1437#D-#==================================================
1438#D-function IGCM_sys_projectAccounting
1439#D-* Purpose: store project accounting information in a file
1440#D-* Examples:
1441#D-
1442function IGCM_sys_projectAccounting {
1443  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1444  if ( $DEBUG_sys ) ; then
1445    echo "IGCM_sys_check_quota"
1446  fi
1447
1448  cpt > $1
1449
1450  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1451}
1452
1453#D-#==================================================
1454#D-function IGCM_sys_getJobSchedulerID
1455#D-* Purpose: Get the job ID during execution
1456#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1457#D-
1458function IGCM_sys_getJobSchedulerID {
1459  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1460  if ( $DEBUG_sys ) ; then
1461    echo "IGCM_sys_getJobSchedulerID"
1462  fi
1463
1464  eval ${1}=$( echo $LOADL_STEP_ID | awk -F. '{print $4}' )
1465
1466  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1467}
1468
1469#D-#==================================================
1470#D-function IGCM_sys_GetJobID
1471#D-* Purpose: Get the job ID from the JobName
1472#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1473#D-
1474function IGCM_sys_GetJobID {
1475  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1476  if ( $DEBUG_sys ) ; then
1477    echo "IGCM_sys_GetJobID"
1478  fi
1479
1480  # Print only the full (-W) JobID (%id) and JobName (%jn)
1481  ID=$( llq -u $2 -W -f %id %jn | \
1482    gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1483
1484  eval ${3}=${ID}
1485  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1486}
1487
1488#D-#==================================================
1489#D-function IGCM_sys_CountJobInQueue
1490#D-* Purpose: Count number of users job
1491#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1492#D-
1493function IGCM_sys_CountJobInQueue {
1494  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1495  if ( $DEBUG_sys ) ; then
1496    echo "IGCM_sys_CountJobInQueue"
1497  fi
1498
1499  # Print only the full (-W) JobName (%jn)
1500  NbRun=$( llq -W -f %jn | grep -c "$1" )
1501
1502  eval ${2}=${NbRun}
1503
1504  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1505}
1506
1507#D-#==================================================
1508#D-function IGCM_sys_ListJobInQueue
1509#D-* Purpose: Produce a list of users jobs
1510#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1511#D-
1512function IGCM_sys_ListJobInQueue {
1513  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1514  if ( $DEBUG_sys ) ; then
1515    echo "IGCM_sys_ListJobInQueue"
1516  fi
1517
1518  # With -W option, column width is as large as necessary
1519  set -A JobList $( llq -u $1 -W -f %jn | head -n -2 | tail -n +3 | \
1520    gawk '( $1 != /TS/      && \
1521                            $1 !~ /PACK/    && \
1522                            $1 !~ /REBUILD/ && \
1523                            $1 !~ /pack/ )     \
1524                          { print $1 }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1525
1526  eval set -A ${2} ${JobList[*]}
1527
1528  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1529}
1530
1531#D-#==================================================
1532#D-function IGCM_sys_atlas
1533#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1534#D-* Examples:
1535#D-
1536function IGCM_sys_atlas {
1537  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1538  if ( $DEBUG_sys ) ; then
1539    echo "IGCM_sys_atlas :" $@
1540  fi
1541
1542  typeset status
1543
1544  \atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1545  status=$?
1546  if [ ${status} -gt 0 ] ; then
1547    echo "IGCM_sys_atlas : error code ${status}"
1548    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1549    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1550    IGCM_debug_PopStack "IGCM_sys_atlas"
1551    return 1
1552  else
1553    IGCM_debug_PopStack "IGCM_sys_atlas"
1554    return 0
1555  fi
1556
1557  IGCM_debug_PopStack "IGCM_sys_atlas"
1558}
1559
Note: See TracBrowser for help on using the repository browser.