source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_mesoipsl.ksh @ 1624

Last change on this file since 1624 was 1624, checked in by jgipsl, 4 months ago

Added possibility to declare a restart file as optional which means that libIGCM want stop if the file is not found. Only a warning will be written.

For example, in the following case, if the routing_restart file is not available when starting the model or if it is not produced, libIGCM will not stop. But if the file sechiba_rest is not available, libIGCM will stop as usual.

[RestartFiles]
List=   (sechiba_rest_out.nc, sechiba_rest.nc, sechiba_rest_in.nc) ,\
        (routing_restart.nc, routing_restart.nc, routing_start.nc, OPTIONAL)
  • Read optional argument OPTIONAL on the line where the restart file is declared.
  • Each time when a restart file is not found for coping in or out, if the argument OPTIONAL is set, then just continue. If not set, the file is considered as MANDATORY and libIGCM will exist if it is missing as before.
  • on irene-amd and jean-zay, corrected a bug for the case in config.card DataProject?=DEFAULT


  • Property svn:keywords set to Revision Date Author
File size: 42.8 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for IPSL ESPRI MESO cluster: spirit and spiritx
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
49# -------------------------------------------------------------------------------------
50# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
51# |          |  Cp/Exe/param/files |            |         |                           |
52# |          |  Chmod Qsub         |            |         |                           |
53# -------------------------------------------------------------------------------------
54# |    0     |       yes           |    yes     |  yes    |      yes                  |
55# -------------------------------------------------------------------------------------
56# |    1     |       yes           |    yes     |  yes    |      no                   |
57# -------------------------------------------------------------------------------------
58# |    2     |       yes           |    yes     |  no     |      no                   |
59# -------------------------------------------------------------------------------------
60# |    3     |       yes           |    no      |  no     |      no                   |
61# -------------------------------------------------------------------------------------
62# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
63# -------------------------------------------------------------------------------------
64
65#=====================================================
66# Global Variables :
67#=====================================================
68# Language : "fr" or "en"
69typeset -r MYLANG="fr"
70
71#=====================================================
72# Host user names project maxCpuTime
73# $hostname ou hostname
74typeset HOST=${HOST:=$( hostname )}
75# $username ou whoami
76typeset LOGIN=${LOGIN:=$( whoami )}
77# $hostname of the MASTER job
78typeset MASTER=${MASTER:=$( hostname )}
79# project name
80typeset PROJECT=NONE
81# jobWarningDelay in seconds
82#typeset jobWarningDelay=${PBS_WALLTIME}
83
84#D-
85#D-#==================================================
86#D-Program used in libIGCM
87#D-#==================================================
88
89# Submit command
90typeset SUBMIT=${SUBMIT:=sbatch}
91# rsync with path
92typeset -r RSYNC=/usr/bin/rsync
93# RSYNC_opt args to rsync
94typeset -r RSYNC_opt="-va"
95# ie storage filesystem
96typeset -r STOREHOST=${MASTER}
97
98#====================================================
99# Access to module command
100#====================================================
101. /etc/profile.d/modules.sh
102
103#====================================================
104# Set environment tools (ferret, nco, cdo, rebuild, ...)
105#====================================================
106if [ X${TaskType} = Xcomputing ] ; then
107    IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
108else
109    if [ $CENTER == spirit ] ; then
110        # At spirit
111        . /home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
112    elif [ $CENTER == spiritx ] ; then
113        # At spiritx
114        . /ciclad-home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
115    fi
116fi
117
118[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
119[ ! X${TaskType} = Xchecking ] && module list
120
121#====================================================
122# Host specific DIRECTORIES
123#====================================================
124# For rebuild
125if [ $CENTER == spirit ] ; then
126    export PATH=${PATH}:/home/igcmg/rebuild/spirit/modipsl/modeles/IOIPSL/bin/
127else
128    export PATH=${PATH}:/ciclad-home/igcmg/rebuild/spirit/modipsl/modeles/IOIPSL/bin/
129fi
130#====================================================
131#- MirrorlibIGCM for frontend
132typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
133
134#====================================================
135#- libIGCM_POST for frontend
136typeset -r libIGCM_POST=${libIGCM}
137
138#====================================================
139#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
140typeset -r R_EXE="${MODIPSL}/bin"
141
142#====================================================
143#- SUBMIT_DIR : submission dir
144if [ X${SLURM_SUMBIT_DIR} != X ] ; then
145  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${SLURM_SUBMIT_DIR}}
146else
147  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
148fi
149
150#====================================================
151#- IN
152typeset -r R_IN=${R_IN:=/projsu/igcmg/IGCM}
153
154#====================================================
155#- RUN_DIR_PATH : Temporary working directory
156
157if [ $CENTER == spirit ] ; then
158    RUN_DIR_PATH=${RUN_DIR_PATH:=/scratchu/${LOGIN}/RUN_DIR}
159else
160    RUN_DIR_PATH=${RUN_DIR_PATH:=/scratchx/${LOGIN}/RUN_DIR}
161fi
162if [ X${SLURM_JOBID} != X ] ; then
163    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH}/${SLURM_JOBID}_${$}
164else
165    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH}/${$}
166fi
167
168#====================================================
169#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
170# Note that typeset -r can not be set at spirit because it sets the path as read-only
171typeset OUTCOMMAND_PATH=/tmp
172
173#====================================================
174#- HOST_MPIRUN_COMMAND
175typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
176
177#====================================================
178#- Max number of arguments passed to nco operator or demigration command
179UNIX_MAX_LIMIT=360
180
181#====================================================
182#- set PackDefault to false on meso-ipsl
183PackDefault=false
184
185#====================================================
186#- Default number of MPI task for IPSL coupled model
187#- required for backward compatibility
188#-
189DEFAULT_NUM_PROC_OCE=5
190DEFAULT_NUM_PROC_CPL=1
191(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
192
193#D-#==================================================
194#D-function IGCM_sys_defineArchives
195#D-* Purpose:
196#D-* Define ARCHIVE : Dedicated to large files
197#D-* Define STORAGE : Dedicated to small/medium files
198#D-* Define R_OUT   : Output tree located on ARCHIVE
199#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
200#D-* Define R_BUF   : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
201#D-* if SpaceName=TEST nothing special will hapen
202#D-* Examples:
203#D-
204function IGCM_sys_defineArchives {
205  IGCM_debug_PushStack "IGCM_sys_defineArchives"
206
207  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
208    #====================================================
209    #- ARCHIVE (dedicated to large files)
210    ARCHIVE=${config_UserChoices_ARCHIVE}
211  else
212    #====================================================
213    #- ARCHIVE (dedicated to large files)
214    if [ $CENTER == spirit ] ; then
215        # At spirit
216        ARCHIVE=${ARCHIVE:=/data/${LOGIN}}
217    else
218        # At spiritx
219        ARCHIVE=${ARCHIVE:=/homedata/${LOGIN}}
220    fi
221  fi
222
223  if [ ! X${config_UserChoices_STORAGE} = X ]; then
224    #====================================================
225    #- STORAGE (dedicated to small/medium files)
226    STORAGE=${config_UserChoices_STORAGE}
227  else
228    #====================================================
229    #- STORAGE (dedicated to small/medium files)
230    STORAGE=${ARCHIVE}
231  fi
232
233  # ON MESO-IPSL NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
234
235  #====================================================
236  #- R_OUT
237  R_OUT=${ARCHIVE}/IGCM_OUT
238
239  #====================================================
240  #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
241  R_FIG=${STORAGE}/IGCM_OUT
242
243  #====================================================
244  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
245  CMIP6_BUF=${STORAGE}/IGCM_OUT
246
247  #====================================================
248  #- R_BUF (ONLY FOR double copy an scratch)
249  R_BUF=${STORAGE}/IGCM_OUT
250
251  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
252  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
253  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
254
255  IGCM_debug_PopStack "IGCM_sys_defineArchives"
256}
257
258#D-#==================================================
259#D-function IGCM_sys_RshArchive
260#D-* Purpose: Archive rsh command
261#D-* Examples:
262#D-
263function IGCM_sys_RshArchive {
264  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
265  /bin/ksh <<-EOF
266    ${@}
267EOF
268  status=$?
269  if [ ${status} -gt 0 ] ; then
270    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
271    IGCM_debug_Exit "IGCM_sys_RshArchive"
272  fi
273  IGCM_debug_PopStack "IGCM_sys_RshArchive"
274}
275
276#D-#==================================================
277#D-function IGCM_sys_RshArchive_NoError
278#D-* Purpose: Archive rsh command, without error
279#D-*          used only in monitoring.job
280#D-* Examples:
281#D-
282function IGCM_sys_RshArchive_NoError {
283  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
284  /bin/ksh <<-EOF
285    ${@} 2> /dev/null
286EOF
287  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
288}
289
290#D-#==================================================
291#D-function IGCM_sys_MkdirArchive
292#D-* Purpose: Mkdir on Archive
293#D-* Examples:
294#D-
295function IGCM_sys_MkdirArchive {
296  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
297  if ( $DEBUG_sys ) ; then
298    echo "IGCM_sys_MkdirArchive :" $@
299  fi
300  #- creation de repertoire sur le serveur fichier
301  if [ ! -d ${1} ]; then
302    \mkdir -p $1
303    status=$?
304
305    if [ ${status} -gt 0 ] ; then
306      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
307      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
308    fi
309  fi
310  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
311}
312
313#D-#==================================================
314#D-function IGCM_sys_TestDirArchive
315#D-* Purpose: Test Directory that must exists on Archive
316#D-* Examples:
317#D-
318function IGCM_sys_TestDirArchive {
319  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
320  if ( $DEBUG_sys ) ; then
321    echo "IGCM_sys_TestDirArchive :" $@
322  fi
323  typeset ExistFlag
324  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
325  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
326  return ${ExistFlag}
327}
328
329#D-#==================================================
330#D-function IGCM_sys_IsFileArchived
331#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
332#D-* Examples:
333#D-
334function IGCM_sys_IsFileArchived {
335  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
336  if ( $DEBUG_sys ) ; then
337    echo "IGCM_sys_IsFileArchived :" $@
338  fi
339  typeset IsArchivedFlag
340  # Never archived for this system
341  IsArchivedFlag=1
342  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
343
344  return ${IsArchivedFlag}
345}
346
347#D-#==================================================
348#D-function IGCM_sys_TestFileArchive
349#D-* Purpose: Test file that must NOT EXISTS on Archive
350#D-* Examples:
351#D-
352function IGCM_sys_TestFileArchive {
353  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
354  typeset ExistFlag
355  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
356  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
357
358  return ${ExistFlag}
359}
360
361#D-#==================================================
362#D-function IGCM_sys_CountFileArchive
363#D-* Purpose: Count files on Archive filesystem
364#D-* Examples:
365#D-
366function IGCM_sys_CountFileArchive {
367  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
368  ls ${@} 2>/dev/null | wc -l
369  if [ $? -gt 0 ] ; then
370    echo "IGCM_sys_CountFileArchive : erreur."
371  fi
372  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
373}
374
375#D-#==================================================
376#D-function IGCM_sys_Tree
377#D-* Purpose: Tree directories with files on ${ARCHIVE}
378#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
379#D-
380function IGCM_sys_Tree {
381  IGCM_debug_PushStack "IGCM_sys_Tree" $@
382  if ( $DEBUG_sys ) ; then
383    echo "IGCM_sys_Tree :" $@
384  fi
385
386  \ls -lR ${@}
387
388  IGCM_debug_PopStack "IGCM_sys_Tree"
389}
390
391#D-#==================================================
392#D-function IGCM_sys_Qsub
393#D-* Purpose: Qsub new job
394#D-* Examples:
395#D-
396function IGCM_sys_Qsub {
397  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
398  if ( $DEBUG_sys ) ; then
399    echo "IGCM_sys_Qsub :" $@
400  fi
401  typeset options status
402  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
403  /usr/bin/time sbatch ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
404  status=$?
405
406  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
407  if [ ${status} -gt 0 ] ; then
408    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
409    IGCM_debug_Exit "IGCM_sys_Qsub"
410  else
411    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
412  fi
413  IGCM_debug_PopStack "IGCM_sys_Qsub"
414}
415
416#D-#==================================================
417#D-function IGCM_sys_QsubPost
418#D-* Purpose: Qsub new job on scalaire
419#D-* Examples:
420#D-
421function IGCM_sys_QsubPost {
422  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
423  if ( $DEBUG_sys ) ; then
424    echo "IGCM_sys_QsubPost :" $@
425  fi
426  typeset options status
427  options="-o ${POST_DIR}/${Script_Post_Output}.out  -e ${POST_DIR}/${Script_Post_Output}.out"
428  sbatch ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
429  status=$?
430
431  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
432  if [ ${status} -gt 0 ] ; then
433    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
434    IGCM_debug_Exit "IGCM_sys_QsubPost"
435  else
436    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
437  fi
438  IGCM_debug_PopStack "IGCM_sys_QsubPost"
439}
440
441#D-*************************
442#D- File transfer functions
443#D-*************************
444#D-
445
446#D-#==================================================
447#D-function IGCM_sys_RmRunDir
448#D-* Purpose: rm tmpdir (dummy function most of the time batch
449#D-                      scheduler will do the job)
450#D-* Examples:
451#D-
452function IGCM_sys_RmRunDir {
453  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
454  if ( $DEBUG_sys ) ; then
455    echo "IGCM_sys_RmRunDir :" $@
456  fi
457
458  typeset status
459
460  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
461  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
462  status=$?
463
464  if [ ${status} -gt 0 ] ; then
465    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
466    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
467    IGCM_debug_Exit "IGCM_sys_RmRunDir"
468  else
469    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
470  fi
471  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
472}
473
474#D-#==================================================
475#D-function IGCM_sys_Put_Dir
476#D-* Purpose: Copy a complete directory on $(ARCHIVE)
477#D-* Examples:
478#D-
479function IGCM_sys_Put_Dir {
480  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
481  if ( $DEBUG_sys ) ; then
482    echo "IGCM_sys_Put_Dir :" $@
483  fi
484  if [ $DRYRUN = 0 ]; then
485    if [ ! -d ${1} ] ; then
486      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
487      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
488      return
489    fi
490
491    typeset status
492
493    # Only if we use rsync
494    #IGCM_sys_TestDirArchive $( dirname $2 )
495    #
496    #USUAL WAY
497    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
498    status=$?
499
500    if [ ${status} -gt 0 ] ; then
501      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
502      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
503      IGCM_debug_Exit "IGCM_sys_Put_Dir"
504    else
505      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
506    fi
507  fi
508  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
509}
510
511#D-#==================================================
512#D-function IGCM_sys_Get_Dir
513#D-* Purpose: Copy a complete directory from ${ARCHIVE}
514#D-* Examples:
515#D-
516function IGCM_sys_Get_Dir {
517  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
518  if ( $DEBUG_sys ) ; then
519    echo "IGCM_sys_Get_Dir :" $@
520  fi
521  if [ $DRYRUN = 0 ]; then
522    typeset NB_ESSAI DELAI status i
523    # number of tentative
524    NB_ESSAI=3
525    # time delay between tentative
526    DELAI=2
527
528    #
529    # USUAL WAY
530    i=0
531    while [ $i -lt $NB_ESSAI ] ; do
532      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
533      status=$?
534      if [ ${status} -gt 0 ] ; then
535        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
536        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
537        sleep $DELAI
538      else
539        break
540      fi
541      (( i = i + 1 ))
542    done
543
544    if [ ${status} -gt 0 ] ; then
545      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
546      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
547      IGCM_debug_Exit "IGCM_sys_Get_Dir"
548    else
549      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
550    fi
551  fi
552  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
553}
554
555#D-#==================================================
556#D-function IGCM_sys_Put_Rest
557#D-* Purpose: Put computied restarts on ${ARCHIVE}.
558#D-           File and target directory must exist.
559#D-* Examples:
560#D-
561function IGCM_sys_Put_Rest {
562  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
563  if ( $DEBUG_sys ) ; then
564    echo "IGCM_sys_Put_Rest :" $@
565  fi
566  if [ $DRYRUN = 0 ]; then
567    if [ ! -f ${1} ] ; then
568        # The file is missing     
569        if [ X${3} = X'OPTIONAL' ] ; then
570            echo "WARINING : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
571            IGCM_debug_Print 1 "IGCM_sys_Put_Rest: Optional restart file ${1} is missing, now continue for the next"
572        else
573            echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
574            IGCM_debug_Exit "IGCM_sys_Put_Rest"
575        fi
576
577    else
578      # The file exist
579
580    typeset status
581    #
582    # USUAL WAY
583    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
584    status=$?
585
586#       #RSYNC WITH NETWORK SSH CALL
587#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
588#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
589
590#       #RSYNC WITH NFS USE
591#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
592#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
593
594#       status=$?
595#       IGCM_sys_Rsync_out $status
596
597#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
598#       (( status=status+$? ))
599
600    if [ ${status} -gt 0 ] ; then
601      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
602      [ -f ${1} ] && ls -l ${1}
603      [ -f ${2} ] && ls -l ${2}
604      [ -f ${2}/${1} ] && ls -l ${2}/${1}
605      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
606      IGCM_debug_Exit "IGCM_sys_Put_Rest"
607    else
608
609      if [ X${JobType} = XRUN ] ; then
610        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
611        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
612      fi
613
614      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
615    fi
616    fi
617  fi
618  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
619}
620
621#D-#==================================================
622#D-function IGCM_sys_Put_Out
623#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
624#D-* Examples:
625#D-
626function IGCM_sys_Put_Out {
627  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
628  if ( $DEBUG_sys ) ; then
629    echo "IGCM_sys_Put_Out :" $@
630  fi
631
632  typeset status
633
634  if [ $DRYRUN = 0 ]; then
635    if [ ! -f ${1} ] ; then
636      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
637      IGCM_debug_PopStack "IGCM_sys_Put_Out"
638      return 1
639    fi
640    #
641    IGCM_sys_MkdirArchive $( dirname $2 )
642    #
643    if [ X${JobType} = XRUN ] ; then
644      if [ X${3} = X ] ; then
645        IGCM_sys_Chmod 444 ${1}
646      fi
647    fi
648    #
649
650    echo ${RSYNC} ${RSYNC_opt} $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
651    ${RSYNC} ${RSYNC_opt} $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
652    status=$?
653    IGCM_sys_Rsync_out $status
654
655    ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
656    (( status=status+$? ))
657
658    if [ ${status} -gt 0 ] ; then
659      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
660      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
661      IGCM_debug_Exit "IGCM_sys_Put_Out"
662    else
663      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
664    fi
665  fi
666  IGCM_debug_PopStack "IGCM_sys_Put_Out"
667  return 0
668}
669
670#D-#==================================================
671#D-function IGCM_sys_Get
672#D-* Purpose: Get a file from ${ARCHIVE}
673#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
674#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
675function IGCM_sys_Get {
676  IGCM_debug_PushStack "IGCM_sys_Get" $@
677
678  typeset DEST status dm_liste
679
680  if ( $DEBUG_sys ) ; then
681    echo "IGCM_sys_Get :" $@
682  fi
683  if [ $DRYRUN -le 2 ]; then
684    if [ X${1} = X'/l' ] ; then
685      eval set +A dm_liste \${${2}}
686    else
687      dm_liste=${1}
688    fi
689    eval DEST=\${${#}}
690
691    # test if the (first) file is present in the old computation :
692    IGCM_sys_TestFileArchive ${dm_liste[0]}
693    status=$?
694    if [ ${status} -gt 0 ] ; then
695      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
696      IGCM_debug_Exit "IGCM_sys_Get"
697      return
698    fi
699
700    #USUAL WAY
701    \cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
702    status=$?
703
704    if [ ${status} -gt 0 ] ; then
705      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
706      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
707      IGCM_debug_Exit "IGCM_sys_Get"
708    else
709      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
710    fi
711  fi
712  IGCM_debug_PopStack "IGCM_sys_Get"
713}
714
715#D-#==================================================
716#D-function IGCM_sys_GetDate_Monitoring
717#D-* Purpose: get the last year for which the monitoring has been computed
718#D-* Examples:
719#D-
720function IGCM_sys_GetDate_Monitoring {
721  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
722  if ( $DEBUG_sys ) ; then
723    echo "IGCM_sys_GetDate_Monitoring :" $@
724  fi
725
726  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
727
728  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
729}
730
731#D-#==================================================
732#D-function IGCM_sys_Dods_Rm
733#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
734#D-* Examples:
735#D-
736function IGCM_sys_Dods_Rm {
737  if ( $DEBUG_sys ) ; then
738    echo "IGCM_sys_Dods_Rm :" $@
739  fi
740  return 0
741}
742
743#D-#==================================================
744#D-function IGCM_sys_Dods_Cp
745#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
746#D-* Examples:
747#D-
748function IGCM_sys_Dods_Cp {
749  if ( $DEBUG_sys ) ; then
750    echo "IGCM_sys_Dods_Cp :" $@
751  fi
752  return 0
753}
754
755#D-#==================================================
756#D-function IGCM_sys_Put_Dods
757#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
758#D-* Examples:
759#D-
760function IGCM_sys_Put_Dods {
761  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
762  if ( $DEBUG_sys ) ; then
763    echo "IGCM_sys_Put_Dods :" $@
764  fi
765  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
766}
767
768##############################################################
769# REBUILD OPERATOR
770
771#D-#==================================================
772#D-function IGCM_sys_sync
773#D-* Purpose: flush buffer on disk (dummy function on Ciclad)
774#D-* Examples:
775#D-
776function IGCM_sys_sync {
777  IGCM_debug_PushStack "IGCM_sys_sync" $@
778  if ( $DEBUG_sys ) ; then
779    echo "IGCM_sys_sync :" $@
780    echo "Dummy call, let the system do that."
781  fi
782  IGCM_debug_PopStack "IGCM_sys_sync"
783}
784
785############################################################
786# Activate Running Environnment Variables
787
788#D-#==================================================
789#D-function IGCM_sys_activ_variables
790#D-* Purpose: set environement variables prior to execution
791#D-* Examples:
792#D-
793function IGCM_sys_activ_variables {
794  IGCM_debug_PushStack "IGCM_sys_activ_variables"
795  if ( $DEBUG_sys ) ; then
796    echo "IGCM_sys_activ_variables"
797  fi
798
799# --------------------------------------------------------------------
800#D- MODULE specifications
801# --------------------------------------------------------------------
802  # Source the file EnvFile containing all module load needed to run the model.
803  # EnvFile can be specified in config.card. If this is the case and if the file exists,
804  # it will be used. Otherwise a default file will be used.
805  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
806      # EnvFile is set in config.card and the file exists
807      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
808      EnvFile=${config_UserChoices_EnvFile}
809  else
810      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
811      if [ $CENTER == spirit ] ; then
812          # At spirit
813          EnvFile=/home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
814      else
815          # At spirit
816          EnvFile=/ciclad-home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
817      fi
818  fi
819  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
820  IGCM_debug_Print 1 ${EnvFile}
821  . ${EnvFile}
822
823
824  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for meso-ipsl . "
825  module list
826 
827
828# --------------------------------------------------------------------
829#D- MPI specifications
830# --------------------------------------------------------------------
831
832# --------------------------------------------------------------------
833#D- Other specifications
834# --------------------------------------------------------------------
835
836  ulimit -s unlimited
837
838  IGCM_debug_PopStack "IGCM_sys_activ_variables"
839}
840
841############################################################
842# Desactivate Running Environnment Variables
843
844#D-#==================================================
845#D-function IGCM_sys_desactiv_variables
846#D-* Purpose: unset environement variables after execution
847#D-* Examples:
848#D-
849function IGCM_sys_desactiv_variables {
850  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
851  if ( $DEBUG_sys ) ; then
852    echo "IGCM_sys_desactiv_variables"
853  fi
854# --------------------------------------------------------------------
855#D- MPI specifications
856# --------------------------------------------------------------------
857
858# --------------------------------------------------------------------
859#D- Other specifications
860# --------------------------------------------------------------------
861
862  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
863}
864
865############################################################
866# Update job headers to be used by the scheduler
867
868#D-#==================================================
869#D-function IGCM_sys_updateHeaders
870#D-* Purpose: Update job headers to be used by the scheduler
871#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
872#D-
873function IGCM_sys_updateHeaders {
874  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
875  if ( $DEBUG_sys ) ; then
876    echo "IGCM_sys_updateHeaders"
877  fi
878  typeset file
879  file=$1
880
881  if [ ${executionType} -eq 1 ] ; then
882    # MPMD + MPI
883    sed -e "/::openMPthreads::/d"                  \
884        -e "s/::JobNumProcTot::/${coreNumber}/"    \
885      ${file} > ${file}.tmp
886
887  elif [ ${executionType} -eq 2 ] ; then
888    # MPMD + MPI + OMP
889      (( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
890      [ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
891      sed -e "/::openMPthreads::/d"                 \
892        -e "s/::JobNumProcTot::/${mpiTasks}/"       \
893        -e "/ntasks/i\#SBATCH --nodes=${nodeNumber}"\
894        -e "/ntasks/i\#SBATCH --exclusive"          \
895         ${file} > ${file}.tmp
896  elif [ ${executionType} -eq 3 ] ; then
897    # SPMD + MPI/OMP
898    sed -e "s/::openMPthreads::/${openMPthreads}/" \
899        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
900      ${file} > ${file}.tmp
901
902  elif [ ${executionType} -eq 4 ] ; then
903    # SPMD + MPI only
904    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
905        -e "/::openMPthreads::/d"                  \
906      ${file} > ${file}.tmp
907
908  elif [ ${executionType} -eq 5 ] ; then
909    # SPMD + OMP only
910    sed -e "s/::openMPthreads::/${openMPthreads}/" \
911        -e "/::JobNumProcTot::/d"                  \
912      ${file} > ${file}.tmp
913
914  elif [ ${executionType} -eq 6 ] ; then
915    # SEQUENTIAL THEN
916    sed -e "s/::JobNumProcTot::/1/"                \
917        -e "/::openMPthreads::/d"                  \
918      ${file} > ${file}.tmp
919
920  fi
921
922  IGCM_sys_Mv ${file}.tmp ${file}
923
924  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
925}
926
927############################################################
928# Build MPI/OMP scripts run file (dummy function)
929
930#D-#==================================================
931#D-function IGCM_sys_build_run_file
932#D-* Purpose: build run file (deprecated)
933#D-* Examples:
934#D-
935function IGCM_sys_build_run_file {
936  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
937}
938
939############################################################
940# Build MPI/OMP scripts
941
942#D-#==================================================
943#D-function IGCM_sys_build_execution_scripts
944#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
945#D-* Examples:
946#D-
947function IGCM_sys_build_execution_scripts
948{
949  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
950  if ( $DEBUG_sys ) ; then
951    echo "IGCM_sys_build_execution_scripts " $@
952  fi
953  IGCM_debug_Print 1 "executionType= ${executionType}"
954  EXECUTION=${HOST_MPIRUN_COMMAND}
955
956  if ( ${OK_PARA_MPMD} ) ; then
957
958    # MPMD mode
959    # 1 MPI only : executionType=1
960    # 2 MPI/OpenMP : executionType=2
961
962    if [ -f run_file ] ; then
963      IGCM_sys_Rm -f run_file
964    fi
965    touch run_file
966
967    # case 1 : Only MPI (MPMD)
968    if  ( ! ${OK_PARA_OMP} ) ; then
969      # Build run_file
970      current_core=0
971      # First loop on the components for the coupler ie oasis (only if oasis3)
972      # the coupler ie oasis3 must be the first one
973      for comp in ${config_ListOfComponents[*]} ; do
974
975        eval ExeNameIn=\${config_Executable_${comp}[0]}
976        eval ExeNameOut=\${config_Executable_${comp}[1]}
977
978        # Only if the component has an executable
979        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
980
981#          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
982#          (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
983#          echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
984#          (( current_core = ${end_core} + 1 ))
985          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
986          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
987          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
988
989        fi
990      done
991
992      EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
993
994      IGCM_sys_Chmod u+x run_file
995      if ( $DEBUG_sys ) ; then
996        echo "run_file contains : "
997        cat run_file
998      fi
999
1000    else 
1001
1002    # 2 MPI/OpenMP : executionType=2
1003
1004    # MPI-OpenMP (MPMD)
1005    # export SLURM_HOSTFILE=./hostlist
1006    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
1007    # example of  hostlist file :
1008    # r3i3n33
1009    # r3i3n33
1010    # ...
1011    # example of run_file :
1012    # 0-70 ./prog_lmdz.x.sh %o %t
1013    # 71-430 ./prog_opa.xx.sh %o %t
1014    # 431-431 ./prog_xios.x.sh %o %t
1015    # examples of prog_file :
1016    # prog_lmdz.x.sh :
1017    # (( init = 0 + $1 ))
1018    # (( index = init * 10 ))
1019    # (( slot = index % 40 ))
1020    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
1021    # that will become
1022    # taskset -c 0-9 ./script_lmdz.x.ksh
1023    # ...
1024    # with script_lmdz.x.ksh
1025    # export OMP_STACKSIZE=3g
1026    # export OMP_PLACES=cores
1027    # export OMP_NUM_THREADS=10
1028    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
1029
1030    #  Hosts treatment
1031    _bkIFS=$IFS;
1032        IFS=$'\n'; set -f
1033        listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1034        IFS=$_bkIFS; set +f
1035        rm -f hostlist
1036
1037     # Loop on the components to build run_file and script_exec files
1038        rank=0
1039        current_core=0
1040        current_core_mpi=0
1041
1042      for comp in ${config_ListOfComponents[*]} ; do
1043
1044        eval ExeNameIn=\${config_Executable_${comp}[0]}
1045        eval ExeNameOut=\${config_Executable_${comp}[1]}
1046
1047        # Not possible if oasis has an executable (i.e old version of oasis3)
1048        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1049          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1050          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1051          IGCM_debug_Verif_Exit
1052        fi
1053
1054        # Only if we really have an executable for the component :
1055        if [ "X${ExeNameOut}" != X\"\" ] ; then
1056
1057          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1058          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1059          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1060     
1061
1062          # Build script files
1063
1064          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1065          echo ""  >> script_${ExeNameOut}.ksh
1066          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1067          # Check if the number of threads is correct
1068              case ${comp_proc_omp_loc} in
1069                  2|4|5|10|20)
1070                      IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1071                      ;;
1072                  *)
1073                      IGCM_debug_Exit "ERROR with OMP parameters !"
1074                      IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1075                      IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1076                      IGCM_debug_Verif_Exit
1077                      ;;
1078              esac
1079            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1080            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1081            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
1082
1083          fi
1084
1085          # to have out/err per process on different files
1086          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1087
1088          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1089
1090        # Build run_file
1091        # Only if the component has an executable
1092        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1093
1094          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1095          (( end_core = ${current_core_mpi} + ${comp_proc_mpi_loc} - 1 ))
1096          echo "${current_core_mpi}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
1097          (( current_core_mpi = ${end_core} + 1 ))
1098        fi
1099
1100          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1101              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1102          else
1103              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1104          fi
1105
1106          # Build configuration file
1107 
1108              echo "#!/bin/sh" > prog_${ExeNameOut}.sh
1109          echo "(( init = $current_core + \$1 ))" >> prog_${ExeNameOut}.sh
1110          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
1111          echo "(( slot = index % 40 ))" >> prog_${ExeNameOut}.sh
1112              echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
1113          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
1114
1115          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
1116
1117        # Build hostlist file
1118
1119          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1120             (( index_host = current_core / NB_CORE_PER_NODE ))
1121             host_value=${listnodes[${index_host}]}
1122               echo "$host_value" >> hostlist
1123             (( current_core = current_core + offset_comp_proc_loc ))
1124          done
1125        fi
1126      done
1127
1128      ## variable added to stop after 60s instead of 600s by default.
1129      ## This is used when no error comes from executables and when something stopped an executable without notice.
1130      export SLURM_WAIT=60
1131
1132      EXECUTION="${HOST_MPIRUN_COMMAND} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1133
1134      IGCM_sys_Chmod u+x run_file
1135      if ( $DEBUG_sys ) ; then
1136        echo "run_file contains : "
1137        cat run_file
1138      fi
1139
1140    fi # if ${OK_PARA_MPMD}
1141
1142  else
1143  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
1144
1145    for comp in ${config_ListOfComponents[*]} ; do
1146
1147      # Only if we really have an executable for the component :
1148      eval ExeNameOut=\${config_Executable_${comp}[1]}
1149      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1150
1151        # Build script files
1152
1153        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1154        echo ""  >> script_${ExeNameOut}.ksh
1155        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1156
1157        if ( ${OK_PARA_OMP} ) ; then
1158            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1159            # Check if the number of threads is correct
1160            case ${comp_proc_omp_loc} in
1161                2|4|5|10|20)
1162                    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1163                    ;;
1164                *)
1165                    IGCM_debug_Exit "ERROR with OMP parameters !"
1166                    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1167                    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1168                    IGCM_debug_Verif_Exit
1169                    ;;
1170            esac
1171            echo ""  >> script_${ExeNameOut}.ksh
1172            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1173            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1174            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1175        fi
1176
1177        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1178       
1179        # To have out/err per process on different files
1180        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1181        EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1182
1183        IGCM_debug_Print 1 "sys Jean-Zay : script_${ExeNameOut}.ksh contains"
1184        cat script_${ExeNameOut}.ksh
1185
1186      fi
1187
1188    done
1189
1190  fi # ${OK_PARA_MPMD}
1191
1192  IGCM_debug_Print 1 "sys meso-ipsl : execution command is "
1193  IGCM_debug_Print 1 "$EXECUTION"
1194
1195  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1196}
1197
1198#D-#==================================================
1199#D-function IGCM_sys_check_path
1200#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1201#D-* do not point to an important use directory. Stop immediately in that case.
1202#D-* Examples:
1203#D-
1204function IGCM_sys_check_path {
1205  IGCM_debug_PushStack "IGCM_sys_check_path"
1206  if ( $DEBUG_sys ) ; then
1207    echo "IGCM_sys_check_path"
1208  fi
1209
1210  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1211    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1212    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1213    IGCM_debug_Exit "This will stop the job"
1214  fi
1215  IGCM_debug_PopStack "IGCM_sys_check_path"
1216}
1217
1218#D-#==================================================
1219#D-function IGCM_sys_check_quota. Dummy call here
1220#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1221#D-* Examples:
1222#D-
1223function IGCM_sys_check_quota {
1224  IGCM_debug_PushStack "IGCM_sys_check_quota"
1225  if ( $DEBUG_sys ) ; then
1226    echo "IGCM_sys_check_quota"
1227  fi
1228
1229  # TO BE IMPLEMENTED
1230
1231  IGCM_debug_PopStack "IGCM_sys_check_quota"
1232}
1233
1234#D-#==================================================
1235#D-function IGCM_sys_projectAccounting
1236#D-* Purpose: store project accounting information in a file
1237#D-* Examples:
1238#D-
1239function IGCM_sys_projectAccounting {
1240  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1241  if ( $DEBUG_sys ) ; then
1242    echo "IGCM_sys_check_quota"
1243  fi
1244
1245  touch $1
1246
1247  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1248}
1249
1250#D-#==================================================
1251#D-function IGCM_sys_getJobSchedulerID
1252#D-* Purpose: Get the job ID during execution
1253#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1254#D-
1255function IGCM_sys_getJobSchedulerID {
1256  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1257  if ( $DEBUG_sys ) ; then
1258    echo "IGCM_sys_getJobSchedulerID"
1259  fi
1260
1261  eval ${1}=$( echo ${PBS_JOBID} | awk -F. '{print $1}' )
1262
1263  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1264}
1265
1266#D-#==================================================
1267#D-function IGCM_sys_GetJobID
1268#D-* Purpose: Get the job ID from the JobName
1269#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1270#D-
1271function IGCM_sys_GetJobID {
1272  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1273  if ( $DEBUG_sys ) ; then
1274    echo "IGCM_sys_GetJobID"
1275  fi
1276
1277  # With -f option, the full job name is given in the last column
1278  ID="$( qstat -u $2 | grep -w $1 | gawk '-F ' '{print $10}' )"
1279
1280  eval ${3}=${ID}
1281  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1282}
1283
1284#D-#==================================================
1285#D-function IGCM_sys_CountJobInQueue
1286#D-* Purpose: Count number of users job
1287#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1288#D-
1289function IGCM_sys_CountJobInQueue {
1290  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1291  if ( $DEBUG_sys ) ; then
1292    echo "IGCM_sys_CountJobInQueue"
1293  fi
1294  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1295}
1296
1297#D-#==================================================
1298#D-function IGCM_sys_atlas
1299#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1300#D-* Examples:
1301#D-
1302function IGCM_sys_atlas {
1303  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1304  if ( $DEBUG_sys ) ; then
1305    echo "IGCM_sys_atlas :" $@
1306  fi
1307
1308  typeset status
1309
1310  \atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1311  status=$?
1312  if [ ${status} -gt 0 ] ; then
1313    echo "IGCM_sys_atlas : error code ${status}"
1314    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1315    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1316    IGCM_debug_PopStack "IGCM_sys_atlas"
1317    return 1
1318  else
1319    IGCM_debug_PopStack "IGCM_sys_atlas"
1320    return 0
1321  fi
1322
1323  IGCM_debug_PopStack "IGCM_sys_atlas"
1324}
1325
1326#D-#==================================================
1327#D-function IGCM_sys_rebuild_nemo
1328#D-* Purpose: rebuild nemo parallel files with general rebuild
1329#D-* Examples:
1330#D-
1331
1332function IGCM_sys_rebuild_nemo {
1333  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1334  if ( $DEBUG_sys ) ; then
1335    echo "IGCM_sys_rebuild_nemo :" $@
1336  fi
1337
1338  nemo_generic_restart_file_name_out=${1}
1339  nemo_extension_out=${3}
1340  shift ; shift ; shift
1341
1342  IGCM_sys_rebuild -o ${nemo_generic_restart_file_name_out}.${nemo_extension_out} $@
1343
1344  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1345}
1346
1347
1348
Note: See TracBrowser for help on using the repository browser.