source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_mesoipsl.ksh @ 1632

Last change on this file since 1632 was 1632, checked in by jgipsl, 4 months ago

Set jobWarningDelay so that the accounting mail can be done now also for obelix.

Correction in conversion from min to sec in sys_mesoipsl and sys_jeanzay since previous commit [1631].
  • Property svn:keywords set to Revision Date Author
File size: 42.9 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for IPSL ESPRI MESO cluster: spirit and spiritx
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
49# -------------------------------------------------------------------------------------
50# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
51# |          |  Cp/Exe/param/files |            |         |                           |
52# |          |  Chmod Qsub         |            |         |                           |
53# -------------------------------------------------------------------------------------
54# |    0     |       yes           |    yes     |  yes    |      yes                  |
55# -------------------------------------------------------------------------------------
56# |    1     |       yes           |    yes     |  yes    |      no                   |
57# -------------------------------------------------------------------------------------
58# |    2     |       yes           |    yes     |  no     |      no                   |
59# -------------------------------------------------------------------------------------
60# |    3     |       yes           |    no      |  no     |      no                   |
61# -------------------------------------------------------------------------------------
62# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
63# -------------------------------------------------------------------------------------
64
65#=====================================================
66# Global Variables :
67#=====================================================
68# Language : "fr" or "en"
69typeset -r MYLANG="fr"
70
71#=====================================================
72# Host user names project maxCpuTime
73# $hostname ou hostname
74typeset HOST=${HOST:=$( hostname )}
75# $username ou whoami
76typeset LOGIN=${LOGIN:=$( whoami )}
77# $hostname of the MASTER job
78typeset MASTER=${MASTER:=$( hostname )}
79# project name
80typeset PROJECT=NONE
81# jobWarningDelay in seconds
82typeset jobWarningDelayMin=$(squeue -j $SLURM_JOB_ID -h --Format TimeLimit | awk -F: '{print $1}')
83typeset jobWarningDelay=$( echo "scale=6;${jobWarningDelayMin=}*60" | bc )
84
85#D-
86#D-#==================================================
87#D-Program used in libIGCM
88#D-#==================================================
89
90# Submit command
91typeset SUBMIT=${SUBMIT:=sbatch}
92# rsync with path
93typeset -r RSYNC=/usr/bin/rsync
94# RSYNC_opt args to rsync
95typeset -r RSYNC_opt="-va"
96# ie storage filesystem
97typeset -r STOREHOST=${MASTER}
98
99#====================================================
100# Access to module command
101#====================================================
102. /etc/profile.d/modules.sh
103
104#====================================================
105# Set environment tools (ferret, nco, cdo, rebuild, ...)
106#====================================================
107if [ X${TaskType} = Xcomputing ] ; then
108    IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
109else
110    if [ $CENTER == spirit ] ; then
111        # At spirit
112        . /home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
113    elif [ $CENTER == spiritx ] ; then
114        # At spiritx
115        . /ciclad-home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
116    fi
117fi
118
119[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
120[ ! X${TaskType} = Xchecking ] && module list
121
122#====================================================
123# Host specific DIRECTORIES
124#====================================================
125# For rebuild
126if [ $CENTER == spirit ] ; then
127    export PATH=${PATH}:/home/igcmg/rebuild/spirit/modipsl/modeles/IOIPSL/bin/
128else
129    export PATH=${PATH}:/ciclad-home/igcmg/rebuild/spirit/modipsl/modeles/IOIPSL/bin/
130fi
131#====================================================
132#- MirrorlibIGCM for frontend
133typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
134
135#====================================================
136#- libIGCM_POST for frontend
137typeset -r libIGCM_POST=${libIGCM}
138
139#====================================================
140#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
141typeset -r R_EXE="${MODIPSL}/bin"
142
143#====================================================
144#- SUBMIT_DIR : submission dir
145if [ X${SLURM_SUMBIT_DIR} != X ] ; then
146  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${SLURM_SUBMIT_DIR}}
147else
148  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
149fi
150
151#====================================================
152#- IN
153typeset -r R_IN=${R_IN:=/projsu/igcmg/IGCM}
154
155#====================================================
156#- RUN_DIR_PATH : Temporary working directory
157
158if [ $CENTER == spirit ] ; then
159    RUN_DIR_PATH=${RUN_DIR_PATH:=/scratchu/${LOGIN}/RUN_DIR}
160else
161    RUN_DIR_PATH=${RUN_DIR_PATH:=/scratchx/${LOGIN}/RUN_DIR}
162fi
163if [ X${SLURM_JOBID} != X ] ; then
164    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH}/${SLURM_JOBID}_${$}
165else
166    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH}/${$}
167fi
168
169#====================================================
170#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
171# Note that typeset -r can not be set at spirit because it sets the path as read-only
172typeset OUTCOMMAND_PATH=/tmp
173
174#====================================================
175#- HOST_MPIRUN_COMMAND
176typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
177
178#====================================================
179#- Max number of arguments passed to nco operator or demigration command
180UNIX_MAX_LIMIT=360
181
182#====================================================
183#- set PackDefault to false on meso-ipsl
184PackDefault=false
185
186#====================================================
187#- Default number of MPI task for IPSL coupled model
188#- required for backward compatibility
189#-
190DEFAULT_NUM_PROC_OCE=5
191DEFAULT_NUM_PROC_CPL=1
192(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
193
194#D-#==================================================
195#D-function IGCM_sys_defineArchives
196#D-* Purpose:
197#D-* Define ARCHIVE : Dedicated to large files
198#D-* Define STORAGE : Dedicated to small/medium files
199#D-* Define R_OUT   : Output tree located on ARCHIVE
200#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
201#D-* Define R_BUF   : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
202#D-* if SpaceName=TEST nothing special will hapen
203#D-* Examples:
204#D-
205function IGCM_sys_defineArchives {
206  IGCM_debug_PushStack "IGCM_sys_defineArchives"
207
208  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
209    #====================================================
210    #- ARCHIVE (dedicated to large files)
211    ARCHIVE=${config_UserChoices_ARCHIVE}
212  else
213    #====================================================
214    #- ARCHIVE (dedicated to large files)
215    if [ $CENTER == spirit ] ; then
216        # At spirit
217        ARCHIVE=${ARCHIVE:=/data/${LOGIN}}
218    else
219        # At spiritx
220        ARCHIVE=${ARCHIVE:=/homedata/${LOGIN}}
221    fi
222  fi
223
224  if [ ! X${config_UserChoices_STORAGE} = X ]; then
225    #====================================================
226    #- STORAGE (dedicated to small/medium files)
227    STORAGE=${config_UserChoices_STORAGE}
228  else
229    #====================================================
230    #- STORAGE (dedicated to small/medium files)
231    STORAGE=${ARCHIVE}
232  fi
233
234  # ON MESO-IPSL NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
235
236  #====================================================
237  #- R_OUT
238  R_OUT=${ARCHIVE}/IGCM_OUT
239
240  #====================================================
241  #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
242  R_FIG=${STORAGE}/IGCM_OUT
243
244  #====================================================
245  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
246  CMIP6_BUF=${STORAGE}/IGCM_OUT
247
248  #====================================================
249  #- R_BUF (ONLY FOR double copy an scratch)
250  R_BUF=${STORAGE}/IGCM_OUT
251
252  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
253  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
254  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
255
256  IGCM_debug_PopStack "IGCM_sys_defineArchives"
257}
258
259#D-#==================================================
260#D-function IGCM_sys_RshArchive
261#D-* Purpose: Archive rsh command
262#D-* Examples:
263#D-
264function IGCM_sys_RshArchive {
265  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
266  /bin/ksh <<-EOF
267    ${@}
268EOF
269  status=$?
270  if [ ${status} -gt 0 ] ; then
271    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
272    IGCM_debug_Exit "IGCM_sys_RshArchive"
273  fi
274  IGCM_debug_PopStack "IGCM_sys_RshArchive"
275}
276
277#D-#==================================================
278#D-function IGCM_sys_RshArchive_NoError
279#D-* Purpose: Archive rsh command, without error
280#D-*          used only in monitoring.job
281#D-* Examples:
282#D-
283function IGCM_sys_RshArchive_NoError {
284  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
285  /bin/ksh <<-EOF
286    ${@} 2> /dev/null
287EOF
288  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
289}
290
291#D-#==================================================
292#D-function IGCM_sys_MkdirArchive
293#D-* Purpose: Mkdir on Archive
294#D-* Examples:
295#D-
296function IGCM_sys_MkdirArchive {
297  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
298  if ( $DEBUG_sys ) ; then
299    echo "IGCM_sys_MkdirArchive :" $@
300  fi
301  #- creation de repertoire sur le serveur fichier
302  if [ ! -d ${1} ]; then
303    \mkdir -p $1
304    status=$?
305
306    if [ ${status} -gt 0 ] ; then
307      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
308      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
309    fi
310  fi
311  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
312}
313
314#D-#==================================================
315#D-function IGCM_sys_TestDirArchive
316#D-* Purpose: Test Directory that must exists on Archive
317#D-* Examples:
318#D-
319function IGCM_sys_TestDirArchive {
320  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
321  if ( $DEBUG_sys ) ; then
322    echo "IGCM_sys_TestDirArchive :" $@
323  fi
324  typeset ExistFlag
325  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
326  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
327  return ${ExistFlag}
328}
329
330#D-#==================================================
331#D-function IGCM_sys_IsFileArchived
332#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
333#D-* Examples:
334#D-
335function IGCM_sys_IsFileArchived {
336  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
337  if ( $DEBUG_sys ) ; then
338    echo "IGCM_sys_IsFileArchived :" $@
339  fi
340  typeset IsArchivedFlag
341  # Never archived for this system
342  IsArchivedFlag=1
343  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
344
345  return ${IsArchivedFlag}
346}
347
348#D-#==================================================
349#D-function IGCM_sys_TestFileArchive
350#D-* Purpose: Test file that must NOT EXISTS on Archive
351#D-* Examples:
352#D-
353function IGCM_sys_TestFileArchive {
354  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
355  typeset ExistFlag
356  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
357  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
358
359  return ${ExistFlag}
360}
361
362#D-#==================================================
363#D-function IGCM_sys_CountFileArchive
364#D-* Purpose: Count files on Archive filesystem
365#D-* Examples:
366#D-
367function IGCM_sys_CountFileArchive {
368  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
369  ls ${@} 2>/dev/null | wc -l
370  if [ $? -gt 0 ] ; then
371    echo "IGCM_sys_CountFileArchive : erreur."
372  fi
373  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
374}
375
376#D-#==================================================
377#D-function IGCM_sys_Tree
378#D-* Purpose: Tree directories with files on ${ARCHIVE}
379#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
380#D-
381function IGCM_sys_Tree {
382  IGCM_debug_PushStack "IGCM_sys_Tree" $@
383  if ( $DEBUG_sys ) ; then
384    echo "IGCM_sys_Tree :" $@
385  fi
386
387  \ls -lR ${@}
388
389  IGCM_debug_PopStack "IGCM_sys_Tree"
390}
391
392#D-#==================================================
393#D-function IGCM_sys_Qsub
394#D-* Purpose: Qsub new job
395#D-* Examples:
396#D-
397function IGCM_sys_Qsub {
398  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
399  if ( $DEBUG_sys ) ; then
400    echo "IGCM_sys_Qsub :" $@
401  fi
402  typeset options status
403  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
404  /usr/bin/time sbatch ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
405  status=$?
406
407  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
408  if [ ${status} -gt 0 ] ; then
409    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
410    IGCM_debug_Exit "IGCM_sys_Qsub"
411  else
412    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
413  fi
414  IGCM_debug_PopStack "IGCM_sys_Qsub"
415}
416
417#D-#==================================================
418#D-function IGCM_sys_QsubPost
419#D-* Purpose: Qsub new job on scalaire
420#D-* Examples:
421#D-
422function IGCM_sys_QsubPost {
423  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
424  if ( $DEBUG_sys ) ; then
425    echo "IGCM_sys_QsubPost :" $@
426  fi
427  typeset options status
428  options="-o ${POST_DIR}/${Script_Post_Output}.out  -e ${POST_DIR}/${Script_Post_Output}.out"
429  sbatch ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
430  status=$?
431
432  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
433  if [ ${status} -gt 0 ] ; then
434    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
435    IGCM_debug_Exit "IGCM_sys_QsubPost"
436  else
437    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
438  fi
439  IGCM_debug_PopStack "IGCM_sys_QsubPost"
440}
441
442#D-*************************
443#D- File transfer functions
444#D-*************************
445#D-
446
447#D-#==================================================
448#D-function IGCM_sys_RmRunDir
449#D-* Purpose: rm tmpdir (dummy function most of the time batch
450#D-                      scheduler will do the job)
451#D-* Examples:
452#D-
453function IGCM_sys_RmRunDir {
454  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
455  if ( $DEBUG_sys ) ; then
456    echo "IGCM_sys_RmRunDir :" $@
457  fi
458
459  typeset status
460
461  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
462  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
463  status=$?
464
465  if [ ${status} -gt 0 ] ; then
466    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
467    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
468    IGCM_debug_Exit "IGCM_sys_RmRunDir"
469  else
470    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
471  fi
472  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
473}
474
475#D-#==================================================
476#D-function IGCM_sys_Put_Dir
477#D-* Purpose: Copy a complete directory on $(ARCHIVE)
478#D-* Examples:
479#D-
480function IGCM_sys_Put_Dir {
481  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
482  if ( $DEBUG_sys ) ; then
483    echo "IGCM_sys_Put_Dir :" $@
484  fi
485  if [ $DRYRUN = 0 ]; then
486    if [ ! -d ${1} ] ; then
487      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
488      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
489      return
490    fi
491
492    typeset status
493
494    # Only if we use rsync
495    #IGCM_sys_TestDirArchive $( dirname $2 )
496    #
497    #USUAL WAY
498    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
499    status=$?
500
501    if [ ${status} -gt 0 ] ; then
502      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
503      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
504      IGCM_debug_Exit "IGCM_sys_Put_Dir"
505    else
506      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
507    fi
508  fi
509  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
510}
511
512#D-#==================================================
513#D-function IGCM_sys_Get_Dir
514#D-* Purpose: Copy a complete directory from ${ARCHIVE}
515#D-* Examples:
516#D-
517function IGCM_sys_Get_Dir {
518  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
519  if ( $DEBUG_sys ) ; then
520    echo "IGCM_sys_Get_Dir :" $@
521  fi
522  if [ $DRYRUN = 0 ]; then
523    typeset NB_ESSAI DELAI status i
524    # number of tentative
525    NB_ESSAI=3
526    # time delay between tentative
527    DELAI=2
528
529    #
530    # USUAL WAY
531    i=0
532    while [ $i -lt $NB_ESSAI ] ; do
533      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
534      status=$?
535      if [ ${status} -gt 0 ] ; then
536        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
537        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
538        sleep $DELAI
539      else
540        break
541      fi
542      (( i = i + 1 ))
543    done
544
545    if [ ${status} -gt 0 ] ; then
546      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
547      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
548      IGCM_debug_Exit "IGCM_sys_Get_Dir"
549    else
550      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
551    fi
552  fi
553  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
554}
555
556#D-#==================================================
557#D-function IGCM_sys_Put_Rest
558#D-* Purpose: Put computied restarts on ${ARCHIVE}.
559#D-           File and target directory must exist.
560#D-* Examples:
561#D-
562function IGCM_sys_Put_Rest {
563  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
564  if ( $DEBUG_sys ) ; then
565    echo "IGCM_sys_Put_Rest :" $@
566  fi
567  if [ $DRYRUN = 0 ]; then
568    if [ ! -f ${1} ] ; then
569        # The file is missing     
570        if [ X${3} = X'OPTIONAL' ] ; then
571            echo "WARINING : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
572            IGCM_debug_Print 1 "IGCM_sys_Put_Rest: Optional restart file ${1} is missing, now continue for the next"
573        else
574            echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
575            IGCM_debug_Exit "IGCM_sys_Put_Rest"
576        fi
577
578    else
579      # The file exist
580
581    typeset status
582    #
583    # USUAL WAY
584    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
585    status=$?
586
587#       #RSYNC WITH NETWORK SSH CALL
588#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
589#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
590
591#       #RSYNC WITH NFS USE
592#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
593#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
594
595#       status=$?
596#       IGCM_sys_Rsync_out $status
597
598#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
599#       (( status=status+$? ))
600
601    if [ ${status} -gt 0 ] ; then
602      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
603      [ -f ${1} ] && ls -l ${1}
604      [ -f ${2} ] && ls -l ${2}
605      [ -f ${2}/${1} ] && ls -l ${2}/${1}
606      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
607      IGCM_debug_Exit "IGCM_sys_Put_Rest"
608    else
609
610      if [ X${JobType} = XRUN ] ; then
611        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
612        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
613      fi
614
615      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
616    fi
617    fi
618  fi
619  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
620}
621
622#D-#==================================================
623#D-function IGCM_sys_Put_Out
624#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
625#D-* Examples:
626#D-
627function IGCM_sys_Put_Out {
628  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
629  if ( $DEBUG_sys ) ; then
630    echo "IGCM_sys_Put_Out :" $@
631  fi
632
633  typeset status
634
635  if [ $DRYRUN = 0 ]; then
636    if [ ! -f ${1} ] ; then
637      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
638      IGCM_debug_PopStack "IGCM_sys_Put_Out"
639      return 1
640    fi
641    #
642    IGCM_sys_MkdirArchive $( dirname $2 )
643    #
644    if [ X${JobType} = XRUN ] ; then
645      if [ X${3} = X ] ; then
646        IGCM_sys_Chmod 444 ${1}
647      fi
648    fi
649    #
650
651    echo ${RSYNC} ${RSYNC_opt} $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
652    ${RSYNC} ${RSYNC_opt} $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
653    status=$?
654    IGCM_sys_Rsync_out $status
655
656    ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
657    (( status=status+$? ))
658
659    if [ ${status} -gt 0 ] ; then
660      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
661      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
662      IGCM_debug_Exit "IGCM_sys_Put_Out"
663    else
664      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
665    fi
666  fi
667  IGCM_debug_PopStack "IGCM_sys_Put_Out"
668  return 0
669}
670
671#D-#==================================================
672#D-function IGCM_sys_Get
673#D-* Purpose: Get a file from ${ARCHIVE}
674#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
675#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
676function IGCM_sys_Get {
677  IGCM_debug_PushStack "IGCM_sys_Get" $@
678
679  typeset DEST status dm_liste
680
681  if ( $DEBUG_sys ) ; then
682    echo "IGCM_sys_Get :" $@
683  fi
684  if [ $DRYRUN -le 2 ]; then
685    if [ X${1} = X'/l' ] ; then
686      eval set +A dm_liste \${${2}}
687    else
688      dm_liste=${1}
689    fi
690    eval DEST=\${${#}}
691
692    # test if the (first) file is present in the old computation :
693    IGCM_sys_TestFileArchive ${dm_liste[0]}
694    status=$?
695    if [ ${status} -gt 0 ] ; then
696      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
697      IGCM_debug_Exit "IGCM_sys_Get"
698      return
699    fi
700
701    #USUAL WAY
702    \cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
703    status=$?
704
705    if [ ${status} -gt 0 ] ; then
706      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
707      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
708      IGCM_debug_Exit "IGCM_sys_Get"
709    else
710      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
711    fi
712  fi
713  IGCM_debug_PopStack "IGCM_sys_Get"
714}
715
716#D-#==================================================
717#D-function IGCM_sys_GetDate_Monitoring
718#D-* Purpose: get the last year for which the monitoring has been computed
719#D-* Examples:
720#D-
721function IGCM_sys_GetDate_Monitoring {
722  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
723  if ( $DEBUG_sys ) ; then
724    echo "IGCM_sys_GetDate_Monitoring :" $@
725  fi
726
727  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
728
729  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
730}
731
732#D-#==================================================
733#D-function IGCM_sys_Dods_Rm
734#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
735#D-* Examples:
736#D-
737function IGCM_sys_Dods_Rm {
738  if ( $DEBUG_sys ) ; then
739    echo "IGCM_sys_Dods_Rm :" $@
740  fi
741  return 0
742}
743
744#D-#==================================================
745#D-function IGCM_sys_Dods_Cp
746#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
747#D-* Examples:
748#D-
749function IGCM_sys_Dods_Cp {
750  if ( $DEBUG_sys ) ; then
751    echo "IGCM_sys_Dods_Cp :" $@
752  fi
753  return 0
754}
755
756#D-#==================================================
757#D-function IGCM_sys_Put_Dods
758#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
759#D-* Examples:
760#D-
761function IGCM_sys_Put_Dods {
762  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
763  if ( $DEBUG_sys ) ; then
764    echo "IGCM_sys_Put_Dods :" $@
765  fi
766  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
767}
768
769##############################################################
770# REBUILD OPERATOR
771
772#D-#==================================================
773#D-function IGCM_sys_sync
774#D-* Purpose: flush buffer on disk (dummy function on Ciclad)
775#D-* Examples:
776#D-
777function IGCM_sys_sync {
778  IGCM_debug_PushStack "IGCM_sys_sync" $@
779  if ( $DEBUG_sys ) ; then
780    echo "IGCM_sys_sync :" $@
781    echo "Dummy call, let the system do that."
782  fi
783  IGCM_debug_PopStack "IGCM_sys_sync"
784}
785
786############################################################
787# Activate Running Environnment Variables
788
789#D-#==================================================
790#D-function IGCM_sys_activ_variables
791#D-* Purpose: set environement variables prior to execution
792#D-* Examples:
793#D-
794function IGCM_sys_activ_variables {
795  IGCM_debug_PushStack "IGCM_sys_activ_variables"
796  if ( $DEBUG_sys ) ; then
797    echo "IGCM_sys_activ_variables"
798  fi
799
800# --------------------------------------------------------------------
801#D- MODULE specifications
802# --------------------------------------------------------------------
803  # Source the file EnvFile containing all module load needed to run the model.
804  # EnvFile can be specified in config.card. If this is the case and if the file exists,
805  # it will be used. Otherwise a default file will be used.
806  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
807      # EnvFile is set in config.card and the file exists
808      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
809      EnvFile=${config_UserChoices_EnvFile}
810  else
811      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
812      if [ $CENTER == spirit ] ; then
813          # At spirit
814          EnvFile=/home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
815      else
816          # At spirit
817          EnvFile=/ciclad-home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
818      fi
819  fi
820  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
821  IGCM_debug_Print 1 ${EnvFile}
822  . ${EnvFile}
823
824
825  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for meso-ipsl . "
826  module list
827 
828
829# --------------------------------------------------------------------
830#D- MPI specifications
831# --------------------------------------------------------------------
832
833# --------------------------------------------------------------------
834#D- Other specifications
835# --------------------------------------------------------------------
836
837  ulimit -s unlimited
838
839  IGCM_debug_PopStack "IGCM_sys_activ_variables"
840}
841
842############################################################
843# Desactivate Running Environnment Variables
844
845#D-#==================================================
846#D-function IGCM_sys_desactiv_variables
847#D-* Purpose: unset environement variables after execution
848#D-* Examples:
849#D-
850function IGCM_sys_desactiv_variables {
851  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
852  if ( $DEBUG_sys ) ; then
853    echo "IGCM_sys_desactiv_variables"
854  fi
855# --------------------------------------------------------------------
856#D- MPI specifications
857# --------------------------------------------------------------------
858
859# --------------------------------------------------------------------
860#D- Other specifications
861# --------------------------------------------------------------------
862
863  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
864}
865
866############################################################
867# Update job headers to be used by the scheduler
868
869#D-#==================================================
870#D-function IGCM_sys_updateHeaders
871#D-* Purpose: Update job headers to be used by the scheduler
872#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
873#D-
874function IGCM_sys_updateHeaders {
875  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
876  if ( $DEBUG_sys ) ; then
877    echo "IGCM_sys_updateHeaders"
878  fi
879  typeset file
880  file=$1
881
882  if [ ${executionType} -eq 1 ] ; then
883    # MPMD + MPI
884    sed -e "/::openMPthreads::/d"                  \
885        -e "s/::JobNumProcTot::/${coreNumber}/"    \
886      ${file} > ${file}.tmp
887
888  elif [ ${executionType} -eq 2 ] ; then
889    # MPMD + MPI + OMP
890      (( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
891      [ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
892      sed -e "/::openMPthreads::/d"                 \
893        -e "s/::JobNumProcTot::/${mpiTasks}/"       \
894        -e "/ntasks/i\#SBATCH --nodes=${nodeNumber}"\
895        -e "/ntasks/i\#SBATCH --exclusive"          \
896         ${file} > ${file}.tmp
897  elif [ ${executionType} -eq 3 ] ; then
898    # SPMD + MPI/OMP
899    sed -e "s/::openMPthreads::/${openMPthreads}/" \
900        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
901      ${file} > ${file}.tmp
902
903  elif [ ${executionType} -eq 4 ] ; then
904    # SPMD + MPI only
905    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
906        -e "/::openMPthreads::/d"                  \
907      ${file} > ${file}.tmp
908
909  elif [ ${executionType} -eq 5 ] ; then
910    # SPMD + OMP only
911    sed -e "s/::openMPthreads::/${openMPthreads}/" \
912        -e "/::JobNumProcTot::/d"                  \
913      ${file} > ${file}.tmp
914
915  elif [ ${executionType} -eq 6 ] ; then
916    # SEQUENTIAL THEN
917    sed -e "s/::JobNumProcTot::/1/"                \
918        -e "/::openMPthreads::/d"                  \
919      ${file} > ${file}.tmp
920
921  fi
922
923  IGCM_sys_Mv ${file}.tmp ${file}
924
925  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
926}
927
928############################################################
929# Build MPI/OMP scripts run file (dummy function)
930
931#D-#==================================================
932#D-function IGCM_sys_build_run_file
933#D-* Purpose: build run file (deprecated)
934#D-* Examples:
935#D-
936function IGCM_sys_build_run_file {
937  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
938}
939
940############################################################
941# Build MPI/OMP scripts
942
943#D-#==================================================
944#D-function IGCM_sys_build_execution_scripts
945#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
946#D-* Examples:
947#D-
948function IGCM_sys_build_execution_scripts
949{
950  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
951  if ( $DEBUG_sys ) ; then
952    echo "IGCM_sys_build_execution_scripts " $@
953  fi
954  IGCM_debug_Print 1 "executionType= ${executionType}"
955  EXECUTION=${HOST_MPIRUN_COMMAND}
956
957  if ( ${OK_PARA_MPMD} ) ; then
958
959    # MPMD mode
960    # 1 MPI only : executionType=1
961    # 2 MPI/OpenMP : executionType=2
962
963    if [ -f run_file ] ; then
964      IGCM_sys_Rm -f run_file
965    fi
966    touch run_file
967
968    # case 1 : Only MPI (MPMD)
969    if  ( ! ${OK_PARA_OMP} ) ; then
970      # Build run_file
971      current_core=0
972      # First loop on the components for the coupler ie oasis (only if oasis3)
973      # the coupler ie oasis3 must be the first one
974      for comp in ${config_ListOfComponents[*]} ; do
975
976        eval ExeNameIn=\${config_Executable_${comp}[0]}
977        eval ExeNameOut=\${config_Executable_${comp}[1]}
978
979        # Only if the component has an executable
980        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
981
982#          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
983#          (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
984#          echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
985#          (( current_core = ${end_core} + 1 ))
986          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
987          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
988          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
989
990        fi
991      done
992
993      EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
994
995      IGCM_sys_Chmod u+x run_file
996      if ( $DEBUG_sys ) ; then
997        echo "run_file contains : "
998        cat run_file
999      fi
1000
1001    else 
1002
1003    # 2 MPI/OpenMP : executionType=2
1004
1005    # MPI-OpenMP (MPMD)
1006    # export SLURM_HOSTFILE=./hostlist
1007    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
1008    # example of  hostlist file :
1009    # r3i3n33
1010    # r3i3n33
1011    # ...
1012    # example of run_file :
1013    # 0-70 ./prog_lmdz.x.sh %o %t
1014    # 71-430 ./prog_opa.xx.sh %o %t
1015    # 431-431 ./prog_xios.x.sh %o %t
1016    # examples of prog_file :
1017    # prog_lmdz.x.sh :
1018    # (( init = 0 + $1 ))
1019    # (( index = init * 10 ))
1020    # (( slot = index % 40 ))
1021    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
1022    # that will become
1023    # taskset -c 0-9 ./script_lmdz.x.ksh
1024    # ...
1025    # with script_lmdz.x.ksh
1026    # export OMP_STACKSIZE=3g
1027    # export OMP_PLACES=cores
1028    # export OMP_NUM_THREADS=10
1029    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
1030
1031    #  Hosts treatment
1032    _bkIFS=$IFS;
1033        IFS=$'\n'; set -f
1034        listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1035        IFS=$_bkIFS; set +f
1036        rm -f hostlist
1037
1038     # Loop on the components to build run_file and script_exec files
1039        rank=0
1040        current_core=0
1041        current_core_mpi=0
1042
1043      for comp in ${config_ListOfComponents[*]} ; do
1044
1045        eval ExeNameIn=\${config_Executable_${comp}[0]}
1046        eval ExeNameOut=\${config_Executable_${comp}[1]}
1047
1048        # Not possible if oasis has an executable (i.e old version of oasis3)
1049        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1050          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1051          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1052          IGCM_debug_Verif_Exit
1053        fi
1054
1055        # Only if we really have an executable for the component :
1056        if [ "X${ExeNameOut}" != X\"\" ] ; then
1057
1058          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1059          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1060          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1061     
1062
1063          # Build script files
1064
1065          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1066          echo ""  >> script_${ExeNameOut}.ksh
1067          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1068          # Check if the number of threads is correct
1069              case ${comp_proc_omp_loc} in
1070                  2|4|5|10|20)
1071                      IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1072                      ;;
1073                  *)
1074                      IGCM_debug_Exit "ERROR with OMP parameters !"
1075                      IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1076                      IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1077                      IGCM_debug_Verif_Exit
1078                      ;;
1079              esac
1080            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1081            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1082            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
1083
1084          fi
1085
1086          # to have out/err per process on different files
1087          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1088
1089          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1090
1091        # Build run_file
1092        # Only if the component has an executable
1093        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1094
1095          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1096          (( end_core = ${current_core_mpi} + ${comp_proc_mpi_loc} - 1 ))
1097          echo "${current_core_mpi}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
1098          (( current_core_mpi = ${end_core} + 1 ))
1099        fi
1100
1101          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1102              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1103          else
1104              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1105          fi
1106
1107          # Build configuration file
1108 
1109              echo "#!/bin/sh" > prog_${ExeNameOut}.sh
1110          echo "(( init = $current_core + \$1 ))" >> prog_${ExeNameOut}.sh
1111          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
1112          echo "(( slot = index % 40 ))" >> prog_${ExeNameOut}.sh
1113              echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
1114          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
1115
1116          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
1117
1118        # Build hostlist file
1119
1120          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1121             (( index_host = current_core / NB_CORE_PER_NODE ))
1122             host_value=${listnodes[${index_host}]}
1123               echo "$host_value" >> hostlist
1124             (( current_core = current_core + offset_comp_proc_loc ))
1125          done
1126        fi
1127      done
1128
1129      ## variable added to stop after 60s instead of 600s by default.
1130      ## This is used when no error comes from executables and when something stopped an executable without notice.
1131      export SLURM_WAIT=60
1132
1133      EXECUTION="${HOST_MPIRUN_COMMAND} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1134
1135      IGCM_sys_Chmod u+x run_file
1136      if ( $DEBUG_sys ) ; then
1137        echo "run_file contains : "
1138        cat run_file
1139      fi
1140
1141    fi # if ${OK_PARA_MPMD}
1142
1143  else
1144  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
1145
1146    for comp in ${config_ListOfComponents[*]} ; do
1147
1148      # Only if we really have an executable for the component :
1149      eval ExeNameOut=\${config_Executable_${comp}[1]}
1150      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1151
1152        # Build script files
1153
1154        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1155        echo ""  >> script_${ExeNameOut}.ksh
1156        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1157
1158        if ( ${OK_PARA_OMP} ) ; then
1159            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1160            # Check if the number of threads is correct
1161            case ${comp_proc_omp_loc} in
1162                2|4|5|10|20)
1163                    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1164                    ;;
1165                *)
1166                    IGCM_debug_Exit "ERROR with OMP parameters !"
1167                    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1168                    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1169                    IGCM_debug_Verif_Exit
1170                    ;;
1171            esac
1172            echo ""  >> script_${ExeNameOut}.ksh
1173            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1174            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1175            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1176        fi
1177
1178        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1179       
1180        # To have out/err per process on different files
1181        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1182        EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1183
1184        IGCM_debug_Print 1 "sys Jean-Zay : script_${ExeNameOut}.ksh contains"
1185        cat script_${ExeNameOut}.ksh
1186
1187      fi
1188
1189    done
1190
1191  fi # ${OK_PARA_MPMD}
1192
1193  IGCM_debug_Print 1 "sys meso-ipsl : execution command is "
1194  IGCM_debug_Print 1 "$EXECUTION"
1195
1196  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1197}
1198
1199#D-#==================================================
1200#D-function IGCM_sys_check_path
1201#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1202#D-* do not point to an important use directory. Stop immediately in that case.
1203#D-* Examples:
1204#D-
1205function IGCM_sys_check_path {
1206  IGCM_debug_PushStack "IGCM_sys_check_path"
1207  if ( $DEBUG_sys ) ; then
1208    echo "IGCM_sys_check_path"
1209  fi
1210
1211  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1212    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1213    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1214    IGCM_debug_Exit "This will stop the job"
1215  fi
1216  IGCM_debug_PopStack "IGCM_sys_check_path"
1217}
1218
1219#D-#==================================================
1220#D-function IGCM_sys_check_quota. Dummy call here
1221#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1222#D-* Examples:
1223#D-
1224function IGCM_sys_check_quota {
1225  IGCM_debug_PushStack "IGCM_sys_check_quota"
1226  if ( $DEBUG_sys ) ; then
1227    echo "IGCM_sys_check_quota"
1228  fi
1229
1230  # TO BE IMPLEMENTED
1231
1232  IGCM_debug_PopStack "IGCM_sys_check_quota"
1233}
1234
1235#D-#==================================================
1236#D-function IGCM_sys_projectAccounting
1237#D-* Purpose: store project accounting information in a file
1238#D-* Examples:
1239#D-
1240function IGCM_sys_projectAccounting {
1241  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1242  if ( $DEBUG_sys ) ; then
1243    echo "IGCM_sys_check_quota"
1244  fi
1245
1246  touch $1
1247
1248  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1249}
1250
1251#D-#==================================================
1252#D-function IGCM_sys_getJobSchedulerID
1253#D-* Purpose: Get the job ID during execution
1254#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1255#D-
1256function IGCM_sys_getJobSchedulerID {
1257  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1258  if ( $DEBUG_sys ) ; then
1259    echo "IGCM_sys_getJobSchedulerID"
1260  fi
1261
1262  eval ${1}=$( echo ${PBS_JOBID} | awk -F. '{print $1}' )
1263
1264  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1265}
1266
1267#D-#==================================================
1268#D-function IGCM_sys_GetJobID
1269#D-* Purpose: Get the job ID from the JobName
1270#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1271#D-
1272function IGCM_sys_GetJobID {
1273  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1274  if ( $DEBUG_sys ) ; then
1275    echo "IGCM_sys_GetJobID"
1276  fi
1277
1278  # With -f option, the full job name is given in the last column
1279  ID="$( qstat -u $2 | grep -w $1 | gawk '-F ' '{print $10}' )"
1280
1281  eval ${3}=${ID}
1282  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1283}
1284
1285#D-#==================================================
1286#D-function IGCM_sys_CountJobInQueue
1287#D-* Purpose: Count number of users job
1288#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1289#D-
1290function IGCM_sys_CountJobInQueue {
1291  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1292  if ( $DEBUG_sys ) ; then
1293    echo "IGCM_sys_CountJobInQueue"
1294  fi
1295  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1296}
1297
1298#D-#==================================================
1299#D-function IGCM_sys_atlas
1300#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1301#D-* Examples:
1302#D-
1303function IGCM_sys_atlas {
1304  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1305  if ( $DEBUG_sys ) ; then
1306    echo "IGCM_sys_atlas :" $@
1307  fi
1308
1309  typeset status
1310
1311  \atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1312  status=$?
1313  if [ ${status} -gt 0 ] ; then
1314    echo "IGCM_sys_atlas : error code ${status}"
1315    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1316    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1317    IGCM_debug_PopStack "IGCM_sys_atlas"
1318    return 1
1319  else
1320    IGCM_debug_PopStack "IGCM_sys_atlas"
1321    return 0
1322  fi
1323
1324  IGCM_debug_PopStack "IGCM_sys_atlas"
1325}
1326
1327#D-#==================================================
1328#D-function IGCM_sys_rebuild_nemo
1329#D-* Purpose: rebuild nemo parallel files with general rebuild
1330#D-* Examples:
1331#D-
1332
1333function IGCM_sys_rebuild_nemo {
1334  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1335  if ( $DEBUG_sys ) ; then
1336    echo "IGCM_sys_rebuild_nemo :" $@
1337  fi
1338
1339  nemo_generic_restart_file_name_out=${1}
1340  nemo_extension_out=${3}
1341  shift ; shift ; shift
1342
1343  IGCM_sys_rebuild -o ${nemo_generic_restart_file_name_out}.${nemo_extension_out} $@
1344
1345  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1346}
1347
1348
1349
Note: See TracBrowser for help on using the repository browser.