source: branches/libIGCM_CESMEP/libIGCM_sys/libIGCM_sys_irene-amd.ksh @ 1586

Last change on this file since 1586 was 1585, checked in by ssenesi, 13 months ago

merge branch libIGCM_CESMEP with trunk that handles RedHat8 on Irene
File size: 66.2 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision:: 1512                                     $ Revision of last commit
7# $Author:: acosce                                     $ Author of last commit
8# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene-amd
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
49# -------------------------------------------------------------------------------------
50# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
51# |          |  Cp/Exe/param/files |            |         |                           |
52# |          |  Chmod Qsub         |            |         |                           |
53# -------------------------------------------------------------------------------------
54# |    0     |       yes           |    yes     |  yes    |      yes                  |
55# -------------------------------------------------------------------------------------
56# |    1     |       yes           |    yes     |  yes    |      no                   |
57# -------------------------------------------------------------------------------------
58# |    2     |       yes           |    yes     |  no     |      no                   |
59# -------------------------------------------------------------------------------------
60# |    3     |       yes           |    no      |  no     |      no                   |
61# -------------------------------------------------------------------------------------
62# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
63# -------------------------------------------------------------------------------------
64
65
66#=====================================================
67# Global Variables :
68#=====================================================
69# Language : "fr" or "en"
70typeset -r MYLANG="fr"
71
72#=====================================================
73# Host user names project maxCpuTime
74# $hostname ou hostname
75typeset HOST=${HOST:=$( hostname )}
76# $username ou whoami
77typeset LOGIN=${LOGIN:=$( whoami )}
78# $hostname of the MASTER job
79typeset MASTER=irene-amd
80# add default project on irene-amd
81if [ ! -z $BRIDGE_MSUB_PROJECT ]; then
82    typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT} | cut -d@ -f1 )
83else
84    typeset PROJECT=$(echo ${CCCHOME} | cut -d/ -f5 )
85fi
86# jobWarningDelay in seconds
87typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
88
89#D-
90#D-#==================================================
91#D-Program used in libIGCM
92#D-#==================================================
93
94# Submit command
95typeset SUBMIT=${SUBMIT:=ccc_msub}
96# rsync with path
97typeset -r RSYNC=/usr/bin/rsync
98# RSYNC_opt args to rsync
99typeset -r RSYNC_opt="-va"
100# ie storage filesystem
101typeset -r STOREHOST=${MASTER}
102typeset -r REMOTE_RSYNC=/usr/bin/rsync
103
104#====================================================
105# Set environment tools (ferret, nco, cdo, rebuild, ...)
106#====================================================
107typeset CCCOS=${CCCOS:=$( ccc_os )}
108
109if [ X${TaskType} = Xcomputing ] ; then
110  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
111else
112  module purge > /dev/null 2>&1
113  if [ X${CCCOS} = X"Rhel_8__x86_64" ] ; then
114      . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
115  else
116      . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_irene-amd > /dev/null 2>&1
117      . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_atlas_irene-amd > /dev/null 2>&1
118  fi
119
120  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
121  export UVCDAT_ANONYMOUS_LOG=FALSE
122fi
123
124# FYI
125[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
126[ ! X${TaskType} = Xchecking ] && module list
127
128# For AddNoise and AddPertu3DOCE
129export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
130
131# For rebuild
132export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl/bin
133
134#====================================================
135# Host specific DIRECTORIES
136#====================================================
137
138# ============ CESIUM START ============ #
139
140#====================================================
141#- Mirror libIGCM from titane to cesium if needed
142#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
143#if [ ! ${ROOTSYS} = "home" ] ; then
144#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
145#else
146#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
147#fi
148
149#====================================================
150#- libIGCM_POST
151#if ( ${MirrorlibIGCM} ) ; then
152#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
153#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
154#else
155#  typeset -r libIGCM_POST=${libIGCM}
156#fi
157
158# ============ CESIUM  END  ============ #
159
160#====================================================
161#- MirrorlibIGCM for frontend
162typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
163
164#====================================================
165#- libIGCM_POST for frontend
166typeset -r libIGCM_POST=${libIGCM}
167
168#====================================================
169#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
170typeset -r R_EXE="${MODIPSL}/bin"
171
172#====================================================
173#- SUBMIT_DIR : submission dir
174typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
175
176#====================================================
177#- IN
178typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
179
180#====================================================
181#- RUN_DIR_PATH : Temporary working directory (=> TMP)
182typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
183
184#====================================================
185#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
186typeset -r OUTCOMMAND_PATH=/tmp
187
188#====================================================
189#- HOST_MPIRUN_COMMAND
190typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
191
192#====================================================
193#- Max number of arguments passed to nco operator or demigration command
194UNIX_MAX_LIMIT=360
195
196#====================================================
197#- set PackDefault to true on irene-amd
198PackDefault=true
199
200#====================================================
201#- Default number of MPI task for IPSL coupled model
202#- required for backward compatibility
203#-
204DEFAULT_NUM_PROC_OCE=5
205DEFAULT_NUM_PROC_CPL=1
206(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
207
208#====================================================
209#- Number of computing cores per node
210#-
211NB_CORE_PER_NODE=128
212
213#D-#==================================================
214#D-function IGCM_sys_defineArchives
215#D-* Purpose:
216#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
217#D-* Define ARCHIVE : Dedicated to large files
218#D-* Define STORAGE : Dedicated to small/medium files
219#D-* Define R_OUT   : Output tree located on ARCHIVE
220#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
221#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
222#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
223#D-* Examples:
224#D-
225function IGCM_sys_defineArchives {
226  IGCM_debug_PushStack "IGCM_sys_defineArchives"
227
228  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
229  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then
230    # Default option: Change the dfldatadir according to the project used for submission
231    # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
232    if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then
233      module switch dfldatadir dfldatadir/gencmip6
234      IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
235    else
236      module switch dfldatadir dfldatadir/${PROJECT}
237      IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
238    fi
239  else
240    # Use the project set by the variable DataProject in section UserChoices in config.card
241    module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
242    IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
243  fi
244
245 #====================================================
246 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
247 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
248
249  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
250    #====================================================
251    #- ARCHIVE (dedicated to large files)
252    ARCHIVE=${config_UserChoices_ARCHIVE}
253  else
254    #====================================================
255    #- ARCHIVE (dedicated to large files)
256    ARCHIVE=${CCCSTOREDIR}
257  fi
258
259  if [ ! X${config_UserChoices_STORAGE} = X ]; then
260    #====================================================
261    #- STORAGE (dedicated to small/medium files)
262    STORAGE=${config_UserChoices_STORAGE}
263  else
264    #====================================================
265    #- STORAGE (dedicated to small/medium files)
266    STORAGE=${CCCWORKDIR}
267  fi
268
269  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
270    #====================================================
271    #- R_OUT
272    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
273
274    #====================================================
275    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
276    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
277
278    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
279
280  else
281    #====================================================
282    #- R_OUT
283    R_OUT=${ARCHIVE}/IGCM_OUT
284
285    #====================================================
286    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
287    R_FIG=${STORAGE}/IGCM_OUT
288  fi
289
290  #====================================================
291  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
292  CMIP6_BUF=${STORAGE}/IGCM_OUT
293
294  #====================================================
295  #- R_BUF (ONLY FOR double copy an scratch)
296  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
297
298  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
299  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
300  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
301
302  IGCM_debug_PopStack "IGCM_sys_defineArchives"
303}
304
305#D-#==================================================
306#D-function IGCM_sys_RshArchive
307#D-* Purpose: Archive rsh command
308#D-* Examples:
309#D-
310function IGCM_sys_RshArchive {
311  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
312  /bin/ksh <<-EOF
313    ${@}
314EOF
315  status=$?
316  if [ ${status} -gt 0 ] ; then
317    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
318    IGCM_debug_Exit "IGCM_sys_RshArchive"
319  fi
320  IGCM_debug_PopStack "IGCM_sys_RshArchive"
321}
322
323#D-#==================================================
324#D-function IGCM_sys_RshArchive_NoError
325#D-* Purpose: Archive rsh command, without error
326#D-*          used only in monitoring.job
327#D-* Examples:
328#D-
329function IGCM_sys_RshArchive_NoError {
330  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
331  /bin/ksh <<-EOF
332    ${@} 2> /dev/null
333EOF
334  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
335}
336
337#D-#==================================================
338#D-function IGCM_sys_MkdirArchive
339#D-* Purpose: Mkdir on Archive
340#D-* Examples:
341#D-
342function IGCM_sys_MkdirArchive {
343  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
344  if ( $DEBUG_sys ) ; then
345    echo "IGCM_sys_MkdirArchive :" $@
346  fi
347  #- creation de repertoire sur le serveur fichier
348  if [ ! -d ${1} ]; then
349    \mkdir -p $1
350    status=$?
351
352    if [ ${status} -gt 0 ] ; then
353      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
354      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
355    fi
356  fi
357  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
358}
359
360#D-#==================================================
361#D-function IGCM_sys_TestDirArchive
362#D-* Purpose: Test Directory that must exists on Archive
363#D-* Examples:
364#D-
365function IGCM_sys_TestDirArchive {
366  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
367  if ( $DEBUG_sys ) ; then
368    echo "IGCM_sys_TestDirArchive :" $@
369  fi
370  typeset ExistFlag
371  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
372  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
373  return ${ExistFlag}
374}
375
376#D-#==================================================
377#D-function IGCM_sys_IsFileArchived
378#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
379#D-* Examples:
380#D-
381function IGCM_sys_IsFileArchived {
382  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
383  if ( $DEBUG_sys ) ; then
384    echo "IGCM_sys_IsFileArchived :" $@
385  fi
386  typeset IsArchivedFlag
387  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
388  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
389
390  return ${IsArchivedFlag}
391}
392
393#D-#==================================================
394#D-function IGCM_sys_TestFileArchive
395#D-* Purpose: Test file that must NOT EXISTS on Archive
396#D-* Examples:
397#D-
398function IGCM_sys_TestFileArchive {
399  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
400  typeset ExistFlag
401  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
402  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
403
404  return ${ExistFlag}
405}
406
407#D-#==================================================
408#D-function IGCM_sys_CountFileArchive
409#D-* Purpose: Count files on Archive filesystem
410#D-* Examples:
411#D-
412function IGCM_sys_CountFileArchive {
413  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
414  ls ${@} 2>/dev/null | wc -l
415  if [ $? -gt 0 ] ; then
416    echo "IGCM_sys_CountFileArchive : erreur."
417  fi
418  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
419}
420
421#D-#==================================================
422#D-function IGCM_sys_Tree
423#D-* Purpose: Tree directories with files on ${ARCHIVE}
424#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
425#D-
426function IGCM_sys_Tree {
427  IGCM_debug_PushStack "IGCM_sys_Tree" $@
428  if ( $DEBUG_sys ) ; then
429    echo "IGCM_sys_Tree :" $@
430  fi
431
432  \ls -lR ${@}
433
434  IGCM_debug_PopStack "IGCM_sys_Tree"
435}
436
437#D-#==================================================
438#D-function IGCM_sys_Qsub
439#D-* Purpose: Qsub new job
440#D-* Examples:
441#D-
442function IGCM_sys_Qsub {
443  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
444  if ( $DEBUG_sys ) ; then
445    echo "IGCM_sys_Qsub :" $@
446  fi
447  typeset options status
448  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
449
450  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
451  status=$?
452
453  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
454  if [ ${status} -gt 0 ] ; then
455    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
456    IGCM_debug_Exit "IGCM_sys_Qsub"
457  else
458    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
459  fi
460  IGCM_debug_PopStack "IGCM_sys_Qsub"
461}
462
463#D-#==================================================
464#D-function IGCM_sys_QsubPost
465#D-* Purpose: Qsub new job on scalaire
466#D-* Examples:
467#D-
468function IGCM_sys_QsubPost {
469  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
470  if ( $DEBUG_sys ) ; then
471    echo "IGCM_sys_QsubPost :" $@
472  fi
473  typeset options status
474  options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
475
476  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
477  status=$?
478
479  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
480  if [ ${status} -gt 0 ] ; then
481    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
482    IGCM_debug_Exit "IGCM_sys_QsubPost"
483  else
484    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
485  fi
486  IGCM_debug_PopStack "IGCM_sys_QsubPost"
487}
488
489#D-*************************
490#D- File transfer functions
491#D-*************************
492#D-
493
494#D-#==================================================
495#D-function IGCM_sys_RmRunDir
496#D-* Purpose: rm tmpdir (dummy function most of the time batch
497#D-                      scheduler will do the job)
498#D-* Examples:
499#D-
500function IGCM_sys_RmRunDir {
501  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
502  if ( $DEBUG_sys ) ; then
503    echo "IGCM_sys_RmRunDir :" $@
504  fi
505
506  typeset status
507
508  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
509  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
510  status=$?
511
512  if [ ${status} -gt 0 ] ; then
513    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
514    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
515    IGCM_debug_Exit "IGCM_sys_RmRunDir"
516  else
517    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
518  fi
519  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
520}
521
522#D-#==================================================
523#D-function IGCM_sys_Put_Dir
524#D-* Purpose: Copy a complete directory on $(ARCHIVE)
525#D-* Examples:
526#D-
527function IGCM_sys_Put_Dir {
528  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
529  if ( $DEBUG_sys ) ; then
530    echo "IGCM_sys_Put_Dir :" $@
531  fi
532  if [ $DRYRUN = 0 ]; then
533    if [ ! -d ${1} ] ; then
534      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
535      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
536      return
537    fi
538
539    typeset status
540
541    # Only if we use rsync
542    #IGCM_sys_TestDirArchive $( dirname $2 )
543    #
544    #USUAL WAY
545    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
546    status=$?
547
548    if [ ${status} -gt 0 ] ; then
549      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
550      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
551      IGCM_debug_Exit "IGCM_sys_Put_Dir"
552    else
553      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
554    fi
555  fi
556  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
557}
558
559#D-#==================================================
560#D-function IGCM_sys_Get_Dir
561#D-* Purpose: Copy a complete directory from ${ARCHIVE}
562#D-* Examples:
563#D-
564function IGCM_sys_Get_Dir {
565  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
566  if ( $DEBUG_sys ) ; then
567    echo "IGCM_sys_Get_Dir :" $@
568  fi
569  if [ $DRYRUN = 0 ]; then
570    typeset NB_ESSAI DELAI status i
571    # number of tentative
572    NB_ESSAI=3
573    # time delay between tentative
574    DELAI=2
575
576    #
577    # USUAL WAY
578    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
579    ccc_hsm get -r $1
580
581    i=0
582    while [ $i -lt $NB_ESSAI ] ; do
583      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
584      status=$?
585      if [ ${status} -gt 0 ] ; then
586        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
587        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
588        sleep $DELAI
589      else
590        break
591      fi
592      (( i = i + 1 ))
593    done
594
595    if [ ${status} -gt 0 ] ; then
596      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
597      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
598      IGCM_debug_Exit "IGCM_sys_Get_Dir"
599    else
600      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
601    fi
602  fi
603  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
604}
605
606#D-#==================================================
607#D-function IGCM_sys_Put_Rest
608#D-* Purpose: Put computied restarts on ${ARCHIVE}.
609#D-           File and target directory must exist.
610#D-* Examples:
611#D-
612function IGCM_sys_Put_Rest {
613  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
614  if ( $DEBUG_sys ) ; then
615    echo "IGCM_sys_Put_Rest :" $@
616  fi
617  if [ $DRYRUN = 0 ]; then
618    if [ ! -f ${1} ] ; then
619      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
620      IGCM_debug_Exit "IGCM_sys_Put_Rest"
621    fi
622
623    typeset status
624    #
625    # USUAL WAY
626    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
627    status=$?
628
629#       #RSYNC WITH NETWORK SSH CALL
630#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
632
633#       #RSYNC WITH NFS USE
634#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
635#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
636
637#       status=$?
638#       IGCM_sys_Rsync_out $status
639
640#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
641#       (( status=status+$? ))
642
643    if [ ${status} -gt 0 ] ; then
644      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
645      [ -f ${1} ] && ls -l ${1}
646      [ -f ${2} ] && ls -l ${2}
647      [ -f ${2}/${1} ] && ls -l ${2}/${1}
648      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
649      IGCM_debug_Exit "IGCM_sys_Put_Rest"
650    else
651
652      if [ X${JobType} = XRUN ] ; then
653        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
654        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
655      fi
656
657      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
658    fi
659  fi
660  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
661}
662
663#D-#==================================================
664#D-function IGCM_sys_Put_Out
665#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
666#D-* Examples:
667#D-
668function IGCM_sys_Put_Out {
669  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
670  if ( $DEBUG_sys ) ; then
671    echo "IGCM_sys_Put_Out :" $@
672  fi
673
674  typeset NB_ESSAI DELAI status i exist skip
675  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
676
677  # number of tentative
678  NB_ESSAI=3
679  # time delay between tentative
680  DELAI=2
681
682  if [ $DRYRUN = 0 ]; then
683    if [ ! -f ${1} ] ; then
684      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
685      IGCM_debug_PopStack "IGCM_sys_Put_Out"
686      return 1
687    fi
688    #
689    IGCM_sys_MkdirArchive $( dirname $2 )
690    #
691    exist=false
692    skip=false
693    if [ -f $2 ] ; then
694      IGCM_debug_Print 1 "$2 already exist"
695      ccc_hsm get $2
696      exist=true
697      if [ "X$( diff $1 $2 )" = X ] ; then
698        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
699        skip=true
700      else
701        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
702        skip=false
703      fi
704    fi
705    #
706    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
707      IGCM_sys_Chmod u+w $2
708    fi
709
710    if [ X${skip} = Xfalse ] ; then
711      i=0
712      while [ $i -lt $NB_ESSAI ] ; do
713        # Identify file system
714        fileDeviceNumberInHex=$( stat -c %d $1 )
715        status=$?
716        if [ ${status} -gt 0 ] ; then
717          IGCM_debug_Exit "IGCM_sys_Put_Out"
718        fi
719        # Identify file system
720        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
721        status=$?
722        if [ ${status} -gt 0 ] ; then
723          IGCM_debug_Exit "IGCM_sys_Put_Out"
724        fi
725
726        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
727          # They are not on the same device. USUAL WAY
728          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
729          status=$?
730        else
731          # They are on the same device. NOT SO USUAL WAY
732          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
733          status=$?
734        fi
735        if [ ${status} -gt 0 ]; then
736          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
737          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
738          [ -f ${1} ] && ls -l ${1}
739          [ -f ${2} ] && ls -l ${2}
740          [ -f ${2}/${1} ] && ls -l ${2}/${1}
741          sleep $DELAI
742        else
743          break
744        fi
745        (( i = i + 1 ))
746      done
747    fi
748
749#       #RSYNC WITH NETWORK SSH CALL
750#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
751#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
752
753#       #RSYNC WITH NFS USE
754#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
755#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
756
757#       status=$?
758#       IGCM_sys_Rsync_out $status
759
760#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
761#       (( status=status+$? ))
762
763    if [ ${status} -gt 0 ] ; then
764      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
765      [ -f ${1} ] && ls -l ${1}
766      [ -f ${2} ] && ls -l ${2}
767      [ -f ${2}/${1} ] && ls -l ${2}/${1}
768      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
769      IGCM_debug_Exit "IGCM_sys_Put_Out"
770    else
771
772      if [ X${JobType} = XRUN ] ; then
773        if [ X${3} = X ] ; then
774          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
775          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
776        fi
777      fi
778
779      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
780    fi
781  fi
782  IGCM_debug_PopStack "IGCM_sys_Put_Out"
783  return 0
784}
785
786#D-#==================================================
787#D-function IGCM_sys_Get
788#D-* Purpose: Get a file from ${ARCHIVE}
789#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
790#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
791function IGCM_sys_Get {
792  IGCM_debug_PushStack "IGCM_sys_Get" $@
793
794  typeset DEST dm_liste target file_work
795  typeset NB_ESSAI DELAI status i
796
797  if ( $DEBUG_sys ) ; then
798    echo "IGCM_sys_Get :" $@
799  fi
800
801  # number of tentative
802  NB_ESSAI=3
803  # time delay between tentative
804  DELAI=2
805
806  if [ $DRYRUN -le 2 ]; then
807    if [ X${1} = X'/l' ] ; then
808      eval set +A dm_liste \${${2}}
809    else
810      eval set +A dm_liste ${1}
811    fi
812    eval DEST=\${${#}}
813    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
814    status=$?
815    if [ ${status} -gt 0 ] ; then
816      echo "WARNING IGCM_sys_Get : error code ${status}"
817      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
818      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
819    fi
820
821    #if [ ${status} -gt 0 ] ; then
822    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
823    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
824    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
825    #    sleep 30
826    #    echo "We try another time"
827    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
828    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
829    #    status=$?
830    #    if [ ${status} -gt 0 ] ; then
831    #      echo "ERROR IGCM_sys_Get : again demigration error :"
832    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
833    #      IGCM_debug_Exit "IGCM_sys_Get"
834    #    fi
835    #  else
836    #    echo "ERROR IGCM_sys_Get : demigration error :"
837    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
838    #    IGCM_debug_Exit "IGCM_sys_Get"
839    #  fi
840    #fi
841
842    #   #RSYNC WITH NETWORK SSH CALL
843    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
844    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
845
846    #   #RSYNC WITH NFS USE
847    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
848    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
849
850    #   status=$?
851    #   IGCM_sys_Rsync_out $status
852
853    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
854    #   (( status=status+$? ))
855
856    #USUAL WAY
857    if [ X${1} = X'/l' ] ; then
858      for target in ${dm_liste[*]} ; do
859        local_file=$( basename ${target} )
860        # test if the target file is present before the loop
861        IGCM_sys_TestFileArchive ${target}
862        status=$?
863        if [ ${status} -gt 0 ] ; then
864          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
865          IGCM_debug_Exit "IGCM_sys_Get"
866        else
867          i=0
868          while [ $i -lt $NB_ESSAI ] ; do
869            #if [ X${DoLink} = Xtrue ] ; then
870            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
871            #  status=$?
872            #  else
873            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
874            #  status=$?
875            #fi
876            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
877            status=$?
878            if [ ${status} -gt 0 ]; then
879              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
880              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
881              sleep $DELAI
882            else
883              break
884            fi
885            (( i = i + 1 ))
886          done
887          if [ ${status} -gt 0 ] ; then
888            echo "IGCM_sys_Get : error"
889            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
890            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
891            IGCM_debug_Exit "IGCM_sys_Get"
892          else
893            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
894          fi
895        fi
896      done
897    else
898      i=0
899      while [ $i -lt $NB_ESSAI ] ; do
900        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
901        status=$?
902        if [ ${status} -gt 0 ]; then
903          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
904          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
905          sleep $DELAI
906        else
907          break
908        fi
909        (( i = i + 1 ))
910      done
911      if [ ${status} -gt 0 ] ; then
912        echo "IGCM_sys_Get : error"
913        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
914        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
915        IGCM_debug_Exit "IGCM_sys_Get"
916      else
917        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
918      fi
919    fi
920  fi
921  IGCM_debug_PopStack "IGCM_sys_Get"
922}
923
924#D-#==================================================
925#D-function IGCM_sys_GetDate_Monitoring
926#D-* Purpose: get the last year for which the monitoring has been computed
927#D-* Examples:
928#D-
929function IGCM_sys_GetDate_Monitoring {
930  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
931  if ( $DEBUG_sys ) ; then
932    echo "IGCM_sys_GetDate_Monitoring :" $@
933  fi
934
935  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
936
937  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
938}
939
940#D-#==================================================
941#D-function IGCM_sys_Dods_Rm
942#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
943#D-* Examples:
944#D-
945function IGCM_sys_Dods_Rm {
946  if ( $DEBUG_sys ) ; then
947    echo "IGCM_sys_Dods_Rm :" $@
948  fi
949  typeset status
950  if [ $DRYRUN = 0 ]; then
951
952#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
953#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
954#      echo "Nothing has been done."
955#      return
956#    fi
957
958    if [ "$#" -eq 1 ]; then
959      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
960      status=$?
961    else
962      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
963      status=$?
964    fi
965
966#    if [ ${status} -gt 0 ] ; then
967#      echo "IGCM_sys_Dods_Rm : error."
968#      cat out_dods_rm
969#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
970#    else
971#      rm out_dods_rm
972#    fi
973
974  fi
975  return $status
976}
977
978#D-#==================================================
979#D-function IGCM_sys_Dods_Cp
980#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
981#D-* Examples:
982#D-
983function IGCM_sys_Dods_Cp {
984  if ( $DEBUG_sys ) ; then
985    echo "IGCM_sys_Dods_Cp :" $@
986  fi
987  typeset status
988  if [ $DRYRUN = 0 ]; then
989
990#    if [ ! -d ${R_SAVE}/${1} ] ; then
991#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
992#      echo "Nothing has been done."
993#      return
994#    fi
995
996    $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
997    status=$?
998
999#       if [ ${status} -gt 0 ] ; then
1000#           echo "IGCM_sys_Dods_Cp : error."
1001#           cat out_dods_cp
1002#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1003#       else
1004#           rm out_dods_cp
1005#       fi
1006
1007  fi
1008  return $status
1009}
1010
1011#D-#==================================================
1012#D-function IGCM_sys_Put_Dods
1013#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1014#D-* Examples:
1015#D-
1016function IGCM_sys_Put_Dods {
1017  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1018  if ( $DEBUG_sys ) ; then
1019    echo "IGCM_sys_Put_Dods :" $@
1020  fi
1021  typeset status
1022  if [ $DRYRUN = 0 ]; then
1023    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1024      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1025      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1026      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1027      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1028      return
1029    fi
1030
1031    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1032      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1033      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1034      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1035      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1036      return
1037    fi
1038    #
1039    if [ -d ${R_SAVE}/${1} ] ; then
1040      cd ${R_SAVE}
1041    elif [ -d ${R_FIGR}/${1} ] ; then
1042      cd ${R_FIGR}
1043    fi
1044
1045    IGCM_sys_Dods_Rm ${1}
1046    IGCM_sys_Dods_Cp ${1}
1047    status=0
1048
1049    if [ ${status} -gt 0 ] ; then
1050      echo "IGCM_sys_Put_Dods : error."
1051      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1052    fi
1053  fi
1054  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1055}
1056
1057##############################################################
1058# REBUILD OPERATOR
1059
1060#D-#==================================================
1061#D-function IGCM_sys_sync
1062#D-* Purpose: flush buffer on disk
1063#D-* Examples:
1064#D-
1065function IGCM_sys_sync {
1066  IGCM_debug_PushStack "IGCM_sys_sync" $@
1067  if ( $DEBUG_sys ) ; then
1068    echo "IGCM_sys_sync :" $@
1069  fi
1070
1071  /bin/sync
1072
1073  IGCM_debug_PopStack "IGCM_sys_sync"
1074}
1075
1076############################################################
1077# Activate Running Environnment Variables
1078
1079#D-#==================================================
1080#D-function IGCM_sys_activ_variables
1081#D-* Purpose: set environement variables prior to execution
1082#D-* Examples:
1083#D-
1084function IGCM_sys_activ_variables {
1085  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1086  if ( $DEBUG_sys ) ; then
1087    echo "IGCM_sys_activ_variables"
1088  fi
1089
1090# --------------------------------------------------------------------
1091#D- MODULE specifications
1092# --------------------------------------------------------------------
1093  # Source the file EnvFile containing all module load needed to run the model.
1094  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1095  # it will be used. Otherwise a default file will be used.
1096  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1097      # EnvFile is set in config.card and the file exists
1098      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1099      EnvFile=${config_UserChoices_EnvFile}
1100  else
1101      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1102      module purge > /dev/null 2>&1
1103      if [ X${compilerVersion} = XV19 ] ; then
1104          IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V19"
1105          EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
1106      else
1107          IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1108      fi
1109  fi
1110  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1111  IGCM_debug_Print 1 ${EnvFile}
1112  . ${EnvFile}
1113
1114  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene-amd. "
1115  module list
1116
1117# --------------------------------------------------------------------
1118#D- MPI specifications
1119# --------------------------------------------------------------------
1120
1121# --------------------------------------------------------------------
1122#D- Other specifications
1123# --------------------------------------------------------------------
1124
1125  ulimit -s unlimited
1126
1127  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1128}
1129
1130############################################################
1131# Desactivate Running Environnment Variables
1132
1133#D-#==================================================
1134#D-function IGCM_sys_desactiv_variables
1135#D-* Purpose: unset environement variables after execution
1136#D-* Examples:
1137#D-
1138function IGCM_sys_desactiv_variables {
1139  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1140  if ( $DEBUG_sys ) ; then
1141    echo "IGCM_sys_desactiv_variables"
1142  fi
1143# --------------------------------------------------------------------
1144#D- MPI specifications
1145# --------------------------------------------------------------------
1146
1147# --------------------------------------------------------------------
1148#D- Other specifications
1149# --------------------------------------------------------------------
1150
1151  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1152}
1153
1154############################################################
1155# Update job headers to be used by the scheduler
1156
1157#D-#==================================================
1158#D-function IGCM_sys_updateHeaders
1159#D-* Purpose: Update job headers to be used by the scheduler
1160#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1161#D-
1162function IGCM_sys_updateHeaders {
1163  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1164  if ( $DEBUG_sys ) ; then
1165    echo "IGCM_sys_updateHeaders"
1166  fi
1167  typeset file
1168  file=$1
1169
1170   # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1171   # Compute the number of core to add to force the use of a new node for xios servers
1172
1173   if [ "X${IOS_OK_DEDICATED}" == "X" ] ; then
1174       (( IOS_OK_DEDICATED = 0 ))
1175   fi
1176
1177
1178   if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1179      (( current_core_noxios = coreNumber - IOS_PROC_MPI * IOS_PROC_OMP * IOS_PROC_DEP ))
1180      (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1181      (( second_comp_proc_mpi_loc = IOS_PROC_MPI - 1 ))
1182      (( coreNumber = current_core_noxios - 1 + first_comp_proc_dep_loc * 2 + second_comp_proc_mpi_loc * IOS_PROC_OMP * IOS_PROC_DEP ))
1183   fi
1184
1185  if [ ${executionType} -eq 1 ] ; then
1186    # MPMD + MPI
1187    sed -e "/::openMPthreads::/d"                  \
1188        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1189      ${file} > ${file}.tmp
1190
1191  elif [ ${executionType} -eq 2 ] ; then
1192    # MPMD + MPI + OMP :ccc_mprun
1193    sed -e "/::openMPthreads::/d"                  \
1194        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1195      ${file} > ${file}.tmp
1196
1197  elif [ ${executionType} -eq 3 ] ; then
1198    # SPMD + MPI/OMP
1199    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1200        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1201        -e "/#MSUB -x/d"                           \
1202      ${file} > ${file}.tmp
1203
1204  elif [ ${executionType} -eq 4 ] ; then
1205    # SPMD + MPI only
1206    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1207        -e "/::openMPthreads::/d"                  \
1208        -e "/#MSUB -x/d"                           \
1209      ${file} > ${file}.tmp
1210
1211  elif [ ${executionType} -eq 5 ] ; then
1212    # SPMD + OMP only
1213    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1214        -e "/::JobNumProcTot::/d"                  \
1215        -e "/#MSUB -x/d"                           \
1216      ${file} > ${file}.tmp
1217
1218  elif [ ${executionType} -eq 6 ] ; then
1219    # SEQUENTIAL THEN
1220    sed -e "s/::JobNumProcTot::/1/"                \
1221        -e "/::openMPthreads::/d"                  \
1222        -e "/#MSUB -x/d"                           \
1223      ${file} > ${file}.tmp
1224
1225  fi
1226
1227  IGCM_sys_Mv ${file}.tmp ${file}
1228
1229  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1230}
1231
1232############################################################
1233# Build MPI/OMP scripts run file (dummy function)
1234
1235#D-#==================================================
1236#D-function IGCM_sys_build_run_file
1237#D-* Purpose: build run file (deprecated)
1238#D-* Examples:
1239#D-
1240function IGCM_sys_build_run_file {
1241  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1242}
1243
1244############################################################
1245# Build MPI/OMP scripts
1246
1247#D-#==================================================
1248#D-function IGCM_sys_build_execution_scripts
1249#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1250#D-* Examples:
1251#D-
1252function IGCM_sys_build_execution_scripts
1253{
1254  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1255  if ( $DEBUG_sys ) ; then
1256    echo "IGCM_sys_build_execution_scripts " $@
1257  fi
1258
1259  EXECUTION=${HOST_MPIRUN_COMMAND}
1260
1261  if ( ${OK_PARA_MPMD} ) ; then
1262
1263    # MPMD mode
1264    # 1 MPI only
1265    # 2 MPI/OpenMP mpirun method
1266    # 3 MPI/OpenMP ccc_mprun method
1267
1268    if [ -f run_file ] ; then
1269      IGCM_sys_Rm -f run_file
1270    fi
1271    touch run_file
1272
1273    # case 1 : Only MPI (MPMD)
1274    if  ( ! ${OK_PARA_OMP} ) ; then
1275
1276      # Build run_file
1277
1278      # First loop on the components for the coupler ie oasis (only if oasis3)
1279      # the coupler ie oasis3 must be the first one
1280      for comp in ${config_ListOfComponents[*]} ; do
1281
1282        eval ExeNameIn=\${config_Executable_${comp}[0]}
1283        eval ExeNameOut=\${config_Executable_${comp}[1]}
1284
1285        # for CPL component only
1286        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1287
1288          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1289          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1290          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1291        fi
1292      done
1293
1294      # Then second loop on the components
1295      for comp in ${config_ListOfComponents[*]} ; do
1296
1297        eval ExeNameIn=\${config_Executable_${comp}[0]}
1298        eval ExeNameOut=\${config_Executable_${comp}[1]}
1299
1300        # Only if we really have an executable for the component and not the coupler ie oasis:
1301        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1302
1303          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1304          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1305          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1306
1307          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1308          # Compute the number of core to add to force the use of a new node for xios servers
1309         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1310            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1311              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_dep_loc ))
1312              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1313              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1314              if [ ${comp_proc_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1315                 echo "1-${first_comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1316                 if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1317                    echo "${second_comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1318                 fi
1319              else
1320                 echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1321              fi
1322           else
1323              echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1324           fi
1325         else
1326          echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1327         fi
1328
1329
1330        fi
1331      done
1332
1333      ## module advised by TGCC (instead of 2 variables)
1334      module load feature/bridge/heterogenous_mpmd
1335
1336      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1337
1338      IGCM_sys_Chmod u+x run_file
1339      if ( $DEBUG_sys ) ; then
1340        echo "run_file contains : "
1341        cat run_file
1342      fi
1343
1344    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1345
1346    # 2 MPI/OpenMP mpirun method
1347
1348      # Use of mpirun instead of ccc_mprun
1349      EXECUTION="time mpirun"
1350
1351      #  Hosts treatment
1352      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1353
1354      i=0
1355      rm -f hosts rankfile
1356      IGCM_debug_Print 1 "sys Irene-amd, Hosts available :"
1357      for nodes in `cat hosts.tmp`
1358      do
1359        host[$i]=$nodes
1360        echo "${host[$i]}" >> hosts
1361        IGCM_debug_Print 1 ${host[$i]}
1362        i=$((i+1))
1363      done
1364      rm -f hosts.tmp
1365
1366      listnodes=${host[*]}
1367
1368      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1369
1370      # Initialisation
1371      rank=0
1372      current_core=0
1373      init_exec=n
1374
1375      # Loop on the components
1376      for comp in ${config_ListOfComponents[*]} ; do
1377
1378        eval ExeNameIn=\${config_Executable_${comp}[0]}
1379        eval ExeNameOut=\${config_Executable_${comp}[1]}
1380
1381        # Not possible if oasis has an executable (i.e old version of oasis3)
1382        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1383          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1384          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1385          IGCM_debug_Verif_Exit
1386        fi
1387
1388        # Only if we really have an executable for the component :
1389        if [ "X${ExeNameOut}" != X\"\" ] ; then
1390
1391          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1392          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1393          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1394
1395          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1396          echo ""  >> script_${ExeNameOut}.ksh
1397          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1398
1399            # Check if the number of threads is correct
1400            case ${comp_proc_omp_loc} in
1401            2|4|8|16)
1402              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1403              ;;
1404            *)
1405              IGCM_debug_Exit "ERROR with OMP parameters !"
1406              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1407              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1408              IGCM_debug_Verif_Exit
1409              ;;
1410            esac
1411            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1412            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1413            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1414            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1415          fi
1416
1417          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1418          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1419          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1420          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1421          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1422
1423          if [ ${init_exec} = y ] ; then
1424            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1425          else
1426            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1427            init_exec=y
1428          fi
1429
1430          # Build rankfile : method used to assign cores and nodes for the MPI process
1431          # Ex :
1432          #rank 0=curie5296 slot=0,1,2,3
1433          #rank 1=curie5296 slot=4,5,6,7
1434          # Example of final command :
1435          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1436          # with script_lmdz.x.ksh :
1437          # #!/bin/ksh
1438          #export KMP_STACKSIZE=3g
1439          #export KMP_LIBRARY=turnaround
1440          #export MKL_SERIAL=YES
1441          #OMP_NUM_THREADS=4
1442          #./lmdz.x
1443          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1444              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1445          else
1446              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1447          fi
1448          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1449             (( index_host = current_core / NB_CORE_PER_NODE ))
1450             host_value=${host[${index_host}]}
1451             (( slot =  current_core % NB_CORE_PER_NODE ))
1452             virg=","
1453             string_final=""
1454             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1455                 string=$index$virg
1456                 string_final=$string_final$string
1457             done
1458             string_final=$( echo $string_final | sed "s/.$//" )
1459             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1460             (( rank = rank + 1 ))
1461             (( current_core = current_core + offset_comp_proc_loc ))
1462          done
1463        fi
1464      done
1465
1466    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1467
1468    # 3 MPI/OpenMP ccc_mprun method
1469
1470    # MPI-OpenMP (MPMD)
1471    # example of run_file
1472    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1473    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1474    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1475
1476      # Loop on the components
1477      for comp in ${config_ListOfComponents[*]} ; do
1478
1479        eval ExeNameIn=\${config_Executable_${comp}[0]}
1480        eval ExeNameOut=\${config_Executable_${comp}[1]}
1481
1482        # Not possible if oasis has an executable (i.e old version of oasis3)
1483        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1484          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1485          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1486          IGCM_debug_Verif_Exit
1487        fi
1488
1489        # Only if we really have an executable for the component :
1490        if [ "X${ExeNameOut}" != X\"\" ] ; then
1491
1492          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1493          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1494          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1495
1496          # Compute number of cores per process
1497          (( comp_proc_omp_dep_loc = comp_proc_omp_loc * comp_proc_dep_loc )) 
1498
1499          # Build script files
1500
1501          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1502          echo ""  >> script_${ExeNameOut}.ksh
1503          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1504
1505            # Check if the number of threads is correct
1506            case ${comp_proc_omp_loc} in
1507            2|4|8|16)
1508              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1509              ;;
1510            *)
1511              IGCM_debug_Exit "ERROR with OMP parameters !"
1512              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1513              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1514              IGCM_debug_Verif_Exit
1515              ;;
1516            esac
1517            ### only for LMDZ?
1518            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1519            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1520            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1521            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1522
1523          fi
1524
1525          # to have out/err per process on different files
1526          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1527          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1528
1529          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1530
1531          # Complete run_file
1532
1533          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1534          # Compute the number of core to add to force the use of a new node for xios servers
1535         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1536            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1537              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_omp_dep_loc )) 
1538              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1539              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1540              if [ ${comp_proc_omp_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1541                 echo "1-${first_comp_proc_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1542                  if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1543                    echo "${second_comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1544                  fi
1545              else
1546                 echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1547              fi
1548           else
1549              echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1550           fi
1551         else
1552          echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1553         fi
1554
1555        fi
1556      done
1557
1558      ## variable added to stop after 60s instead of 600s by default.
1559      ## This is used when no error comes from executables and when something stopped an executable without notice.
1560      export SLURM_WAIT=60
1561
1562      ## module advised by TGCC (instead of 2 variables)
1563      module load feature/bridge/heterogenous_mpmd
1564
1565      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1566
1567      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1568
1569      IGCM_sys_Chmod u+x run_file
1570      if ( $DEBUG_sys ) ; then
1571        echo "run_file contains : "
1572        cat run_file
1573      fi
1574
1575    else
1576
1577      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1578
1579    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1580
1581  else
1582  # Only one executable (SPMD mode).
1583
1584    for comp in ${config_ListOfComponents[*]} ; do
1585
1586      # Only if we really have an executable for the component :
1587      eval ExeNameOut=\${config_Executable_${comp}[1]}
1588      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1589
1590        # Build script files
1591
1592        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1593        echo ""  >> script_${ExeNameOut}.ksh
1594        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1595
1596        if ( ${OK_PARA_OMP} ) ; then
1597          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1598          echo ""  >> script_${ExeNameOut}.ksh
1599          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1600          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1601          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1602          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1603        fi
1604
1605        if  ( ${OK_PARA_MPI} ) ; then
1606          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1607          # Default : ccc_mprun used if nb_proc gt 1
1608          # to have out/err per process on different files
1609          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1610          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1611          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1612        else
1613          # Default : ccc_mprun is NOT used if nb_proc eq 1
1614          # to have out/err per process on different files
1615          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1616          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1617          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1618        fi
1619
1620        IGCM_debug_Print 1 "sys Irene-amd : script_${ExeNameOut}.ksh contains"
1621        cat script_${ExeNameOut}.ksh
1622
1623      fi
1624
1625    done
1626
1627  fi # ${OK_PARA_MPMD}
1628
1629  IGCM_debug_Print 1 "sys Irene-amd : execution command is "
1630  IGCM_debug_Print 1 "$EXECUTION"
1631
1632  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1633}
1634
1635#D-#==================================================
1636#D-function IGCM_sys_check_path
1637#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1638#D-* do not point to an important use directory. Stop immediately in that case.
1639#D-* Examples:
1640#D-
1641function IGCM_sys_check_path {
1642  IGCM_debug_PushStack "IGCM_sys_check_path"
1643  if ( $DEBUG_sys ) ; then
1644    echo "IGCM_sys_check_path"
1645  fi
1646
1647  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1648    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1649    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1650    IGCM_debug_Exit "This will stop the job"
1651  fi
1652  IGCM_debug_PopStack "IGCM_sys_check_path"
1653}
1654
1655#D-#==================================================
1656#D-function IGCM_sys_check_quota
1657#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1658#D-* Examples:
1659#D-
1660function IGCM_sys_check_quota {
1661  IGCM_debug_PushStack "IGCM_sys_check_quota"
1662  if ( $DEBUG_sys ) ; then
1663    echo "IGCM_sys_check_quota"
1664  fi
1665  # Limit of quota (in %)
1666  limit_quota=90
1667
1668  # Check of the volume
1669  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1670  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1671
1672  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1673
1674    unit_avail=${volume_avail: -1}
1675    unit_quota=${volume_quota: -1}
1676
1677    if [ "${unit_quota}" = "*" ] ; then
1678      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1679      IGCM_debug_Print 1 "More than 100% of your quota is used"
1680      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1681      IGCM_debug_Print 1 "You must have more than 10% available to run"
1682      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1683      IGCM_debug_Verif_Exit
1684    fi
1685
1686    temp_avail=${volume_avail%%${volume_avail: -1}*}
1687    temp_quota=${volume_quota%%${volume_quota: -1}*}
1688
1689    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1690
1691    # Convertion
1692      if [ ${volume_avail: -1} = "T" ] ; then
1693        (( temp_avail = temp_avail * 1000000000000 ))
1694      elif [ ${volume_avail: -1} = "G" ] ; then
1695        (( temp_avail = temp_avail * 1000000000 ))
1696      elif [ ${volume_avail: -1} = "M" ] ; then
1697        (( temp_avail = temp_avail * 1000000 ))
1698      elif [ ${volume_avail: -1} = "k" ] ; then
1699        (( temp_avail = temp_avail * 1000 ))
1700      else
1701        (( temp_avail = volume_avail ))
1702      fi
1703      if [ ${volume_quota: -1} = "T" ] ; then
1704        (( temp_quota = temp_quota * 1000000000000 ))
1705      elif [ ${volume_quota: -1} = "G" ] ; then
1706        (( temp_quota = temp_quota * 1000000000 ))
1707      elif [ ${volume_quota: -1} = "M" ] ; then
1708        (( temp_quota = temp_quota * 1000000 ))
1709      elif [ ${volume_quota: -1} = "k" ] ; then
1710        (( temp_quota = temp_quota * 1000 ))
1711      else
1712        (( temp_quota = volume_quota ))
1713      fi
1714    fi
1715
1716    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1717#    echo "volume ratio is " $quota_volume
1718
1719    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1720      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1721      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1722      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1723      IGCM_debug_Print 1 "You must have more than 10% available to run"
1724      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1725      IGCM_debug_Verif_Exit
1726    fi
1727
1728  fi
1729
1730# Check of the number of inodes
1731
1732  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1733  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1734
1735  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1736
1737    unit_avail=${inode_avail: -1}
1738    unit_quota=${inode_quota: -1}
1739
1740    if [ "${unit_quota}" = "*" ] ; then
1741      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1742      IGCM_debug_Print 1 "More than 100% of your quota is used"
1743      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1744      IGCM_debug_Print 1 "You must have more than 10% available to run"
1745      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1746      IGCM_debug_Verif_Exit
1747    fi
1748
1749    temp_avail=${inode_avail%%${inode_avail: -1}*}
1750    temp_quota=${inode_quota%%${inode_quota: -1}*}
1751
1752    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1753
1754    # Convertion
1755      if [ ${inode_avail: -1} = "T" ] ; then
1756        (( temp_avail = temp_avail * 1000000000000 ))
1757      elif [ ${inode_avail: -1} = "G" ] ; then
1758        (( temp_avail = temp_avail * 1000000000 ))
1759      elif [ ${inode_avail: -1} = "M" ] ; then
1760        (( temp_avail = temp_avail * 1000000 ))
1761      elif [ ${inode_avail: -1} = "k" ] ; then
1762        (( temp_avail = temp_avail * 1000 ))
1763      else
1764        (( temp_avail = inode_avail ))
1765      fi
1766
1767      if [ ${inode_quota: -1} = "T" ] ; then
1768        (( temp_quota = temp_quota * 1000000000000 ))
1769      elif [ ${inode_quota: -1} = "G" ] ; then
1770        (( temp_quota = temp_quota * 1000000000 ))
1771      elif [ ${inode_quota: -1} = "M" ] ; then
1772        (( temp_quota = temp_quota * 1000000 ))
1773      elif [ ${inode_quota: -1} = "k" ] ; then
1774        (( temp_quota = temp_quota * 1000 ))
1775      else
1776        (( temp_quota = inode_quota ))
1777      fi
1778    fi
1779    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1780#    echo "inode ratio is " $quota_inode
1781
1782    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1783      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1784      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1785      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1786      IGCM_debug_Print 1 "You must have more than 10% available to run"
1787      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1788      IGCM_debug_Verif_Exit
1789    fi
1790  fi
1791  IGCM_debug_PopStack "IGCM_sys_check_quota"
1792}
1793
1794#D-#==================================================
1795#D-function IGCM_sys_projectAccounting
1796#D-* Purpose: store project accounting information in a file
1797#D-* Examples:
1798#D-
1799function IGCM_sys_projectAccounting {
1800  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1801  if ( $DEBUG_sys ) ; then
1802    echo "IGCM_sys_check_quota"
1803  fi
1804
1805  ssh irene194 /usr/bin/ccc_myproject > $1
1806
1807  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1808}
1809
1810#D-#==================================================
1811#D-function IGCM_sys_getJobSchedulerID
1812#D-* Purpose: Get the job ID during execution
1813#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1814#D-
1815function IGCM_sys_getJobSchedulerID {
1816  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1817  if ( $DEBUG_sys ) ; then
1818    echo "IGCM_sys_getJobSchedulerID"
1819  fi
1820  eval ${1}=${BRIDGE_MSUB_JOBID}
1821
1822  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1823}
1824
1825#D-#==================================================
1826#D-function IGCM_sys_GetJobID
1827#D-* Purpose: Get the job ID from the JobName
1828#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1829#D-
1830function IGCM_sys_GetJobID {
1831  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1832  if ( $DEBUG_sys ) ; then
1833    echo "IGCM_sys_GetJobID"
1834  fi
1835
1836  # With -f option, the full job name is given in the last column
1837  ID=$( ccc_mstat -f -u $2 | \
1838        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1839
1840  eval ${3}=${ID}
1841  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1842}
1843
1844#D-#==================================================
1845#D-function IGCM_sys_CountJobInQueue
1846#D-* Purpose: Count number of users job
1847#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1848#D-
1849function IGCM_sys_CountJobInQueue {
1850  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1851  if ( $DEBUG_sys ) ; then
1852    echo "IGCM_sys_CountJobInQueue"
1853  fi
1854
1855  # With -f option, the full job name is given in the last column
1856  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1857
1858  eval ${2}=${NbRun}
1859
1860  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1861}
1862
1863#D-#==================================================
1864#D-function IGCM_sys_ListJobInQueue
1865#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1866#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1867#D-
1868function IGCM_sys_ListJobInQueue {
1869  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1870  if ( $DEBUG_sys ) ; then
1871    echo "IGCM_sys_ListJobInQueue"
1872  fi
1873
1874  # With -f option, the full job name is given in the last column
1875  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1876                                        '( $2  == User      && \
1877                                           $NF != /TS/      && \
1878                                           $NF !~ /PACK/    && \
1879                                           $NF !~ /REBUILD/ && \
1880                                           $NF !~ /pack/ )     \
1881                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1882
1883  eval set -A ${2} ${JobList[*]}
1884
1885  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1886}
1887
1888#D-#==================================================
1889#D-function IGCM_sys_atlas
1890#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1891#D-* Examples:
1892#D-
1893function IGCM_sys_atlas {
1894  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1895  if ( $DEBUG_sys ) ; then
1896    echo "IGCM_sys_atlas :" $@
1897  fi
1898
1899  typeset status
1900
1901  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1902  status=$?
1903  if [ ${status} -gt 0 ] ; then
1904    echo "IGCM_sys_atlas : error code ${status}"
1905    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1906    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1907    IGCM_debug_PopStack "IGCM_sys_atlas"
1908    return 1
1909  else
1910    IGCM_debug_PopStack "IGCM_sys_atlas"
1911    return 0
1912  fi
1913
1914  IGCM_debug_PopStack "IGCM_sys_atlas"
1915}
1916
1917#D-#==================================================
1918#D-function IGCM_sys_rebuild_nemo
1919#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1920#D-* Examples:
1921#D-
1922
1923function IGCM_sys_rebuild_nemo {
1924  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1925  if ( $DEBUG_sys ) ; then
1926    echo "IGCM_sys_rebuild_nemo :" $@
1927  fi
1928
1929  $( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
1930
1931  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1932}
1933
Note: See TracBrowser for help on using the repository browser.