source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene-amd.ksh @ 1528

Last change on this file since 1528 was 1528, checked in by aclsce, 4 years ago

Irene-amd : added 2 functionalities to increase computing performances :

  • increase this number of cores per MPI process (OpenMP thread) by using DEP keyword in config.card
  • use of dedicated nodes for XIOS servers by using DEDICATED keyword in config.card

Example of config.card :
ATM= (gcm.e, lmdz.x, 71MPI, 8OMP, 2DEP)
SRF= ("" ,"" )
SBG= ("" ,"" )
OCE= (opa, opa.xx, 360MPI, 2DEP)
ICE= ("" ,"" )
MBG= ("" ,"" )
CPL= ("", "" )
IOS= (xios_server.exe, xios.x, 12MPI,2DEP, DEDICATED)
File size: 65.6 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision:: 1512                                     $ Revision of last commit
7# $Author:: acosce                                     $ Author of last commit
8# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene-amd
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=irene-amd
71# add default project on irene-amd
72typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} | cut -d@ -f1 )
73# jobWarningDelay in seconds
74typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=ccc_msub}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94if [ X${TaskType} = Xcomputing ] ; then
95  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
96else
97  module purge > /dev/null 2>&1
98  . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
99  . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_atlas_irene-amd > /dev/null 2>&1
100
101  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
102  export UVCDAT_ANONYMOUS_LOG=FALSE
103fi
104
105# FYI
106[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
107[ ! X${TaskType} = Xchecking ] && module list
108
109# For AddNoise and AddPertu3DOCE
110export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene-amd/bin
111
112# For rebuild
113export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene-amd/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
114
115#====================================================
116# Host specific DIRECTORIES
117#====================================================
118
119# ============ CESIUM START ============ #
120
121#====================================================
122#- Mirror libIGCM from titane to cesium if needed
123#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
124#if [ ! ${ROOTSYS} = "home" ] ; then
125#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
126#else
127#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
128#fi
129
130#====================================================
131#- libIGCM_POST
132#if ( ${MirrorlibIGCM} ) ; then
133#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
134#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
135#else
136#  typeset -r libIGCM_POST=${libIGCM}
137#fi
138
139# ============ CESIUM  END  ============ #
140
141#====================================================
142#- MirrorlibIGCM for frontend
143typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
144
145#====================================================
146#- libIGCM_POST for frontend
147typeset -r libIGCM_POST=${libIGCM}
148
149#====================================================
150#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
151typeset -r R_EXE="${MODIPSL}/bin"
152
153#====================================================
154#- SUBMIT_DIR : submission dir
155typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
156
157#====================================================
158#- IN
159typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
160
161#====================================================
162#- RUN_DIR_PATH : Temporary working directory (=> TMP)
163typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
164
165#====================================================
166#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
167typeset -r OUTCOMMAND_PATH=/tmp
168
169#====================================================
170#- HOST_MPIRUN_COMMAND
171typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
172
173#====================================================
174#- Max number of arguments passed to nco operator or demigration command
175UNIX_MAX_LIMIT=360
176
177#====================================================
178#- set PackDefault to true on irene-amd
179PackDefault=true
180
181#====================================================
182#- Default number of MPI task for IPSL coupled model
183#- required for backward compatibility
184#-
185DEFAULT_NUM_PROC_OCE=5
186DEFAULT_NUM_PROC_CPL=1
187(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
188
189#====================================================
190#- Number of computing cores per node
191#-
192NB_CORE_PER_NODE=128
193
194#D-#==================================================
195#D-function IGCM_sys_defineArchives
196#D-* Purpose:
197#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
198#D-* Define ARCHIVE : Dedicated to large files
199#D-* Define STORAGE : Dedicated to small/medium files
200#D-* Define R_OUT   : Output tree located on ARCHIVE
201#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
202#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
203#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
204#D-* Examples:
205#D-
206function IGCM_sys_defineArchives {
207  IGCM_debug_PushStack "IGCM_sys_defineArchives"
208
209  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
210  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then
211    # Default option: Change the dfldatadir according to the project used for submission
212    # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
213    if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then
214      module switch dfldatadir dfldatadir/gencmip6
215      IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
216    else
217      module switch dfldatadir dfldatadir/${PROJECT}
218      IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
219    fi
220  else
221    # Use the project set by the variable DataProject in section UserChoices in config.card
222    module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
223    IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
224  fi
225
226 #====================================================
227 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
228 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
229
230  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
231    #====================================================
232    #- ARCHIVE (dedicated to large files)
233    ARCHIVE=${config_UserChoices_ARCHIVE}
234  else
235    #====================================================
236    #- ARCHIVE (dedicated to large files)
237    ARCHIVE=${CCCSTOREDIR}
238  fi
239
240  if [ ! X${config_UserChoices_STORAGE} = X ]; then
241    #====================================================
242    #- STORAGE (dedicated to small/medium files)
243    STORAGE=${config_UserChoices_STORAGE}
244  else
245    #====================================================
246    #- STORAGE (dedicated to small/medium files)
247    STORAGE=${CCCWORKDIR}
248  fi
249
250  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
251    #====================================================
252    #- R_OUT
253    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
254
255    #====================================================
256    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
257    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
258
259    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
260
261  else
262    #====================================================
263    #- R_OUT
264    R_OUT=${ARCHIVE}/IGCM_OUT
265
266    #====================================================
267    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
268    R_FIG=${STORAGE}/IGCM_OUT
269  fi
270
271  #====================================================
272  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
273  CMIP6_BUF=${STORAGE}/IGCM_OUT
274
275  #====================================================
276  #- R_BUF (ONLY FOR double copy an scratch)
277  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
278
279  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
280  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
281  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
282
283  IGCM_debug_PopStack "IGCM_sys_defineArchives"
284}
285
286#D-#==================================================
287#D-function IGCM_sys_RshArchive
288#D-* Purpose: Archive rsh command
289#D-* Examples:
290#D-
291function IGCM_sys_RshArchive {
292  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
293  /bin/ksh <<-EOF
294    ${@}
295EOF
296  status=$?
297  if [ ${status} -gt 0 ] ; then
298    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
299    IGCM_debug_Exit "IGCM_sys_RshArchive"
300  fi
301  IGCM_debug_PopStack "IGCM_sys_RshArchive"
302}
303
304#D-#==================================================
305#D-function IGCM_sys_RshArchive_NoError
306#D-* Purpose: Archive rsh command, without error
307#D-*          used only in monitoring.job
308#D-* Examples:
309#D-
310function IGCM_sys_RshArchive_NoError {
311  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
312  /bin/ksh <<-EOF
313    ${@} 2> /dev/null
314EOF
315  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
316}
317
318#D-#==================================================
319#D-function IGCM_sys_MkdirArchive
320#D-* Purpose: Mkdir on Archive
321#D-* Examples:
322#D-
323function IGCM_sys_MkdirArchive {
324  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
325  if ( $DEBUG_sys ) ; then
326    echo "IGCM_sys_MkdirArchive :" $@
327  fi
328  #- creation de repertoire sur le serveur fichier
329  if [ ! -d ${1} ]; then
330    \mkdir -p $1
331    status=$?
332
333    if [ ${status} -gt 0 ] ; then
334      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
335      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
336    fi
337  fi
338  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
339}
340
341#D-#==================================================
342#D-function IGCM_sys_TestDirArchive
343#D-* Purpose: Test Directory that must exists on Archive
344#D-* Examples:
345#D-
346function IGCM_sys_TestDirArchive {
347  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
348  if ( $DEBUG_sys ) ; then
349    echo "IGCM_sys_TestDirArchive :" $@
350  fi
351  typeset ExistFlag
352  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
353  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
354  return ${ExistFlag}
355}
356
357#D-#==================================================
358#D-function IGCM_sys_IsFileArchived
359#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
360#D-* Examples:
361#D-
362function IGCM_sys_IsFileArchived {
363  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
364  if ( $DEBUG_sys ) ; then
365    echo "IGCM_sys_IsFileArchived :" $@
366  fi
367  typeset IsArchivedFlag
368  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
369  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
370
371  return ${IsArchivedFlag}
372}
373
374#D-#==================================================
375#D-function IGCM_sys_TestFileArchive
376#D-* Purpose: Test file that must NOT EXISTS on Archive
377#D-* Examples:
378#D-
379function IGCM_sys_TestFileArchive {
380  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
381  typeset ExistFlag
382  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
383  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
384
385  return ${ExistFlag}
386}
387
388#D-#==================================================
389#D-function IGCM_sys_CountFileArchive
390#D-* Purpose: Count files on Archive filesystem
391#D-* Examples:
392#D-
393function IGCM_sys_CountFileArchive {
394  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
395  ls ${@} 2>/dev/null | wc -l
396  if [ $? -gt 0 ] ; then
397    echo "IGCM_sys_CountFileArchive : erreur."
398  fi
399  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
400}
401
402#D-#==================================================
403#D-function IGCM_sys_Tree
404#D-* Purpose: Tree directories with files on ${ARCHIVE}
405#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
406#D-
407function IGCM_sys_Tree {
408  IGCM_debug_PushStack "IGCM_sys_Tree" $@
409  if ( $DEBUG_sys ) ; then
410    echo "IGCM_sys_Tree :" $@
411  fi
412
413  \ls -lR ${@}
414
415  IGCM_debug_PopStack "IGCM_sys_Tree"
416}
417
418#D-#==================================================
419#D-function IGCM_sys_Qsub
420#D-* Purpose: Qsub new job
421#D-* Examples:
422#D-
423function IGCM_sys_Qsub {
424  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
425  if ( $DEBUG_sys ) ; then
426    echo "IGCM_sys_Qsub :" $@
427  fi
428  typeset options status
429  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
430
431  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
432  status=$?
433
434  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
435  if [ ${status} -gt 0 ] ; then
436    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
437    IGCM_debug_Exit "IGCM_sys_Qsub"
438  else
439    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
440  fi
441  IGCM_debug_PopStack "IGCM_sys_Qsub"
442}
443
444#D-#==================================================
445#D-function IGCM_sys_QsubPost
446#D-* Purpose: Qsub new job on scalaire
447#D-* Examples:
448#D-
449function IGCM_sys_QsubPost {
450  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
451  if ( $DEBUG_sys ) ; then
452    echo "IGCM_sys_QsubPost :" $@
453  fi
454  typeset options status
455  options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
456
457  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
458  status=$?
459
460  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
461  if [ ${status} -gt 0 ] ; then
462    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
463    IGCM_debug_Exit "IGCM_sys_QsubPost"
464  else
465    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
466  fi
467  IGCM_debug_PopStack "IGCM_sys_QsubPost"
468}
469
470#D-*************************
471#D- File transfer functions
472#D-*************************
473#D-
474
475#D-#==================================================
476#D-function IGCM_sys_RmRunDir
477#D-* Purpose: rm tmpdir (dummy function most of the time batch
478#D-                      scheduler will do the job)
479#D-* Examples:
480#D-
481function IGCM_sys_RmRunDir {
482  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
483  if ( $DEBUG_sys ) ; then
484    echo "IGCM_sys_RmRunDir :" $@
485  fi
486
487  typeset status
488
489  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
490  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
491  status=$?
492
493  if [ ${status} -gt 0 ] ; then
494    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
495    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
496    IGCM_debug_Exit "IGCM_sys_RmRunDir"
497  else
498    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
499  fi
500  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
501}
502
503#D-#==================================================
504#D-function IGCM_sys_Put_Dir
505#D-* Purpose: Copy a complete directory on $(ARCHIVE)
506#D-* Examples:
507#D-
508function IGCM_sys_Put_Dir {
509  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
510  if ( $DEBUG_sys ) ; then
511    echo "IGCM_sys_Put_Dir :" $@
512  fi
513  if [ $DRYRUN = 0 ]; then
514    if [ ! -d ${1} ] ; then
515      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
516      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
517      return
518    fi
519
520    typeset status
521
522    # Only if we use rsync
523    #IGCM_sys_TestDirArchive $( dirname $2 )
524    #
525    #USUAL WAY
526    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
527    status=$?
528
529    if [ ${status} -gt 0 ] ; then
530      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
531      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
532      IGCM_debug_Exit "IGCM_sys_Put_Dir"
533    else
534      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
535    fi
536  fi
537  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
538}
539
540#D-#==================================================
541#D-function IGCM_sys_Get_Dir
542#D-* Purpose: Copy a complete directory from ${ARCHIVE}
543#D-* Examples:
544#D-
545function IGCM_sys_Get_Dir {
546  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
547  if ( $DEBUG_sys ) ; then
548    echo "IGCM_sys_Get_Dir :" $@
549  fi
550  if [ $DRYRUN = 0 ]; then
551    typeset NB_ESSAI DELAI status i
552    # number of tentative
553    NB_ESSAI=3
554    # time delay between tentative
555    DELAI=2
556
557    #
558    # USUAL WAY
559    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
560    ccc_hsm get -r $1
561
562    i=0
563    while [ $i -lt $NB_ESSAI ] ; do
564      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
565      status=$?
566      if [ ${status} -gt 0 ] ; then
567        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
568        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
569        sleep $DELAI
570      else
571        break
572      fi
573      (( i = i + 1 ))
574    done
575
576    if [ ${status} -gt 0 ] ; then
577      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
578      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
579      IGCM_debug_Exit "IGCM_sys_Get_Dir"
580    else
581      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
582    fi
583  fi
584  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
585}
586
587#D-#==================================================
588#D-function IGCM_sys_Put_Rest
589#D-* Purpose: Put computied restarts on ${ARCHIVE}.
590#D-           File and target directory must exist.
591#D-* Examples:
592#D-
593function IGCM_sys_Put_Rest {
594  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
595  if ( $DEBUG_sys ) ; then
596    echo "IGCM_sys_Put_Rest :" $@
597  fi
598  if [ $DRYRUN = 0 ]; then
599    if [ ! -f ${1} ] ; then
600      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
601      IGCM_debug_Exit "IGCM_sys_Put_Rest"
602    fi
603
604    typeset status
605    #
606    # USUAL WAY
607    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
608    status=$?
609
610#       #RSYNC WITH NETWORK SSH CALL
611#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
612#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
613
614#       #RSYNC WITH NFS USE
615#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
616#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
617
618#       status=$?
619#       IGCM_sys_Rsync_out $status
620
621#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
622#       (( status=status+$? ))
623
624    if [ ${status} -gt 0 ] ; then
625      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
626      [ -f ${1} ] && ls -l ${1}
627      [ -f ${2} ] && ls -l ${2}
628      [ -f ${2}/${1} ] && ls -l ${2}/${1}
629      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
630      IGCM_debug_Exit "IGCM_sys_Put_Rest"
631    else
632
633      if [ X${JobType} = XRUN ] ; then
634        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
635        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
636      fi
637
638      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
639    fi
640  fi
641  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
642}
643
644#D-#==================================================
645#D-function IGCM_sys_Put_Out
646#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
647#D-* Examples:
648#D-
649function IGCM_sys_Put_Out {
650  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
651  if ( $DEBUG_sys ) ; then
652    echo "IGCM_sys_Put_Out :" $@
653  fi
654
655  typeset NB_ESSAI DELAI status i exist skip
656  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
657
658  # number of tentative
659  NB_ESSAI=3
660  # time delay between tentative
661  DELAI=2
662
663  if [ $DRYRUN = 0 ]; then
664    if [ ! -f ${1} ] ; then
665      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
666      IGCM_debug_PopStack "IGCM_sys_Put_Out"
667      return 1
668    fi
669    #
670    IGCM_sys_MkdirArchive $( dirname $2 )
671    #
672    exist=false
673    skip=false
674    if [ -f $2 ] ; then
675      IGCM_debug_Print 1 "$2 already exist"
676      ccc_hsm get $2
677      exist=true
678      if [ "X$( diff $1 $2 )" = X ] ; then
679        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
680        skip=true
681      else
682        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
683        skip=false
684      fi
685    fi
686    #
687    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
688      IGCM_sys_Chmod u+w $2
689    fi
690
691    if [ X${skip} = Xfalse ] ; then
692      i=0
693      while [ $i -lt $NB_ESSAI ] ; do
694        # Identify file system
695        fileDeviceNumberInHex=$( stat -c %d $1 )
696        status=$?
697        if [ ${status} -gt 0 ] ; then
698          IGCM_debug_Exit "IGCM_sys_Put_Out"
699        fi
700        # Identify file system
701        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
702        status=$?
703        if [ ${status} -gt 0 ] ; then
704          IGCM_debug_Exit "IGCM_sys_Put_Out"
705        fi
706
707        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
708          # They are not on the same device. USUAL WAY
709          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
710          status=$?
711        else
712          # They are on the same device. NOT SO USUAL WAY
713          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
714          status=$?
715        fi
716        if [ ${status} -gt 0 ]; then
717          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
718          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
719          [ -f ${1} ] && ls -l ${1}
720          [ -f ${2} ] && ls -l ${2}
721          [ -f ${2}/${1} ] && ls -l ${2}/${1}
722          sleep $DELAI
723        else
724          break
725        fi
726        (( i = i + 1 ))
727      done
728    fi
729
730#       #RSYNC WITH NETWORK SSH CALL
731#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
732#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
733
734#       #RSYNC WITH NFS USE
735#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
736#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
737
738#       status=$?
739#       IGCM_sys_Rsync_out $status
740
741#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
742#       (( status=status+$? ))
743
744    if [ ${status} -gt 0 ] ; then
745      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
746      [ -f ${1} ] && ls -l ${1}
747      [ -f ${2} ] && ls -l ${2}
748      [ -f ${2}/${1} ] && ls -l ${2}/${1}
749      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
750      IGCM_debug_Exit "IGCM_sys_Put_Out"
751    else
752
753      if [ X${JobType} = XRUN ] ; then
754        if [ X${3} = X ] ; then
755          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
756          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
757        fi
758      fi
759
760      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
761    fi
762  fi
763  IGCM_debug_PopStack "IGCM_sys_Put_Out"
764  return 0
765}
766
767#D-#==================================================
768#D-function IGCM_sys_Get
769#D-* Purpose: Get a file from ${ARCHIVE}
770#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
771#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
772function IGCM_sys_Get {
773  IGCM_debug_PushStack "IGCM_sys_Get" $@
774
775  typeset DEST dm_liste target file_work
776  typeset NB_ESSAI DELAI status i
777
778  if ( $DEBUG_sys ) ; then
779    echo "IGCM_sys_Get :" $@
780  fi
781
782  # number of tentative
783  NB_ESSAI=3
784  # time delay between tentative
785  DELAI=2
786
787  if [ $DRYRUN -le 2 ]; then
788    if [ X${1} = X'/l' ] ; then
789      eval set +A dm_liste \${${2}}
790    else
791      eval set +A dm_liste ${1}
792    fi
793    eval DEST=\${${#}}
794    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
795    status=$?
796    if [ ${status} -gt 0 ] ; then
797      echo "WARNING IGCM_sys_Get : error code ${status}"
798      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
799      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
800    fi
801
802    #if [ ${status} -gt 0 ] ; then
803    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
804    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
805    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
806    #    sleep 30
807    #    echo "We try another time"
808    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
809    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
810    #    status=$?
811    #    if [ ${status} -gt 0 ] ; then
812    #      echo "ERROR IGCM_sys_Get : again demigration error :"
813    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
814    #      IGCM_debug_Exit "IGCM_sys_Get"
815    #    fi
816    #  else
817    #    echo "ERROR IGCM_sys_Get : demigration error :"
818    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
819    #    IGCM_debug_Exit "IGCM_sys_Get"
820    #  fi
821    #fi
822
823    #   #RSYNC WITH NETWORK SSH CALL
824    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
825    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
826
827    #   #RSYNC WITH NFS USE
828    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
829    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
830
831    #   status=$?
832    #   IGCM_sys_Rsync_out $status
833
834    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
835    #   (( status=status+$? ))
836
837    #USUAL WAY
838    if [ X${1} = X'/l' ] ; then
839      for target in ${dm_liste[*]} ; do
840        local_file=$( basename ${target} )
841        # test if the target file is present before the loop
842        IGCM_sys_TestFileArchive ${target}
843        status=$?
844        if [ ${status} -gt 0 ] ; then
845          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
846          IGCM_debug_Exit "IGCM_sys_Get"
847        else
848          i=0
849          while [ $i -lt $NB_ESSAI ] ; do
850            #if [ X${DoLink} = Xtrue ] ; then
851            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
852            #  status=$?
853            #  else
854            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
855            #  status=$?
856            #fi
857            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
858            status=$?
859            if [ ${status} -gt 0 ]; then
860              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
861              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
862              sleep $DELAI
863            else
864              break
865            fi
866            (( i = i + 1 ))
867          done
868          if [ ${status} -gt 0 ] ; then
869            echo "IGCM_sys_Get : error"
870            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
871            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
872            IGCM_debug_Exit "IGCM_sys_Get"
873          else
874            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
875          fi
876        fi
877      done
878    else
879      i=0
880      while [ $i -lt $NB_ESSAI ] ; do
881        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
882        status=$?
883        if [ ${status} -gt 0 ]; then
884          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
885          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
886          sleep $DELAI
887        else
888          break
889        fi
890        (( i = i + 1 ))
891      done
892      if [ ${status} -gt 0 ] ; then
893        echo "IGCM_sys_Get : error"
894        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
895        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
896        IGCM_debug_Exit "IGCM_sys_Get"
897      else
898        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
899      fi
900    fi
901  fi
902  IGCM_debug_PopStack "IGCM_sys_Get"
903}
904
905#D-#==================================================
906#D-function IGCM_sys_GetDate_Monitoring
907#D-* Purpose: get the last year for which the monitoring has been computed
908#D-* Examples:
909#D-
910function IGCM_sys_GetDate_Monitoring {
911  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
912  if ( $DEBUG_sys ) ; then
913    echo "IGCM_sys_GetDate_Monitoring :" $@
914  fi
915
916  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
917
918  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
919}
920
921#D-#==================================================
922#D-function IGCM_sys_Dods_Rm
923#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
924#D-* Examples:
925#D-
926function IGCM_sys_Dods_Rm {
927  if ( $DEBUG_sys ) ; then
928    echo "IGCM_sys_Dods_Rm :" $@
929  fi
930  typeset status
931  if [ $DRYRUN = 0 ]; then
932
933#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
934#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
935#      echo "Nothing has been done."
936#      return
937#    fi
938
939    if [ "$#" -eq 1 ]; then
940      $(ccc_home -u igcmg)/Tools/irene-amd/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
941      status=$?
942    else
943      $(ccc_home -u igcmg)/Tools/irene-amd/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
944      status=$?
945    fi
946
947#    if [ ${status} -gt 0 ] ; then
948#      echo "IGCM_sys_Dods_Rm : error."
949#      cat out_dods_rm
950#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
951#    else
952#      rm out_dods_rm
953#    fi
954
955  fi
956  return $status
957}
958
959#D-#==================================================
960#D-function IGCM_sys_Dods_Cp
961#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
962#D-* Examples:
963#D-
964function IGCM_sys_Dods_Cp {
965  if ( $DEBUG_sys ) ; then
966    echo "IGCM_sys_Dods_Cp :" $@
967  fi
968  typeset status
969  if [ $DRYRUN = 0 ]; then
970
971#    if [ ! -d ${R_SAVE}/${1} ] ; then
972#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
973#      echo "Nothing has been done."
974#      return
975#    fi
976
977    $(ccc_home -u igcmg)/Tools/irene-amd/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
978    status=$?
979
980#       if [ ${status} -gt 0 ] ; then
981#           echo "IGCM_sys_Dods_Cp : error."
982#           cat out_dods_cp
983#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
984#       else
985#           rm out_dods_cp
986#       fi
987
988  fi
989  return $status
990}
991
992#D-#==================================================
993#D-function IGCM_sys_Put_Dods
994#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
995#D-* Examples:
996#D-
997function IGCM_sys_Put_Dods {
998  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
999  if ( $DEBUG_sys ) ; then
1000    echo "IGCM_sys_Put_Dods :" $@
1001  fi
1002  typeset status
1003  if [ $DRYRUN = 0 ]; then
1004    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1005      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1006      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1007      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1008      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1009      return
1010    fi
1011
1012    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1013      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1014      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1015      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1016      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1017      return
1018    fi
1019    #
1020    if [ -d ${R_SAVE}/${1} ] ; then
1021      cd ${R_SAVE}
1022    elif [ -d ${R_FIGR}/${1} ] ; then
1023      cd ${R_FIGR}
1024    fi
1025
1026    IGCM_sys_Dods_Rm ${1}
1027    IGCM_sys_Dods_Cp ${1}
1028    status=0
1029
1030    if [ ${status} -gt 0 ] ; then
1031      echo "IGCM_sys_Put_Dods : error."
1032      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1033    fi
1034  fi
1035  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1036}
1037
1038##############################################################
1039# REBUILD OPERATOR
1040
1041#D-#==================================================
1042#D-function IGCM_sys_sync
1043#D-* Purpose: flush buffer on disk
1044#D-* Examples:
1045#D-
1046function IGCM_sys_sync {
1047  IGCM_debug_PushStack "IGCM_sys_sync" $@
1048  if ( $DEBUG_sys ) ; then
1049    echo "IGCM_sys_sync :" $@
1050  fi
1051
1052  /bin/sync
1053
1054  IGCM_debug_PopStack "IGCM_sys_sync"
1055}
1056
1057############################################################
1058# Activate Running Environnment Variables
1059
1060#D-#==================================================
1061#D-function IGCM_sys_activ_variables
1062#D-* Purpose: set environement variables prior to execution
1063#D-* Examples:
1064#D-
1065function IGCM_sys_activ_variables {
1066  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1067  if ( $DEBUG_sys ) ; then
1068    echo "IGCM_sys_activ_variables"
1069  fi
1070
1071# --------------------------------------------------------------------
1072#D- MODULE specifications
1073# --------------------------------------------------------------------
1074  # Source the file EnvFile containing all module load needed to run the model.
1075  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1076  # it will be used. Otherwise a default file will be used.
1077  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1078      # EnvFile is set in config.card and the file exists
1079      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1080      EnvFile=${config_UserChoices_EnvFile}
1081  else
1082      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1083      module purge > /dev/null 2>&1
1084      if [ X${compilerVersion} = XV19 ] ; then
1085          IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V19"
1086          EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
1087      else
1088          IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1089      fi
1090  fi
1091  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1092  IGCM_debug_Print 1 ${EnvFile}
1093  . ${EnvFile}
1094
1095  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene-amd. "
1096  module list
1097
1098# --------------------------------------------------------------------
1099#D- MPI specifications
1100# --------------------------------------------------------------------
1101
1102# --------------------------------------------------------------------
1103#D- Other specifications
1104# --------------------------------------------------------------------
1105
1106  ulimit -s unlimited
1107
1108  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1109}
1110
1111############################################################
1112# Desactivate Running Environnment Variables
1113
1114#D-#==================================================
1115#D-function IGCM_sys_desactiv_variables
1116#D-* Purpose: unset environement variables after execution
1117#D-* Examples:
1118#D-
1119function IGCM_sys_desactiv_variables {
1120  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1121  if ( $DEBUG_sys ) ; then
1122    echo "IGCM_sys_desactiv_variables"
1123  fi
1124# --------------------------------------------------------------------
1125#D- MPI specifications
1126# --------------------------------------------------------------------
1127
1128# --------------------------------------------------------------------
1129#D- Other specifications
1130# --------------------------------------------------------------------
1131
1132  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1133}
1134
1135############################################################
1136# Update job headers to be used by the scheduler
1137
1138#D-#==================================================
1139#D-function IGCM_sys_updateHeaders
1140#D-* Purpose: Update job headers to be used by the scheduler
1141#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1142#D-
1143function IGCM_sys_updateHeaders {
1144  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1145  if ( $DEBUG_sys ) ; then
1146    echo "IGCM_sys_updateHeaders"
1147  fi
1148  typeset file
1149  file=$1
1150
1151   # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1152   # Compute the number of core to add to force the use of a new node for xios servers
1153   if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1154      (( current_core_noxios = coreNumber - IOS_PROC_MPI * IOS_PROC_OMP * IOS_PROC_DEP ))
1155      (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1156      (( second_comp_proc_mpi_loc = IOS_PROC_MPI - 1 ))
1157      (( coreNumber = current_core_noxios + first_comp_proc_dep_loc + second_comp_proc_mpi_loc * IOS_PROC_OMP * IOS_PROC_DEP ))
1158   fi
1159
1160  if [ ${executionType} -eq 1 ] ; then
1161    # MPMD + MPI
1162    sed -e "/::openMPthreads::/d"                  \
1163        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1164      ${file} > ${file}.tmp
1165
1166  elif [ ${executionType} -eq 2 ] ; then
1167    # MPMD + MPI + OMP :ccc_mprun
1168    sed -e "/::openMPthreads::/d"                  \
1169        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1170      ${file} > ${file}.tmp
1171
1172  elif [ ${executionType} -eq 3 ] ; then
1173    # SPMD + MPI/OMP
1174    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1175        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1176        -e "/#MSUB -x/d"                           \
1177      ${file} > ${file}.tmp
1178
1179  elif [ ${executionType} -eq 4 ] ; then
1180    # SPMD + MPI only
1181    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1182        -e "/::openMPthreads::/d"                  \
1183        -e "/#MSUB -x/d"                           \
1184      ${file} > ${file}.tmp
1185
1186  elif [ ${executionType} -eq 5 ] ; then
1187    # SPMD + OMP only
1188    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1189        -e "/::JobNumProcTot::/d"                  \
1190        -e "/#MSUB -x/d"                           \
1191      ${file} > ${file}.tmp
1192
1193  elif [ ${executionType} -eq 6 ] ; then
1194    # SEQUENTIAL THEN
1195    sed -e "s/::JobNumProcTot::/1/"                \
1196        -e "/::openMPthreads::/d"                  \
1197        -e "/#MSUB -x/d"                           \
1198      ${file} > ${file}.tmp
1199
1200  fi
1201
1202  IGCM_sys_Mv ${file}.tmp ${file}
1203
1204  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1205}
1206
1207############################################################
1208# Build MPI/OMP scripts run file (dummy function)
1209
1210#D-#==================================================
1211#D-function IGCM_sys_build_run_file
1212#D-* Purpose: build run file (deprecated)
1213#D-* Examples:
1214#D-
1215function IGCM_sys_build_run_file {
1216  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1217}
1218
1219############################################################
1220# Build MPI/OMP scripts
1221
1222#D-#==================================================
1223#D-function IGCM_sys_build_execution_scripts
1224#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1225#D-* Examples:
1226#D-
1227function IGCM_sys_build_execution_scripts
1228{
1229  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1230  if ( $DEBUG_sys ) ; then
1231    echo "IGCM_sys_build_execution_scripts " $@
1232  fi
1233
1234  EXECUTION=${HOST_MPIRUN_COMMAND}
1235
1236  if ( ${OK_PARA_MPMD} ) ; then
1237
1238    # MPMD mode
1239    # 1 MPI only
1240    # 2 MPI/OpenMP mpirun method
1241    # 3 MPI/OpenMP ccc_mprun method
1242
1243    if [ -f run_file ] ; then
1244      IGCM_sys_Rm -f run_file
1245    fi
1246    touch run_file
1247
1248    # case 1 : Only MPI (MPMD)
1249    if  ( ! ${OK_PARA_OMP} ) ; then
1250
1251      # Build run_file
1252
1253      # First loop on the components for the coupler ie oasis (only if oasis3)
1254      # the coupler ie oasis3 must be the first one
1255      for comp in ${config_ListOfComponents[*]} ; do
1256
1257        eval ExeNameIn=\${config_Executable_${comp}[0]}
1258        eval ExeNameOut=\${config_Executable_${comp}[1]}
1259
1260        # for CPL component only
1261        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1262
1263          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1264          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1265          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1266        fi
1267      done
1268
1269      # Then second loop on the components
1270      for comp in ${config_ListOfComponents[*]} ; do
1271
1272        eval ExeNameIn=\${config_Executable_${comp}[0]}
1273        eval ExeNameOut=\${config_Executable_${comp}[1]}
1274
1275        # Only if we really have an executable for the component and not the coupler ie oasis:
1276        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1277
1278          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1279          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1280          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1281
1282          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1283          # Compute the number of core to add to force the use of a new node for xios servers
1284         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1285            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1286              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_dep_loc ))
1287              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1288              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1289              if [ ${comp_proc_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1290                 echo "1-${first_comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1291                 if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1292                    echo "${second_comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1293                 fi
1294              else
1295                 echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1296              fi
1297           else
1298              echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1299           fi
1300         else
1301          echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1302         fi
1303
1304
1305        fi
1306      done
1307
1308      ## module advised by TGCC (instead of 2 variables)
1309      module load feature/bridge/heterogenous_mpmd
1310
1311      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1312
1313      IGCM_sys_Chmod u+x run_file
1314      if ( $DEBUG_sys ) ; then
1315        echo "run_file contains : "
1316        cat run_file
1317      fi
1318
1319    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1320
1321    # 2 MPI/OpenMP mpirun method
1322
1323      # Use of mpirun instead of ccc_mprun
1324      EXECUTION="time mpirun"
1325
1326      #  Hosts treatment
1327      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1328
1329      i=0
1330      rm -f hosts rankfile
1331      IGCM_debug_Print 1 "sys Irene-amd, Hosts available :"
1332      for nodes in `cat hosts.tmp`
1333      do
1334        host[$i]=$nodes
1335        echo "${host[$i]}" >> hosts
1336        IGCM_debug_Print 1 ${host[$i]}
1337        i=$((i+1))
1338      done
1339      rm -f hosts.tmp
1340
1341      listnodes=${host[*]}
1342
1343      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1344
1345      # Initialisation
1346      rank=0
1347      current_core=0
1348      init_exec=n
1349
1350      # Loop on the components
1351      for comp in ${config_ListOfComponents[*]} ; do
1352
1353        eval ExeNameIn=\${config_Executable_${comp}[0]}
1354        eval ExeNameOut=\${config_Executable_${comp}[1]}
1355
1356        # Not possible if oasis has an executable (i.e old version of oasis3)
1357        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1358          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1359          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1360          IGCM_debug_Verif_Exit
1361        fi
1362
1363        # Only if we really have an executable for the component :
1364        if [ "X${ExeNameOut}" != X\"\" ] ; then
1365
1366          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1367          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1368          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1369
1370          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1371          echo ""  >> script_${ExeNameOut}.ksh
1372          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1373
1374            # Check if the number of threads is correct
1375            case ${comp_proc_omp_loc} in
1376            2|4|8|16)
1377              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1378              ;;
1379            *)
1380              IGCM_debug_Exit "ERROR with OMP parameters !"
1381              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1382              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1383              IGCM_debug_Verif_Exit
1384              ;;
1385            esac
1386            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1387            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1388            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1389            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1390          fi
1391
1392          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1393          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1394          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1395          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1396          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1397
1398          if [ ${init_exec} = y ] ; then
1399            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1400          else
1401            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1402            init_exec=y
1403          fi
1404
1405          # Build rankfile : method used to assign cores and nodes for the MPI process
1406          # Ex :
1407          #rank 0=curie5296 slot=0,1,2,3
1408          #rank 1=curie5296 slot=4,5,6,7
1409          # Example of final command :
1410          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1411          # with script_lmdz.x.ksh :
1412          # #!/bin/ksh
1413          #export KMP_STACKSIZE=3g
1414          #export KMP_LIBRARY=turnaround
1415          #export MKL_SERIAL=YES
1416          #OMP_NUM_THREADS=4
1417          #./lmdz.x
1418          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1419              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1420          else
1421              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1422          fi
1423          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1424             (( index_host = current_core / NB_CORE_PER_NODE ))
1425             host_value=${host[${index_host}]}
1426             (( slot =  current_core % NB_CORE_PER_NODE ))
1427             virg=","
1428             string_final=""
1429             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1430                 string=$index$virg
1431                 string_final=$string_final$string
1432             done
1433             string_final=$( echo $string_final | sed "s/.$//" )
1434             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1435             (( rank = rank + 1 ))
1436             (( current_core = current_core + offset_comp_proc_loc ))
1437          done
1438        fi
1439      done
1440
1441    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1442
1443    # 3 MPI/OpenMP ccc_mprun method
1444
1445    # MPI-OpenMP (MPMD)
1446    # example of run_file
1447    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1448    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1449    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1450
1451      # Loop on the components
1452      for comp in ${config_ListOfComponents[*]} ; do
1453
1454        eval ExeNameIn=\${config_Executable_${comp}[0]}
1455        eval ExeNameOut=\${config_Executable_${comp}[1]}
1456
1457        # Not possible if oasis has an executable (i.e old version of oasis3)
1458        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1459          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1460          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1461          IGCM_debug_Verif_Exit
1462        fi
1463
1464        # Only if we really have an executable for the component :
1465        if [ "X${ExeNameOut}" != X\"\" ] ; then
1466
1467          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1468          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1469          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1470
1471          # Compute number of cores per process
1472          (( comp_proc_omp_dep_loc = comp_proc_omp_loc * comp_proc_dep_loc )) 
1473
1474          # Build script files
1475
1476          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1477          echo ""  >> script_${ExeNameOut}.ksh
1478          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1479
1480            # Check if the number of threads is correct
1481            case ${comp_proc_omp_loc} in
1482            2|4|8|16)
1483              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1484              ;;
1485            *)
1486              IGCM_debug_Exit "ERROR with OMP parameters !"
1487              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1488              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1489              IGCM_debug_Verif_Exit
1490              ;;
1491            esac
1492            ### only for LMDZ?
1493            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1494            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1495            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1496            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1497
1498          fi
1499
1500          # to have out/err per process on different files
1501          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1502          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1503
1504          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1505
1506          # Complete run_file
1507
1508          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1509          # Compute the number of core to add to force the use of a new node for xios servers
1510         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1511            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1512              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_omp_dep_loc )) 
1513              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1514              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1515              if [ ${comp_proc_omp_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1516                 echo "1-${first_comp_proc_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1517                  if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1518                    echo "${second_comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} ./${ExeNameOut}" >>run_file
1519                  fi
1520              else
1521                 echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1522              fi
1523           else
1524              echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1525           fi
1526         else
1527          echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1528         fi
1529
1530        fi
1531      done
1532
1533      ## variable added to stop after 60s instead of 600s by default.
1534      ## This is used when no error comes from executables and when something stopped an executable without notice.
1535      export SLURM_WAIT=60
1536
1537      ## module advised by TGCC (instead of 2 variables)
1538      module load feature/bridge/heterogenous_mpmd
1539
1540      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1541
1542      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1543
1544      IGCM_sys_Chmod u+x run_file
1545      if ( $DEBUG_sys ) ; then
1546        echo "run_file contains : "
1547        cat run_file
1548      fi
1549
1550    else
1551
1552      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1553
1554    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1555
1556  else
1557  # Only one executable (SPMD mode).
1558
1559    for comp in ${config_ListOfComponents[*]} ; do
1560
1561      # Only if we really have an executable for the component :
1562      eval ExeNameOut=\${config_Executable_${comp}[1]}
1563      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1564
1565        # Build script files
1566
1567        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1568        echo ""  >> script_${ExeNameOut}.ksh
1569        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1570
1571        if ( ${OK_PARA_OMP} ) ; then
1572          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1573          echo ""  >> script_${ExeNameOut}.ksh
1574          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1575          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1576          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1577          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1578        fi
1579
1580        if  ( ${OK_PARA_MPI} ) ; then
1581          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1582          # Default : ccc_mprun used if nb_proc gt 1
1583          # to have out/err per process on different files
1584          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1585          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1586          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1587        else
1588          # Default : ccc_mprun is NOT used if nb_proc eq 1
1589          # to have out/err per process on different files
1590          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1591          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1592          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1593        fi
1594
1595        IGCM_debug_Print 1 "sys Irene-amd : script_${ExeNameOut}.ksh contains"
1596        cat script_${ExeNameOut}.ksh
1597
1598      fi
1599
1600    done
1601
1602  fi # ${OK_PARA_MPMD}
1603
1604  IGCM_debug_Print 1 "sys Irene-amd : execution command is "
1605  IGCM_debug_Print 1 "$EXECUTION"
1606
1607  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1608}
1609
1610#D-#==================================================
1611#D-function IGCM_sys_check_path
1612#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1613#D-* do not point to an important use directory. Stop immediately in that case.
1614#D-* Examples:
1615#D-
1616function IGCM_sys_check_path {
1617  IGCM_debug_PushStack "IGCM_sys_check_path"
1618  if ( $DEBUG_sys ) ; then
1619    echo "IGCM_sys_check_path"
1620  fi
1621
1622  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1623    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1624    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1625    IGCM_debug_Exit "This will stop the job"
1626  fi
1627  IGCM_debug_PopStack "IGCM_sys_check_path"
1628}
1629
1630#D-#==================================================
1631#D-function IGCM_sys_check_quota
1632#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1633#D-* Examples:
1634#D-
1635function IGCM_sys_check_quota {
1636  IGCM_debug_PushStack "IGCM_sys_check_quota"
1637  if ( $DEBUG_sys ) ; then
1638    echo "IGCM_sys_check_quota"
1639  fi
1640  # Limit of quota (in %)
1641  limit_quota=90
1642
1643  # Check of the volume
1644  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1645  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1646
1647  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1648
1649    unit_avail=${volume_avail: -1}
1650    unit_quota=${volume_quota: -1}
1651
1652    if [ "${unit_quota}" = "*" ] ; then
1653      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1654      IGCM_debug_Print 1 "More than 100% of your quota is used"
1655      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1656      IGCM_debug_Print 1 "You must have more than 10% available to run"
1657      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1658      IGCM_debug_Verif_Exit
1659    fi
1660
1661    temp_avail=${volume_avail%%${volume_avail: -1}*}
1662    temp_quota=${volume_quota%%${volume_quota: -1}*}
1663
1664    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1665
1666    # Convertion
1667      if [ ${volume_avail: -1} = "T" ] ; then
1668        (( temp_avail = temp_avail * 1000000000000 ))
1669      elif [ ${volume_avail: -1} = "G" ] ; then
1670        (( temp_avail = temp_avail * 1000000000 ))
1671      elif [ ${volume_avail: -1} = "M" ] ; then
1672        (( temp_avail = temp_avail * 1000000 ))
1673      elif [ ${volume_avail: -1} = "k" ] ; then
1674        (( temp_avail = temp_avail * 1000 ))
1675      else
1676        (( temp_avail = volume_avail ))
1677      fi
1678      if [ ${volume_quota: -1} = "T" ] ; then
1679        (( temp_quota = temp_quota * 1000000000000 ))
1680      elif [ ${volume_quota: -1} = "G" ] ; then
1681        (( temp_quota = temp_quota * 1000000000 ))
1682      elif [ ${volume_quota: -1} = "M" ] ; then
1683        (( temp_quota = temp_quota * 1000000 ))
1684      elif [ ${volume_quota: -1} = "k" ] ; then
1685        (( temp_quota = temp_quota * 1000 ))
1686      else
1687        (( temp_quota = volume_quota ))
1688      fi
1689    fi
1690
1691    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1692#    echo "volume ratio is " $quota_volume
1693
1694    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1695      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1696      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1697      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1698      IGCM_debug_Print 1 "You must have more than 10% available to run"
1699      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1700      IGCM_debug_Verif_Exit
1701    fi
1702
1703  fi
1704
1705# Check of the number of inodes
1706
1707  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1708  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1709
1710  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1711
1712    unit_avail=${inode_avail: -1}
1713    unit_quota=${inode_quota: -1}
1714
1715    if [ "${unit_quota}" = "*" ] ; then
1716      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1717      IGCM_debug_Print 1 "More than 100% of your quota is used"
1718      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1719      IGCM_debug_Print 1 "You must have more than 10% available to run"
1720      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1721      IGCM_debug_Verif_Exit
1722    fi
1723
1724    temp_avail=${inode_avail%%${inode_avail: -1}*}
1725    temp_quota=${inode_quota%%${inode_quota: -1}*}
1726
1727    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1728
1729    # Convertion
1730      if [ ${inode_avail: -1} = "T" ] ; then
1731        (( temp_avail = temp_avail * 1000000000000 ))
1732      elif [ ${inode_avail: -1} = "G" ] ; then
1733        (( temp_avail = temp_avail * 1000000000 ))
1734      elif [ ${inode_avail: -1} = "M" ] ; then
1735        (( temp_avail = temp_avail * 1000000 ))
1736      elif [ ${inode_avail: -1} = "k" ] ; then
1737        (( temp_avail = temp_avail * 1000 ))
1738      else
1739        (( temp_avail = inode_avail ))
1740      fi
1741
1742      if [ ${inode_quota: -1} = "T" ] ; then
1743        (( temp_quota = temp_quota * 1000000000000 ))
1744      elif [ ${inode_quota: -1} = "G" ] ; then
1745        (( temp_quota = temp_quota * 1000000000 ))
1746      elif [ ${inode_quota: -1} = "M" ] ; then
1747        (( temp_quota = temp_quota * 1000000 ))
1748      elif [ ${inode_quota: -1} = "k" ] ; then
1749        (( temp_quota = temp_quota * 1000 ))
1750      else
1751        (( temp_quota = inode_quota ))
1752      fi
1753    fi
1754    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1755#    echo "inode ratio is " $quota_inode
1756
1757    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1758      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1759      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1760      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1761      IGCM_debug_Print 1 "You must have more than 10% available to run"
1762      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1763      IGCM_debug_Verif_Exit
1764    fi
1765  fi
1766  IGCM_debug_PopStack "IGCM_sys_check_quota"
1767}
1768
1769#D-#==================================================
1770#D-function IGCM_sys_projectAccounting
1771#D-* Purpose: store project accounting information in a file
1772#D-* Examples:
1773#D-
1774function IGCM_sys_projectAccounting {
1775  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1776  if ( $DEBUG_sys ) ; then
1777    echo "IGCM_sys_check_quota"
1778  fi
1779
1780  ssh irene194 /usr/bin/ccc_myproject > $1
1781
1782  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1783}
1784
1785#D-#==================================================
1786#D-function IGCM_sys_getJobSchedulerID
1787#D-* Purpose: Get the job ID during execution
1788#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1789#D-
1790function IGCM_sys_getJobSchedulerID {
1791  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1792  if ( $DEBUG_sys ) ; then
1793    echo "IGCM_sys_getJobSchedulerID"
1794  fi
1795  eval ${1}=${BRIDGE_MSUB_JOBID}
1796
1797  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1798}
1799
1800#D-#==================================================
1801#D-function IGCM_sys_GetJobID
1802#D-* Purpose: Get the job ID from the JobName
1803#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1804#D-
1805function IGCM_sys_GetJobID {
1806  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1807  if ( $DEBUG_sys ) ; then
1808    echo "IGCM_sys_GetJobID"
1809  fi
1810
1811  # With -f option, the full job name is given in the last column
1812  ID=$( ccc_mstat -f -u $2 | \
1813        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1814
1815  eval ${3}=${ID}
1816  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1817}
1818
1819#D-#==================================================
1820#D-function IGCM_sys_CountJobInQueue
1821#D-* Purpose: Count number of users job
1822#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1823#D-
1824function IGCM_sys_CountJobInQueue {
1825  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1826  if ( $DEBUG_sys ) ; then
1827    echo "IGCM_sys_CountJobInQueue"
1828  fi
1829
1830  # With -f option, the full job name is given in the last column
1831  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1832
1833  eval ${2}=${NbRun}
1834
1835  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1836}
1837
1838#D-#==================================================
1839#D-function IGCM_sys_ListJobInQueue
1840#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1841#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1842#D-
1843function IGCM_sys_ListJobInQueue {
1844  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1845  if ( $DEBUG_sys ) ; then
1846    echo "IGCM_sys_ListJobInQueue"
1847  fi
1848
1849  # With -f option, the full job name is given in the last column
1850  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1851                                        '( $2  == User      && \
1852                                           $NF != /TS/      && \
1853                                           $NF !~ /PACK/    && \
1854                                           $NF !~ /REBUILD/ && \
1855                                           $NF !~ /pack/ )     \
1856                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1857
1858  eval set -A ${2} ${JobList[*]}
1859
1860  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1861}
1862
1863#D-#==================================================
1864#D-function IGCM_sys_atlas
1865#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1866#D-* Examples:
1867#D-
1868function IGCM_sys_atlas {
1869  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1870  if ( $DEBUG_sys ) ; then
1871    echo "IGCM_sys_atlas :" $@
1872  fi
1873
1874  typeset status
1875
1876  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1877  status=$?
1878  if [ ${status} -gt 0 ] ; then
1879    echo "IGCM_sys_atlas : error code ${status}"
1880    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1881    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1882    IGCM_debug_PopStack "IGCM_sys_atlas"
1883    return 1
1884  else
1885    IGCM_debug_PopStack "IGCM_sys_atlas"
1886    return 0
1887  fi
1888
1889  IGCM_debug_PopStack "IGCM_sys_atlas"
1890}
1891
1892#D-#==================================================
1893#D-function IGCM_sys_rebuild_nemo
1894#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1895#D-* Examples:
1896#D-
1897
1898function IGCM_sys_rebuild_nemo {
1899  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1900  if ( $DEBUG_sys ) ; then
1901    echo "IGCM_sys_rebuild_nemo :" $@
1902  fi
1903
1904  $( ccc_home -u igcmg)/Tools/irene-amd/bin/rebuild_nemo ${1} ${2}
1905
1906  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1907}
1908
Note: See TracBrowser for help on using the repository browser.