source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene-amd.ksh @ 1573

Last change on this file since 1573 was 1569, checked in by aclsce, 17 months ago

Added fonctionality to create RUN_DIR and Job_debug to be launched directly in the RUN_DIR directory.
File size: 66.0 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision:: 1512                                     $ Revision of last commit
7# $Author:: acosce                                     $ Author of last commit
8# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene-amd
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X\"\" ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
49# -------------------------------------------------------------------------------------
50# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
51# |          |  Cp/Exe/param/files |            |         |                           |
52# |          |  Chmod Qsub         |            |         |                           |
53# -------------------------------------------------------------------------------------
54# |    0     |       yes           |    yes     |  yes    |      yes                  |
55# -------------------------------------------------------------------------------------
56# |    1     |       yes           |    yes     |  yes    |      no                   |
57# -------------------------------------------------------------------------------------
58# |    2     |       yes           |    yes     |  no     |      no                   |
59# -------------------------------------------------------------------------------------
60# |    3     |       yes           |    no      |  no     |      no                   |
61# -------------------------------------------------------------------------------------
62# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
63# -------------------------------------------------------------------------------------
64
65
66#=====================================================
67# Global Variables :
68#=====================================================
69# Language : "fr" or "en"
70typeset -r MYLANG="fr"
71
72#=====================================================
73# Host user names project maxCpuTime
74# $hostname ou hostname
75typeset HOST=${HOST:=$( hostname )}
76# $username ou whoami
77typeset LOGIN=${LOGIN:=$( whoami )}
78# $hostname of the MASTER job
79typeset MASTER=irene-amd
80# add default project on irene-amd
81typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} | cut -d@ -f1 )
82# jobWarningDelay in seconds
83typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
84
85#D-
86#D-#==================================================
87#D-Program used in libIGCM
88#D-#==================================================
89
90# Submit command
91typeset SUBMIT=${SUBMIT:=ccc_msub}
92# rsync with path
93typeset -r RSYNC=/usr/bin/rsync
94# RSYNC_opt args to rsync
95typeset -r RSYNC_opt="-va"
96# ie storage filesystem
97typeset -r STOREHOST=${MASTER}
98typeset -r REMOTE_RSYNC=/usr/bin/rsync
99
100#====================================================
101# Set environment tools (ferret, nco, cdo, rebuild, ...)
102#====================================================
103if [ X${TaskType} = Xcomputing ] ; then
104  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
105else
106  module purge > /dev/null 2>&1
107  . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
108  . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_atlas_irene-amd > /dev/null 2>&1
109
110  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
111  export UVCDAT_ANONYMOUS_LOG=FALSE
112fi
113
114# FYI
115[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
116[ ! X${TaskType} = Xchecking ] && module list
117
118# For AddNoise and AddPertu3DOCE
119export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene-amd/bin
120
121# For rebuild
122export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene-amd/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
123
124#====================================================
125# Host specific DIRECTORIES
126#====================================================
127
128# ============ CESIUM START ============ #
129
130#====================================================
131#- Mirror libIGCM from titane to cesium if needed
132#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
133#if [ ! ${ROOTSYS} = "home" ] ; then
134#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
135#else
136#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
137#fi
138
139#====================================================
140#- libIGCM_POST
141#if ( ${MirrorlibIGCM} ) ; then
142#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
143#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
144#else
145#  typeset -r libIGCM_POST=${libIGCM}
146#fi
147
148# ============ CESIUM  END  ============ #
149
150#====================================================
151#- MirrorlibIGCM for frontend
152typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
153
154#====================================================
155#- libIGCM_POST for frontend
156typeset -r libIGCM_POST=${libIGCM}
157
158#====================================================
159#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
160typeset -r R_EXE="${MODIPSL}/bin"
161
162#====================================================
163#- SUBMIT_DIR : submission dir
164typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
165
166#====================================================
167#- IN
168typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
169
170#====================================================
171#- RUN_DIR_PATH : Temporary working directory (=> TMP)
172typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
173
174#====================================================
175#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
176typeset -r OUTCOMMAND_PATH=/tmp
177
178#====================================================
179#- HOST_MPIRUN_COMMAND
180typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
181
182#====================================================
183#- Max number of arguments passed to nco operator or demigration command
184UNIX_MAX_LIMIT=360
185
186#====================================================
187#- set PackDefault to true on irene-amd
188PackDefault=true
189
190#====================================================
191#- Default number of MPI task for IPSL coupled model
192#- required for backward compatibility
193#-
194DEFAULT_NUM_PROC_OCE=5
195DEFAULT_NUM_PROC_CPL=1
196(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
197
198#====================================================
199#- Number of computing cores per node
200#-
201NB_CORE_PER_NODE=128
202
203#D-#==================================================
204#D-function IGCM_sys_defineArchives
205#D-* Purpose:
206#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
207#D-* Define ARCHIVE : Dedicated to large files
208#D-* Define STORAGE : Dedicated to small/medium files
209#D-* Define R_OUT   : Output tree located on ARCHIVE
210#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
211#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
212#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
213#D-* Examples:
214#D-
215function IGCM_sys_defineArchives {
216  IGCM_debug_PushStack "IGCM_sys_defineArchives"
217
218  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
219  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then
220    # Default option: Change the dfldatadir according to the project used for submission
221    # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
222    if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then
223      module switch dfldatadir dfldatadir/gencmip6
224      IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
225    else
226      module switch dfldatadir dfldatadir/${PROJECT}
227      IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
228    fi
229  else
230    # Use the project set by the variable DataProject in section UserChoices in config.card
231    module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
232    IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
233  fi
234
235 #====================================================
236 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
237 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
238
239  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
240    #====================================================
241    #- ARCHIVE (dedicated to large files)
242    ARCHIVE=${config_UserChoices_ARCHIVE}
243  else
244    #====================================================
245    #- ARCHIVE (dedicated to large files)
246    ARCHIVE=${CCCSTOREDIR}
247  fi
248
249  if [ ! X${config_UserChoices_STORAGE} = X ]; then
250    #====================================================
251    #- STORAGE (dedicated to small/medium files)
252    STORAGE=${config_UserChoices_STORAGE}
253  else
254    #====================================================
255    #- STORAGE (dedicated to small/medium files)
256    STORAGE=${CCCWORKDIR}
257  fi
258
259  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
260    #====================================================
261    #- R_OUT
262    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
263
264    #====================================================
265    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
266    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
267
268    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
269
270  else
271    #====================================================
272    #- R_OUT
273    R_OUT=${ARCHIVE}/IGCM_OUT
274
275    #====================================================
276    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
277    R_FIG=${STORAGE}/IGCM_OUT
278  fi
279
280  #====================================================
281  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
282  CMIP6_BUF=${STORAGE}/IGCM_OUT
283
284  #====================================================
285  #- R_BUF (ONLY FOR double copy an scratch)
286  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
287
288  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
289  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
290  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
291
292  IGCM_debug_PopStack "IGCM_sys_defineArchives"
293}
294
295#D-#==================================================
296#D-function IGCM_sys_RshArchive
297#D-* Purpose: Archive rsh command
298#D-* Examples:
299#D-
300function IGCM_sys_RshArchive {
301  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
302  /bin/ksh <<-EOF
303    ${@}
304EOF
305  status=$?
306  if [ ${status} -gt 0 ] ; then
307    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
308    IGCM_debug_Exit "IGCM_sys_RshArchive"
309  fi
310  IGCM_debug_PopStack "IGCM_sys_RshArchive"
311}
312
313#D-#==================================================
314#D-function IGCM_sys_RshArchive_NoError
315#D-* Purpose: Archive rsh command, without error
316#D-*          used only in monitoring.job
317#D-* Examples:
318#D-
319function IGCM_sys_RshArchive_NoError {
320  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
321  /bin/ksh <<-EOF
322    ${@} 2> /dev/null
323EOF
324  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
325}
326
327#D-#==================================================
328#D-function IGCM_sys_MkdirArchive
329#D-* Purpose: Mkdir on Archive
330#D-* Examples:
331#D-
332function IGCM_sys_MkdirArchive {
333  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
334  if ( $DEBUG_sys ) ; then
335    echo "IGCM_sys_MkdirArchive :" $@
336  fi
337  #- creation de repertoire sur le serveur fichier
338  if [ ! -d ${1} ]; then
339    \mkdir -p $1
340    status=$?
341
342    if [ ${status} -gt 0 ] ; then
343      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
344      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
345    fi
346  fi
347  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
348}
349
350#D-#==================================================
351#D-function IGCM_sys_TestDirArchive
352#D-* Purpose: Test Directory that must exists on Archive
353#D-* Examples:
354#D-
355function IGCM_sys_TestDirArchive {
356  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
357  if ( $DEBUG_sys ) ; then
358    echo "IGCM_sys_TestDirArchive :" $@
359  fi
360  typeset ExistFlag
361  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
362  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
363  return ${ExistFlag}
364}
365
366#D-#==================================================
367#D-function IGCM_sys_IsFileArchived
368#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
369#D-* Examples:
370#D-
371function IGCM_sys_IsFileArchived {
372  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
373  if ( $DEBUG_sys ) ; then
374    echo "IGCM_sys_IsFileArchived :" $@
375  fi
376  typeset IsArchivedFlag
377  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
378  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
379
380  return ${IsArchivedFlag}
381}
382
383#D-#==================================================
384#D-function IGCM_sys_TestFileArchive
385#D-* Purpose: Test file that must NOT EXISTS on Archive
386#D-* Examples:
387#D-
388function IGCM_sys_TestFileArchive {
389  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
390  typeset ExistFlag
391  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
392  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
393
394  return ${ExistFlag}
395}
396
397#D-#==================================================
398#D-function IGCM_sys_CountFileArchive
399#D-* Purpose: Count files on Archive filesystem
400#D-* Examples:
401#D-
402function IGCM_sys_CountFileArchive {
403  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
404  ls ${@} 2>/dev/null | wc -l
405  if [ $? -gt 0 ] ; then
406    echo "IGCM_sys_CountFileArchive : erreur."
407  fi
408  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
409}
410
411#D-#==================================================
412#D-function IGCM_sys_Tree
413#D-* Purpose: Tree directories with files on ${ARCHIVE}
414#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
415#D-
416function IGCM_sys_Tree {
417  IGCM_debug_PushStack "IGCM_sys_Tree" $@
418  if ( $DEBUG_sys ) ; then
419    echo "IGCM_sys_Tree :" $@
420  fi
421
422  \ls -lR ${@}
423
424  IGCM_debug_PopStack "IGCM_sys_Tree"
425}
426
427#D-#==================================================
428#D-function IGCM_sys_Qsub
429#D-* Purpose: Qsub new job
430#D-* Examples:
431#D-
432function IGCM_sys_Qsub {
433  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
434  if ( $DEBUG_sys ) ; then
435    echo "IGCM_sys_Qsub :" $@
436  fi
437  typeset options status
438  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
439
440  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
441  status=$?
442
443  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
444  if [ ${status} -gt 0 ] ; then
445    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
446    IGCM_debug_Exit "IGCM_sys_Qsub"
447  else
448    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
449  fi
450  IGCM_debug_PopStack "IGCM_sys_Qsub"
451}
452
453#D-#==================================================
454#D-function IGCM_sys_QsubPost
455#D-* Purpose: Qsub new job on scalaire
456#D-* Examples:
457#D-
458function IGCM_sys_QsubPost {
459  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
460  if ( $DEBUG_sys ) ; then
461    echo "IGCM_sys_QsubPost :" $@
462  fi
463  typeset options status
464  options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
465
466  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
467  status=$?
468
469  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
470  if [ ${status} -gt 0 ] ; then
471    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
472    IGCM_debug_Exit "IGCM_sys_QsubPost"
473  else
474    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
475  fi
476  IGCM_debug_PopStack "IGCM_sys_QsubPost"
477}
478
479#D-*************************
480#D- File transfer functions
481#D-*************************
482#D-
483
484#D-#==================================================
485#D-function IGCM_sys_RmRunDir
486#D-* Purpose: rm tmpdir (dummy function most of the time batch
487#D-                      scheduler will do the job)
488#D-* Examples:
489#D-
490function IGCM_sys_RmRunDir {
491  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
492  if ( $DEBUG_sys ) ; then
493    echo "IGCM_sys_RmRunDir :" $@
494  fi
495
496  typeset status
497
498  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
499  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
500  status=$?
501
502  if [ ${status} -gt 0 ] ; then
503    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
504    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
505    IGCM_debug_Exit "IGCM_sys_RmRunDir"
506  else
507    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
508  fi
509  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
510}
511
512#D-#==================================================
513#D-function IGCM_sys_Put_Dir
514#D-* Purpose: Copy a complete directory on $(ARCHIVE)
515#D-* Examples:
516#D-
517function IGCM_sys_Put_Dir {
518  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
519  if ( $DEBUG_sys ) ; then
520    echo "IGCM_sys_Put_Dir :" $@
521  fi
522  if [ $DRYRUN = 0 ]; then
523    if [ ! -d ${1} ] ; then
524      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
525      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
526      return
527    fi
528
529    typeset status
530
531    # Only if we use rsync
532    #IGCM_sys_TestDirArchive $( dirname $2 )
533    #
534    #USUAL WAY
535    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
536    status=$?
537
538    if [ ${status} -gt 0 ] ; then
539      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
540      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
541      IGCM_debug_Exit "IGCM_sys_Put_Dir"
542    else
543      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
544    fi
545  fi
546  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
547}
548
549#D-#==================================================
550#D-function IGCM_sys_Get_Dir
551#D-* Purpose: Copy a complete directory from ${ARCHIVE}
552#D-* Examples:
553#D-
554function IGCM_sys_Get_Dir {
555  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
556  if ( $DEBUG_sys ) ; then
557    echo "IGCM_sys_Get_Dir :" $@
558  fi
559  if [ $DRYRUN = 0 ]; then
560    typeset NB_ESSAI DELAI status i
561    # number of tentative
562    NB_ESSAI=3
563    # time delay between tentative
564    DELAI=2
565
566    #
567    # USUAL WAY
568    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
569    ccc_hsm get -r $1
570
571    i=0
572    while [ $i -lt $NB_ESSAI ] ; do
573      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
574      status=$?
575      if [ ${status} -gt 0 ] ; then
576        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
577        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
578        sleep $DELAI
579      else
580        break
581      fi
582      (( i = i + 1 ))
583    done
584
585    if [ ${status} -gt 0 ] ; then
586      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
587      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
588      IGCM_debug_Exit "IGCM_sys_Get_Dir"
589    else
590      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
591    fi
592  fi
593  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
594}
595
596#D-#==================================================
597#D-function IGCM_sys_Put_Rest
598#D-* Purpose: Put computied restarts on ${ARCHIVE}.
599#D-           File and target directory must exist.
600#D-* Examples:
601#D-
602function IGCM_sys_Put_Rest {
603  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
604  if ( $DEBUG_sys ) ; then
605    echo "IGCM_sys_Put_Rest :" $@
606  fi
607  if [ $DRYRUN = 0 ]; then
608    if [ ! -f ${1} ] ; then
609      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
610      IGCM_debug_Exit "IGCM_sys_Put_Rest"
611    fi
612
613    typeset status
614    #
615    # USUAL WAY
616    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
617    status=$?
618
619#       #RSYNC WITH NETWORK SSH CALL
620#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
621#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
622
623#       #RSYNC WITH NFS USE
624#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
625#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
626
627#       status=$?
628#       IGCM_sys_Rsync_out $status
629
630#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
631#       (( status=status+$? ))
632
633    if [ ${status} -gt 0 ] ; then
634      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
635      [ -f ${1} ] && ls -l ${1}
636      [ -f ${2} ] && ls -l ${2}
637      [ -f ${2}/${1} ] && ls -l ${2}/${1}
638      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
639      IGCM_debug_Exit "IGCM_sys_Put_Rest"
640    else
641
642      if [ X${JobType} = XRUN ] ; then
643        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
644        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
645      fi
646
647      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
648    fi
649  fi
650  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
651}
652
653#D-#==================================================
654#D-function IGCM_sys_Put_Out
655#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
656#D-* Examples:
657#D-
658function IGCM_sys_Put_Out {
659  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
660  if ( $DEBUG_sys ) ; then
661    echo "IGCM_sys_Put_Out :" $@
662  fi
663
664  typeset NB_ESSAI DELAI status i exist skip
665  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
666
667  # number of tentative
668  NB_ESSAI=3
669  # time delay between tentative
670  DELAI=2
671
672  if [ $DRYRUN = 0 ]; then
673    if [ ! -f ${1} ] ; then
674      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
675      IGCM_debug_PopStack "IGCM_sys_Put_Out"
676      return 1
677    fi
678    #
679    IGCM_sys_MkdirArchive $( dirname $2 )
680    #
681    exist=false
682    skip=false
683    if [ -f $2 ] ; then
684      IGCM_debug_Print 1 "$2 already exist"
685      ccc_hsm get $2
686      exist=true
687      if [ "X$( diff $1 $2 )" = X ] ; then
688        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
689        skip=true
690      else
691        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
692        skip=false
693      fi
694    fi
695    #
696    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
697      IGCM_sys_Chmod u+w $2
698    fi
699
700    if [ X${skip} = Xfalse ] ; then
701      i=0
702      while [ $i -lt $NB_ESSAI ] ; do
703        # Identify file system
704        fileDeviceNumberInHex=$( stat -c %d $1 )
705        status=$?
706        if [ ${status} -gt 0 ] ; then
707          IGCM_debug_Exit "IGCM_sys_Put_Out"
708        fi
709        # Identify file system
710        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
711        status=$?
712        if [ ${status} -gt 0 ] ; then
713          IGCM_debug_Exit "IGCM_sys_Put_Out"
714        fi
715
716        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
717          # They are not on the same device. USUAL WAY
718          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
719          status=$?
720        else
721          # They are on the same device. NOT SO USUAL WAY
722          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
723          status=$?
724        fi
725        if [ ${status} -gt 0 ]; then
726          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
727          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
728          [ -f ${1} ] && ls -l ${1}
729          [ -f ${2} ] && ls -l ${2}
730          [ -f ${2}/${1} ] && ls -l ${2}/${1}
731          sleep $DELAI
732        else
733          break
734        fi
735        (( i = i + 1 ))
736      done
737    fi
738
739#       #RSYNC WITH NETWORK SSH CALL
740#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
741#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
742
743#       #RSYNC WITH NFS USE
744#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
745#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
746
747#       status=$?
748#       IGCM_sys_Rsync_out $status
749
750#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
751#       (( status=status+$? ))
752
753    if [ ${status} -gt 0 ] ; then
754      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
755      [ -f ${1} ] && ls -l ${1}
756      [ -f ${2} ] && ls -l ${2}
757      [ -f ${2}/${1} ] && ls -l ${2}/${1}
758      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
759      IGCM_debug_Exit "IGCM_sys_Put_Out"
760    else
761
762      if [ X${JobType} = XRUN ] ; then
763        if [ X${3} = X ] ; then
764          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
765          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
766        fi
767      fi
768
769      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
770    fi
771  fi
772  IGCM_debug_PopStack "IGCM_sys_Put_Out"
773  return 0
774}
775
776#D-#==================================================
777#D-function IGCM_sys_Get
778#D-* Purpose: Get a file from ${ARCHIVE}
779#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
780#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
781function IGCM_sys_Get {
782  IGCM_debug_PushStack "IGCM_sys_Get" $@
783
784  typeset DEST dm_liste target file_work
785  typeset NB_ESSAI DELAI status i
786
787  if ( $DEBUG_sys ) ; then
788    echo "IGCM_sys_Get :" $@
789  fi
790
791  # number of tentative
792  NB_ESSAI=3
793  # time delay between tentative
794  DELAI=2
795
796  if [ $DRYRUN -le 2 ]; then
797    if [ X${1} = X'/l' ] ; then
798      eval set +A dm_liste \${${2}}
799    else
800      eval set +A dm_liste ${1}
801    fi
802    eval DEST=\${${#}}
803    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
804    status=$?
805    if [ ${status} -gt 0 ] ; then
806      echo "WARNING IGCM_sys_Get : error code ${status}"
807      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
808      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
809    fi
810
811    #if [ ${status} -gt 0 ] ; then
812    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
813    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
814    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
815    #    sleep 30
816    #    echo "We try another time"
817    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
818    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
819    #    status=$?
820    #    if [ ${status} -gt 0 ] ; then
821    #      echo "ERROR IGCM_sys_Get : again demigration error :"
822    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
823    #      IGCM_debug_Exit "IGCM_sys_Get"
824    #    fi
825    #  else
826    #    echo "ERROR IGCM_sys_Get : demigration error :"
827    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
828    #    IGCM_debug_Exit "IGCM_sys_Get"
829    #  fi
830    #fi
831
832    #   #RSYNC WITH NETWORK SSH CALL
833    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
834    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
835
836    #   #RSYNC WITH NFS USE
837    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
838    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
839
840    #   status=$?
841    #   IGCM_sys_Rsync_out $status
842
843    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
844    #   (( status=status+$? ))
845
846    #USUAL WAY
847    if [ X${1} = X'/l' ] ; then
848      for target in ${dm_liste[*]} ; do
849        local_file=$( basename ${target} )
850        # test if the target file is present before the loop
851        IGCM_sys_TestFileArchive ${target}
852        status=$?
853        if [ ${status} -gt 0 ] ; then
854          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
855          IGCM_debug_Exit "IGCM_sys_Get"
856        else
857          i=0
858          while [ $i -lt $NB_ESSAI ] ; do
859            #if [ X${DoLink} = Xtrue ] ; then
860            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
861            #  status=$?
862            #  else
863            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
864            #  status=$?
865            #fi
866            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
867            status=$?
868            if [ ${status} -gt 0 ]; then
869              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
870              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
871              sleep $DELAI
872            else
873              break
874            fi
875            (( i = i + 1 ))
876          done
877          if [ ${status} -gt 0 ] ; then
878            echo "IGCM_sys_Get : error"
879            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
880            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
881            IGCM_debug_Exit "IGCM_sys_Get"
882          else
883            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
884          fi
885        fi
886      done
887    else
888      i=0
889      while [ $i -lt $NB_ESSAI ] ; do
890        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
891        status=$?
892        if [ ${status} -gt 0 ]; then
893          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
894          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
895          sleep $DELAI
896        else
897          break
898        fi
899        (( i = i + 1 ))
900      done
901      if [ ${status} -gt 0 ] ; then
902        echo "IGCM_sys_Get : error"
903        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
904        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
905        IGCM_debug_Exit "IGCM_sys_Get"
906      else
907        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
908      fi
909    fi
910  fi
911  IGCM_debug_PopStack "IGCM_sys_Get"
912}
913
914#D-#==================================================
915#D-function IGCM_sys_GetDate_Monitoring
916#D-* Purpose: get the last year for which the monitoring has been computed
917#D-* Examples:
918#D-
919function IGCM_sys_GetDate_Monitoring {
920  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
921  if ( $DEBUG_sys ) ; then
922    echo "IGCM_sys_GetDate_Monitoring :" $@
923  fi
924
925  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
926
927  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
928}
929
930#D-#==================================================
931#D-function IGCM_sys_Dods_Rm
932#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
933#D-* Examples:
934#D-
935function IGCM_sys_Dods_Rm {
936  if ( $DEBUG_sys ) ; then
937    echo "IGCM_sys_Dods_Rm :" $@
938  fi
939  typeset status
940  if [ $DRYRUN = 0 ]; then
941
942#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
943#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
944#      echo "Nothing has been done."
945#      return
946#    fi
947
948    if [ "$#" -eq 1 ]; then
949      $(ccc_home -u igcmg)/Tools/irene-amd/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
950      status=$?
951    else
952      $(ccc_home -u igcmg)/Tools/irene-amd/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
953      status=$?
954    fi
955
956#    if [ ${status} -gt 0 ] ; then
957#      echo "IGCM_sys_Dods_Rm : error."
958#      cat out_dods_rm
959#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
960#    else
961#      rm out_dods_rm
962#    fi
963
964  fi
965  return $status
966}
967
968#D-#==================================================
969#D-function IGCM_sys_Dods_Cp
970#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
971#D-* Examples:
972#D-
973function IGCM_sys_Dods_Cp {
974  if ( $DEBUG_sys ) ; then
975    echo "IGCM_sys_Dods_Cp :" $@
976  fi
977  typeset status
978  if [ $DRYRUN = 0 ]; then
979
980#    if [ ! -d ${R_SAVE}/${1} ] ; then
981#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
982#      echo "Nothing has been done."
983#      return
984#    fi
985
986    $(ccc_home -u igcmg)/Tools/irene-amd/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
987    status=$?
988
989#       if [ ${status} -gt 0 ] ; then
990#           echo "IGCM_sys_Dods_Cp : error."
991#           cat out_dods_cp
992#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
993#       else
994#           rm out_dods_cp
995#       fi
996
997  fi
998  return $status
999}
1000
1001#D-#==================================================
1002#D-function IGCM_sys_Put_Dods
1003#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1004#D-* Examples:
1005#D-
1006function IGCM_sys_Put_Dods {
1007  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1008  if ( $DEBUG_sys ) ; then
1009    echo "IGCM_sys_Put_Dods :" $@
1010  fi
1011  typeset status
1012  if [ $DRYRUN = 0 ]; then
1013    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1014      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1015      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1016      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1017      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1018      return
1019    fi
1020
1021    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1022      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1023      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1024      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1025      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1026      return
1027    fi
1028    #
1029    if [ -d ${R_SAVE}/${1} ] ; then
1030      cd ${R_SAVE}
1031    elif [ -d ${R_FIGR}/${1} ] ; then
1032      cd ${R_FIGR}
1033    fi
1034
1035    IGCM_sys_Dods_Rm ${1}
1036    IGCM_sys_Dods_Cp ${1}
1037    status=0
1038
1039    if [ ${status} -gt 0 ] ; then
1040      echo "IGCM_sys_Put_Dods : error."
1041      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1042    fi
1043  fi
1044  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1045}
1046
1047##############################################################
1048# REBUILD OPERATOR
1049
1050#D-#==================================================
1051#D-function IGCM_sys_sync
1052#D-* Purpose: flush buffer on disk
1053#D-* Examples:
1054#D-
1055function IGCM_sys_sync {
1056  IGCM_debug_PushStack "IGCM_sys_sync" $@
1057  if ( $DEBUG_sys ) ; then
1058    echo "IGCM_sys_sync :" $@
1059  fi
1060
1061  /bin/sync
1062
1063  IGCM_debug_PopStack "IGCM_sys_sync"
1064}
1065
1066############################################################
1067# Activate Running Environnment Variables
1068
1069#D-#==================================================
1070#D-function IGCM_sys_activ_variables
1071#D-* Purpose: set environement variables prior to execution
1072#D-* Examples:
1073#D-
1074function IGCM_sys_activ_variables {
1075  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1076  if ( $DEBUG_sys ) ; then
1077    echo "IGCM_sys_activ_variables"
1078  fi
1079
1080# --------------------------------------------------------------------
1081#D- MODULE specifications
1082# --------------------------------------------------------------------
1083  # Source the file EnvFile containing all module load needed to run the model.
1084  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1085  # it will be used. Otherwise a default file will be used.
1086  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1087      # EnvFile is set in config.card and the file exists
1088      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1089      EnvFile=${config_UserChoices_EnvFile}
1090  else
1091      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1092      module purge > /dev/null 2>&1
1093      if [ X${compilerVersion} = XV19 ] ; then
1094          IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V19"
1095          EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
1096      else
1097          IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1098      fi
1099  fi
1100  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1101  IGCM_debug_Print 1 ${EnvFile}
1102  . ${EnvFile}
1103
1104  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene-amd. "
1105  module list
1106
1107# --------------------------------------------------------------------
1108#D- MPI specifications
1109# --------------------------------------------------------------------
1110
1111# --------------------------------------------------------------------
1112#D- Other specifications
1113# --------------------------------------------------------------------
1114
1115  ulimit -s unlimited
1116
1117  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1118}
1119
1120############################################################
1121# Desactivate Running Environnment Variables
1122
1123#D-#==================================================
1124#D-function IGCM_sys_desactiv_variables
1125#D-* Purpose: unset environement variables after execution
1126#D-* Examples:
1127#D-
1128function IGCM_sys_desactiv_variables {
1129  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1130  if ( $DEBUG_sys ) ; then
1131    echo "IGCM_sys_desactiv_variables"
1132  fi
1133# --------------------------------------------------------------------
1134#D- MPI specifications
1135# --------------------------------------------------------------------
1136
1137# --------------------------------------------------------------------
1138#D- Other specifications
1139# --------------------------------------------------------------------
1140
1141  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1142}
1143
1144############################################################
1145# Update job headers to be used by the scheduler
1146
1147#D-#==================================================
1148#D-function IGCM_sys_updateHeaders
1149#D-* Purpose: Update job headers to be used by the scheduler
1150#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1151#D-
1152function IGCM_sys_updateHeaders {
1153  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1154  if ( $DEBUG_sys ) ; then
1155    echo "IGCM_sys_updateHeaders"
1156  fi
1157  typeset file
1158  file=$1
1159
1160   # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1161   # Compute the number of core to add to force the use of a new node for xios servers
1162
1163   if [ "X${IOS_OK_DEDICATED}" == "X" ] ; then
1164       (( IOS_OK_DEDICATED = 0 ))
1165   fi
1166
1167
1168   if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1169      (( current_core_noxios = coreNumber - IOS_PROC_MPI * IOS_PROC_OMP * IOS_PROC_DEP ))
1170      (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1171      (( second_comp_proc_mpi_loc = IOS_PROC_MPI - 1 ))
1172      (( coreNumber = current_core_noxios - 1 + first_comp_proc_dep_loc * 2 + second_comp_proc_mpi_loc * IOS_PROC_OMP * IOS_PROC_DEP ))
1173   fi
1174
1175  if [ ${executionType} -eq 1 ] ; then
1176    # MPMD + MPI
1177    sed -e "/::openMPthreads::/d"                  \
1178        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1179      ${file} > ${file}.tmp
1180
1181  elif [ ${executionType} -eq 2 ] ; then
1182    # MPMD + MPI + OMP :ccc_mprun
1183    sed -e "/::openMPthreads::/d"                  \
1184        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1185      ${file} > ${file}.tmp
1186
1187  elif [ ${executionType} -eq 3 ] ; then
1188    # SPMD + MPI/OMP
1189    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1190        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1191        -e "/#MSUB -x/d"                           \
1192      ${file} > ${file}.tmp
1193
1194  elif [ ${executionType} -eq 4 ] ; then
1195    # SPMD + MPI only
1196    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1197        -e "/::openMPthreads::/d"                  \
1198        -e "/#MSUB -x/d"                           \
1199      ${file} > ${file}.tmp
1200
1201  elif [ ${executionType} -eq 5 ] ; then
1202    # SPMD + OMP only
1203    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1204        -e "/::JobNumProcTot::/d"                  \
1205        -e "/#MSUB -x/d"                           \
1206      ${file} > ${file}.tmp
1207
1208  elif [ ${executionType} -eq 6 ] ; then
1209    # SEQUENTIAL THEN
1210    sed -e "s/::JobNumProcTot::/1/"                \
1211        -e "/::openMPthreads::/d"                  \
1212        -e "/#MSUB -x/d"                           \
1213      ${file} > ${file}.tmp
1214
1215  fi
1216
1217  IGCM_sys_Mv ${file}.tmp ${file}
1218
1219  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1220}
1221
1222############################################################
1223# Build MPI/OMP scripts run file (dummy function)
1224
1225#D-#==================================================
1226#D-function IGCM_sys_build_run_file
1227#D-* Purpose: build run file (deprecated)
1228#D-* Examples:
1229#D-
1230function IGCM_sys_build_run_file {
1231  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1232}
1233
1234############################################################
1235# Build MPI/OMP scripts
1236
1237#D-#==================================================
1238#D-function IGCM_sys_build_execution_scripts
1239#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1240#D-* Examples:
1241#D-
1242function IGCM_sys_build_execution_scripts
1243{
1244  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1245  if ( $DEBUG_sys ) ; then
1246    echo "IGCM_sys_build_execution_scripts " $@
1247  fi
1248
1249  EXECUTION=${HOST_MPIRUN_COMMAND}
1250
1251  if ( ${OK_PARA_MPMD} ) ; then
1252
1253    # MPMD mode
1254    # 1 MPI only
1255    # 2 MPI/OpenMP mpirun method
1256    # 3 MPI/OpenMP ccc_mprun method
1257
1258    if [ -f run_file ] ; then
1259      IGCM_sys_Rm -f run_file
1260    fi
1261    touch run_file
1262
1263    # case 1 : Only MPI (MPMD)
1264    if  ( ! ${OK_PARA_OMP} ) ; then
1265
1266      # Build run_file
1267
1268      # First loop on the components for the coupler ie oasis (only if oasis3)
1269      # the coupler ie oasis3 must be the first one
1270      for comp in ${config_ListOfComponents[*]} ; do
1271
1272        eval ExeNameIn=\${config_Executable_${comp}[0]}
1273        eval ExeNameOut=\${config_Executable_${comp}[1]}
1274
1275        # for CPL component only
1276        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1277
1278          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1279          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1280          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1281        fi
1282      done
1283
1284      # Then second loop on the components
1285      for comp in ${config_ListOfComponents[*]} ; do
1286
1287        eval ExeNameIn=\${config_Executable_${comp}[0]}
1288        eval ExeNameOut=\${config_Executable_${comp}[1]}
1289
1290        # Only if we really have an executable for the component and not the coupler ie oasis:
1291        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1292
1293          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1294          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1295          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1296
1297          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1298          # Compute the number of core to add to force the use of a new node for xios servers
1299         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1300            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1301              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_dep_loc ))
1302              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1303              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1304              if [ ${comp_proc_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1305                 echo "1-${first_comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1306                 if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1307                    echo "${second_comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1308                 fi
1309              else
1310                 echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1311              fi
1312           else
1313              echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1314           fi
1315         else
1316          echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1317         fi
1318
1319
1320        fi
1321      done
1322
1323      ## module advised by TGCC (instead of 2 variables)
1324      module load feature/bridge/heterogenous_mpmd
1325
1326      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1327
1328      IGCM_sys_Chmod u+x run_file
1329      if ( $DEBUG_sys ) ; then
1330        echo "run_file contains : "
1331        cat run_file
1332      fi
1333
1334    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1335
1336    # 2 MPI/OpenMP mpirun method
1337
1338      # Use of mpirun instead of ccc_mprun
1339      EXECUTION="time mpirun"
1340
1341      #  Hosts treatment
1342      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1343
1344      i=0
1345      rm -f hosts rankfile
1346      IGCM_debug_Print 1 "sys Irene-amd, Hosts available :"
1347      for nodes in `cat hosts.tmp`
1348      do
1349        host[$i]=$nodes
1350        echo "${host[$i]}" >> hosts
1351        IGCM_debug_Print 1 ${host[$i]}
1352        i=$((i+1))
1353      done
1354      rm -f hosts.tmp
1355
1356      listnodes=${host[*]}
1357
1358      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1359
1360      # Initialisation
1361      rank=0
1362      current_core=0
1363      init_exec=n
1364
1365      # Loop on the components
1366      for comp in ${config_ListOfComponents[*]} ; do
1367
1368        eval ExeNameIn=\${config_Executable_${comp}[0]}
1369        eval ExeNameOut=\${config_Executable_${comp}[1]}
1370
1371        # Not possible if oasis has an executable (i.e old version of oasis3)
1372        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1373          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1374          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1375          IGCM_debug_Verif_Exit
1376        fi
1377
1378        # Only if we really have an executable for the component :
1379        if [ "X${ExeNameOut}" != X\"\" ] ; then
1380
1381          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1382          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1383          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1384
1385          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1386          echo ""  >> script_${ExeNameOut}.ksh
1387          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1388
1389            # Check if the number of threads is correct
1390            case ${comp_proc_omp_loc} in
1391            2|4|8|16)
1392              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1393              ;;
1394            *)
1395              IGCM_debug_Exit "ERROR with OMP parameters !"
1396              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1397              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1398              IGCM_debug_Verif_Exit
1399              ;;
1400            esac
1401            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1402            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1403            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1404            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1405          fi
1406
1407          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1408          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1409          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1410          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1411          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1412
1413          if [ ${init_exec} = y ] ; then
1414            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1415          else
1416            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1417            init_exec=y
1418          fi
1419
1420          # Build rankfile : method used to assign cores and nodes for the MPI process
1421          # Ex :
1422          #rank 0=curie5296 slot=0,1,2,3
1423          #rank 1=curie5296 slot=4,5,6,7
1424          # Example of final command :
1425          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1426          # with script_lmdz.x.ksh :
1427          # #!/bin/ksh
1428          #export KMP_STACKSIZE=3g
1429          #export KMP_LIBRARY=turnaround
1430          #export MKL_SERIAL=YES
1431          #OMP_NUM_THREADS=4
1432          #./lmdz.x
1433          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1434              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1435          else
1436              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1437          fi
1438          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1439             (( index_host = current_core / NB_CORE_PER_NODE ))
1440             host_value=${host[${index_host}]}
1441             (( slot =  current_core % NB_CORE_PER_NODE ))
1442             virg=","
1443             string_final=""
1444             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1445                 string=$index$virg
1446                 string_final=$string_final$string
1447             done
1448             string_final=$( echo $string_final | sed "s/.$//" )
1449             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1450             (( rank = rank + 1 ))
1451             (( current_core = current_core + offset_comp_proc_loc ))
1452          done
1453        fi
1454      done
1455
1456    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1457
1458    # 3 MPI/OpenMP ccc_mprun method
1459
1460    # MPI-OpenMP (MPMD)
1461    # example of run_file
1462    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1463    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1464    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1465
1466      # Loop on the components
1467      for comp in ${config_ListOfComponents[*]} ; do
1468
1469        eval ExeNameIn=\${config_Executable_${comp}[0]}
1470        eval ExeNameOut=\${config_Executable_${comp}[1]}
1471
1472        # Not possible if oasis has an executable (i.e old version of oasis3)
1473        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1474          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1475          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1476          IGCM_debug_Verif_Exit
1477        fi
1478
1479        # Only if we really have an executable for the component :
1480        if [ "X${ExeNameOut}" != X\"\" ] ; then
1481
1482          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1483          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1484          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1485
1486          # Compute number of cores per process
1487          (( comp_proc_omp_dep_loc = comp_proc_omp_loc * comp_proc_dep_loc )) 
1488
1489          # Build script files
1490
1491          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1492          echo ""  >> script_${ExeNameOut}.ksh
1493          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1494
1495            # Check if the number of threads is correct
1496            case ${comp_proc_omp_loc} in
1497            2|4|8|16)
1498              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1499              ;;
1500            *)
1501              IGCM_debug_Exit "ERROR with OMP parameters !"
1502              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1503              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1504              IGCM_debug_Verif_Exit
1505              ;;
1506            esac
1507            ### only for LMDZ?
1508            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1509            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1510            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1511            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1512
1513          fi
1514
1515          # to have out/err per process on different files
1516          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1517          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1518
1519          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1520
1521          # Complete run_file
1522
1523          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1524          # Compute the number of core to add to force the use of a new node for xios servers
1525         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1526            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1527              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_omp_dep_loc )) 
1528              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1529              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1530              if [ ${comp_proc_omp_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1531                 echo "1-${first_comp_proc_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1532                  if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1533                    echo "${second_comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1534                  fi
1535              else
1536                 echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1537              fi
1538           else
1539              echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1540           fi
1541         else
1542          echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1543         fi
1544
1545        fi
1546      done
1547
1548      ## variable added to stop after 60s instead of 600s by default.
1549      ## This is used when no error comes from executables and when something stopped an executable without notice.
1550      export SLURM_WAIT=60
1551
1552      ## module advised by TGCC (instead of 2 variables)
1553      module load feature/bridge/heterogenous_mpmd
1554
1555      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1556
1557      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1558
1559      IGCM_sys_Chmod u+x run_file
1560      if ( $DEBUG_sys ) ; then
1561        echo "run_file contains : "
1562        cat run_file
1563      fi
1564
1565    else
1566
1567      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1568
1569    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1570
1571  else
1572  # Only one executable (SPMD mode).
1573
1574    for comp in ${config_ListOfComponents[*]} ; do
1575
1576      # Only if we really have an executable for the component :
1577      eval ExeNameOut=\${config_Executable_${comp}[1]}
1578      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1579
1580        # Build script files
1581
1582        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1583        echo ""  >> script_${ExeNameOut}.ksh
1584        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1585
1586        if ( ${OK_PARA_OMP} ) ; then
1587          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1588          echo ""  >> script_${ExeNameOut}.ksh
1589          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1590          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1591          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1592          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1593        fi
1594
1595        if  ( ${OK_PARA_MPI} ) ; then
1596          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1597          # Default : ccc_mprun used if nb_proc gt 1
1598          # to have out/err per process on different files
1599          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1600          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1601          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1602        else
1603          # Default : ccc_mprun is NOT used if nb_proc eq 1
1604          # to have out/err per process on different files
1605          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1606          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1607          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1608        fi
1609
1610        IGCM_debug_Print 1 "sys Irene-amd : script_${ExeNameOut}.ksh contains"
1611        cat script_${ExeNameOut}.ksh
1612
1613      fi
1614
1615    done
1616
1617  fi # ${OK_PARA_MPMD}
1618
1619  IGCM_debug_Print 1 "sys Irene-amd : execution command is "
1620  IGCM_debug_Print 1 "$EXECUTION"
1621
1622  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1623}
1624
1625#D-#==================================================
1626#D-function IGCM_sys_check_path
1627#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1628#D-* do not point to an important use directory. Stop immediately in that case.
1629#D-* Examples:
1630#D-
1631function IGCM_sys_check_path {
1632  IGCM_debug_PushStack "IGCM_sys_check_path"
1633  if ( $DEBUG_sys ) ; then
1634    echo "IGCM_sys_check_path"
1635  fi
1636
1637  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1638    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1639    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1640    IGCM_debug_Exit "This will stop the job"
1641  fi
1642  IGCM_debug_PopStack "IGCM_sys_check_path"
1643}
1644
1645#D-#==================================================
1646#D-function IGCM_sys_check_quota
1647#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1648#D-* Examples:
1649#D-
1650function IGCM_sys_check_quota {
1651  IGCM_debug_PushStack "IGCM_sys_check_quota"
1652  if ( $DEBUG_sys ) ; then
1653    echo "IGCM_sys_check_quota"
1654  fi
1655  # Limit of quota (in %)
1656  limit_quota=90
1657
1658  # Check of the volume
1659  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1660  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1661
1662  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1663
1664    unit_avail=${volume_avail: -1}
1665    unit_quota=${volume_quota: -1}
1666
1667    if [ "${unit_quota}" = "*" ] ; then
1668      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1669      IGCM_debug_Print 1 "More than 100% of your quota is used"
1670      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1671      IGCM_debug_Print 1 "You must have more than 10% available to run"
1672      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1673      IGCM_debug_Verif_Exit
1674    fi
1675
1676    temp_avail=${volume_avail%%${volume_avail: -1}*}
1677    temp_quota=${volume_quota%%${volume_quota: -1}*}
1678
1679    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1680
1681    # Convertion
1682      if [ ${volume_avail: -1} = "T" ] ; then
1683        (( temp_avail = temp_avail * 1000000000000 ))
1684      elif [ ${volume_avail: -1} = "G" ] ; then
1685        (( temp_avail = temp_avail * 1000000000 ))
1686      elif [ ${volume_avail: -1} = "M" ] ; then
1687        (( temp_avail = temp_avail * 1000000 ))
1688      elif [ ${volume_avail: -1} = "k" ] ; then
1689        (( temp_avail = temp_avail * 1000 ))
1690      else
1691        (( temp_avail = volume_avail ))
1692      fi
1693      if [ ${volume_quota: -1} = "T" ] ; then
1694        (( temp_quota = temp_quota * 1000000000000 ))
1695      elif [ ${volume_quota: -1} = "G" ] ; then
1696        (( temp_quota = temp_quota * 1000000000 ))
1697      elif [ ${volume_quota: -1} = "M" ] ; then
1698        (( temp_quota = temp_quota * 1000000 ))
1699      elif [ ${volume_quota: -1} = "k" ] ; then
1700        (( temp_quota = temp_quota * 1000 ))
1701      else
1702        (( temp_quota = volume_quota ))
1703      fi
1704    fi
1705
1706    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1707#    echo "volume ratio is " $quota_volume
1708
1709    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1710      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1711      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1712      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1713      IGCM_debug_Print 1 "You must have more than 10% available to run"
1714      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1715      IGCM_debug_Verif_Exit
1716    fi
1717
1718  fi
1719
1720# Check of the number of inodes
1721
1722  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1723  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1724
1725  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1726
1727    unit_avail=${inode_avail: -1}
1728    unit_quota=${inode_quota: -1}
1729
1730    if [ "${unit_quota}" = "*" ] ; then
1731      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1732      IGCM_debug_Print 1 "More than 100% of your quota is used"
1733      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1734      IGCM_debug_Print 1 "You must have more than 10% available to run"
1735      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1736      IGCM_debug_Verif_Exit
1737    fi
1738
1739    temp_avail=${inode_avail%%${inode_avail: -1}*}
1740    temp_quota=${inode_quota%%${inode_quota: -1}*}
1741
1742    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1743
1744    # Convertion
1745      if [ ${inode_avail: -1} = "T" ] ; then
1746        (( temp_avail = temp_avail * 1000000000000 ))
1747      elif [ ${inode_avail: -1} = "G" ] ; then
1748        (( temp_avail = temp_avail * 1000000000 ))
1749      elif [ ${inode_avail: -1} = "M" ] ; then
1750        (( temp_avail = temp_avail * 1000000 ))
1751      elif [ ${inode_avail: -1} = "k" ] ; then
1752        (( temp_avail = temp_avail * 1000 ))
1753      else
1754        (( temp_avail = inode_avail ))
1755      fi
1756
1757      if [ ${inode_quota: -1} = "T" ] ; then
1758        (( temp_quota = temp_quota * 1000000000000 ))
1759      elif [ ${inode_quota: -1} = "G" ] ; then
1760        (( temp_quota = temp_quota * 1000000000 ))
1761      elif [ ${inode_quota: -1} = "M" ] ; then
1762        (( temp_quota = temp_quota * 1000000 ))
1763      elif [ ${inode_quota: -1} = "k" ] ; then
1764        (( temp_quota = temp_quota * 1000 ))
1765      else
1766        (( temp_quota = inode_quota ))
1767      fi
1768    fi
1769    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1770#    echo "inode ratio is " $quota_inode
1771
1772    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1773      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1774      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1775      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1776      IGCM_debug_Print 1 "You must have more than 10% available to run"
1777      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1778      IGCM_debug_Verif_Exit
1779    fi
1780  fi
1781  IGCM_debug_PopStack "IGCM_sys_check_quota"
1782}
1783
1784#D-#==================================================
1785#D-function IGCM_sys_projectAccounting
1786#D-* Purpose: store project accounting information in a file
1787#D-* Examples:
1788#D-
1789function IGCM_sys_projectAccounting {
1790  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1791  if ( $DEBUG_sys ) ; then
1792    echo "IGCM_sys_check_quota"
1793  fi
1794
1795  ssh irene194 /usr/bin/ccc_myproject > $1
1796
1797  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1798}
1799
1800#D-#==================================================
1801#D-function IGCM_sys_getJobSchedulerID
1802#D-* Purpose: Get the job ID during execution
1803#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1804#D-
1805function IGCM_sys_getJobSchedulerID {
1806  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1807  if ( $DEBUG_sys ) ; then
1808    echo "IGCM_sys_getJobSchedulerID"
1809  fi
1810  eval ${1}=${BRIDGE_MSUB_JOBID}
1811
1812  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1813}
1814
1815#D-#==================================================
1816#D-function IGCM_sys_GetJobID
1817#D-* Purpose: Get the job ID from the JobName
1818#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1819#D-
1820function IGCM_sys_GetJobID {
1821  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1822  if ( $DEBUG_sys ) ; then
1823    echo "IGCM_sys_GetJobID"
1824  fi
1825
1826  # With -f option, the full job name is given in the last column
1827  ID=$( ccc_mstat -f -u $2 | \
1828        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1829
1830  eval ${3}=${ID}
1831  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1832}
1833
1834#D-#==================================================
1835#D-function IGCM_sys_CountJobInQueue
1836#D-* Purpose: Count number of users job
1837#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1838#D-
1839function IGCM_sys_CountJobInQueue {
1840  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1841  if ( $DEBUG_sys ) ; then
1842    echo "IGCM_sys_CountJobInQueue"
1843  fi
1844
1845  # With -f option, the full job name is given in the last column
1846  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1847
1848  eval ${2}=${NbRun}
1849
1850  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1851}
1852
1853#D-#==================================================
1854#D-function IGCM_sys_ListJobInQueue
1855#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1856#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1857#D-
1858function IGCM_sys_ListJobInQueue {
1859  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1860  if ( $DEBUG_sys ) ; then
1861    echo "IGCM_sys_ListJobInQueue"
1862  fi
1863
1864  # With -f option, the full job name is given in the last column
1865  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1866                                        '( $2  == User      && \
1867                                           $NF != /TS/      && \
1868                                           $NF !~ /PACK/    && \
1869                                           $NF !~ /REBUILD/ && \
1870                                           $NF !~ /pack/ )     \
1871                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1872
1873  eval set -A ${2} ${JobList[*]}
1874
1875  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1876}
1877
1878#D-#==================================================
1879#D-function IGCM_sys_atlas
1880#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1881#D-* Examples:
1882#D-
1883function IGCM_sys_atlas {
1884  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1885  if ( $DEBUG_sys ) ; then
1886    echo "IGCM_sys_atlas :" $@
1887  fi
1888
1889  typeset status
1890
1891  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1892  status=$?
1893  if [ ${status} -gt 0 ] ; then
1894    echo "IGCM_sys_atlas : error code ${status}"
1895    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1896    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1897    IGCM_debug_PopStack "IGCM_sys_atlas"
1898    return 1
1899  else
1900    IGCM_debug_PopStack "IGCM_sys_atlas"
1901    return 0
1902  fi
1903
1904  IGCM_debug_PopStack "IGCM_sys_atlas"
1905}
1906
1907#D-#==================================================
1908#D-function IGCM_sys_rebuild_nemo
1909#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1910#D-* Examples:
1911#D-
1912
1913function IGCM_sys_rebuild_nemo {
1914  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1915  if ( $DEBUG_sys ) ; then
1916    echo "IGCM_sys_rebuild_nemo :" $@
1917  fi
1918
1919  $( ccc_home -u igcmg)/Tools/irene-amd/bin/rebuild_nemo ${1} ${2}
1920
1921  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1922}
1923
Note: See TracBrowser for help on using the repository browser.