source: CONFIG_DEVT/IPSLCM6.2.2_ENSEMBLES/libIGCM/libIGCM_sys/libIGCM_sys_irene.ksh @ 6204

Last change on this file since 6204 was 6204, checked in by aclsce, 2 years ago

First import of IPSLCM6.2.2_work_ENSEMBLES working configuration

File size: 63.7 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision:: 1512                                     $ Revision of last commit
7# $Author:: acosce                                     $ Author of last commit
8# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=irene
71# add default project on irene
72typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=devcmip6} | cut -d@ -f1 )
73# jobWarningDelay in seconds
74typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=ccc_msub}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94if [ X${TaskType} = Xcomputing ] ; then
95  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
96else
97  module purge > /dev/null 2>&1
98  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
99  . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
100  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
101  . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
102
103  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
104  export UVCDAT_ANONYMOUS_LOG=FALSE
105fi
106
107# FYI
108[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
109[ ! X${TaskType} = Xchecking ] && module list
110
111# For AddNoise and AddPertu3DOCE
112##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/curie/bin
113export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
114
115# For rebuild
116##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
117export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
118
119#====================================================
120# Host specific DIRECTORIES
121#====================================================
122
123# ============ CESIUM START ============ #
124
125#====================================================
126#- Mirror libIGCM from titane to cesium if needed
127#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
128#if [ ! ${ROOTSYS} = "home" ] ; then
129#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
130#else
131#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
132#fi
133
134#====================================================
135#- libIGCM_POST
136#if ( ${MirrorlibIGCM} ) ; then
137#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
138#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
139#else
140#  typeset -r libIGCM_POST=${libIGCM}
141#fi
142
143# ============ CESIUM  END  ============ #
144
145#====================================================
146#- MirrorlibIGCM for frontend
147typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
148
149#====================================================
150#- libIGCM_POST for frontend
151typeset -r libIGCM_POST=${libIGCM}
152
153#====================================================
154#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
155typeset -r R_EXE="${MODIPSL}/bin"
156
157#====================================================
158#- SUBMIT_DIR : submission dir
159typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
160
161#====================================================
162#- IN
163typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
164
165#====================================================
166#- RUN_DIR_PATH : Temporary working directory (=> TMP)
167typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
168
169#====================================================
170#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
171typeset -r OUTCOMMAND_PATH=/tmp
172
173#====================================================
174#- HOST_MPIRUN_COMMAND
175typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
176
177#====================================================
178#- Max number of arguments passed to nco operator or demigration command
179UNIX_MAX_LIMIT=360
180
181#====================================================
182#- set PackDefault to true on irene
183PackDefault=true
184
185#====================================================
186#- Default number of MPI task for IPSL coupled model
187#- required for backward compatibility
188#-
189DEFAULT_NUM_PROC_OCE=5
190DEFAULT_NUM_PROC_CPL=1
191(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
192
193#====================================================
194#- Number of computing cores per node
195#-
196NB_CORE_PER_NODE=48
197
198#D-#==================================================
199#D-function IGCM_sys_defineArchives
200#D-* Purpose:
201#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
202#D-* Define ARCHIVE : Dedicated to large files
203#D-* Define STORAGE : Dedicated to small/medium files
204#D-* Define R_OUT   : Output tree located on ARCHIVE
205#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
206#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
207#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
208#D-* Examples:
209#D-
210function IGCM_sys_defineArchives {
211  IGCM_debug_PushStack "IGCM_sys_defineArchives"
212
213  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
214  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then
215    # Default option: Change the dfldatadir according to the project used for submission
216    # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
217    if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then
218      module switch dfldatadir dfldatadir/gencmip6
219      IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
220    else
221      module switch dfldatadir dfldatadir/${PROJECT}
222      IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
223    fi
224  else
225    # Use the project set by the variable DataProject in section UserChoices in config.card
226    module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
227    IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
228  fi
229
230 #====================================================
231 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
232 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
233
234  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
235    #====================================================
236    #- ARCHIVE (dedicated to large files)
237    ARCHIVE=${config_UserChoices_ARCHIVE}
238  else
239    #====================================================
240    #- ARCHIVE (dedicated to large files)
241    ARCHIVE=${CCCSTOREDIR}
242  fi
243
244  if [ ! X${config_UserChoices_STORAGE} = X ]; then
245    #====================================================
246    #- STORAGE (dedicated to small/medium files)
247    STORAGE=${config_UserChoices_STORAGE}
248  else
249    #====================================================
250    #- STORAGE (dedicated to small/medium files)
251    STORAGE=${CCCWORKDIR}
252  fi
253
254  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
255    #====================================================
256    #- R_OUT
257    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
258
259    #====================================================
260    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
261    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
262
263    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
264
265  else
266    #====================================================
267    #- R_OUT
268    R_OUT=${ARCHIVE}/IGCM_OUT
269
270    #====================================================
271    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
272    R_FIG=${STORAGE}/IGCM_OUT
273  fi
274
275  #====================================================
276  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
277  CMIP6_BUF=${STORAGE}/IGCM_OUT
278
279  #====================================================
280  #- R_BUF (ONLY FOR double copy an scratch)
281  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
282
283  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
284  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
285  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
286
287  IGCM_debug_PopStack "IGCM_sys_defineArchives"
288}
289
290#D-#==================================================
291#D-function IGCM_sys_RshArchive
292#D-* Purpose: Archive rsh command
293#D-* Examples:
294#D-
295function IGCM_sys_RshArchive {
296  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
297  /bin/ksh <<-EOF
298    ${@}
299EOF
300  status=$?
301  if [ ${status} -gt 0 ] ; then
302    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
303    IGCM_debug_Exit "IGCM_sys_RshArchive"
304  fi
305  IGCM_debug_PopStack "IGCM_sys_RshArchive"
306}
307
308#D-#==================================================
309#D-function IGCM_sys_RshArchive_NoError
310#D-* Purpose: Archive rsh command, without error
311#D-*          used only in monitoring.job
312#D-* Examples:
313#D-
314function IGCM_sys_RshArchive_NoError {
315  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
316  /bin/ksh <<-EOF
317    ${@} 2> /dev/null
318EOF
319  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
320}
321
322#D-#==================================================
323#D-function IGCM_sys_MkdirArchive
324#D-* Purpose: Mkdir on Archive
325#D-* Examples:
326#D-
327function IGCM_sys_MkdirArchive {
328  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
329  if ( $DEBUG_sys ) ; then
330    echo "IGCM_sys_MkdirArchive :" $@
331  fi
332  #- creation de repertoire sur le serveur fichier
333  if [ ! -d ${1} ]; then
334    \mkdir -p $1
335    status=$?
336
337    if [ ${status} -gt 0 ] ; then
338      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
339      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
340    fi
341  fi
342  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
343}
344
345#D-#==================================================
346#D-function IGCM_sys_TestDirArchive
347#D-* Purpose: Test Directory that must exists on Archive
348#D-* Examples:
349#D-
350function IGCM_sys_TestDirArchive {
351  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
352  if ( $DEBUG_sys ) ; then
353    echo "IGCM_sys_TestDirArchive :" $@
354  fi
355  typeset ExistFlag
356  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
357  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
358  return ${ExistFlag}
359}
360
361#D-#==================================================
362#D-function IGCM_sys_IsFileArchived
363#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
364#D-* Examples:
365#D-
366function IGCM_sys_IsFileArchived {
367  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
368  if ( $DEBUG_sys ) ; then
369    echo "IGCM_sys_IsFileArchived :" $@
370  fi
371  typeset IsArchivedFlag
372  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
373  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
374
375  return ${IsArchivedFlag}
376}
377
378#D-#==================================================
379#D-function IGCM_sys_TestFileArchive
380#D-* Purpose: Test file that must NOT EXISTS on Archive
381#D-* Examples:
382#D-
383function IGCM_sys_TestFileArchive {
384  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
385  typeset ExistFlag
386  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
387  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
388
389  return ${ExistFlag}
390}
391
392#D-#==================================================
393#D-function IGCM_sys_CountFileArchive
394#D-* Purpose: Count files on Archive filesystem
395#D-* Examples:
396#D-
397function IGCM_sys_CountFileArchive {
398  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
399  ls ${@} 2>/dev/null | wc -l
400  if [ $? -gt 0 ] ; then
401    echo "IGCM_sys_CountFileArchive : erreur."
402  fi
403  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
404}
405
406#D-#==================================================
407#D-function IGCM_sys_Tree
408#D-* Purpose: Tree directories with files on ${ARCHIVE}
409#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
410#D-
411function IGCM_sys_Tree {
412  IGCM_debug_PushStack "IGCM_sys_Tree" $@
413  if ( $DEBUG_sys ) ; then
414    echo "IGCM_sys_Tree :" $@
415  fi
416
417  \ls -lR ${@}
418
419  IGCM_debug_PopStack "IGCM_sys_Tree"
420}
421
422#D-#==================================================
423#D-function IGCM_sys_Qsub
424#D-* Purpose: Qsub new job
425#D-* Examples:
426#D-
427function IGCM_sys_Qsub {
428  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
429  if ( $DEBUG_sys ) ; then
430    echo "IGCM_sys_Qsub :" $@
431  fi
432  typeset options status
433  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
434
435  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
436  status=$?
437
438  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
439  if [ ${status} -gt 0 ] ; then
440    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
441    IGCM_debug_Exit "IGCM_sys_Qsub"
442  else
443    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
444  fi
445  IGCM_debug_PopStack "IGCM_sys_Qsub"
446}
447
448#D-#==================================================
449#D-function IGCM_sys_QsubPost
450#D-* Purpose: Qsub new job on scalaire
451#D-* Examples:
452#D-
453function IGCM_sys_QsubPost {
454  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
455  if ( $DEBUG_sys ) ; then
456    echo "IGCM_sys_QsubPost :" $@
457  fi
458  typeset options status
459  options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
460
461  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
462  status=$?
463
464  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
465  if [ ${status} -gt 0 ] ; then
466    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
467    IGCM_debug_Exit "IGCM_sys_QsubPost"
468  else
469    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
470  fi
471  IGCM_debug_PopStack "IGCM_sys_QsubPost"
472}
473
474#D-*************************
475#D- File transfer functions
476#D-*************************
477#D-
478
479#D-#==================================================
480#D-function IGCM_sys_RmRunDir
481#D-* Purpose: rm tmpdir (dummy function most of the time batch
482#D-                      scheduler will do the job)
483#D-* Examples:
484#D-
485function IGCM_sys_RmRunDir {
486  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
487  if ( $DEBUG_sys ) ; then
488    echo "IGCM_sys_RmRunDir :" $@
489  fi
490
491  typeset status
492
493  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
494  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
495  status=$?
496
497  if [ ${status} -gt 0 ] ; then
498    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
499    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
500    IGCM_debug_Exit "IGCM_sys_RmRunDir"
501  else
502    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
503  fi
504  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
505}
506
507#D-#==================================================
508#D-function IGCM_sys_Put_Dir
509#D-* Purpose: Copy a complete directory on $(ARCHIVE)
510#D-* Examples:
511#D-
512function IGCM_sys_Put_Dir {
513  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
514  if ( $DEBUG_sys ) ; then
515    echo "IGCM_sys_Put_Dir :" $@
516  fi
517  if [ $DRYRUN = 0 ]; then
518    if [ ! -d ${1} ] ; then
519      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
520      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
521      return
522    fi
523
524    typeset status
525
526    # Only if we use rsync
527    #IGCM_sys_TestDirArchive $( dirname $2 )
528    #
529    #USUAL WAY
530    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
531    status=$?
532
533    if [ ${status} -gt 0 ] ; then
534      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
535      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
536      IGCM_debug_Exit "IGCM_sys_Put_Dir"
537    else
538      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
539    fi
540  fi
541  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
542}
543
544#D-#==================================================
545#D-function IGCM_sys_Get_Dir
546#D-* Purpose: Copy a complete directory from ${ARCHIVE}
547#D-* Examples:
548#D-
549function IGCM_sys_Get_Dir {
550  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
551  if ( $DEBUG_sys ) ; then
552    echo "IGCM_sys_Get_Dir :" $@
553  fi
554  if [ $DRYRUN = 0 ]; then
555    typeset NB_ESSAI DELAI status i
556    # number of tentative
557    NB_ESSAI=3
558    # time delay between tentative
559    DELAI=2
560
561    #
562    # USUAL WAY
563    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
564    ccc_hsm get -r $1
565
566    i=0
567    while [ $i -lt $NB_ESSAI ] ; do
568      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
569      status=$?
570      if [ ${status} -gt 0 ] ; then
571        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
572        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
573        sleep $DELAI
574      else
575        break
576      fi
577      (( i = i + 1 ))
578    done
579
580    if [ ${status} -gt 0 ] ; then
581      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
582      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
583      IGCM_debug_Exit "IGCM_sys_Get_Dir"
584    else
585      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
586    fi
587  fi
588  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
589}
590
591#D-#==================================================
592#D-function IGCM_sys_Put_Rest
593#D-* Purpose: Put computied restarts on ${ARCHIVE}.
594#D-           File and target directory must exist.
595#D-* Examples:
596#D-
597function IGCM_sys_Put_Rest {
598  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
599  if ( $DEBUG_sys ) ; then
600    echo "IGCM_sys_Put_Rest :" $@
601  fi
602  if [ $DRYRUN = 0 ]; then
603    if [ ! -f ${1} ] ; then
604      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
605      IGCM_debug_Exit "IGCM_sys_Put_Rest"
606    fi
607
608    typeset status
609    #
610    # USUAL WAY
611    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
612    status=$?
613
614#       #RSYNC WITH NETWORK SSH CALL
615#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
616#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
617
618#       #RSYNC WITH NFS USE
619#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
620#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
621
622#       status=$?
623#       IGCM_sys_Rsync_out $status
624
625#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
626#       (( status=status+$? ))
627
628    if [ ${status} -gt 0 ] ; then
629      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
630      [ -f ${1} ] && ls -l ${1}
631      [ -f ${2} ] && ls -l ${2}
632      [ -f ${2}/${1} ] && ls -l ${2}/${1}
633      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
634      IGCM_debug_Exit "IGCM_sys_Put_Rest"
635    else
636
637      if [ X${JobType} = XRUN ] ; then
638        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
639        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
640      fi
641
642      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
643    fi
644  fi
645  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
646}
647
648#D-#==================================================
649#D-function IGCM_sys_Put_Out
650#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
651#D-* Examples:
652#D-
653function IGCM_sys_Put_Out {
654  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
655  if ( $DEBUG_sys ) ; then
656    echo "IGCM_sys_Put_Out :" $@
657  fi
658
659  typeset NB_ESSAI DELAI status i exist skip
660  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
661
662  # number of tentative
663  NB_ESSAI=3
664  # time delay between tentative
665  DELAI=2
666
667  if [ $DRYRUN = 0 ]; then
668    if [ ! -f ${1} ] ; then
669      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
670      IGCM_debug_PopStack "IGCM_sys_Put_Out"
671      return 1
672    fi
673    #
674    IGCM_sys_MkdirArchive $( dirname $2 )
675    #
676    exist=false
677    skip=false
678    if [ -f $2 ] ; then
679      IGCM_debug_Print 1 "$2 already exist"
680      ccc_hsm get $2
681      exist=true
682      if [ "X$( diff $1 $2 )" = X ] ; then
683        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
684        skip=true
685      else
686        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
687        skip=false
688      fi
689    fi
690    #
691    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
692      IGCM_sys_Chmod u+w $2
693    fi
694
695    if [ X${skip} = Xfalse ] ; then
696      i=0
697      while [ $i -lt $NB_ESSAI ] ; do
698        # Identify file system
699        fileDeviceNumberInHex=$( stat -c %d $1 )
700        status=$?
701        if [ ${status} -gt 0 ] ; then
702          IGCM_debug_Exit "IGCM_sys_Put_Out"
703        fi
704        # Identify file system
705        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
706        status=$?
707        if [ ${status} -gt 0 ] ; then
708          IGCM_debug_Exit "IGCM_sys_Put_Out"
709        fi
710
711        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
712          # They are not on the same device. USUAL WAY
713          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
714          status=$?
715        else
716          # They are on the same device. NOT SO USUAL WAY
717          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
718          status=$?
719        fi
720        if [ ${status} -gt 0 ]; then
721          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
722          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
723          [ -f ${1} ] && ls -l ${1}
724          [ -f ${2} ] && ls -l ${2}
725          [ -f ${2}/${1} ] && ls -l ${2}/${1}
726          sleep $DELAI
727        else
728          break
729        fi
730        (( i = i + 1 ))
731      done
732    fi
733
734#       #RSYNC WITH NETWORK SSH CALL
735#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
736#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
737
738#       #RSYNC WITH NFS USE
739#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
740#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
741
742#       status=$?
743#       IGCM_sys_Rsync_out $status
744
745#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
746#       (( status=status+$? ))
747
748    if [ ${status} -gt 0 ] ; then
749      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
750      [ -f ${1} ] && ls -l ${1}
751      [ -f ${2} ] && ls -l ${2}
752      [ -f ${2}/${1} ] && ls -l ${2}/${1}
753      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
754      IGCM_debug_Exit "IGCM_sys_Put_Out"
755    else
756
757      if [ X${JobType} = XRUN ] ; then
758        if [ X${3} = X ] ; then
759          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
760          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
761        fi
762      fi
763
764      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
765    fi
766  fi
767  IGCM_debug_PopStack "IGCM_sys_Put_Out"
768  return 0
769}
770
771#D-#==================================================
772#D-function IGCM_sys_Get
773#D-* Purpose: Get a file from ${ARCHIVE}
774#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
775#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
776function IGCM_sys_Get {
777  IGCM_debug_PushStack "IGCM_sys_Get" $@
778
779  typeset DEST dm_liste target file_work
780  typeset NB_ESSAI DELAI status i
781
782  if ( $DEBUG_sys ) ; then
783    echo "IGCM_sys_Get :" $@
784  fi
785
786  # number of tentative
787  NB_ESSAI=3
788  # time delay between tentative
789  DELAI=2
790
791  if [ $DRYRUN -le 2 ]; then
792    if [ X${1} = X'/l' ] ; then
793      eval set +A dm_liste \${${2}}
794    else
795      eval set +A dm_liste ${1}
796    fi
797    eval DEST=\${${#}}
798    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
799    status=$?
800    if [ ${status} -gt 0 ] ; then
801      echo "WARNING IGCM_sys_Get : error code ${status}"
802      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
803      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
804    fi
805
806    #if [ ${status} -gt 0 ] ; then
807    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
808    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
809    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
810    #    sleep 30
811    #    echo "We try another time"
812    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
813    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
814    #    status=$?
815    #    if [ ${status} -gt 0 ] ; then
816    #      echo "ERROR IGCM_sys_Get : again demigration error :"
817    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
818    #      IGCM_debug_Exit "IGCM_sys_Get"
819    #    fi
820    #  else
821    #    echo "ERROR IGCM_sys_Get : demigration error :"
822    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
823    #    IGCM_debug_Exit "IGCM_sys_Get"
824    #  fi
825    #fi
826
827    #   #RSYNC WITH NETWORK SSH CALL
828    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
829    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
830
831    #   #RSYNC WITH NFS USE
832    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
833    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
834
835    #   status=$?
836    #   IGCM_sys_Rsync_out $status
837
838    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
839    #   (( status=status+$? ))
840
841    #USUAL WAY
842    if [ X${1} = X'/l' ] ; then
843      for target in ${dm_liste[*]} ; do
844        local_file=$( basename ${target} )
845        # test if the target file is present before the loop
846        IGCM_sys_TestFileArchive ${target}
847        status=$?
848        if [ ${status} -gt 0 ] ; then
849          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
850          IGCM_debug_Exit "IGCM_sys_Get"
851        else
852          i=0
853          while [ $i -lt $NB_ESSAI ] ; do
854            #if [ X${DoLink} = Xtrue ] ; then
855            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
856            #  status=$?
857            #  else
858            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
859            #  status=$?
860            #fi
861            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
862            status=$?
863            if [ ${status} -gt 0 ]; then
864              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
865              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
866              sleep $DELAI
867            else
868              break
869            fi
870            (( i = i + 1 ))
871          done
872          if [ ${status} -gt 0 ] ; then
873            echo "IGCM_sys_Get : error"
874            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
875            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
876            IGCM_debug_Exit "IGCM_sys_Get"
877          else
878            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
879          fi
880        fi
881      done
882    else
883      i=0
884      while [ $i -lt $NB_ESSAI ] ; do
885        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
886        status=$?
887        if [ ${status} -gt 0 ]; then
888          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
889          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
890          sleep $DELAI
891        else
892          break
893        fi
894        (( i = i + 1 ))
895      done
896      if [ ${status} -gt 0 ] ; then
897        echo "IGCM_sys_Get : error"
898        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
899        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
900        IGCM_debug_Exit "IGCM_sys_Get"
901      else
902        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
903      fi
904    fi
905  fi
906  IGCM_debug_PopStack "IGCM_sys_Get"
907}
908
909#D-#==================================================
910#D-function IGCM_sys_GetDate_Monitoring
911#D-* Purpose: get the last year for which the monitoring has been computed
912#D-* Examples:
913#D-
914function IGCM_sys_GetDate_Monitoring {
915  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
916  if ( $DEBUG_sys ) ; then
917    echo "IGCM_sys_GetDate_Monitoring :" $@
918  fi
919
920  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
921
922  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
923}
924
925#D-#==================================================
926#D-function IGCM_sys_Dods_Rm
927#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
928#D-* Examples:
929#D-
930function IGCM_sys_Dods_Rm {
931  if ( $DEBUG_sys ) ; then
932    echo "IGCM_sys_Dods_Rm :" $@
933  fi
934  typeset status
935  if [ $DRYRUN = 0 ]; then
936
937#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
938#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
939#      echo "Nothing has been done."
940#      return
941#    fi
942
943    if [ "$#" -eq 1 ]; then
944      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
945      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
946      status=$?
947    else
948      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
949      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
950      status=$?
951    fi
952
953#    if [ ${status} -gt 0 ] ; then
954#      echo "IGCM_sys_Dods_Rm : error."
955#      cat out_dods_rm
956#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
957#    else
958#      rm out_dods_rm
959#    fi
960
961  fi
962  return $status
963}
964
965#D-#==================================================
966#D-function IGCM_sys_Dods_Cp
967#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
968#D-* Examples:
969#D-
970function IGCM_sys_Dods_Cp {
971  if ( $DEBUG_sys ) ; then
972    echo "IGCM_sys_Dods_Cp :" $@
973  fi
974  typeset status
975  if [ $DRYRUN = 0 ]; then
976
977#    if [ ! -d ${R_SAVE}/${1} ] ; then
978#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
979#      echo "Nothing has been done."
980#      return
981#    fi
982
983    ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/bin/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
984    $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
985    status=$?
986
987#       if [ ${status} -gt 0 ] ; then
988#           echo "IGCM_sys_Dods_Cp : error."
989#           cat out_dods_cp
990#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
991#       else
992#           rm out_dods_cp
993#       fi
994
995  fi
996  return $status
997}
998
999#D-#==================================================
1000#D-function IGCM_sys_Put_Dods
1001#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1002#D-* Examples:
1003#D-
1004function IGCM_sys_Put_Dods {
1005  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1006  if ( $DEBUG_sys ) ; then
1007    echo "IGCM_sys_Put_Dods :" $@
1008  fi
1009  typeset status
1010  if [ $DRYRUN = 0 ]; then
1011    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1012      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1013      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1014      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1015      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1016      return
1017    fi
1018
1019    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1020      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1021      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1022      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1023      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1024      return
1025    fi
1026    #
1027    if [ -d ${R_SAVE}/${1} ] ; then
1028      cd ${R_SAVE}
1029    elif [ -d ${R_FIGR}/${1} ] ; then
1030      cd ${R_FIGR}
1031    fi
1032
1033    IGCM_sys_Dods_Rm ${1}
1034    IGCM_sys_Dods_Cp ${1}
1035    status=0
1036
1037    if [ ${status} -gt 0 ] ; then
1038      echo "IGCM_sys_Put_Dods : error."
1039      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1040    fi
1041  fi
1042  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1043}
1044
1045##############################################################
1046# REBUILD OPERATOR
1047
1048#D-#==================================================
1049#D-function IGCM_sys_sync
1050#D-* Purpose: flush buffer on disk
1051#D-* Examples:
1052#D-
1053function IGCM_sys_sync {
1054  IGCM_debug_PushStack "IGCM_sys_sync" $@
1055  if ( $DEBUG_sys ) ; then
1056    echo "IGCM_sys_sync :" $@
1057  fi
1058
1059  /bin/sync
1060
1061  IGCM_debug_PopStack "IGCM_sys_sync"
1062}
1063
1064############################################################
1065# Activate Running Environnment Variables
1066
1067#D-#==================================================
1068#D-function IGCM_sys_activ_variables
1069#D-* Purpose: set environement variables prior to execution
1070#D-* Examples:
1071#D-
1072function IGCM_sys_activ_variables {
1073  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1074  if ( $DEBUG_sys ) ; then
1075    echo "IGCM_sys_activ_variables"
1076  fi
1077
1078# --------------------------------------------------------------------
1079#D- MODULE specifications
1080# --------------------------------------------------------------------
1081  # Source the file EnvFile containing all module load needed to run the model.
1082  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1083  # it will be used. Otherwise a default file will be used.
1084  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1085      # EnvFile is set in config.card and the file exists
1086      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1087      EnvFile=${config_UserChoices_EnvFile}
1088  else
1089      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1090      module purge > /dev/null 2>&1
1091      if [ X${compilerVersion} = XV17 ] ; then
1092          IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17"
1093          ## EnvFile=/ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1094          EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1095      else
1096          IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1097      fi
1098  fi
1099  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1100  IGCM_debug_Print 1 ${EnvFile}
1101  . ${EnvFile}
1102
1103  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene. "
1104  module list
1105
1106# --------------------------------------------------------------------
1107#D- MPI specifications
1108# --------------------------------------------------------------------
1109
1110# --------------------------------------------------------------------
1111#D- Other specifications
1112# --------------------------------------------------------------------
1113
1114  ulimit -s unlimited
1115
1116  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1117}
1118
1119############################################################
1120# Desactivate Running Environnment Variables
1121
1122#D-#==================================================
1123#D-function IGCM_sys_desactiv_variables
1124#D-* Purpose: unset environement variables after execution
1125#D-* Examples:
1126#D-
1127function IGCM_sys_desactiv_variables {
1128  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1129  if ( $DEBUG_sys ) ; then
1130    echo "IGCM_sys_desactiv_variables"
1131  fi
1132# --------------------------------------------------------------------
1133#D- MPI specifications
1134# --------------------------------------------------------------------
1135
1136# --------------------------------------------------------------------
1137#D- Other specifications
1138# --------------------------------------------------------------------
1139
1140  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1141}
1142
1143############################################################
1144# Update job headers to be used by the scheduler
1145
1146#D-#==================================================
1147#D-function IGCM_sys_updateHeaders
1148#D-* Purpose: Update job headers to be used by the scheduler
1149#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1150#D-
1151function IGCM_sys_updateHeaders {
1152  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1153  if ( $DEBUG_sys ) ; then
1154    echo "IGCM_sys_updateHeaders"
1155  fi
1156  typeset file
1157  file=$1
1158
1159  if [ ${executionType} -eq 1 ] ; then
1160    # MPMD + MPI
1161    sed -e "/::openMPthreads::/d"                  \
1162        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1163        -e "/#MSUB -x/d"                           \
1164        -e "/--cpu_bind=none/d"                    \
1165      ${file} > ${file}.tmp
1166
1167  elif [ ${executionType} -eq 2 ] ; then
1168    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
1169    if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1170      sed -e "/::openMPthreads::/d"                  \
1171          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1172        ${file} > ${file}.tmp
1173    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1174      sed -e "/::openMPthreads::/d"                  \
1175          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1176          -e "/--cpu_bind=none/d"                    \
1177        ${file} > ${file}.tmp
1178    else
1179     IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
1180     IGCM_debug_Exit "IGCM_sys_updateHeaders"
1181    fi
1182  elif [ ${executionType} -eq 3 ] ; then
1183    # SPMD + MPI/OMP
1184    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1185        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1186        -e "/#MSUB -x/d"                           \
1187        -e "/--cpu_bind=none/d"                    \
1188      ${file} > ${file}.tmp
1189
1190  elif [ ${executionType} -eq 4 ] ; then
1191    # SPMD + MPI only
1192    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1193        -e "/::openMPthreads::/d"                  \
1194        -e "/#MSUB -x/d"                           \
1195        -e "/--cpu_bind=none/d"                    \
1196      ${file} > ${file}.tmp
1197
1198  elif [ ${executionType} -eq 5 ] ; then
1199    # SPMD + OMP only
1200    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1201        -e "/::JobNumProcTot::/d"                  \
1202        -e "/#MSUB -x/d"                           \
1203        -e "/--cpu_bind=none/d"                    \
1204      ${file} > ${file}.tmp
1205
1206  elif [ ${executionType} -eq 6 ] ; then
1207    # SEQUENTIAL THEN
1208    sed -e "s/::JobNumProcTot::/1/"                \
1209        -e "/::openMPthreads::/d"                  \
1210        -e "/#MSUB -x/d"                           \
1211        -e "/--cpu_bind=none/d"                    \
1212      ${file} > ${file}.tmp
1213
1214  fi
1215
1216  IGCM_sys_Mv ${file}.tmp ${file}
1217
1218  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1219}
1220
1221############################################################
1222# Build MPI/OMP scripts run file (dummy function)
1223
1224#D-#==================================================
1225#D-function IGCM_sys_build_run_file
1226#D-* Purpose: build run file (deprecated)
1227#D-* Examples:
1228#D-
1229function IGCM_sys_build_run_file {
1230  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1231}
1232
1233############################################################
1234# Build MPI/OMP scripts
1235
1236#D-#==================================================
1237#D-function IGCM_sys_build_execution_scripts
1238#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1239#D-* Examples:
1240#D-
1241function IGCM_sys_build_execution_scripts
1242{
1243  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1244  if ( $DEBUG_sys ) ; then
1245    echo "IGCM_sys_build_execution_scripts " $@
1246  fi
1247
1248  EXECUTION=${HOST_MPIRUN_COMMAND}
1249
1250  if ( ${OK_PARA_MPMD} ) ; then
1251
1252    # MPMD mode
1253    # 1 MPI only
1254    # 2 MPI/OpenMP mpirun method
1255    # 3 MPI/OpenMP ccc_mprun method
1256
1257    if [ -f run_file ] ; then
1258      IGCM_sys_Rm -f run_file
1259    fi
1260    touch run_file
1261
1262    # case 1 : Only MPI (MPMD)
1263    if  ( ! ${OK_PARA_OMP} ) ; then
1264
1265      # Build run_file
1266
1267      # First loop on the components for the coupler ie oasis (only if oasis3)
1268      # the coupler ie oasis3 must be the first one
1269      for comp in ${config_ListOfComponents[*]} ; do
1270
1271        eval ExeNameIn=\${config_Executable_${comp}[0]}
1272        eval ExeNameOut=\${config_Executable_${comp}[1]}
1273
1274        # for CPL component only
1275        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1276
1277          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1278          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1279          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1280        fi
1281      done
1282
1283      # Then second loop on the components
1284      for comp in ${config_ListOfComponents[*]} ; do
1285
1286        eval ExeNameIn=\${config_Executable_${comp}[0]}
1287        eval ExeNameOut=\${config_Executable_${comp}[1]}
1288
1289        # Only if we really have an executable for the component and not the coupler ie oasis:
1290        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1291
1292          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1293          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1294          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1295        fi
1296      done
1297
1298      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1299
1300      IGCM_sys_Chmod u+x run_file
1301      if ( $DEBUG_sys ) ; then
1302        echo "run_file contains : "
1303        cat run_file
1304      fi
1305
1306    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1307
1308    # 2 MPI/OpenMP mpirun method
1309
1310      # Use of mpirun instead of ccc_mprun
1311      EXECUTION="time mpirun"
1312
1313      #  Hosts treatment
1314      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1315
1316      i=0
1317      rm -f hosts rankfile
1318      IGCM_debug_Print 1 "sys Irene, Hosts available :"
1319      for nodes in `cat hosts.tmp`
1320      do
1321        host[$i]=$nodes
1322        echo "${host[$i]}" >> hosts
1323        IGCM_debug_Print 1 ${host[$i]}
1324        i=$((i+1))
1325      done
1326      rm -f hosts.tmp
1327
1328      listnodes=${host[*]}
1329
1330      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1331
1332      # Initialisation
1333      rank=0
1334      current_core=0
1335      init_exec=n
1336
1337      # Loop on the components
1338      for comp in ${config_ListOfComponents[*]} ; do
1339
1340        eval ExeNameIn=\${config_Executable_${comp}[0]}
1341        eval ExeNameOut=\${config_Executable_${comp}[1]}
1342
1343        # Not possible if oasis has an executable (i.e old version of oasis3)
1344        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1345          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1346          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1347          IGCM_debug_Verif_Exit
1348        fi
1349
1350        # Only if we really have an executable for the component :
1351        if [ "X${ExeNameOut}" != X\"\" ] ; then
1352
1353          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1354          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1355          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1356
1357          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1358          echo ""  >> script_${ExeNameOut}.ksh
1359          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1360
1361            # Check if the number of threads is correct
1362            case ${comp_proc_omp_loc} in
1363            2|4|8|16)
1364              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1365              ;;
1366            *)
1367              IGCM_debug_Exit "ERROR with OMP parameters !"
1368              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1369              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1370              IGCM_debug_Verif_Exit
1371              ;;
1372            esac
1373            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1374            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1375            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1376            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1377          fi
1378
1379          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1380          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1381          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1382          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1383          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1384
1385          if [ ${init_exec} = y ] ; then
1386            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1387          else
1388            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1389            init_exec=y
1390          fi
1391
1392          # Build rankfile : method used to assign cores and nodes for the MPI process
1393          # Ex :
1394          #rank 0=curie5296 slot=0,1,2,3
1395          #rank 1=curie5296 slot=4,5,6,7
1396          # Example of final command :
1397          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1398          # with script_lmdz.x.ksh :
1399          # #!/bin/ksh
1400          #export KMP_STACKSIZE=3g
1401          #export KMP_LIBRARY=turnaround
1402          #export MKL_SERIAL=YES
1403          #OMP_NUM_THREADS=4
1404          #./lmdz.x
1405          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1406              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1407          else
1408              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1409          fi
1410          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1411             (( index_host = current_core / NB_CORE_PER_NODE ))
1412             host_value=${host[${index_host}]}
1413             (( slot =  current_core % NB_CORE_PER_NODE ))
1414             virg=","
1415             string_final=""
1416             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1417                 string=$index$virg
1418                 string_final=$string_final$string
1419             done
1420             string_final=$( echo $string_final | sed "s/.$//" )
1421             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1422             (( rank = rank + 1 ))
1423             (( current_core = current_core + offset_comp_proc_loc ))
1424          done
1425        fi
1426      done
1427
1428    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1429
1430    # 3 MPI/OpenMP ccc_mprun method
1431
1432    # MPI-OpenMP (MPMD)
1433    # example of run_file
1434    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1435    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1436    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1437
1438      # Loop on the components
1439      for comp in ${config_ListOfComponents[*]} ; do
1440
1441        eval ExeNameIn=\${config_Executable_${comp}[0]}
1442        eval ExeNameOut=\${config_Executable_${comp}[1]}
1443
1444        # Not possible if oasis has an executable (i.e old version of oasis3)
1445        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1446          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1447          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1448          IGCM_debug_Verif_Exit
1449        fi
1450
1451        # Only if we really have an executable for the component :
1452        if [ "X${ExeNameOut}" != X\"\" ] ; then
1453
1454          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1455          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1456
1457          # Build script files
1458
1459          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1460          echo ""  >> script_${ExeNameOut}.ksh
1461          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1462
1463            # Check if the number of threads is correct
1464            case ${comp_proc_omp_loc} in
1465            2|4|8|12|16|24)
1466              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1467              ;;
1468            *)
1469              IGCM_debug_Exit "ERROR with OMP parameters !"
1470              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1471              IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible "
1472              IGCM_debug_Verif_Exit
1473              ;;
1474            esac
1475            ### only for LMDZ?
1476            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1477            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1478            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1479            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1480
1481          fi
1482
1483          # to have out/err per process on different files
1484          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1485          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1486
1487          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1488
1489          # Complete run_file
1490
1491          echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1492
1493        fi
1494      done
1495
1496      ## variable added to stop after 60s instead of 600s by default.
1497      ## This is used when no error comes from executables and when something stopped an executable without notice.
1498      export SLURM_WAIT=60
1499
1500      ## module advised by TGCC (instead of 2 variables)
1501      module load feature/bridge/heterogenous_mpmd
1502
1503      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1504
1505      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1506
1507      IGCM_sys_Chmod u+x run_file
1508      if ( $DEBUG_sys ) ; then
1509        echo "run_file contains : "
1510        cat run_file
1511      fi
1512
1513    else
1514
1515      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1516
1517    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1518
1519  else
1520  # Only one executable (SPMD mode).
1521
1522    for comp in ${config_ListOfComponents[*]} ; do
1523
1524      # Only if we really have an executable for the component :
1525      eval ExeNameOut=\${config_Executable_${comp}[1]}
1526      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1527
1528        # Build script files
1529
1530        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1531        echo ""  >> script_${ExeNameOut}.ksh
1532        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1533
1534        if ( ${OK_PARA_OMP} ) ; then
1535          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1536          echo ""  >> script_${ExeNameOut}.ksh
1537          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1538          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1539          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1540          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1541        fi
1542
1543        if  ( ${OK_PARA_MPI} ) ; then
1544          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1545          # Default : ccc_mprun used if nb_proc gt 1
1546          # to have out/err per process on different files
1547          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1548          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1549          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1550        else
1551          # Default : ccc_mprun is NOT used if nb_proc eq 1
1552          # to have out/err per process on different files
1553          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1554          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1555          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1556        fi
1557
1558        IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
1559        cat script_${ExeNameOut}.ksh
1560
1561      fi
1562
1563    done
1564
1565  fi # ${OK_PARA_MPMD}
1566
1567  IGCM_debug_Print 1 "sys Irene : execution command is "
1568  IGCM_debug_Print 1 "$EXECUTION"
1569
1570  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1571}
1572
1573#D-#==================================================
1574#D-function IGCM_sys_check_path
1575#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1576#D-* do not point to an important use directory. Stop immediately in that case.
1577#D-* Examples:
1578#D-
1579function IGCM_sys_check_path {
1580  IGCM_debug_PushStack "IGCM_sys_check_path"
1581  if ( $DEBUG_sys ) ; then
1582    echo "IGCM_sys_check_path"
1583  fi
1584
1585  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1586    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1587    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1588    IGCM_debug_Exit "This will stop the job"
1589  fi
1590  IGCM_debug_PopStack "IGCM_sys_check_path"
1591}
1592
1593#D-#==================================================
1594#D-function IGCM_sys_check_quota
1595#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1596#D-* Examples:
1597#D-
1598function IGCM_sys_check_quota {
1599  IGCM_debug_PushStack "IGCM_sys_check_quota"
1600  if ( $DEBUG_sys ) ; then
1601    echo "IGCM_sys_check_quota"
1602  fi
1603  # Limit of quota (in %)
1604  limit_quota=90
1605
1606  # Check of the volume
1607  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1608  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1609
1610  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1611
1612    unit_avail=${volume_avail: -1}
1613    unit_quota=${volume_quota: -1}
1614
1615    if [ "${unit_quota}" = "*" ] ; then
1616      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1617      IGCM_debug_Print 1 "More than 100% of your quota is used"
1618      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1619      IGCM_debug_Print 1 "You must have more than 10% available to run"
1620      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1621      IGCM_debug_Verif_Exit
1622    fi
1623
1624    temp_avail=${volume_avail%%${volume_avail: -1}*}
1625    temp_quota=${volume_quota%%${volume_quota: -1}*}
1626
1627    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1628
1629    # Convertion
1630      if [ ${volume_avail: -1} = "T" ] ; then
1631        (( temp_avail = temp_avail * 1000000000000 ))
1632      elif [ ${volume_avail: -1} = "G" ] ; then
1633        (( temp_avail = temp_avail * 1000000000 ))
1634      elif [ ${volume_avail: -1} = "M" ] ; then
1635        (( temp_avail = temp_avail * 1000000 ))
1636      elif [ ${volume_avail: -1} = "k" ] ; then
1637        (( temp_avail = temp_avail * 1000 ))
1638      else
1639        (( temp_avail = volume_avail ))
1640      fi
1641      if [ ${volume_quota: -1} = "T" ] ; then
1642        (( temp_quota = temp_quota * 1000000000000 ))
1643      elif [ ${volume_quota: -1} = "G" ] ; then
1644        (( temp_quota = temp_quota * 1000000000 ))
1645      elif [ ${volume_quota: -1} = "M" ] ; then
1646        (( temp_quota = temp_quota * 1000000 ))
1647      elif [ ${volume_quota: -1} = "k" ] ; then
1648        (( temp_quota = temp_quota * 1000 ))
1649      else
1650        (( temp_quota = volume_quota ))
1651      fi
1652    fi
1653
1654    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1655#    echo "volume ratio is " $quota_volume
1656
1657    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1658      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1659      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1660      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1661      IGCM_debug_Print 1 "You must have more than 10% available to run"
1662      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1663      IGCM_debug_Verif_Exit
1664    fi
1665
1666  fi
1667
1668# Check of the number of inodes
1669
1670  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1671  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1672
1673  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1674
1675    unit_avail=${inode_avail: -1}
1676    unit_quota=${inode_quota: -1}
1677
1678    if [ "${unit_quota}" = "*" ] ; then
1679      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1680      IGCM_debug_Print 1 "More than 100% of your quota is used"
1681      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1682      IGCM_debug_Print 1 "You must have more than 10% available to run"
1683      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1684      IGCM_debug_Verif_Exit
1685    fi
1686
1687    temp_avail=${inode_avail%%${inode_avail: -1}*}
1688    temp_quota=${inode_quota%%${inode_quota: -1}*}
1689
1690    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1691
1692    # Convertion
1693      if [ ${inode_avail: -1} = "T" ] ; then
1694        (( temp_avail = temp_avail * 1000000000000 ))
1695      elif [ ${inode_avail: -1} = "G" ] ; then
1696        (( temp_avail = temp_avail * 1000000000 ))
1697      elif [ ${inode_avail: -1} = "M" ] ; then
1698        (( temp_avail = temp_avail * 1000000 ))
1699      elif [ ${inode_avail: -1} = "k" ] ; then
1700        (( temp_avail = temp_avail * 1000 ))
1701      else
1702        (( temp_avail = inode_avail ))
1703      fi
1704
1705      if [ ${inode_quota: -1} = "T" ] ; then
1706        (( temp_quota = temp_quota * 1000000000000 ))
1707      elif [ ${inode_quota: -1} = "G" ] ; then
1708        (( temp_quota = temp_quota * 1000000000 ))
1709      elif [ ${inode_quota: -1} = "M" ] ; then
1710        (( temp_quota = temp_quota * 1000000 ))
1711      elif [ ${inode_quota: -1} = "k" ] ; then
1712        (( temp_quota = temp_quota * 1000 ))
1713      else
1714        (( temp_quota = inode_quota ))
1715      fi
1716    fi
1717    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1718#    echo "inode ratio is " $quota_inode
1719
1720    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1721      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1722      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1723      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1724      IGCM_debug_Print 1 "You must have more than 10% available to run"
1725      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1726      IGCM_debug_Verif_Exit
1727    fi
1728  fi
1729  IGCM_debug_PopStack "IGCM_sys_check_quota"
1730}
1731
1732#D-#==================================================
1733#D-function IGCM_sys_projectAccounting
1734#D-* Purpose: store project accounting information in a file
1735#D-* Examples:
1736#D-
1737function IGCM_sys_projectAccounting {
1738  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1739  if ( $DEBUG_sys ) ; then
1740    echo "IGCM_sys_check_quota"
1741  fi
1742
1743  ssh irene191 /usr/bin/ccc_myproject > $1
1744
1745  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1746}
1747
1748#D-#==================================================
1749#D-function IGCM_sys_getJobSchedulerID
1750#D-* Purpose: Get the job ID during execution
1751#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1752#D-
1753function IGCM_sys_getJobSchedulerID {
1754  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1755  if ( $DEBUG_sys ) ; then
1756    echo "IGCM_sys_getJobSchedulerID"
1757  fi
1758
1759  eval ${1}=${BRIDGE_MSUB_JOBID}
1760
1761  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1762}
1763
1764#D-#==================================================
1765#D-function IGCM_sys_GetJobID
1766#D-* Purpose: Get the job ID from the JobName
1767#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1768#D-
1769function IGCM_sys_GetJobID {
1770  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1771  if ( $DEBUG_sys ) ; then
1772    echo "IGCM_sys_GetJobID"
1773  fi
1774
1775  # With -f option, the full job name is given in the last column
1776  ID=$( ccc_mstat -f -u $2 | \
1777        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1778
1779  eval ${3}=${ID}
1780  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1781}
1782
1783#D-#==================================================
1784#D-function IGCM_sys_CountJobInQueue
1785#D-* Purpose: Count number of users job
1786#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1787#D-
1788function IGCM_sys_CountJobInQueue {
1789  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1790  if ( $DEBUG_sys ) ; then
1791    echo "IGCM_sys_CountJobInQueue"
1792  fi
1793
1794  # With -f option, the full job name is given in the last column
1795  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1796
1797  eval ${2}=${NbRun}
1798
1799  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1800}
1801
1802#D-#==================================================
1803#D-function IGCM_sys_ListJobInQueue
1804#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1805#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1806#D-
1807function IGCM_sys_ListJobInQueue {
1808  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1809  if ( $DEBUG_sys ) ; then
1810    echo "IGCM_sys_ListJobInQueue"
1811  fi
1812
1813  # With -f option, the full job name is given in the last column
1814  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1815                                        '( $2  == User      && \
1816                                           $NF != /TS/      && \
1817                                           $NF !~ /PACK/    && \
1818                                           $NF !~ /REBUILD/ && \
1819                                           $NF !~ /pack/ )     \
1820                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1821
1822  eval set -A ${2} ${JobList[*]}
1823
1824  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1825}
1826
1827#D-#==================================================
1828#D-function IGCM_sys_atlas
1829#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1830#D-* Examples:
1831#D-
1832function IGCM_sys_atlas {
1833  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1834  if ( $DEBUG_sys ) ; then
1835    echo "IGCM_sys_atlas :" $@
1836  fi
1837
1838  typeset status
1839
1840  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1841  status=$?
1842  if [ ${status} -gt 0 ] ; then
1843    echo "IGCM_sys_atlas : error code ${status}"
1844    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1845    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1846    IGCM_debug_PopStack "IGCM_sys_atlas"
1847    return 1
1848  else
1849    IGCM_debug_PopStack "IGCM_sys_atlas"
1850    return 0
1851  fi
1852
1853  IGCM_debug_PopStack "IGCM_sys_atlas"
1854}
1855
1856#D-#==================================================
1857#D-function IGCM_sys_rebuild_nemo
1858#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1859#D-* Examples:
1860#D-
1861
1862function IGCM_sys_rebuild_nemo {
1863  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1864  if ( $DEBUG_sys ) ; then
1865    echo "IGCM_sys_rebuild_nemo :" $@
1866  fi
1867
1868  $( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
1869
1870  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1871}
1872
Note: See TracBrowser for help on using the repository browser.